From 9d56592cadf7e047fb059e5a8746188595a08b43 Mon Sep 17 00:00:00 2001 From: kaotei Date: Wed, 2 Sep 2026 09:50:12 +0800 Subject: [PATCH] CM93 MySQL importer validated baseline --- .env.example | 9 + .gitignore | 46 + .python-version | 1 + AGENTS.md | 61 + MANIFEST.sha256 | 37 + README.md | 236 +++ V10_1_结果摘要_v10_2.json | 96 + V10_1_结果摘要_v10_2.txt | 64 + ...ormalization_source_audit_standalone_v1.py | 569 ++++++ ...importer_normalization_source_audit_v12.py | 880 +++++++++ ..._importer_production_source_snapshot_v1.py | 613 ++++++ cm93_mysql_audit.py | 734 +++++++ cm93_mysql_audit_v2.py | 719 +++++++ cm93_mysql_fillrule_audit_v8.py | 1162 +++++++++++ cm93_mysql_final_geometry_audit_v1.py | 182 ++ cm93_mysql_invalid_area_audit_v4.py | 977 ++++++++++ cm93_mysql_invalid_trace_v5.py | 1041 ++++++++++ cm93_mysql_non_geometry_final_audit_v1.py | 1272 ++++++++++++ cm93_mysql_normalization_canonical_v10.py | 1633 ++++++++++++++++ cm93_mysql_normalization_canonical_v10_1.py | 1651 ++++++++++++++++ cm93_mysql_normalization_dryrun_v9.py | 1713 +++++++++++++++++ cm93_mysql_ogc_validity_audit_v7.py | 1179 ++++++++++++ cm93_mysql_raw_edge_audit_v6.py | 1461 ++++++++++++++ cm93_mysql_spatial_audit_v3.py | 1570 +++++++++++++++ ...rmalization_implementation_precheck_v11.py | 1270 ++++++++++++ cm93_v10_1_result_digest_v10_2.py | 475 +++++ codex/10_第一阶段接手任务.md | 87 + codex/20_导入与审计接手.md | 82 + codex/ARCHITECTURE.md | 55 + docs/CM93解析与审计设计.md | 150 ++ docs/Win11环境安装.md | 50 + docs/外部参考.md | 25 + docs/导入与审计.md | 151 ++ docs/报告/.gitkeep | 0 .../报告/CM93_Importer_v0.2.0_离线验证报告.md | 69 + docs/报告/CM93导入器开发验证报告_v0.2.0.md | 69 + docs/数据库设计.md | 50 + docs/架构设计.md | 55 + docs/精度规范.md | 55 + pyproject.toml | 24 + requirements.txt | 3 + samples/README.md | 17 + scripts/audit_cm93.ps1 | 7 + scripts/import_cm93.ps1 | 10 + scripts/init_db.ps1 | 11 + scripts/repair_env.ps1 | 23 + scripts/setup_env.ps1 | 61 + sql/001_schema_mysql57.sql | 281 +++ src/cm93db/__init__.py | 1 + src/cm93db/audit.py | 294 +++ src/cm93db/cell_parser.py | 468 +++++ src/cm93db/cli.py | 189 ++ src/cm93db/cm93_binary.py | 478 +++++ src/cm93db/config.py | 35 + src/cm93db/database.py | 101 + src/cm93db/decoder.py | 38 + src/cm93db/dictionaries.py | 94 + src/cm93db/geometry.py | 870 +++++++++ src/cm93db/importer.py | 1104 +++++++++++ src/cm93db/precision.py | 39 + src/cm93db/scanner.py | 59 + src/cm93db/schema_contract.py | 22 + 62 files changed, 24778 insertions(+) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 .python-version create mode 100644 AGENTS.md create mode 100644 MANIFEST.sha256 create mode 100644 README.md create mode 100644 V10_1_结果摘要_v10_2.json create mode 100644 V10_1_结果摘要_v10_2.txt create mode 100644 cm93_importer_normalization_source_audit_standalone_v1.py create mode 100644 cm93_importer_normalization_source_audit_v12.py create mode 100644 cm93_importer_production_source_snapshot_v1.py create mode 100644 cm93_mysql_audit.py create mode 100644 cm93_mysql_audit_v2.py create mode 100644 cm93_mysql_fillrule_audit_v8.py create mode 100644 cm93_mysql_final_geometry_audit_v1.py create mode 100644 cm93_mysql_invalid_area_audit_v4.py create mode 100644 cm93_mysql_invalid_trace_v5.py create mode 100644 cm93_mysql_non_geometry_final_audit_v1.py create mode 100644 cm93_mysql_normalization_canonical_v10.py create mode 100644 cm93_mysql_normalization_canonical_v10_1.py create mode 100644 cm93_mysql_normalization_dryrun_v9.py create mode 100644 cm93_mysql_ogc_validity_audit_v7.py create mode 100644 cm93_mysql_raw_edge_audit_v6.py create mode 100644 cm93_mysql_spatial_audit_v3.py create mode 100644 cm93_normalization_implementation_precheck_v11.py create mode 100644 cm93_v10_1_result_digest_v10_2.py create mode 100644 codex/10_第一阶段接手任务.md create mode 100644 codex/20_导入与审计接手.md create mode 100644 codex/ARCHITECTURE.md create mode 100644 docs/CM93解析与审计设计.md create mode 100644 docs/Win11环境安装.md create mode 100644 docs/外部参考.md create mode 100644 docs/导入与审计.md create mode 100644 docs/报告/.gitkeep create mode 100644 docs/报告/CM93_Importer_v0.2.0_离线验证报告.md create mode 100644 docs/报告/CM93导入器开发验证报告_v0.2.0.md create mode 100644 docs/数据库设计.md create mode 100644 docs/架构设计.md create mode 100644 docs/精度规范.md create mode 100644 pyproject.toml create mode 100644 requirements.txt create mode 100644 samples/README.md create mode 100644 scripts/audit_cm93.ps1 create mode 100644 scripts/import_cm93.ps1 create mode 100644 scripts/init_db.ps1 create mode 100644 scripts/repair_env.ps1 create mode 100644 scripts/setup_env.ps1 create mode 100644 sql/001_schema_mysql57.sql create mode 100644 src/cm93db/__init__.py create mode 100644 src/cm93db/audit.py create mode 100644 src/cm93db/cell_parser.py create mode 100644 src/cm93db/cli.py create mode 100644 src/cm93db/cm93_binary.py create mode 100644 src/cm93db/config.py create mode 100644 src/cm93db/database.py create mode 100644 src/cm93db/decoder.py create mode 100644 src/cm93db/dictionaries.py create mode 100644 src/cm93db/geometry.py create mode 100644 src/cm93db/importer.py create mode 100644 src/cm93db/precision.py create mode 100644 src/cm93db/scanner.py create mode 100644 src/cm93db/schema_contract.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..9267f33 --- /dev/null +++ b/.env.example @@ -0,0 +1,9 @@ +CM93_DB_HOST=127.0.0.1 +CM93_DB_PORT=3306 +CM93_DB_NAME=cm93_chart +CM93_DB_USER=root +CM93_DB_PASSWORD=change_me +CM93_DB_CHARSET=utf8mb4 + +# CM93 根目录。正式导入器将递归寻找 Z/A~G cell 与字典。 +CM93_SOURCE_DIR=D:\Ship\CM93V2\CM93JAPAN diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fb465e6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,46 @@ +.venv/ +__pycache__/ +*.py[cod] +.env +.pytest_cache/ +.mypy_cache/ +dist/ +build/ +*.egg-info/ +logs/ +tmp/ +.DS_Store +Thumbs.db +.venv/ +venv/ +__pycache__/ +*.pyc +*.pyo +.pytest_cache/ +.mypy_cache/ + +# 本地数据库/环境配置 +.env +.env.* +!.env.example + +# 最终测试产生的临时输出目录 +cm93_final_geometry_audit_*/ +cm93_non_geometry_final_audit_dataset_*/ +cm93_importer_source_audit_*/ +cm93_normalization_dryrun_*/ +cm93_normalization_canonical_*/ +cm93_invalid_area_audit_*/ +cm93_invalid_trace_*/ +cm93_ogc_validity_audit_*/ +cm93_raw_edge_audit_*/ +cm93_spatial_audit_*/ + +# 临时打包文件 +*.zip + +# IDE / OS +.vscode/ +.idea/ +.DS_Store +Thumbs.db diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..3b2cfc0 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.12.10 diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..94311ba --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,61 @@ +# Codex / AI Agent 项目规则 + +## 当前目标 + +第一阶段已经具备 `CM93 → MySQL` 完整导入与审计代码。当前工作是用真实 `CM93JAPAN` 数据运行、诊断和修复实际兼容问题。 + +**不要提前实现 PBF。** + +## 固定基线 + +- Windows 11 +- Python 3.12.10 +- MySQL 5.7.34-log +- CM93 根目录：`D:\Ship\CM93V2\CM93JAPAN` +- DB Schema：`sql/001_schema_mysql57.sql` + +## 不可破坏规则 + +1. 禁止 MySQL 8 专用 SQL。 +2. 禁止 float32 存最终经纬度。 +3. 禁止 importer simplify/snap/quantize/round geometry。 +4. 禁止自动 make_valid 后覆盖原 Polygon。 +5. raw decoded tables、raw feature、raw attribute 必须可保留。 +6. `_wgsox/_wgsoy` 必须保存。 +7. 每个 feature 必须可追溯 dataset/cell/source_record_index。 +8. 未知 object/attribute 不得静默跳过；无法确定长度时 cell 必须 FAILED。 +9. parser 任一 header/count/EOF 不一致时 cell 必须 FAILED。 +10. 每个 cell 独立事务；失败不得破坏其他 DONE cell。 +11. DONE cell 支持续导跳过。 +12. 不复制 OpenCPN GPL 源文件或类实现；保持独立实现。 +13. PBF 不属于当前阶段。 + +## 实际运行顺序 + +```powershell +cm93db env-check +cm93db db-check +cm93db scan +cm93db inspect-dicts +cm93db import --scale G --limit 10 +cm93db audit --scale G --limit 10 --precision-samples 32 +``` + +Pilot PASS 后再全量。 + +## 修复要求 + +发现失败时，优先保留一个具体失败 cell 作为回归输入，并报告： + +- cell 路径 +- parser stage +- byte position/record index +- header source counts +- 实际异常 +- 修改前后审计结果 + +不得通过 catch-and-skip 使任务“看起来完成”。 + +## 文档 + +项目说明、报告、设计记录使用中文；代码稳定标识符、数据库字段使用英文。 diff --git a/MANIFEST.sha256 b/MANIFEST.sha256 new file mode 100644 index 0000000..62e9f60 --- /dev/null +++ b/MANIFEST.sha256 @@ -0,0 +1,37 @@ +eb70074aa5e04f09d8eb0ddca51a4df731e512211912097a03a0356034b70554 ./.env.example +1b4fdd7c2d13638d81c8d477a87a555d604edbc8f206b4f3ea3f86cb2962a1a8 ./.gitignore +5b703ca38d3fd391f3e889f7ba893f94b921ff9e7b7e5d8d6622d5f7daca7049 ./.python-version +8b8a8fa81c0bf2b82b1c5c07adaa8f91943187e34df4bbdd0c802dc1f8fee2c6 ./AGENTS.md +67b929d05d979a0eeebfd81f8bb482e70c8dd2c336fb6798a83bd37f0a61b9f3 ./README.md +c681b8ab9eb555ea79601e421c1a59f1dd97f2a0e6a787fdf1952e0c0bf68f74 ./codex/10_第一阶段接手任务.md +f9540f72e60242c3c6ece6f8e105eeb60458e4828e91f47ba2d8606122ef44de ./codex/ARCHITECTURE.md +f55a5b2cd4a8c95b76860523cd5da000a608d93eec86548aa70cf210ac51faef ./docs/Win11环境安装.md +54052bd183c8d5e021cde70d406f05a64500e19bafd186fa6a3a9311e16c4e47 ./docs/外部参考.md +4f48b830dd2ea32e329781c20870f9f1ba39cf65644d7cf98ab0b43bb3d06a03 ./docs/导入与审计.md +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 ./docs/报告/.gitkeep +e67539ec6ea0173487f90a5b2906c8083a91f13abd3b86ddb0c054f9290f238e ./docs/报告/CM93_Importer_v0.2.0_离线验证报告.md +2e32a838656fda45fcb7a86b574aa174f5dc656733a8281c94d16b75fe39a1de ./docs/数据库设计.md +0041bfc53607e619d752cad94e8d2b6e38c735d4d42b54d9da822dde65b88ccd ./docs/架构设计.md +3353d08b5bfa81af16cb1fc7a4c9a7b02eb971224c9fa6642dfce0d95cde1ef6 ./docs/精度规范.md +0420e31b40c75c4aa96a1d9eb5c6ec68a6d38ea697e07f0ae1818080a26e7dfe ./pyproject.toml +10aa41c34072cfa8b5c0c6cb7ff11de35b6c88951064dc058883da4c6ebbf9c8 ./requirements.txt +44168b993990c13b0c4f127af4e63d517571d42ac699b784bbfc975ccd02c50d ./samples/README.md +2ba8827a3eb730f997edcc696d4f6e3372ab6ee9d53502b363be8eddb681b210 ./scripts/audit_cm93.ps1 +7029d7216c38003e670c4a799ea6c56cee0554154c2638a4767106b68ee2799c ./scripts/import_cm93.ps1 +daf081a2ff981f4d8c2da5df84917f9f18b574c639b040ce3e66d21fbe9e700a ./scripts/init_db.ps1 +6b34552119e6a5ad6af0bdfdfa1ddde01b4fba3f066d5982ea8dd67d4b34f169 ./scripts/repair_env.ps1 +c477b0f32b5b9b636ff2160956cf19d2c4164597ec40687337f488bc3527b8fb ./scripts/setup_env.ps1 +7818c17d282f5cd1f7dfb6faa5f0d3b00246508090f5516d7d08c9fe28497125 ./sql/001_schema_mysql57.sql +667d4a15b970b851e20d17510224670c14646cfb6d5a1e388ca6b9cc6da8bf41 ./src/cm93db/__init__.py +420bc6ea2968415d011c17fd390506990c63249c5fd7a3daa13bd121aeb8f689 ./src/cm93db/audit.py +da2d555eb10ecf3209c19381b540811c650bf88ea9c53e09e66663652abd59dc ./src/cm93db/cell_parser.py +c8fce15a77a42eda7bdb5e1677e7f306872f691779fae0aa72cd310063bc0616 ./src/cm93db/cli.py +f5b6adcbd898daba8a798119c5050792e6e0f836fa6928976937ec9180139213 ./src/cm93db/config.py +3488b6265d343829762b5204283c0fec012b3009b651972ff9d48a162b1f9c10 ./src/cm93db/database.py +0d5aec70e3eabcdf88e13b5e6c79b8ab40b4fed8b4664aaf4e4d4d7a42770838 ./src/cm93db/decoder.py +3d8f4fbdff0e0a3d77a902c11b40d7431fba4d6fd10abcdcc05d77e00359b322 ./src/cm93db/dictionaries.py +fe5db556801d55c47ae81c211f1237c19288adac837db523bc17f727dbf913a1 ./src/cm93db/geometry.py +7ebbf16369fd717d777ad98cc14c5c477e533ddb2b4f8fa7b1c681334b69c8c0 ./src/cm93db/importer.py +8ccffe2edcd8fcc50f7433c587e341e5254e868820260f4479e35e886ca6eef5 ./src/cm93db/precision.py +21347b9d043b87e2e7fe7eb7ae845e53632b3d7e26e58db920cf19c7b6e0ba14 ./src/cm93db/scanner.py +700d0990032ca3021ade98135f0a52756f8f80c2144835d1fa1d53a2ec8d4650 ./src/cm93db/schema_contract.py diff --git a/README.md b/README.md new file mode 100644 index 0000000..c334866 --- /dev/null +++ b/README.md @@ -0,0 +1,236 @@ +# CM93 → MySQL 高精度母数据库导入器 v0.2.0 + +## 目标 + +本工程完成第一阶段： + +**CM93 Z/A/B/C/D/E/F/G → 完整解析 → MySQL 5.7.34 母数据库 → 源/库双重审计。** + +本阶段不生成 PBF。PBF 是第二阶段，可从本母数据库反复生成。 + +## 固定环境 + +- Windows 11 +- pyenv-win +- Python **3.12.10** +- MySQL **5.7.34** +- 当前 CM93 日本数据目录：`D:\Ship\CM93V2\CM93JAPAN` +- 标准化几何：经纬度/WGS84 语义，SRID 4326 +- 坐标计算：Python double / MySQL WKB double + +## 已实现 + +- CM93 256-byte substitution decode +- 10-byte prolog + 128-byte header +- 文件长度完整性检查 +- vector table +- 2D point table +- 3D sounding table +- feature table +- object dictionary +- typed attribute dictionary / attribute blob +- Point / Line / Area / Sounding +- edge reverse flag +- Area contour/ring 组装 +- `_m_sor` coverage +- `_wgsox/_wgsoy` 保存与标准化坐标修正 +- CM93 6378388.0 m Mercator +- typed MySQL attributes +- raw feature / raw attribute +- 解码后的全部 CM93 table 分块保存 +- 每 cell 独立事务 +- DONE 自动跳过 +- FAILED 默认重试 +- 全目录断点续导 +- import job / event 日志 +- 源文件重新解析审计 +- 数据库逐 cell 数量审计 +- raw 数据字节完整性审计 +- SHA-256 审计 +- MySQL WKB 双精度回读审计 + +## 已验证的真实 CM93 小样本 + +开发时对用户提供的 147 个真实 CM93 cell 执行完整 parser 回归： + +- Cell: 147/147 PASS +- Feature: 100,621 +- Attribute: 112,746 +- 未发生 record 越界或文件尾剩余 +- header attribute-byte count 与解析结果一致 +- header geometry-ref count 与解析结果一致 +- header related-pointer count 与解析结果一致 +- decoded raw table 字节合计与原文件长度一致 + +MySQL 写入需要在目标 Windows/MySQL 5.7.34 上做 pilot 验收；本包提供对应命令和审计。 + +--- + +# 1. 升级已有 v0.1.x 工程 + +如果原目录已经有正确的 `.venv` 和 `.env`，把 v0.2.0 ZIP 中整个 `cm93-mysql-importer-starter` 目录覆盖到原工程即可。 + +ZIP 不包含 `.env`，所以你已有的数据库密码不会被覆盖。 + +然后执行： + +```powershell +cd D:\React\cm93-mysql-importer-starter +& ".\.venv\Scripts\python.exe" -m pip install -e . +& ".\.venv\Scripts\cm93db.exe" --help +``` + +确认出现 `import` 和 `audit`。 + +# 2. .env + +确保： + +```text +CM93_DB_HOST=127.0.0.1 +CM93_DB_PORT=3306 +CM93_DB_NAME=cm93_chart +CM93_DB_USER=你的用户 +CM93_DB_PASSWORD=你的密码 +CM93_DB_CHARSET=utf8mb4 +CM93_SOURCE_DIR=D:\Ship\CM93V2\CM93JAPAN +``` + +`scan / inspect-dicts / import / audit` 现在都可以省略路径，自动读取 `CM93_SOURCE_DIR`。 + +# 3. 环境/数据库检查 + +```powershell +& ".\.venv\Scripts\cm93db.exe" env-check +& ".\.venv\Scripts\cm93db.exe" db-check +& ".\.venv\Scripts\cm93db.exe" scan +& ".\.venv\Scripts\cm93db.exe" inspect-dicts +``` + +# 4. 先做 G 层 10-cell pilot + +不要第一次就直接跑完整日本数据。先验证 MySQL 实际写入： + +```powershell +& ".\.venv\Scripts\cm93db.exe" import --scale G --limit 10 +``` + +然后立即审计同一批： + +```powershell +& ".\.venv\Scripts\cm93db.exe" audit --scale G --limit 10 --precision-samples 32 +``` + +目标： + +```text +source_parse_failed : 0 +db_missing_cells : 0 +db_not_done_cells : 0 +count_mismatches : 0 +hash_mismatches : 0 +raw_preservation_bad : 0 +unknown_objects : 0 +unknown_attributes : 0 +precision_failed : 0 +RESULT : PASS +``` + +# 5. 全量导入日本数据 + +Pilot PASS 后： + +```powershell +& ".\.venv\Scripts\cm93db.exe" import +``` + +程序逐 cell 工作。已经 `DONE` 且 **SHA-256 未变化** 的 cell 自动跳过，因此中断后重新执行同一命令即可续导。 + +FAILED cell 默认在下一次执行时重试。 + +# 6. 全量审计 + +```powershell +& ".\.venv\Scripts\cm93db.exe" audit --precision-samples 256 +``` + +这会重新读取 CM93 源文件，并与数据库逐 cell 对比： + +- SHA-256 +- Feature 数 +- Attribute 数 +- Point geometry 数 +- Line geometry 数 +- Area geometry 数 +- `_m_sor` coverage 数 +- Sounding point 数 +- raw table 保存字节数 +- 未知 object / attribute +- 抽样 WKB 双精度 round-trip + +## 更严格的 geometry validity 检查 + +```powershell +& ".\.venv\Scripts\cm93db.exe" audit --precision-samples 256 --validate-geometry +``` + +`--validate-geometry` 会额外用 Shapely 全量检查拓扑，明显更重。CM93 中存在自相交/边界类 contour 并不自动等于解析错误，因此这项结果作为信息单独显示，不会静默修复原 geometry。 + +# 7. 常用控制 + +只处理 F/G： + +```powershell +cm93db import --scale F --scale G +``` + +只处理一个 cell： + +```powershell +cm93db import --cell 04501075.G +cm93db audit --cell 04501075.G --precision-samples 16 +``` + +强制重导： + +```powershell +cm93db import --cell 04501075.G --reimport +``` + +只审计源文件 parser，不访问数据库内容： + +```powershell +cm93db audit --source-only +``` + +正式母库**不建议**使用 `--no-raw-tables`。如果有意不保存 raw table，审计时需显式： + +```powershell +cm93db audit --allow-missing-raw +``` + +--- + +# 精度原则 + +1. CM93 raw X/Y 保持 16-bit 原始语义。 +2. Cell easting/northing header 值使用 double。 +3. CM93 Mercator 半长轴固定 `6378388.0`。 +4. `_wgsox/_wgsoy` 以米为单位，在 Mercator→经纬度之前按 CM93 Transform 语义减去。 +5. MySQL geometry 由 WKB 写入；WKB 坐标为 IEEE-754 8-byte double。 +6. importer 阶段不 simplify、不 quantize、不 snap、不 round。 +7. Polygon 不自动 `make_valid`。 +8. 原始 decoded table + raw feature/attribute 与标准化 geometry 同时保留。 +9. PBF 将来是可重新生成的发布格式，不能反向影响母数据库精度。 + +详见 `docs/精度规范.md` 和 `docs/CM93解析与审计设计.md`。 + +# Codex 接手 + +Codex 不需要重新实现 parser。先读： + +- `AGENTS.md` +- `codex/ARCHITECTURE.md` +- `codex/20_导入与审计接手.md` + +后续主要职责是：在你的 Win11 + MySQL 5.7.34 + CM93JAPAN 实际环境运行 pilot、分析失败 cell、修正真实兼容问题，并保留精度/审计边界。 diff --git a/V10_1_结果摘要_v10_2.json b/V10_1_结果摘要_v10_2.json new file mode 100644 index 0000000..6c36c1b --- /dev/null +++ b/V10_1_结果摘要_v10_2.json @@ -0,0 +1,96 @@ +{ + "version": "v10.2-result-digest", + "generated_at": "2026-09-01T08:32:55", + "input": "D:\\React\\cm93-mysql-importer-starter\\cm93_normalization_canonical_v10_1_dataset_1_20260831_215602", + "files_present": { + "summary": true, + "report": true, + "golden_csv": true, + "golden_json": true, + "variant": true, + "face_order": true, + "failures": true, + "bbox": true + }, + "complete_core_set": true, + "checks": { + "report": { + "overall_pass": true, + "total_polygon_records": 82236, + "valid_polygon_records": 82181, + "invalid_polygon_records": 55, + "valid_policy_changed_count": 0, + "valid_hash_changed_count": 0, + "golden_vector_count": 55, + "golden_record_status_counts": { + "PASS": 55 + }, + "input_variant_case_count": 330, + "input_variant_status_counts": { + "PASS": 330 + }, + "face_order_case_count": 220, + "face_order_hash_mismatch_count": 0, + "canonical_topology_failure_count": 0, + "repeat_hash_failure_count": 0, + "input_variant_failure_record_count": 0, + "future_area_feature_bbox_update_needed_count": 26 + }, + "golden": { + "rows": 55, + "status_counts": { + "PASS": 55 + }, + "unique_hash_count": 45, + "missing_hash_count": 0, + "variant_failure_records": 0, + "face_order_unstable_records": 0, + "repeat_unstable_records": 0, + "canonical_topology_changed_records": 0 + }, + "variant": { + "rows": 330, + "status_counts": { + "PASS": 330 + }, + "fail_rows": 0, + "hash_mismatch_rows": 0, + "topology_mismatch_rows": 0 + }, + "face_order": { + "rows": 220, + "hash_mismatch_rows": 0 + }, + "failures": { + "rows": 0 + }, + "bbox": { + "rows": 55, + "bbox_changed_rows": 27, + "future_feature_bbox_update_needed_rows": 26, + "rows_with_connector": 41 + } + }, + "overall": "PASS", + "pass_conditions": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true + ] +} \ No newline at end of file diff --git a/V10_1_结果摘要_v10_2.txt b/V10_1_结果摘要_v10_2.txt new file mode 100644 index 0000000..9a02dee --- /dev/null +++ b/V10_1_结果摘要_v10_2.txt @@ -0,0 +1,64 @@ +CM93 V10.1 结果汇总与完整性验证 v10.2 +============================================================================== +Input: D:\React\cm93-mysql-importer-starter\cm93_normalization_canonical_v10_1_dataset_1_20260831_215602 +Overall: PASS + +FILES + summary YES normalization_canonical_summary_v10_1.txt + report YES normalization_canonical_report_v10_1.json + golden_csv YES normalization_golden_vectors_v10_1.csv + golden_json YES normalization_golden_vectors_v10_1.json + variant YES normalization_variant_results_v10_1.csv + face_order YES normalization_face_order_results_v10_1.csv + failures YES normalization_canonical_failures_v10_1.csv + bbox YES normalization_bbox_metadata_impact_v10_1.csv + +REPORT + overall_pass: True + total_polygon_records: 82236 + valid_polygon_records: 82181 + invalid_polygon_records: 55 + valid_policy_changed_count: 0 + valid_hash_changed_count: 0 + golden_vector_count: 55 + golden_record_status_counts: {'PASS': 55} + input_variant_case_count: 330 + input_variant_status_counts: {'PASS': 330} + face_order_case_count: 220 + face_order_hash_mismatch_count: 0 + canonical_topology_failure_count: 0 + repeat_hash_failure_count: 0 + input_variant_failure_record_count: 0 + future_area_feature_bbox_update_needed_count: 26 + +GOLDEN + rows: 55 + status_counts: {'PASS': 55} + unique_hash_count: 45 + missing_hash_count: 0 + variant_failure_records: 0 + face_order_unstable_records: 0 + repeat_unstable_records: 0 + canonical_topology_changed_records: 0 + +VARIANT + rows: 330 + status_counts: {'PASS': 330} + fail_rows: 0 + hash_mismatch_rows: 0 + topology_mismatch_rows: 0 + +FACE_ORDER + rows: 220 + hash_mismatch_rows: 0 + +FAILURES + rows: 0 + +BBOX + rows: 55 + bbox_changed_rows: 27 + future_feature_bbox_update_needed_rows: 26 + rows_with_connector: 41 + +PASS：V10.1 canonical normalization 全部门槛通过。 \ No newline at end of file diff --git a/cm93_importer_normalization_source_audit_standalone_v1.py b/cm93_importer_normalization_source_audit_standalone_v1.py new file mode 100644 index 0000000..b991879 --- /dev/null +++ b/cm93_importer_normalization_source_audit_standalone_v1.py @@ -0,0 +1,569 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +CM93 Importer Normalization 独立源码审计 +Version: standalone-v1.0 + +完全独立： +- 不读取 V10/V11/V12 历史结果 +- 不连接 MySQL +- 不修改源码 +- 不生成 patch +- 不开始 PBF + +用途： +从当前 importer 项目源码零开始定位： +1. Area / Coverage geometry 构造 +2. chart_area_geometry / chart_coverage / chart_feature 写库 +3. geometry_issue / geometry missing +4. raw_geotype / geometry_type / META / AREA +5. geometry_invalid_count / geometry_repaired_count +6. bbox 计算与写入 +7. Shapely / GEOS / Polygon / MultiPolygon / WKB +8. raw_feature_blob / chart_raw_record / source_record_index +9. import job / summary / counters +10. 可能的 normalization 接入点 + +运行： + python .\\cm93_importer_normalization_source_audit_standalone_v1.py ` + --project-root "." + +输出自动生成 ZIP。 +""" + +from __future__ import annotations +import argparse +import ast +import csv +import json +import re +import sys +import zipfile +from collections import Counter, defaultdict +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Tuple + +TEXT_EXTENSIONS = { + ".py", ".sql", ".toml", ".ini", ".cfg", ".json", + ".yaml", ".yml", ".md", ".txt", ".env", ".example", +} + +SKIP_DIRS = { + ".git", ".hg", ".svn", "__pycache__", ".pytest_cache", + ".mypy_cache", ".ruff_cache", ".venv", "venv", "env", + "node_modules", "dist", "build", ".idea", ".vscode", +} + +SEARCH_GROUPS = { + "GEOMETRY_BUILD": [ + r"\bPolygon\s*\(", r"\bMultiPolygon\s*\(", r"\bLineString\s*\(", + r"\bPoint\s*\(", r"\bshapely\b", r"\bGEOS\b", r"\bwkb\b", r"\bgeom\b", + ], + "NORMALIZATION": [ + r"\bmake_valid\b", r"\bpolygonize_full\b", r"\bpolygonize\b", + r"\bunary_union\b", r"\bnormalize\b", r"\bis_valid\b", + r"\bexplain_validity\b", r"\bbuffer\s*\(\s*0", + ], + "GEOMETRY_ISSUE": [ + r"geometry_issue", r"geometry missing", r"geometry_missing", + r"invalid geometry", r"geometry_invalid", r"geometry_repaired", + ], + "GEOTYPE": [ + r"raw_geotype", r"\bgeotype\b", r"geometry_type", + r"\bMETA\b", r"\bAREA\b", r"\bSOUNDING\b", r"\bPOINT\b", r"\bLINE\b", + ], + "BBOX": [ + r"bbox_min_lon", r"bbox_min_lat", r"bbox_max_lon", r"bbox_max_lat", + r"\bbbox\b", r"\bbounds\b", r"min_lon", r"max_lon", r"min_lat", r"max_lat", + ], + "COUNTERS": [ + r"geometry_invalid_count", r"geometry_repaired_count", + r"invalid_count", r"repaired_count", r"summary_json", + r"chart_import_job", r"import_job", + ], + "DB_AREA": [r"chart_area_geometry", r"area_geometry"], + "DB_COVERAGE": [r"chart_coverage", r"coverage.*geom"], + "DB_FEATURE": [r"chart_feature", r"feature_attribute"], + "DB_WRITE": [ + r"\bINSERT\s+INTO\b", r"\bUPDATE\b", r"\bexecutemany\s*\(", + r"\bexecute\s*\(", r"\bcommit\s*\(", r"\bflush\s*\(", r"\badd\s*\(", + ], + "RAW_SOURCE": [ + r"raw_feature_blob", r"chart_raw_record", r"source_record_index", + r"raw_blob", r"raw_metadata_json", r"record_kind", + ], + "TOPOLOGY": [ + r"edge_index", r"segment_usage", r"vector_record", r"vector_table", + r"n_elements", r"\bring\b", r"\bcontour\b", r"\bedge\b", r"\bnode\b", + ], + "SRID": [r"normalized_srid", r"\bSRID\b", r"\bsrid\b", r"4326"], +} + +WEIGHTS = { + "GEOMETRY_BUILD": 5, "NORMALIZATION": 8, "GEOMETRY_ISSUE": 10, + "GEOTYPE": 7, "BBOX": 6, "COUNTERS": 9, "DB_AREA": 10, + "DB_COVERAGE": 10, "DB_FEATURE": 7, "DB_WRITE": 6, + "RAW_SOURCE": 5, "TOPOLOGY": 5, "SRID": 3, +} + + +def write_csv(path: Path, rows: Iterable[Dict[str, Any]]) -> None: + rows = list(rows) + keys: List[str] = [] + seen = set() + for row in rows: + for key in row: + if key not in seen: + seen.add(key) + keys.append(key) + + with path.open("w", encoding="utf-8-sig", newline="") as f: + if not keys: + return + w = csv.DictWriter(f, fieldnames=keys, extrasaction="ignore") + w.writeheader() + for row in rows: + out = {} + for key in keys: + val = row.get(key) + if isinstance(val, (dict, list, tuple)): + val = json.dumps(val, ensure_ascii=False, default=str) + out[key] = "" if val is None else val + w.writerow(out) + + +def safe_text(path: Path) -> Optional[str]: + for enc in ("utf-8", "utf-8-sig", "cp932", "latin-1"): + try: + return path.read_text(encoding=enc) + except Exception: + pass + return None + + +def should_skip(path: Path) -> bool: + return any(part in SKIP_DIRS for part in path.parts) + + +def rel(path: Path, root: Path) -> str: + try: + return str(path.resolve().relative_to(root.resolve())) + except Exception: + return str(path) + + +def list_files(root: Path, outdir: Path) -> List[Path]: + result = [] + for p in root.rglob("*"): + if not p.is_file() or should_skip(p): + continue + try: + p.resolve().relative_to(outdir.resolve()) + continue + except Exception: + pass + if p.suffix.lower() in TEXT_EXTENSIONS: + result.append(p) + return sorted(result) + + +def context(lines: List[str], line_no: int, radius: int = 3) -> str: + start = max(1, line_no - radius) + end = min(len(lines), line_no + radius) + return "\n".join(f"{i:05d}: {lines[i-1]}" for i in range(start, end + 1)) + + +def scan_hits(path: Path, text: str, root: Path) -> List[Dict[str, Any]]: + rows = [] + lines = text.splitlines() + compiled = { + group: [re.compile(p, re.I) for p in pats] + for group, pats in SEARCH_GROUPS.items() + } + for no, line in enumerate(lines, 1): + for group, pats in compiled.items(): + matched = [p.pattern for p in pats if p.search(line)] + if matched: + rows.append({ + "file": rel(path, root), + "line": no, + "group": group, + "matched_patterns": matched, + "code": line.strip(), + "context": context(lines, no), + }) + return rows + + +def ast_inventory(path: Path, text: str, root: Path): + funcs, classes, deps, sqls = [], [], [], [] + if path.suffix.lower() != ".py": + return funcs, classes, deps, sqls + try: + tree = ast.parse(text) + except Exception: + return funcs, classes, deps, sqls + + class V(ast.NodeVisitor): + def __init__(self): + self.cls: List[str] = [] + self.fn: List[str] = [] + + def visit_ClassDef(self, node): + classes.append({ + "file": rel(path, root), + "class_name": node.name, + "line": node.lineno, + "end_line": getattr(node, "end_lineno", None), + }) + self.cls.append(node.name) + self.generic_visit(node) + self.cls.pop() + + def visit_FunctionDef(self, node): + funcs.append({ + "file": rel(path, root), + "class_name": ".".join(self.cls), + "function_name": node.name, + "line": node.lineno, + "end_line": getattr(node, "end_lineno", None), + "async": 0, + "args": [a.arg for a in node.args.args], + }) + self.fn.append(node.name) + self.generic_visit(node) + self.fn.pop() + + def visit_AsyncFunctionDef(self, node): + funcs.append({ + "file": rel(path, root), + "class_name": ".".join(self.cls), + "function_name": node.name, + "line": node.lineno, + "end_line": getattr(node, "end_lineno", None), + "async": 1, + "args": [a.arg for a in node.args.args], + }) + self.fn.append(node.name) + self.generic_visit(node) + self.fn.pop() + + def visit_Import(self, node): + for a in node.names: + deps.append({ + "file": rel(path, root), "line": node.lineno, + "type": "import", "module": a.name, "names": "", + }) + + def visit_ImportFrom(self, node): + deps.append({ + "file": rel(path, root), "line": node.lineno, + "type": "from", "module": node.module or "", + "names": [x.name for x in node.names], + }) + + def visit_Constant(self, node): + if isinstance(node.value, str) and re.search( + r"\b(SELECT|INSERT\s+INTO|UPDATE|DELETE\s+FROM|CREATE\s+TABLE|ALTER\s+TABLE)\b", + node.value, re.I + ): + sqls.append({ + "file": rel(path, root), + "line": getattr(node, "lineno", None), + "class_name": ".".join(self.cls), + "function_name": ".".join(self.fn), + "sql": node.value.strip(), + }) + + V().visit(tree) + return funcs, classes, deps, sqls + + +def score_file(path: Path, hits: List[Dict[str, Any]], root: Path) -> Dict[str, Any]: + c = Counter(x["group"] for x in hits) + score = sum(WEIGHTS.get(g, 1) * min(n, 15) for g, n in c.items()) + name = rel(path, root).lower() + for token, bonus in ( + ("import", 10), ("cm93", 10), ("geom", 7), ("coverage", 6), + ("feature", 5), ("parser", 5), ("decoder", 5), ("mysql", 4), + ("database", 4), ("repository", 3), ("model", 2), + ): + if token in name: + score += bonus + row = {"file": rel(path, root), "score": score, "hit_count": len(hits)} + for g in SEARCH_GROUPS: + row[f"hits_{g.lower()}"] = c.get(g, 0) + return row + + +def derive_points(hits: List[Dict[str, Any]]) -> Dict[str, List[Dict[str, Any]]]: + buckets = { + "area_geometry_build": [], "coverage_geometry_build": [], + "area_geometry_write": [], "coverage_geometry_write": [], + "feature_write": [], "geometry_issue_logic": [], + "geotype_logic": [], "invalid_counter_logic": [], + "repaired_counter_logic": [], "bbox_logic": [], + "raw_source_preservation": [], "srid_logic": [], + "normalization_existing": [], + } + + for r in hits: + code = r["code"].lower() + grp = r["group"] + + if grp == "GEOMETRY_BUILD": + if "coverage" in code or "coverage" in r["file"].lower(): + buckets["coverage_geometry_build"].append(r) + else: + buckets["area_geometry_build"].append(r) + + if "chart_area_geometry" in code or "area_geometry" in code: + buckets["area_geometry_write"].append(r) + if "chart_coverage" in code: + buckets["coverage_geometry_write"].append(r) + if "chart_feature" in code and grp in {"DB_FEATURE", "DB_WRITE"}: + buckets["feature_write"].append(r) + if grp == "GEOMETRY_ISSUE": + buckets["geometry_issue_logic"].append(r) + if grp == "GEOTYPE" or "geometry missing" in code: + buckets["geotype_logic"].append(r) + if "geometry_invalid_count" in code: + buckets["invalid_counter_logic"].append(r) + if "geometry_repaired_count" in code: + buckets["repaired_counter_logic"].append(r) + if grp == "BBOX": + buckets["bbox_logic"].append(r) + if grp == "RAW_SOURCE": + buckets["raw_source_preservation"].append(r) + if grp == "SRID": + buckets["srid_logic"].append(r) + if grp == "NORMALIZATION": + buckets["normalization_existing"].append(r) + + for key, rows in buckets.items(): + uniq = {(r["file"], int(r["line"])): r for r in rows} + buckets[key] = sorted( + uniq.values(), + key=lambda x: (x["file"], int(x["line"])) + )[:60] + + return buckets + + +def build_md(candidates, points) -> str: + titles = { + "area_geometry_build": "Area geometry 构造", + "coverage_geometry_build": "Coverage geometry 构造", + "area_geometry_write": "Area geometry 写库", + "coverage_geometry_write": "Coverage geometry 写库", + "feature_write": "Feature 写库", + "geometry_issue_logic": "geometry_issue 判定", + "geotype_logic": "raw_geotype / META / geometry missing", + "invalid_counter_logic": "geometry_invalid_count", + "repaired_counter_logic": "geometry_repaired_count", + "bbox_logic": "bbox 计算与写入", + "raw_source_preservation": "raw source 保留", + "srid_logic": "SRID / normalized_srid", + "normalization_existing": "现有 validity / normalization", + } + + lines = [ + "# CM93 Importer Normalization 独立源码审计 v1", + "", + "本报告仅依据当前项目源码，不依赖任何历史测试结果。", + "", + "## 高优先级候选文件", + "", + ] + for r in candidates[:20]: + lines.append(f"- `{r['file']}` — score={r['score']}, hits={r['hit_count']}") + + for key, title in titles.items(): + lines += ["", f"## {title}", ""] + rows = points.get(key, []) + if not rows: + lines.append("- 未自动定位。") + else: + for r in rows[:30]: + lines.append(f"- `{r['file']}:{r['line']}` — `{r['code']}`") + + lines += [ + "", + "## 下一阶段人工确认重点", + "", + "1. 确认 Area geometry 的唯一最终构造入口。", + "2. 确认 Coverage geometry 的唯一最终构造入口。", + "3. 确认 validity 检查发生在构造后、写库前还是写库后。", + "4. 确认 raw_geotype / META 为什么会进入 geometry missing。", + "5. 确认 geometry_invalid_count 的实际累计条件。", + "6. 确认 bbox 来自 raw topology 还是最终 geometry。", + "7. 确认数据库写入点对 Polygon/MultiPolygon 的处理方式。", + "8. 确认 raw_feature_blob / chart_raw_record 始终保留。", + "", + "本轮不修改源码，不修改数据库，不进入 PBF。", + ] + return "\n".join(lines) + + +def parse_args(): + p = argparse.ArgumentParser(description="CM93 importer 独立源码审计") + p.add_argument("--project-root", default=".") + p.add_argument("--output", default=None) + return p.parse_args() + + +def main(): + args = parse_args() + root = Path(args.project_root).expanduser().resolve() + if not root.is_dir(): + print(f"项目目录不存在：{root}") + sys.exit(2) + + stamp = datetime.now().strftime("%Y%m%d_%H%M%S") + outdir = Path( + args.output or root / f"cm93_importer_source_audit_standalone_v1_{stamp}" + ).expanduser().resolve() + outdir.mkdir(parents=True, exist_ok=True) + + print("[1/6] 枚举源码文件 ...") + files = list_files(root, outdir) + print(f" files={len(files)}") + + file_rows, hits = [], [] + funcs, classes, deps, ast_sql = [], [], [], [] + candidates = [] + + print("[2/6] 关键字 + AST 扫描 ...") + for i, path in enumerate(files, 1): + text = safe_text(path) + file_rows.append({ + "file": rel(path, root), + "suffix": path.suffix.lower(), + "size_bytes": path.stat().st_size, + "readable_text": int(text is not None), + }) + if text is None: + continue + + h = scan_hits(path, text, root) + hits.extend(h) + f, c, d, s = ast_inventory(path, text, root) + funcs.extend(f) + classes.extend(c) + deps.extend(d) + ast_sql.extend(s) + if h: + candidates.append(score_file(path, h, root)) + + if i % 100 == 0: + print(f" scanned={i}/{len(files)}") + + candidates.sort(key=lambda r: (-int(r["score"]), r["file"])) + + print("[3/6] 归类 SQL 与接入点 ...") + sql_hits = [ + r for r in hits + if r["group"] in {"DB_AREA", "DB_COVERAGE", "DB_FEATURE", "DB_WRITE"} + or re.search(r"\b(SELECT|INSERT|UPDATE|DELETE|CREATE|ALTER)\b", r["code"], re.I) + ] + for r in ast_sql: + sql_hits.append({ + "file": r["file"], "line": r["line"], + "group": "AST_SQL_LITERAL", "code": r["sql"], + "context": r["sql"], "class_name": r.get("class_name"), + "function_name": r.get("function_name"), + }) + + points = derive_points(hits) + + print("[4/6] 写结构化输出 ...") + write_csv(outdir / "项目文件清单_v1.csv", file_rows) + write_csv(outdir / "候选源码文件_v1.csv", candidates) + write_csv(outdir / "关键代码命中_v1.csv", hits) + write_csv(outdir / "SQL命中_v1.csv", sql_hits) + write_csv(outdir / "Python函数清单_v1.csv", funcs) + write_csv(outdir / "Python类清单_v1.csv", classes) + write_csv(outdir / "Python依赖清单_v1.csv", deps) + + (outdir / "normalization接入点_v1.md").write_text( + build_md(candidates, points), encoding="utf-8" + ) + + print("[5/6] 写摘要 ...") + group_counts = Counter(r["group"] for r in hits) + report = { + "version": "standalone-v1.0", + "generated_at": datetime.now().isoformat(timespec="seconds"), + "project_root": str(root), + "history_dependency": False, + "database_dependency": False, + "read_only": True, + "pbf": "OUT_OF_SCOPE", + "source_file_count": len(files), + "candidate_file_count": len(candidates), + "code_hit_count": len(hits), + "sql_hit_count": len(sql_hits), + "function_count": len(funcs), + "class_count": len(classes), + "dependency_count": len(deps), + "hit_group_counts": dict(group_counts), + "integration_point_counts": {k: len(v) for k, v in points.items()}, + "top_candidate_files": candidates[:25], + } + + (outdir / "独立源码审计报告_v1.json").write_text( + json.dumps(report, ensure_ascii=False, indent=2, default=str), + encoding="utf-8" + ) + + lines = [ + "CM93 Importer Normalization 独立源码审计 v1", + "=" * 88, + f"Generated: {report['generated_at']}", + f"Project: {root}", + "", + "DEPENDENCY", + " 历史测试依赖: NO", + " 数据库依赖: NO", + " 源码修改: NO", + " PBF: OUT OF SCOPE", + "", + "SCAN", + f" source files: {len(files)}", + f" candidate files: {len(candidates)}", + f" code hits: {len(hits)}", + f" SQL hits: {len(sql_hits)}", + f" Python functions: {len(funcs)}", + f" Python classes: {len(classes)}", + "", + "TOP CANDIDATE FILES", + ] + for r in candidates[:15]: + lines.append(f" {int(r['score']):4d} {r['file']}") + lines += [ + "", + "SOURCE AUDIT COMPLETE: YES", + "NO SOURCE WRITE / NO DATABASE WRITE / NO PBF", + ] + + (outdir / "独立源码审计摘要_v1.txt").write_text( + "\n".join(lines), encoding="utf-8" + ) + + print("[6/6] 压缩 ...") + zpath = outdir.with_suffix(".zip") + with zipfile.ZipFile(zpath, "w", compression=zipfile.ZIP_DEFLATED) as z: + for p in sorted(outdir.rglob("*")): + if p.is_file(): + z.write(p, p.relative_to(outdir.parent)) + + print("") + print("SOURCE AUDIT COMPLETE: YES") + print(f"Output: {outdir}") + print(f"ZIP: {zpath}") + print("把 ZIP 上传给 ChatGPT。") + + +if __name__ == "__main__": + main() diff --git a/cm93_importer_normalization_source_audit_v12.py b/cm93_importer_normalization_source_audit_v12.py new file mode 100644 index 0000000..9a36a61 --- /dev/null +++ b/cm93_importer_normalization_source_audit_v12.py @@ -0,0 +1,880 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +CM93 Importer Normalization 接入点只读调查 +Version: v12.0 + +用途 +==== +V12 不修改数据库，也不修改源码。 + +它只做两件事： + +Gate A： +读取 V11 输出目录或 ZIP，要求： + IMPLEMENTATION PRECHECK PASS = YES +或 report JSON 中： + summary.implementation_precheck_pass = true + +Gate B： +只读扫描当前 importer 项目源码，自动定位： + +1. CM93 Feature / Area geometry 构造入口 +2. chart_area_geometry / chart_coverage 写入位置 +3. geometry_issue = "geometry missing" 或同义逻辑 +4. geometry_invalid_count / geometry_repaired_count 更新位置 +5. raw_geotype / geometry_type 判定位置 +6. bbox_min_lon / bbox_min_lat / bbox_max_lon / bbox_max_lat 计算位置 +7. Shapely / GEOS / WKB / Polygon / MultiPolygon 相关代码 +8. chart_feature / chart_area_geometry / chart_coverage SQL 写入语句 +9. raw_feature_blob / chart_raw_record 保留逻辑 +10. importer job summary / counters + +输出的目标是让下一阶段可以基于当前真实源码生成精准修改版， +而不是凭文件名或架构猜测。 + +重要： +- 不修改项目文件 +- 不修改 MySQL +- 不开始 PBF +- 不生成自动 apply patch +- 只输出调查报告和代码片段 + +执行 +==== +假设当前就在 importer 项目根目录： + + python .\\cm93_importer_normalization_source_audit_v12.py ^ + --v11-input ".\\cm93_normalization_implementation_precheck_v11_dataset_1_20260901_084147" ^ + --project-root "." + +也可以给 V11 ZIP： + + python .\\cm93_importer_normalization_source_audit_v12.py ^ + --v11-input ".\\cm93_normalization_implementation_precheck_v11_dataset_1_20260901_084147.zip" ^ + --project-root "." + +输出 +==== +- importer_source_audit_summary_v12.txt +- importer_source_audit_report_v12.json +- importer_candidate_files_v12.csv +- importer_code_hits_v12.csv +- importer_sql_hits_v12.csv +- importer_function_inventory_v12.csv +- importer_dependency_inventory_v12.csv +- importer_normalization_integration_points_v12.md +- importer_source_audit_v12.zip +""" + +from __future__ import annotations + +import argparse +import ast +import csv +import json +import os +import re +import shutil +import sys +import tempfile +import zipfile +from collections import Counter, defaultdict +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple + + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- + +TEXT_EXTENSIONS = { + ".py", ".sql", ".toml", ".ini", ".cfg", ".json", + ".yaml", ".yml", ".md", ".txt", +} + +SKIP_DIRS = { + ".git", ".hg", ".svn", + "__pycache__", ".pytest_cache", ".mypy_cache", + ".venv", "venv", "env", + "node_modules", "dist", "build", + ".idea", ".vscode", +} + +SEARCH_GROUPS = { + "NORMALIZATION": [ + r"\bmake_valid\b", + r"\bpolygonize_full\b", + r"\bunary_union\b", + r"\bnormalize\b", + r"\bMultiPolygon\b", + r"\bPolygon\b", + r"\bShapely\b", + r"\bshapely\b", + r"\bWKB\b", + r"\bwkb\b", + ], + "GEOMETRY_ISSUE": [ + r"geometry_issue", + r"geometry missing", + r"geometry_missing", + r"invalid geometry", + r"geometry_invalid", + r"geometry_repaired", + ], + "GEOTYPE": [ + r"raw_geotype", + r"geotype", + r"geometry_type", + r"\bMETA\b", + r"\bAREA\b", + ], + "BBOX": [ + r"bbox_min_lon", + r"bbox_min_lat", + r"bbox_max_lon", + r"bbox_max_lat", + r"\bbounds\b", + r"\bbbox\b", + ], + "COUNTERS": [ + r"geometry_invalid_count", + r"geometry_repaired_count", + r"invalid_count", + r"repaired_count", + r"summary_json", + r"chart_import_job", + ], + "DB_WRITE": [ + r"chart_area_geometry", + r"chart_coverage", + r"chart_feature", + r"INSERT\s+INTO", + r"UPDATE\s+", + r"executemany", + r"execute\(", + ], + "RAW_SOURCE": [ + r"raw_feature_blob", + r"chart_raw_record", + r"source_record_index", + r"raw_blob", + r"raw_metadata_json", + ], +} + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def write_csv( + path: Path, + rows: Iterable[Dict[str, Any]], + fieldnames: Optional[List[str]] = None, +): + rows = list(rows) + + if fieldnames is None: + seen = set() + fieldnames = [] + for row in rows: + for k in row.keys(): + if k not in seen: + seen.add(k) + fieldnames.append(k) + + path.parent.mkdir(parents=True, exist_ok=True) + + with path.open("w", encoding="utf-8-sig", newline="") as f: + if not fieldnames: + return + w = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore") + w.writeheader() + for row in rows: + out = {} + for k in fieldnames: + v = row.get(k) + if isinstance(v, (dict, list, tuple)): + v = json.dumps(v, ensure_ascii=False, default=str) + out[k] = "" if v is None else v + w.writerow(out) + + +def safe_read_text(path: Path) -> Optional[str]: + for enc in ("utf-8", "utf-8-sig", "cp932", "latin-1"): + try: + return path.read_text(encoding=enc) + except Exception: + pass + return None + + +def relpath(path: Path, root: Path) -> str: + try: + return str(path.resolve().relative_to(root.resolve())) + except Exception: + return str(path) + + +def should_skip(path: Path) -> bool: + return any(part in SKIP_DIRS for part in path.parts) + + +def list_source_files(root: Path) -> List[Path]: + out = [] + for p in root.rglob("*"): + if not p.is_file(): + continue + if should_skip(p): + continue + if p.suffix.lower() in TEXT_EXTENSIONS: + out.append(p) + return sorted(out) + + +def open_input(src: Path) -> Tuple[Path, Optional[Path]]: + if src.is_dir(): + return src, None + + if src.is_file() and src.suffix.lower() == ".zip": + temp = Path(tempfile.mkdtemp(prefix="cm93_v12_v11_")) + with zipfile.ZipFile(src, "r") as z: + z.extractall(temp) + return temp, temp + + raise ValueError("V11 输入必须是目录或 ZIP") + + +def find_file(root: Path, filename: str) -> Optional[Path]: + direct = root / filename + if direct.exists(): + return direct + hits = list(root.rglob(filename)) + return hits[0] if hits else None + + +# --------------------------------------------------------------------------- +# V11 gate +# --------------------------------------------------------------------------- + +def load_v11_gate(root: Path) -> Dict[str, Any]: + result = { + "source": None, + "implementation_precheck_pass": False, + "db_invalid_count": None, + "golden_revalidation_pass_count": None, + "golden_revalidation_fail_count": None, + "area_geometry_column_type": None, + "coverage_geometry_column_type": None, + "future_area_bbox_update_needed_count": None, + "raw_geotype16_meta_count": None, + "raw_geotype16_with_issue_count": None, + "total_feature_geometry_issue_count": None, + "import_job_geometry_invalid_sum": None, + "errors": [], + } + + report_path = find_file(root, "implementation_precheck_report_v11.json") + summary_path = find_file(root, "implementation_precheck_summary_v11.txt") + + if report_path: + report = json.loads(report_path.read_text(encoding="utf-8")) + s = report.get("summary", {}) + result["source"] = "implementation_precheck_report_v11.json" + result["implementation_precheck_pass"] = bool( + s.get("implementation_precheck_pass") + ) + for key in ( + "db_invalid_count", + "golden_revalidation_pass_count", + "golden_revalidation_fail_count", + "area_geometry_column_type", + "coverage_geometry_column_type", + "future_area_bbox_update_needed_count", + "raw_geotype16_meta_count", + "raw_geotype16_with_issue_count", + "total_feature_geometry_issue_count", + "import_job_geometry_invalid_sum", + ): + result[key] = s.get(key) + + elif summary_path: + text = summary_path.read_text(encoding="utf-8", errors="replace") + result["source"] = "implementation_precheck_summary_v11.txt" + result["implementation_precheck_pass"] = bool( + re.search( + r"IMPLEMENTATION\s+PRECHECK\s+PASS:\s*YES", + text, + flags=re.I, + ) + ) + + else: + result["errors"].append( + "找不到 implementation_precheck_report_v11.json " + "或 implementation_precheck_summary_v11.txt" + ) + + return result + + +# --------------------------------------------------------------------------- +# Source analysis +# --------------------------------------------------------------------------- + +def scan_hits( + path: Path, + text: str, + root: Path, +) -> List[Dict[str, Any]]: + rows = [] + lines = text.splitlines() + + compiled = { + group: [re.compile(pat, re.I) for pat in patterns] + for group, patterns in SEARCH_GROUPS.items() + } + + for lineno, line in enumerate(lines, start=1): + stripped = line.strip() + + for group, patterns in compiled.items(): + matched = [] + for pat in patterns: + if pat.search(line): + matched.append(pat.pattern) + + if not matched: + continue + + start = max(1, lineno - 2) + end = min(len(lines), lineno + 2) + context = "\n".join( + f"{n:05d}: {lines[n-1]}" + for n in range(start, end + 1) + ) + + rows.append({ + "file": relpath(path, root), + "line": lineno, + "group": group, + "matched_patterns": matched, + "code": stripped, + "context": context, + }) + + return rows + + +def python_function_inventory( + path: Path, + text: str, + root: Path, +) -> List[Dict[str, Any]]: + if path.suffix.lower() != ".py": + return [] + + try: + tree = ast.parse(text) + except Exception: + return [] + + rows = [] + + for node in ast.walk(tree): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + rows.append({ + "file": relpath(path, root), + "name": node.name, + "line": node.lineno, + "end_line": getattr(node, "end_lineno", None), + "async": int(isinstance(node, ast.AsyncFunctionDef)), + "args": [ + arg.arg + for arg in ( + list(node.args.posonlyargs) + + list(node.args.args) + + list(node.args.kwonlyargs) + ) + ], + }) + + return rows + + +def python_dependency_inventory( + path: Path, + text: str, + root: Path, +) -> List[Dict[str, Any]]: + if path.suffix.lower() != ".py": + return [] + + try: + tree = ast.parse(text) + except Exception: + return [] + + rows = [] + + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + rows.append({ + "file": relpath(path, root), + "line": node.lineno, + "module": alias.name, + "import_type": "import", + }) + + elif isinstance(node, ast.ImportFrom): + rows.append({ + "file": relpath(path, root), + "line": node.lineno, + "module": node.module or "", + "import_type": "from", + "names": [x.name for x in node.names], + }) + + return rows + + +def candidate_score( + path: Path, + hits: List[Dict[str, Any]], + root: Path, +) -> Dict[str, Any]: + groups = Counter(r["group"] for r in hits) + score = 0 + + weights = { + "GEOMETRY_ISSUE": 8, + "GEOTYPE": 5, + "BBOX": 5, + "COUNTERS": 7, + "DB_WRITE": 7, + "RAW_SOURCE": 4, + "NORMALIZATION": 3, + } + + for group, count in groups.items(): + score += weights.get(group, 1) * min(count, 10) + + name = path.name.lower() + if "import" in name: + score += 8 + if "geometry" in name or "geom" in name: + score += 6 + if "cm93" in name: + score += 6 + if "db" in name or "mysql" in name: + score += 3 + + return { + "file": relpath(path, root), + "score": score, + "hit_count": len(hits), + **{f"hits_{g.lower()}": groups.get(g, 0) for g in SEARCH_GROUPS}, + } + + +def choose_integration_points( + hits: List[Dict[str, Any]] +) -> Dict[str, List[Dict[str, Any]]]: + buckets = { + "area_geometry_build_or_write": [], + "coverage_geometry_build_or_write": [], + "geotype16_issue_logic": [], + "geometry_invalid_counter": [], + "bbox_update": [], + "raw_source_preservation": [], + } + + for r in hits: + code = r["code"].lower() + file = r["file"].lower() + + if ( + "chart_area_geometry" in code + or ( + r["group"] == "NORMALIZATION" + and "area" in file + ) + ): + buckets["area_geometry_build_or_write"].append(r) + + if "chart_coverage" in code: + buckets["coverage_geometry_build_or_write"].append(r) + + if ( + "raw_geotype" in code + or "geotype" in code + or "geometry missing" in code + ): + buckets["geotype16_issue_logic"].append(r) + + if "geometry_invalid_count" in code: + buckets["geometry_invalid_counter"].append(r) + + if ( + "bbox_min_lon" in code + or "bbox_max_lon" in code + or ".bounds" in code + ): + buckets["bbox_update"].append(r) + + if ( + "raw_feature_blob" in code + or "chart_raw_record" in code + ): + buckets["raw_source_preservation"].append(r) + + for key in buckets: + buckets[key] = buckets[key][:30] + + return buckets + + +# --------------------------------------------------------------------------- +# Report +# --------------------------------------------------------------------------- + +def build_integration_md( + gate: Dict[str, Any], + candidates: List[Dict[str, Any]], + integration: Dict[str, List[Dict[str, Any]]], +) -> str: + lines = [] + + lines.append("# CM93 Importer Normalization 接入点调查 v12") + lines.append("") + lines.append("## V11 门控") + lines.append("") + lines.append( + f"- IMPLEMENTATION PRECHECK：" + f"{'PASS' if gate['implementation_precheck_pass'] else 'FAIL'}" + ) + + for key in ( + "db_invalid_count", + "golden_revalidation_pass_count", + "golden_revalidation_fail_count", + "area_geometry_column_type", + "coverage_geometry_column_type", + "future_area_bbox_update_needed_count", + "raw_geotype16_meta_count", + "raw_geotype16_with_issue_count", + "import_job_geometry_invalid_sum", + ): + lines.append(f"- {key}: `{gate.get(key)}`") + + lines.append("") + lines.append("## 高优先级候选源码文件") + lines.append("") + + for row in candidates[:15]: + lines.append( + f"- `{row['file']}` — score={row['score']}, " + f"hits={row['hit_count']}" + ) + + title_map = { + "area_geometry_build_or_write": "Area geometry 构造/写入", + "coverage_geometry_build_or_write": "Coverage geometry 构造/写入", + "geotype16_issue_logic": "raw_geotype=16 / geometry missing 判定", + "geometry_invalid_counter": "geometry_invalid_count 统计", + "bbox_update": "bbox 计算/写入", + "raw_source_preservation": "raw source 保留", + } + + for key, title in title_map.items(): + lines.append("") + lines.append(f"## {title}") + lines.append("") + + rows = integration.get(key, []) + + if not rows: + lines.append("- 未自动定位，需要人工继续查。") + continue + + seen = set() + for row in rows: + marker = (row["file"], row["line"]) + if marker in seen: + continue + seen.add(marker) + lines.append( + f"- `{row['file']}:{row['line']}` " + f"`{row['code']}`" + ) + + lines.append("") + lines.append("## 下一阶段修改边界") + lines.append("") + lines.append("下一阶段只应修改真实定位到的 importer 代码，不应重新设计数据库。") + lines.append("") + lines.append("冻结要求：") + lines.append("") + lines.append("1. valid geometry 原样 PASS-THROUGH。") + lines.append("2. invalid Area/Coverage 才执行 odd-even face selection。") + lines.append("3. normalization 后执行 GEOS normalize canonicalization。") + lines.append("4. 输出必须为 valid Polygon/MultiPolygon。") + lines.append("5. Area normalized 后同步重新计算 feature bbox。") + lines.append("6. retraced connector 不保存为 Area 面积，但 raw source 必须保留。") + lines.append("7. raw_geotype=16 / META 不生成 `geometry missing`。") + lines.append("8. raw_geotype=16 不增加 `geometry_invalid_count`。") + lines.append("9. 不删除 `raw_feature_blob` 或 `chart_raw_record`。") + lines.append("10. 当前仍不进入 PBF。") + + return "\n".join(lines) + + +def parse_args(): + p = argparse.ArgumentParser( + description="CM93 importer normalization source audit v12" + ) + p.add_argument("--v11-input", required=True) + p.add_argument("--project-root", default=".") + p.add_argument("--output", default=None) + return p.parse_args() + + +def main(): + args = parse_args() + + v11_src = Path(args.v11_input).expanduser().resolve() + project_root = Path(args.project_root).expanduser().resolve() + + if not v11_src.exists(): + print(f"V11 输入不存在：{v11_src}") + sys.exit(2) + + if not project_root.is_dir(): + print(f"项目目录不存在：{project_root}") + sys.exit(2) + + temp_v11 = None + + try: + v11_root, temp_v11 = open_input(v11_src) + + print("[1/6] V11 implementation precheck gate ...") + gate = load_v11_gate(v11_root) + + print(json.dumps(gate, ensure_ascii=False, indent=2)) + + if not gate["implementation_precheck_pass"]: + print("") + print("STOP：V11 IMPLEMENTATION PRECHECK 未通过。") + print("V12 不扫描 importer 源码。") + sys.exit(3) + + print(" V11 gate: PASS") + + stamp = datetime.now().strftime("%Y%m%d_%H%M%S") + + outdir = Path( + args.output + or project_root / f"cm93_importer_source_audit_v12_{stamp}" + ) + outdir.mkdir(parents=True, exist_ok=True) + + print("[2/6] 扫描项目文本源码 ...") + files = list_source_files(project_root) + print(f" files={len(files)}") + + all_hits = [] + function_rows = [] + dependency_rows = [] + candidate_rows = [] + + for i, path in enumerate(files, start=1): + # 避免扫描当前输出目录自身 + try: + path.resolve().relative_to(outdir.resolve()) + continue + except Exception: + pass + + text = safe_read_text(path) + if text is None: + continue + + hits = scan_hits(path, text, project_root) + all_hits.extend(hits) + + function_rows.extend( + python_function_inventory(path, text, project_root) + ) + + dependency_rows.extend( + python_dependency_inventory(path, text, project_root) + ) + + if hits: + candidate_rows.append( + candidate_score(path, hits, project_root) + ) + + if i % 100 == 0: + print(f" scanned={i}/{len(files)}") + + candidate_rows.sort( + key=lambda r: (-int(r["score"]), r["file"]) + ) + + print("[3/6] 分类 SQL 命中与函数 ...") + sql_hits = [ + r for r in all_hits + if ( + r["group"] == "DB_WRITE" + or re.search( + r"\b(SELECT|INSERT|UPDATE|DELETE|CREATE|ALTER)\b", + r["code"], + re.I, + ) + ) + ] + + print("[4/6] 推导 normalization 接入点 ...") + integration = choose_integration_points(all_hits) + + print("[5/6] 写报告 ...") + write_csv( + outdir / "importer_candidate_files_v12.csv", + candidate_rows + ) + write_csv( + outdir / "importer_code_hits_v12.csv", + all_hits + ) + write_csv( + outdir / "importer_sql_hits_v12.csv", + sql_hits + ) + write_csv( + outdir / "importer_function_inventory_v12.csv", + function_rows + ) + write_csv( + outdir / "importer_dependency_inventory_v12.csv", + dependency_rows + ) + + integration_md = build_integration_md( + gate, + candidate_rows, + integration, + ) + + ( + outdir / "importer_normalization_integration_points_v12.md" + ).write_text( + integration_md, + encoding="utf-8" + ) + + report = { + "version": "v12.0", + "generated_at": datetime.now().isoformat(timespec="seconds"), + "project_root": str(project_root), + "read_only": True, + "v11_gate": gate, + "source_file_count": len(files), + "candidate_file_count": len(candidate_rows), + "code_hit_count": len(all_hits), + "sql_hit_count": len(sql_hits), + "function_count": len(function_rows), + "dependency_count": len(dependency_rows), + "top_candidate_files": candidate_rows[:25], + "integration_point_counts": { + k: len(v) + for k, v in integration.items() + }, + } + + ( + outdir / "importer_source_audit_report_v12.json" + ).write_text( + json.dumps( + report, + ensure_ascii=False, + indent=2, + default=str, + ), + encoding="utf-8" + ) + + lines = [] + lines.append("CM93 Importer Normalization 接入点只读调查 v12") + lines.append("=" * 88) + lines.append(f"Generated: {report['generated_at']}") + lines.append(f"Project: {project_root}") + lines.append("") + lines.append("V11 GATE") + lines.append(" PASS") + lines.append("") + lines.append("SOURCE SCAN") + lines.append(f" files: {len(files)}") + lines.append(f" candidate files:{len(candidate_rows)}") + lines.append(f" code hits: {len(all_hits)}") + lines.append(f" SQL hits: {len(sql_hits)}") + lines.append(f" functions: {len(function_rows)}") + lines.append("") + lines.append("TOP CANDIDATE FILES") + + for row in candidate_rows[:15]: + lines.append( + f" {row['score']:4d} {row['file']}" + ) + + lines.append("") + lines.append("INTEGRATION POINT COUNTS") + + for key, rows in integration.items(): + lines.append( + f" {key:36s} {len(rows)}" + ) + + lines.append("") + lines.append( + "NO SOURCE WRITE / NO DATABASE WRITE / NO PBF" + ) + + ( + outdir / "importer_source_audit_summary_v12.txt" + ).write_text( + "\n".join(lines), + encoding="utf-8" + ) + + print("[6/6] 压缩 ...") + zpath = outdir.with_suffix(".zip") + + with zipfile.ZipFile( + zpath, + "w", + compression=zipfile.ZIP_DEFLATED + ) as z: + for p in sorted(outdir.rglob("*")): + if p.is_file(): + z.write( + p, + p.relative_to(outdir.parent) + ) + + print("") + print("SOURCE AUDIT PASS: YES") + print(f"ZIP: {zpath.resolve()}") + print("把 V12 ZIP 上传给 ChatGPT。") + + finally: + if temp_v11 and temp_v11.exists(): + shutil.rmtree(temp_v11, ignore_errors=True) + + +if __name__ == "__main__": + main() diff --git a/cm93_importer_production_source_snapshot_v1.py b/cm93_importer_production_source_snapshot_v1.py new file mode 100644 index 0000000..3b3437f --- /dev/null +++ b/cm93_importer_production_source_snapshot_v1.py @@ -0,0 +1,613 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +CM93 Importer 生产源码快照与接入点提取 +Version: production-snapshot-v1 + +目的 +==== +完全独立，不连接数据库，不读取 V10/V11/V12，不扫描历史审计脚本。 + +只针对当前项目真正的生产源码： +- src/cm93db/importer.py +- src/cm93db/geometry.py +- src/cm93db/cm93_binary.py +- src/cm93db/cell_parser.py +- src/cm93db/audit.py +- sql/001_schema_mysql57.sql +- README.md +- AGENTS.md（若存在） + +生成： +1. 上述文件的完整只读快照副本； +2. SHA-256； +3. Python 函数/类清单； +4. 与 normalization / geometry issue / geotype / bbox / + counters / DB write / raw source 相关的完整函数摘录； +5. SQL schema 中关键表完整 CREATE TABLE 片段； +6. 一个小 ZIP，供 ChatGPT 精确分析后生成真正修改版。 + +不做： +- 不修改源码 +- 不连接 MySQL +- 不执行 SQL +- 不生成 patch +- 不生成 PBF +""" + +from __future__ import annotations + +import argparse +import ast +import csv +import hashlib +import json +import re +import shutil +import sys +import zipfile +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Tuple + + +PRODUCTION_FILES = [ + "src/cm93db/importer.py", + "src/cm93db/geometry.py", + "src/cm93db/cm93_binary.py", + "src/cm93db/cell_parser.py", + "src/cm93db/audit.py", + "sql/001_schema_mysql57.sql", + "README.md", + "AGENTS.md", +] + +KEY_PATTERNS = { + "normalization": [ + r"\bis_valid\b", + r"\bmake_valid\b", + r"\bpolygonize\b", + r"\bpolygonize_full\b", + r"\bunary_union\b", + r"\bnormalize\b", + r"\bexplain_validity\b", + r"\bbuffer\s*\(\s*0", + ], + "geometry_issue": [ + r"geometry_issue", + r"geometry missing", + r"geometry_missing", + r"geometry_repaired", + r"invalid geometry", + ], + "geotype": [ + r"raw_geotype", + r"\bgeotype\b", + r"geometry_type", + r"\bMETA\b", + r"\bAREA\b", + ], + "bbox": [ + r"bbox_min_lon", + r"bbox_min_lat", + r"bbox_max_lon", + r"bbox_max_lat", + r"\bbbox\b", + r"\bbounds\b", + ], + "counters": [ + r"geometry_invalid_count", + r"geometry_repaired_count", + r"summary_json", + r"chart_import_job", + ], + "db_write": [ + r"chart_area_geometry", + r"chart_coverage", + r"chart_feature", + r"INSERT\s+INTO", + r"UPDATE\s+", + r"executemany\s*\(", + r"execute\s*\(", + ], + "raw_source": [ + r"raw_feature_blob", + r"chart_raw_record", + r"source_record_index", + r"raw_blob", + r"raw_metadata_json", + ], + "geometry_build": [ + r"\bPolygon\s*\(", + r"\bMultiPolygon\s*\(", + r"\bLineString\s*\(", + r"\bPoint\s*\(", + r"\bwkb\b", + r"\bshapely\b", + ], + "srid": [ + r"\bSRID\b", + r"\bsrid\b", + r"4326", + r"normalized_srid", + ], +} + +SCHEMA_TABLES = [ + "chart_feature", + "chart_area_geometry", + "chart_coverage", + "chart_import_job", + "chart_raw_record", +] + + +def sha256_file(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as f: + while True: + chunk = f.read(1024 * 1024) + if not chunk: + break + h.update(chunk) + return h.hexdigest() + + +def safe_text(path: Path) -> Optional[str]: + for enc in ("utf-8", "utf-8-sig", "cp932", "latin-1"): + try: + return path.read_text(encoding=enc) + except Exception: + pass + return None + + +def write_csv(path: Path, rows: Iterable[Dict[str, Any]]) -> None: + rows = list(rows) + keys: List[str] = [] + seen = set() + + for row in rows: + for key in row: + if key not in seen: + seen.add(key) + keys.append(key) + + with path.open("w", encoding="utf-8-sig", newline="") as f: + if not keys: + return + w = csv.DictWriter(f, fieldnames=keys, extrasaction="ignore") + w.writeheader() + for row in rows: + out = {} + for key in keys: + v = row.get(key) + if isinstance(v, (list, tuple, dict)): + v = json.dumps(v, ensure_ascii=False, default=str) + out[key] = "" if v is None else v + w.writerow(out) + + +def ast_functions(path: Path, text: str, rel: str): + functions: List[Dict[str, Any]] = [] + classes: List[Dict[str, Any]] = [] + + if path.suffix.lower() != ".py": + return functions, classes + + try: + tree = ast.parse(text) + except Exception as e: + functions.append({ + "file": rel, + "function_name": "__AST_PARSE_ERROR__", + "line": None, + "end_line": None, + "error": f"{type(e).__name__}: {e}", + }) + return functions, classes + + class_stack: List[str] = [] + + class Visitor(ast.NodeVisitor): + def visit_ClassDef(self, node: ast.ClassDef): + classes.append({ + "file": rel, + "class_name": node.name, + "line": node.lineno, + "end_line": getattr(node, "end_lineno", None), + }) + class_stack.append(node.name) + self.generic_visit(node) + class_stack.pop() + + def visit_FunctionDef(self, node: ast.FunctionDef): + functions.append({ + "file": rel, + "class_name": ".".join(class_stack), + "function_name": node.name, + "line": node.lineno, + "end_line": getattr(node, "end_lineno", None), + "async": 0, + "args": [a.arg for a in node.args.args], + }) + self.generic_visit(node) + + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef): + functions.append({ + "file": rel, + "class_name": ".".join(class_stack), + "function_name": node.name, + "line": node.lineno, + "end_line": getattr(node, "end_lineno", None), + "async": 1, + "args": [a.arg for a in node.args.args], + }) + self.generic_visit(node) + + Visitor().visit(tree) + return functions, classes + + +def line_hits(rel: str, text: str) -> List[Dict[str, Any]]: + rows = [] + lines = text.splitlines() + + compiled = { + group: [re.compile(p, re.I) for p in pats] + for group, pats in KEY_PATTERNS.items() + } + + for no, line in enumerate(lines, 1): + for group, pats in compiled.items(): + matched = [p.pattern for p in pats if p.search(line)] + if not matched: + continue + start = max(1, no - 3) + end = min(len(lines), no + 3) + context = "\n".join( + f"{i:05d}: {lines[i-1]}" + for i in range(start, end + 1) + ) + rows.append({ + "file": rel, + "line": no, + "group": group, + "code": line.strip(), + "matched_patterns": matched, + "context": context, + }) + + return rows + + +def function_key_groups( + fn: Dict[str, Any], + text: str, +) -> List[str]: + if fn.get("line") is None: + return [] + + lines = text.splitlines() + start = max(1, int(fn["line"])) + end = fn.get("end_line") + if end is None: + end = min(len(lines), start + 80) + else: + end = min(len(lines), int(end)) + + body = "\n".join(lines[start - 1:end]) + + groups = [] + for group, pats in KEY_PATTERNS.items(): + if any(re.search(p, body, re.I) for p in pats): + groups.append(group) + + return groups + + +def make_function_excerpts( + root: Path, + function_rows: List[Dict[str, Any]], + texts: Dict[str, str], +) -> str: + chunks: List[str] = [] + + for fn in sorted( + function_rows, + key=lambda x: ( + x.get("file") or "", + int(x.get("line") or 0), + ) + ): + rel = fn.get("file") + text = texts.get(rel) + if not text or fn.get("line") is None: + continue + + groups = function_key_groups(fn, text) + if not groups: + continue + + lines = text.splitlines() + start = max(1, int(fn["line"])) + end_val = fn.get("end_line") + end = ( + min(len(lines), int(end_val)) + if end_val is not None + else min(len(lines), start + 100) + ) + + # 防止单个超大函数让结果失控，但保留足够上下文。 + if end - start > 260: + end = start + 260 + + cls = fn.get("class_name") or "" + name = fn.get("function_name") or "" + display = f"{cls}.{name}" if cls else name + + chunks.append("=" * 110) + chunks.append( + f"FILE={rel} FUNCTION={display} " + f"LINES={start}-{fn.get('end_line')} " + f"GROUPS={','.join(groups)}" + ) + chunks.append("=" * 110) + + for no in range(start, end + 1): + chunks.append(f"{no:05d}: {lines[no-1]}") + + chunks.append("") + + return "\n".join(chunks) + + +def extract_create_table(sql_text: str, table: str) -> Optional[str]: + # 兼容 CREATE TABLE `name` (...) ENGINE=... + pat = re.compile( + rf"CREATE\s+TABLE\s+`?{re.escape(table)}`?\s*\(", + re.I, + ) + m = pat.search(sql_text) + if not m: + return None + + start = m.start() + pos = m.end() - 1 + depth = 0 + in_single = False + in_double = False + escape = False + + for i in range(pos, len(sql_text)): + ch = sql_text[i] + + if escape: + escape = False + continue + + if ch == "\\": + escape = True + continue + + if ch == "'" and not in_double: + in_single = not in_single + continue + + if ch == '"' and not in_single: + in_double = not in_double + continue + + if in_single or in_double: + continue + + if ch == "(": + depth += 1 + elif ch == ")": + depth -= 1 + if depth == 0: + # 到分号为止 + semi = sql_text.find(";", i) + if semi == -1: + semi = i + return sql_text[start:semi + 1] + + return None + + +def parse_args(): + p = argparse.ArgumentParser( + description="CM93 importer production source snapshot" + ) + p.add_argument("--project-root", default=".") + p.add_argument("--output", default=None) + return p.parse_args() + + +def main(): + args = parse_args() + root = Path(args.project_root).expanduser().resolve() + + if not root.is_dir(): + print(f"项目目录不存在：{root}") + sys.exit(2) + + stamp = datetime.now().strftime("%Y%m%d_%H%M%S") + outdir = Path( + args.output + or root / f"cm93_importer_production_snapshot_v1_{stamp}" + ).expanduser().resolve() + outdir.mkdir(parents=True, exist_ok=True) + + snapshot_dir = outdir / "source_snapshot" + snapshot_dir.mkdir(parents=True, exist_ok=True) + + inventory = [] + all_functions = [] + all_classes = [] + all_hits = [] + texts: Dict[str, str] = {} + + print("[1/5] 收集生产文件 ...") + + for rel in PRODUCTION_FILES: + src = root / rel + + if not src.exists(): + inventory.append({ + "file": rel, + "exists": 0, + "size_bytes": None, + "sha256": None, + }) + continue + + dest = snapshot_dir / rel + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dest) + + inventory.append({ + "file": rel, + "exists": 1, + "size_bytes": src.stat().st_size, + "sha256": sha256_file(src), + }) + + text = safe_text(src) + if text is None: + continue + + texts[rel] = text + + funcs, classes = ast_functions(src, text, rel) + all_functions.extend(funcs) + all_classes.extend(classes) + all_hits.extend(line_hits(rel, text)) + + print("[2/5] 提取相关完整函数 ...") + + excerpts = make_function_excerpts( + root, + all_functions, + texts, + ) + + (outdir / "生产源码关键函数完整摘录_v1.txt").write_text( + excerpts, + encoding="utf-8", + ) + + print("[3/5] 提取关键 schema ...") + + schema_text = texts.get("sql/001_schema_mysql57.sql", "") + schema_chunks = [] + + for table in SCHEMA_TABLES: + ddl = extract_create_table(schema_text, table) + schema_chunks.append("=" * 100) + schema_chunks.append(f"TABLE: {table}") + schema_chunks.append("=" * 100) + schema_chunks.append( + ddl if ddl is not None else "NOT FOUND" + ) + schema_chunks.append("") + + (outdir / "关键表DDL_v1.txt").write_text( + "\n".join(schema_chunks), + encoding="utf-8", + ) + + print("[4/5] 写清单与报告 ...") + + write_csv(outdir / "生产文件SHA256_v1.csv", inventory) + write_csv(outdir / "生产Python函数清单_v1.csv", all_functions) + write_csv(outdir / "生产Python类清单_v1.csv", all_classes) + write_csv(outdir / "生产源码关键命中_v1.csv", all_hits) + + group_counts = {} + for row in all_hits: + group_counts[row["group"]] = ( + group_counts.get(row["group"], 0) + 1 + ) + + report = { + "version": "production-snapshot-v1", + "generated_at": datetime.now().isoformat(timespec="seconds"), + "project_root": str(root), + "database_dependency": False, + "history_dependency": False, + "read_only": True, + "pbf": "OUT_OF_SCOPE", + "production_files": inventory, + "function_count": len(all_functions), + "class_count": len(all_classes), + "key_hit_count": len(all_hits), + "key_hit_group_counts": group_counts, + "snapshot_contains_only_explicit_production_files": True, + } + + (outdir / "生产源码快照报告_v1.json").write_text( + json.dumps( + report, + ensure_ascii=False, + indent=2, + default=str, + ), + encoding="utf-8", + ) + + summary = [ + "CM93 Importer 生产源码快照与接入点提取 v1", + "=" * 90, + f"Project: {root}", + "", + "仅包含明确指定的生产文件，不扫描任何 V* 历史审计脚本。", + "", + "FILES", + ] + + for row in inventory: + summary.append( + f" {'OK' if row['exists'] else 'MISSING':7s} " + f"{row['file']}" + ) + + summary += [ + "", + f"Python functions: {len(all_functions)}", + f"Python classes: {len(all_classes)}", + f"Key hits: {len(all_hits)}", + "", + "NO DATABASE / NO SOURCE MODIFY / NO PBF", + ] + + (outdir / "生产源码快照摘要_v1.txt").write_text( + "\n".join(summary), + encoding="utf-8", + ) + + print("[5/5] 压缩 ...") + + zip_path = outdir.with_suffix(".zip") + + with zipfile.ZipFile( + zip_path, + "w", + compression=zipfile.ZIP_DEFLATED, + ) as z: + for p in sorted(outdir.rglob("*")): + if p.is_file(): + z.write( + p, + p.relative_to(outdir.parent), + ) + + print("") + print("PRODUCTION SNAPSHOT COMPLETE: YES") + print(f"Output: {outdir}") + print(f"ZIP: {zip_path}") + print("把这个 ZIP 上传给 ChatGPT。") + + +if __name__ == "__main__": + main() diff --git a/cm93_mysql_audit.py b/cm93_mysql_audit.py new file mode 100644 index 0000000..ac99b3d --- /dev/null +++ b/cm93_mysql_audit.py @@ -0,0 +1,734 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +CM93 -> MySQL Dataset Audit + Geometry Issue Classification +v1.0 + +目标： + 1. 审计 Dataset ID=1 + 2. 分类 geometry issue + 3. 导出 CSV / JSON / TXT + +安全： + - 只读事务 + - 不执行 INSERT / UPDATE / DELETE / DDL + - 不修 geometry + - 不涉及 PBF + +依赖： + pip install pymysql + +示例： + python cm93_mysql_audit.py --host 127.0.0.1 --port 3306 --user root --database YOUR_DB --dataset-id 1 +""" + +from __future__ import annotations + +import argparse +import csv +import getpass +import json +import os +import re +import sys +import traceback +from collections import Counter, defaultdict +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, List, Optional, Sequence, Tuple + +try: + import pymysql + from pymysql.cursors import DictCursor +except ImportError: + print("缺少 pymysql，请执行: pip install pymysql") + sys.exit(2) + + +CATEGORY_RULES: List[Tuple[str, List[str]]] = [ + ("INVALID_COORDINATE", [ + r"\bnan\b", r"\binf(?:inity)?\b", r"invalid coordinate", + r"coordinate.*out of range", r"latitude.*out of range", + r"longitude.*out of range", r"non[- ]?finite" + ]), + ("INSUFFICIENT_POINTS", [ + r"insufficient point", r"too few point", r"not enough point", + r"minimum.*point", r"less than .* points?", r"requires? .* points?" + ]), + ("RING_NOT_CLOSED", [ + r"ring.*not closed", r"not closed.*ring", r"unclosed ring", + r"closed linestring" + ]), + ("SELF_INTERSECTION", [ + r"self[- ]?intersection", r"self intersect", r"ring self", r"bow[- ]?tie" + ]), + ("INVALID_RING", [ + r"invalid ring", r"ring.*invalid", r"illegal ring", + r"shell.*invalid", r"hole.*invalid", r"interior ring", r"exterior ring" + ]), + ("DEGENERATE_GEOMETRY", [ + r"degenerate", r"zero[- ]?length", r"zero[- ]?area", + r"collapsed", r"duplicate point", r"all points.*same", + r"too few distinct" + ]), + ("MULTIPART_ASSEMBLY", [ + r"multipart", r"multi[- ]?part", r"cannot.*assemble", + r"failed.*assemble", r"assembly failed", r"cannot.*merge", + r"failed.*merge", r"stitch" + ]), + ("TOPOLOGY_MISMATCH", [ + r"topolog", r"missing edge", r"edge.*missing", r"missing node", + r"node.*missing", r"edge reference", r"node reference", + r"primitive reference", r"dangling edge", r"broken chain", r"disconnected" + ]), + ("GEOMETRY_TYPE_MISMATCH", [ + r"type mismatch", r"geometry type", r"expected.*(?:point|line|polygon|area)", + r"unexpected.*(?:point|line|polygon|area)", r"wrong.*geometry" + ]), + ("EMPTY_GEOMETRY", [ + r"empty geometry", r"geometry.*empty", r"no geometry", r"null geometry", + r"geometry is null", r"no coordinates?", r"empty point", + r"empty line", r"empty polygon" + ]), + ("CONVERSION_EXCEPTION", [ + r"\bwkb\b", r"\bwkt\b", r"parse exception", r"conversion exception", + r"geometry exception", r"failed to convert", r"cannot convert", + r"invalid gis data", r"data truncation", r"spatial.*exception" + ]), +] + +STAGE_RULES: List[Tuple[str, List[str]]] = [ + ("MYSQL_WRITE", [r"mysql", r"insert", r"database", r"spatial", r"\bwkb\b", r"\bwkt\b"]), + ("VALIDATION", [r"valid", r"self[- ]?intersect", r"ring", r"geometry.*invalid"]), + ("ASSEMBLY", [r"assemble", r"assembly", r"merge", r"stitch", r"multipart", r"chain"]), + ("TOPOLOGY", [r"topolog", r"edge", r"node", r"primitive"]), + ("DECODE", [r"decode", r"parse", r"binary", r"record"]), + ("SOURCE", [r"source", r"raw", r"cm93"]), +] + + +@dataclass +class TableInfo: + name: str + columns: List[str] + + +def qi(name: str) -> str: + return "`" + name.replace("`", "``") + "`" + + +def text(v: Any) -> str: + if v is None: + return "" + if isinstance(v, (bytes, bytearray)): + return v.decode("utf-8", errors="replace") + return str(v) + + +def score_name(name: str, candidates: Sequence[str]) -> int: + n = name.lower() + best = 0 + for c in candidates: + c = c.lower() + if n == c: + best = max(best, 100) + elif n.endswith("_" + c) or n.startswith(c + "_"): + best = max(best, 45) + elif c in n: + best = max(best, 10) + return best + + +def best_col(cols: Sequence[str], candidates: Sequence[str]) -> Optional[str]: + ranked = sorted(((score_name(c, candidates), c) for c in cols), reverse=True) + return ranked[0][1] if ranked and ranked[0][0] > 0 else None + + +def best_table(tables: List[TableInfo], candidates: Sequence[str], must_have: Sequence[str] = ()) -> Optional[TableInfo]: + ranked = [] + for t in tables: + if must_have and not all(any(x.lower() in c.lower() for c in t.columns) for x in must_have): + continue + ranked.append((score_name(t.name, candidates), t.name, t)) + ranked.sort(reverse=True) + return ranked[0][2] if ranked and ranked[0][0] > 0 else None + + +def discover_tables(cur, db: str) -> List[TableInfo]: + cur.execute(""" + SELECT TABLE_NAME, COLUMN_NAME + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA=%s + ORDER BY TABLE_NAME, ORDINAL_POSITION + """, (db,)) + grouped: Dict[str, List[str]] = defaultdict(list) + for r in cur.fetchall(): + grouped[r["TABLE_NAME"]].append(r["COLUMN_NAME"]) + return [TableInfo(k, v) for k, v in grouped.items()] + + +def resolve_schema(tables: List[TableInfo]) -> Dict[str, Optional[TableInfo]]: + issue_candidates = [] + for t in tables: + n = t.name.lower() + if ("geometry" in n or "geom" in n) and ("issue" in n or "error" in n): + issue_candidates.append(t) + issue_candidates.sort(key=lambda t: (len(t.name), t.name)) + + return { + "dataset": best_table(tables, ["datasets", "dataset"]), + "cell": best_table(tables, ["cells", "cell"]), + "feature": best_table(tables, ["features", "feature"]), + "attribute": best_table(tables, ["feature_attributes", "feature_attribute", "attributes", "attribute"]), + "sounding": best_table(tables, ["sounding_points", "sounding_point", "soundings", "sounding"]), + "geometry_issue": issue_candidates[0] if issue_candidates else best_table(tables, ["geometry_issues", "geometry_issue", "geom_issue"]), + } + + +def scalar(cur, sql: str, params=()) -> Any: + cur.execute(sql, params) + r = cur.fetchone() + return next(iter(r.values())) if r else None + + +def dataset_col(t: Optional[TableInfo]) -> Optional[str]: + if not t: + return None + return best_col(t.columns, ["dataset_id", "datasetid", "dataset"]) + + +def count_table(cur, t: Optional[TableInfo], dataset_id: int) -> Dict[str, Any]: + if not t: + return {"table": None, "count": None, "dataset_column": None, "error": "table not found"} + ds = dataset_col(t) + try: + if ds: + n = scalar(cur, f"SELECT COUNT(*) AS n FROM {qi(t.name)} WHERE {qi(ds)}=%s", (dataset_id,)) + else: + n = scalar(cur, f"SELECT COUNT(*) AS n FROM {qi(t.name)}") + return {"table": t.name, "count": int(n), "dataset_column": ds, "error": None} + except Exception as e: + return {"table": t.name, "count": None, "dataset_column": ds, "error": str(e)} + + +def orphan_count(cur, child: Optional[TableInfo], parent: Optional[TableInfo], child_fk_names, parent_pk_names, dataset_id: int) -> Dict[str, Any]: + out = { + "child": child.name if child else None, + "parent": parent.name if parent else None, + "child_fk": None, + "parent_pk": None, + "count": None, + "error": None, + } + if not child or not parent: + out["error"] = "table not found" + return out + + cfk = best_col(child.columns, child_fk_names) + ppk = best_col(parent.columns, parent_pk_names) + cds = dataset_col(child) + out["child_fk"] = cfk + out["parent_pk"] = ppk + if not cfk or not ppk: + out["error"] = "required columns not found" + return out + + try: + where = [f"c.{qi(cfk)} IS NOT NULL", f"p.{qi(ppk)} IS NULL"] + params = [] + if cds: + where.insert(0, f"c.{qi(cds)}=%s") + params.append(dataset_id) + sql = f""" + SELECT COUNT(*) AS n + FROM {qi(child.name)} c + LEFT JOIN {qi(parent.name)} p + ON c.{qi(cfk)} = p.{qi(ppk)} + WHERE {' AND '.join(where)} + """ + out["count"] = int(scalar(cur, sql, params)) + except Exception as e: + out["error"] = str(e) + return out + + +def geometry_audit(cur, feature: Optional[TableInfo], dataset_id: int) -> Dict[str, Any]: + out: Dict[str, Any] = { + "table": feature.name if feature else None, + "geometry_column": None, + "null_count": None, + "empty_count": None, + "valid_count": None, + "invalid_count": None, + "types": {}, + "errors": [], + } + if not feature: + out["errors"].append("feature table not found") + return out + + geom = best_col(feature.columns, ["geometry", "geom", "shape"]) + ds = dataset_col(feature) + out["geometry_column"] = geom + if not geom: + out["errors"].append("geometry column not found") + return out + + params = [dataset_id] if ds else [] + base = f"{qi(ds)}=%s" if ds else "1=1" + + try: + out["null_count"] = int(scalar(cur, + f"SELECT COUNT(*) AS n FROM {qi(feature.name)} WHERE {base} AND {qi(geom)} IS NULL", params)) + except Exception as e: + out["errors"].append(f"NULL count failed: {e}") + + try: + cur.execute(f""" + SELECT ST_GeometryType({qi(geom)}) AS geometry_type, COUNT(*) AS n + FROM {qi(feature.name)} + WHERE {base} AND {qi(geom)} IS NOT NULL + GROUP BY ST_GeometryType({qi(geom)}) + ORDER BY n DESC + """, params) + out["types"] = {text(r["geometry_type"] or "NULL"): int(r["n"]) for r in cur.fetchall()} + except Exception as e: + out["errors"].append(f"ST_GeometryType failed: {e}") + + try: + cur.execute(f""" + SELECT + SUM(CASE WHEN ST_IsEmpty({qi(geom)}) THEN 1 ELSE 0 END) AS empty_count, + SUM(CASE WHEN ST_IsValid({qi(geom)}) THEN 1 ELSE 0 END) AS valid_count, + SUM(CASE WHEN NOT ST_IsValid({qi(geom)}) THEN 1 ELSE 0 END) AS invalid_count + FROM {qi(feature.name)} + WHERE {base} AND {qi(geom)} IS NOT NULL + """, params) + r = cur.fetchone() or {} + out["empty_count"] = int(r.get("empty_count") or 0) + out["valid_count"] = int(r.get("valid_count") or 0) + out["invalid_count"] = int(r.get("invalid_count") or 0) + except Exception as e: + out["errors"].append(f"ST_IsEmpty/ST_IsValid failed: {e}") + + return out + + +def issue_text(row: Dict[str, Any]) -> str: + first = [] + rest = [] + for k, v in row.items(): + if v is None: + continue + s = text(v).strip() + if not s: + continue + kl = k.lower() + part = f"{k}={s}" + if any(x in kl for x in ["issue", "error", "reason", "message", "detail", "exception", "stage", "type", "status"]): + first.append(part) + else: + rest.append(part) + return " | ".join(first + rest) + + +def classify_issue(s: str) -> str: + for category, patterns in CATEGORY_RULES: + for p in patterns: + if re.search(p, s, re.I): + return category + return "OTHER" + + +def classify_stage(s: str) -> str: + for stage, patterns in STAGE_RULES: + for p in patterns: + if re.search(p, s, re.I): + return stage + return "UNKNOWN" + + +def fetch_issues(cur, issue: Optional[TableInfo], feature: Optional[TableInfo], cell: Optional[TableInfo], dataset_id: int, max_rows: int): + meta = { + "table": issue.name if issue else None, + "dataset_column": None, + "feature_fk": None, + "joined_feature": False, + "joined_cell": False, + "error": None, + } + if not issue: + meta["error"] = "geometry issue table not found" + return [], meta + + ids = dataset_col(issue) + ifk = best_col(issue.columns, ["feature_id", "featureid", "feature"]) + meta["dataset_column"] = ids + meta["feature_fk"] = ifk + + joins = [] + extras = [] + where = [] + params = [] + + fpk = best_col(feature.columns, ["id", "feature_id", "featureid"]) if feature else None + fds = dataset_col(feature) + fcell = best_col(feature.columns, ["cell_id", "cellid", "cell"]) if feature else None + + if feature and ifk and fpk: + joins.append(f"LEFT JOIN {qi(feature.name)} f ON i.{qi(ifk)} = f.{qi(fpk)}") + meta["joined_feature"] = True + for c in feature.columns: + cl = c.lower() + if any(x in cl for x in ["object", "class", "type", "geom", "cell", "feature"]): + extras.append(f"f.{qi(c)} AS {qi('feature__' + c)}") + + if cell and fcell: + cpk = best_col(cell.columns, ["id", "cell_id", "cellid"]) + if cpk: + joins.append(f"LEFT JOIN {qi(cell.name)} c ON f.{qi(fcell)} = c.{qi(cpk)}") + meta["joined_cell"] = True + for cc in cell.columns: + ccl = cc.lower() + if any(x in ccl for x in ["name", "cell", "file", "path"]): + extras.append(f"c.{qi(cc)} AS {qi('cell__' + cc)}") + + if ids: + where.append(f"i.{qi(ids)}=%s") + params.append(dataset_id) + elif meta["joined_feature"] and fds: + where.append(f"f.{qi(fds)}=%s") + params.append(dataset_id) + + select_extra = (", " + ", ".join(extras)) if extras else "" + sql = f""" + SELECT i.*{select_extra} + FROM {qi(issue.name)} i + {' '.join(joins)} + {'WHERE ' + ' AND '.join(where) if where else ''} + LIMIT {int(max_rows)} + """ + try: + cur.execute(sql, params) + return list(cur.fetchall()), meta + except Exception as e: + meta["error"] = str(e) + return [], meta + + +def find_object_class(row: Dict[str, Any]) -> str: + for k, v in row.items(): + kl = k.lower() + if v is not None and "object" in kl and ("class" in kl or "type" in kl or "code" in kl or "name" in kl): + return text(v) + for k, v in row.items(): + if v is not None and "class" in k.lower(): + return text(v) + return "" + + +def find_cell(row: Dict[str, Any]) -> str: + for k, v in row.items(): + kl = k.lower() + if v is not None and kl.startswith("cell__") and any(x in kl for x in ["name", "file", "path", "cell"]): + return text(v) + for k, v in row.items(): + if v is not None and "cell" in k.lower(): + return text(v) + return "" + + +def enrich(rows: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + out = [] + for r in rows: + s = issue_text(r) + x = dict(r) + x["_issue_text"] = s + x["_issue_category"] = classify_issue(s) + x["_issue_stage"] = classify_stage(s) + x["_object_class"] = find_object_class(r) + x["_cell"] = find_cell(r) + out.append(x) + return out + + +def summarize(rows: List[Dict[str, Any]]) -> Dict[str, Any]: + by_cat = Counter(r["_issue_category"] for r in rows) + by_stage = Counter(r["_issue_stage"] for r in rows) + cross = Counter((r["_issue_stage"], r["_issue_category"]) for r in rows) + obj = Counter(r["_object_class"] for r in rows if r["_object_class"]) + cell = Counter(r["_cell"] for r in rows if r["_cell"]) + return { + "total": len(rows), + "by_category": dict(by_cat.most_common()), + "by_stage": dict(by_stage.most_common()), + "by_stage_category": [ + {"stage": s, "category": c, "count": n} + for (s, c), n in cross.most_common() + ], + "by_object_class_top20": [ + {"object_class": k, "count": v} for k, v in obj.most_common(20) + ], + "by_cell_top30": [ + {"cell": k, "count": v} for k, v in cell.most_common(30) + ], + } + + +def duplicate_audit(rows: List[Dict[str, Any]]) -> Dict[str, Any]: + groups = defaultdict(int) + for r in rows: + parts = [] + for k, v in r.items(): + kl = k.lower() + if v is not None and any(x in kl for x in ["feature_id", "featureid", "reason", "message", "detail", "exception", "stage", "issue_type", "error_type"]): + parts.append(f"{k}={text(v)}") + if parts: + groups[tuple(parts)] += 1 + dup = [(k, n) for k, n in groups.items() if n > 1] + dup.sort(key=lambda x: x[1], reverse=True) + return { + "duplicate_groups": len(dup), + "duplicate_rows_beyond_first": sum(n - 1 for _, n in dup), + "examples": [{"count": n, "identity": " | ".join(k)[:1500]} for k, n in dup[:20]], + } + + +def write_csv(path: Path, rows: List[Dict[str, Any]], fields: Optional[List[str]] = None): + path.parent.mkdir(parents=True, exist_ok=True) + if fields is None: + fields = [] + seen = set() + for r in rows: + for k in r: + if k not in seen: + seen.add(k) + fields.append(k) + with path.open("w", encoding="utf-8-sig", newline="") as f: + w = csv.DictWriter(f, fieldnames=fields, extrasaction="ignore") + w.writeheader() + for r in rows: + w.writerow({k: text(r.get(k, "")) for k in fields}) + + +def make_samples(rows: List[Dict[str, Any]], n=5) -> List[Dict[str, Any]]: + buckets = defaultdict(list) + for r in rows: + cat = r["_issue_category"] + if len(buckets[cat]) < n: + buckets[cat].append(r) + result = [] + for cat in sorted(buckets): + result.extend(buckets[cat]) + return result + + +def parse_args(): + p = argparse.ArgumentParser(description="CM93 MySQL Dataset audit + geometry issue classification") + p.add_argument("--host", default=os.getenv("MYSQL_HOST", "127.0.0.1")) + p.add_argument("--port", type=int, default=int(os.getenv("MYSQL_PORT", "3306"))) + p.add_argument("--user", default=os.getenv("MYSQL_USER", "root")) + p.add_argument("--password", default=os.getenv("MYSQL_PASSWORD")) + p.add_argument("--database", default=os.getenv("MYSQL_DATABASE")) + p.add_argument("--dataset-id", type=int, default=1) + p.add_argument("--output", default=None) + p.add_argument("--max-issues", type=int, default=1000000) + return p.parse_args() + + +def main(): + a = parse_args() + if not a.database: + a.database = input("MySQL database name: ").strip() + if not a.password: + a.password = getpass.getpass("MySQL password: ") + + stamp = datetime.now().strftime("%Y%m%d_%H%M%S") + outdir = Path(a.output or f"cm93_audit_dataset_{a.dataset_id}_{stamp}") + outdir.mkdir(parents=True, exist_ok=True) + + report: Dict[str, Any] = { + "generated_at": datetime.now().isoformat(timespec="seconds"), + "database": a.database, + "dataset_id": a.dataset_id, + "read_only": True, + "schema": {}, + "counts": {}, + "relationships": {}, + "geometry": {}, + "issues": {}, + "warnings": [], + } + + conn = None + try: + conn = pymysql.connect( + host=a.host, + port=a.port, + user=a.user, + password=a.password, + database=a.database, + charset="utf8mb4", + cursorclass=DictCursor, + autocommit=False, + connect_timeout=20, + read_timeout=300, + write_timeout=30, + ) + + with conn.cursor() as cur: + try: + cur.execute("SET SESSION TRANSACTION READ ONLY") + except Exception as e: + report["warnings"].append(f"SET SESSION TRANSACTION READ ONLY failed: {e}") + cur.execute("START TRANSACTION READ ONLY") + + print("[1/7] 扫描 schema...") + tables = discover_tables(cur, a.database) + schema = resolve_schema(tables) + report["schema"]["resolved_tables"] = {k: (v.name if v else None) for k, v in schema.items()} + report["schema"]["all_tables"] = {t.name: t.columns for t in tables} + for k, v in report["schema"]["resolved_tables"].items(): + print(f" {k:16s}: {v}") + + print("[2/7] 数量审计...") + for k in ["dataset", "cell", "feature", "attribute", "sounding", "geometry_issue"]: + report["counts"][k] = count_table(cur, schema[k], a.dataset_id) + + print("[3/7] 关系完整性审计...") + report["relationships"]["feature_to_cell"] = orphan_count( + cur, schema["feature"], schema["cell"], + ["cell_id", "cellid", "cell"], ["id", "cell_id", "cellid"], a.dataset_id) + report["relationships"]["attribute_to_feature"] = orphan_count( + cur, schema["attribute"], schema["feature"], + ["feature_id", "featureid", "feature"], ["id", "feature_id", "featureid"], a.dataset_id) + report["relationships"]["sounding_to_feature"] = orphan_count( + cur, schema["sounding"], schema["feature"], + ["feature_id", "featureid", "feature"], ["id", "feature_id", "featureid"], a.dataset_id) + report["relationships"]["issue_to_feature"] = orphan_count( + cur, schema["geometry_issue"], schema["feature"], + ["feature_id", "featureid", "feature"], ["id", "feature_id", "featureid"], a.dataset_id) + + print("[4/7] Geometry 基础审计...") + report["geometry"] = geometry_audit(cur, schema["feature"], a.dataset_id) + + print("[5/7] 读取 geometry issues...") + raw, meta = fetch_issues(cur, schema["geometry_issue"], schema["feature"], schema["cell"], a.dataset_id, a.max_issues) + report["issues"]["meta"] = meta + print(f" loaded={len(raw)}") + + print("[6/7] 分类 geometry issues...") + rows = enrich(raw) + sm = summarize(rows) + dup = duplicate_audit(rows) + report["issues"]["summary"] = sm + report["issues"]["duplicates"] = dup + + write_csv(outdir / "geometry_issues_classified.csv", rows) + write_csv(outdir / "geometry_issue_samples.csv", make_samples(rows, 5)) + write_csv(outdir / "geometry_issue_stage_category.csv", sm["by_stage_category"], ["stage", "category", "count"]) + write_csv(outdir / "geometry_issue_object_class_top20.csv", sm["by_object_class_top20"], ["object_class", "count"]) + write_csv(outdir / "geometry_issue_cell_top30.csv", sm["by_cell_top30"], ["cell", "count"]) + + expected = { + "cell": 784, + "feature": 322465, + "attribute": 405153, + "sounding": 275700, + "geometry_issue": 4406, + } + + lines = [] + lines.append("CM93 -> MySQL Dataset Audit + Geometry Issue Classification") + lines.append("=" * 72) + lines.append(f"Generated: {report['generated_at']}") + lines.append(f"Database: {a.database}") + lines.append(f"Dataset ID: {a.dataset_id}") + lines.append("") + lines.append("Resolved tables:") + for k, v in report["schema"]["resolved_tables"].items(): + lines.append(f" {k:16s}: {v}") + lines.append("") + lines.append("Counts:") + for k, v in report["counts"].items(): + lines.append(f" {k:16s}: count={v['count']} table={v['table']} dataset_col={v['dataset_column']} error={v['error']}") + lines.append("") + lines.append("Relationships:") + for k, v in report["relationships"].items(): + lines.append(f" {k:24s}: orphan={v['count']} error={v['error']}") + lines.append("") + lines.append("Geometry audit:") + lines.append(json.dumps(report["geometry"], ensure_ascii=False, indent=2, default=str)) + lines.append("") + lines.append(f"Geometry issues loaded: {sm['total']}") + lines.append("By category:") + for k, n in sm["by_category"].items(): + pct = n / sm["total"] * 100 if sm["total"] else 0 + lines.append(f" {k:28s} {n:8d} {pct:7.3f}%") + lines.append("") + lines.append("By stage:") + for k, n in sm["by_stage"].items(): + pct = n / sm["total"] * 100 if sm["total"] else 0 + lines.append(f" {k:16s} {n:8d} {pct:7.3f}%") + lines.append("") + lines.append(f"Duplicate groups: {dup['duplicate_groups']}") + lines.append(f"Duplicate rows beyond first: {dup['duplicate_rows_beyond_first']}") + lines.append("") + lines.append("Acceptance checks:") + for k, exp in expected.items(): + actual = report["counts"].get(k, {}).get("count") + lines.append(f" {k:16s}: actual={actual} expected={exp} => {'PASS' if actual == exp else 'CHECK'}") + category_sum = sum(sm["by_category"].values()) + lines.append(f" issue loaded total: actual={sm['total']} expected=4406 => {'PASS' if sm['total'] == 4406 else 'CHECK'}") + lines.append(f" category sum: actual={category_sum} expected={sm['total']} => {'PASS' if category_sum == sm['total'] else 'FAIL'}") + lines.append(f" OTHER count: {sm['by_category'].get('OTHER', 0)}") + lines.append("") + lines.append("说明：") + lines.append(" 1. 本脚本仅根据数据库现有 schema、issue 文本和空间字段进行只读审计。") + lines.append(" 2. OTHER 表示现有文本证据不足，不表示该 issue 可以忽略。") + lines.append(" 3. 本脚本没有修复 geometry，也没有修改任何数据库记录。") + lines.append(" 4. 不包含任何 PBF 逻辑。") + + (outdir / "audit_summary.txt").write_text("\n".join(lines), encoding="utf-8") + (outdir / "audit_report.json").write_text(json.dumps(report, ensure_ascii=False, indent=2, default=str), encoding="utf-8") + + conn.rollback() + + print("[7/7] 完成") + print("") + print("\n".join(lines[-18:])) + print("") + print(f"输出目录: {outdir.resolve()}") + + except KeyboardInterrupt: + if conn: + try: + conn.rollback() + except Exception: + pass + print("已中断。") + sys.exit(130) + except Exception: + if conn: + try: + conn.rollback() + except Exception: + pass + err = traceback.format_exc() + print(err) + try: + (outdir / "fatal_error.txt").write_text(err, encoding="utf-8") + except Exception: + pass + sys.exit(1) + finally: + if conn: + try: + conn.close() + except Exception: + pass + + +if __name__ == "__main__": + main() diff --git a/cm93_mysql_audit_v2.py b/cm93_mysql_audit_v2.py new file mode 100644 index 0000000..d50fd49 --- /dev/null +++ b/cm93_mysql_audit_v2.py @@ -0,0 +1,719 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +CM93 -> MySQL Dataset ID audit + geometry issue classification v2 +Target schema discovered from Dataset 1 audit: + chart_dataset + chart_cell + chart_feature + chart_feature_attribute + chart_sounding_point + chart_point_geometry + chart_line_geometry + chart_area_geometry + chart_coverage + chart_import_job + chart_import_event + +READ ONLY: SELECT only; transaction is opened READ ONLY. +No geometry repair, no re-import, no PBF. + +Dependency: + pip install pymysql + +Example: + python .\\cm93_mysql_audit_v2.py --host 127.0.0.1 --port 3306 --user root --database cm93_chart --dataset-id 1 +""" +from __future__ import annotations + +import argparse +import csv +import getpass +import json +import os +import re +import sys +import traceback +from collections import Counter, defaultdict +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple + +try: + import pymysql + from pymysql.cursors import DictCursor +except ImportError: + print("Missing dependency: pymysql") + print("Install with: pip install pymysql") + sys.exit(2) + +EXPECTED = { + "cells": 784, + "features": 322465, + "attributes": 405153, + "sounding_points": 275700, + "geometry_issues": 4406, +} + +# Conservative text classification. Exact raw issue text is always preserved. +CATEGORY_RULES: List[Tuple[str, Sequence[str]]] = [ + ("INVALID_COORDINATE", [ + r"\bnan\b", r"\binf(?:inity)?\b", r"invalid coordinate", r"coordinate.*out of range", + r"latitude.*out of range", r"longitude.*out of range", r"non[- ]?finite", + ]), + ("INSUFFICIENT_POINTS", [ + r"insufficient point", r"too few point", r"not enough point", r"minimum.*point", + r"less than .* points?", r"requires? .* points?", + ]), + ("RING_NOT_CLOSED", [ + r"ring.*not closed", r"not closed.*ring", r"unclosed ring", r"closed linestring", + ]), + ("SELF_INTERSECTION", [ + r"self[- ]?intersection", r"self intersect", r"ring self", r"bow[- ]?tie", + ]), + ("INVALID_RING", [ + r"invalid ring", r"ring.*invalid", r"illegal ring", r"shell.*invalid", r"hole.*invalid", + r"interior ring", r"exterior ring", + ]), + ("DEGENERATE_GEOMETRY", [ + r"degenerate", r"zero[- ]?length", r"zero[- ]?area", r"collapsed", r"duplicate point", + r"all points.*same", r"too few distinct", + ]), + ("MULTIPART_ASSEMBLY", [ + r"multipart", r"multi[- ]?part", r"cannot.*assemble", r"failed.*assemble", r"assembly failed", + r"cannot.*merge", r"failed.*merge", r"stitch", + ]), + ("TOPOLOGY_MISMATCH", [ + r"topolog", r"missing edge", r"edge.*missing", r"missing node", r"node.*missing", + r"edge reference", r"node reference", r"primitive reference", r"dangling edge", r"broken chain", + r"disconnected", + ]), + ("GEOMETRY_TYPE_MISMATCH", [ + r"type mismatch", r"geometry type", r"expected.*(?:point|line|polygon|area)", + r"unexpected.*(?:point|line|polygon|area)", r"wrong.*geometry", + ]), + ("EMPTY_GEOMETRY", [ + r"empty geometry", r"geometry.*empty", r"no geometry", r"null geometry", r"geometry is null", + r"no coordinates?", r"empty point", r"empty line", r"empty polygon", + ]), + ("CONVERSION_EXCEPTION", [ + r"wkb", r"wkt", r"parse exception", r"conversion exception", r"geometry exception", + r"failed to convert", r"cannot convert", r"invalid gis data", r"spatial.*exception", + ]), +] + +STAGE_RULES: List[Tuple[str, Sequence[str]]] = [ + ("MYSQL_WRITE", [r"mysql", r"database", r"insert", r"spatial", r"wkb", r"wkt"]), + ("VALIDATION", [r"valid", r"self[- ]?intersect", r"ring", r"geometry.*invalid"]), + ("ASSEMBLY", [r"assemble", r"assembly", r"merge", r"stitch", r"multipart", r"chain"]), + ("TOPOLOGY", [r"topolog", r"edge", r"node", r"primitive"]), + ("DECODE", [r"decode", r"parse", r"read", r"binary", r"record"]), + ("SOURCE", [r"source", r"raw", r"cm93"]), +] + + +def qi(s: str) -> str: + return "`" + s.replace("`", "``") + "`" + + +def text(v: Any) -> str: + if v is None: + return "" + if isinstance(v, (bytes, bytearray)): + return v.decode("utf-8", errors="replace") + return str(v) + + +def classify_category(raw: str) -> str: + for category, pats in CATEGORY_RULES: + if any(re.search(p, raw, re.I) for p in pats): + return category + return "OTHER" + + +def classify_stage(raw: str) -> str: + for stage, pats in STAGE_RULES: + if any(re.search(p, raw, re.I) for p in pats): + return stage + return "UNKNOWN" + + +def scalar(cur, sql: str, params: Sequence[Any] = ()) -> Any: + cur.execute(sql, params) + row = cur.fetchone() + return next(iter(row.values())) if row else None + + +def rows(cur, sql: str, params: Sequence[Any] = ()) -> List[Dict[str, Any]]: + cur.execute(sql, params) + return list(cur.fetchall()) + + +def table_exists(cur, db: str, table: str) -> bool: + return bool(scalar(cur, "SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA=%s AND TABLE_NAME=%s", (db, table))) + + +def columns(cur, db: str, table: str) -> Dict[str, Dict[str, Any]]: + out = {} + for r in rows(cur, """ + SELECT COLUMN_NAME, DATA_TYPE, COLUMN_TYPE, IS_NULLABLE, COLUMN_KEY + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA=%s AND TABLE_NAME=%s + ORDER BY ORDINAL_POSITION + """, (db, table)): + out[r["COLUMN_NAME"]] = r + return out + + +def write_csv(path: Path, data: Iterable[Dict[str, Any]], fieldnames: Optional[List[str]] = None) -> None: + data = list(data) + if fieldnames is None: + fieldnames = [] + seen = set() + for r in data: + for k in r: + if k not in seen: + fieldnames.append(k) + seen.add(k) + with path.open("w", encoding="utf-8-sig", newline="") as f: + w = csv.DictWriter(f, fieldnames=fieldnames or [], extrasaction="ignore") + w.writeheader() + for r in data: + w.writerow({k: text(r.get(k)) for k in (fieldnames or [])}) + + +def safe(cur, sql: str, params: Sequence[Any] = ()) -> Dict[str, Any]: + try: + return {"ok": True, "rows": rows(cur, sql, params)} + except Exception as e: + return {"ok": False, "error": f"{type(e).__name__}: {e}"} + + +def normalize_issue_value(v: Any) -> str: + s = text(v).strip() + if not s: + return "" + # If JSON, preserve a stable compact textual form to improve grouping. + try: + obj = json.loads(s) + return json.dumps(obj, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + except Exception: + return s + + +def issue_is_present_sql(col: str) -> str: + # Works for text/varchar/json-like columns; CAST keeps this generic. + return f"{qi(col)} IS NOT NULL AND TRIM(CAST({qi(col)} AS CHAR)) <> '' AND LOWER(TRIM(CAST({qi(col)} AS CHAR))) NOT IN ('0','false','none','null','[]','{{}}')" + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--host", default=os.getenv("MYSQL_HOST", "127.0.0.1")) + ap.add_argument("--port", type=int, default=int(os.getenv("MYSQL_PORT", "3306"))) + ap.add_argument("--user", default=os.getenv("MYSQL_USER", "root")) + ap.add_argument("--password", default=os.getenv("MYSQL_PASSWORD")) + ap.add_argument("--database", default=os.getenv("MYSQL_DATABASE", "cm93_chart")) + ap.add_argument("--dataset-id", type=int, default=1) + ap.add_argument("--output", default=None) + args = ap.parse_args() + + if args.password is None: + args.password = getpass.getpass("MySQL password: ") + + stamp = datetime.now().strftime("%Y%m%d_%H%M%S") + out = Path(args.output or f"cm93_audit_v2_dataset_{args.dataset_id}_{stamp}") + out.mkdir(parents=True, exist_ok=True) + + report: Dict[str, Any] = { + "generated_at": datetime.now().isoformat(timespec="seconds"), + "database": args.database, + "dataset_id": args.dataset_id, + "read_only": True, + "expected": EXPECTED, + "schema": {}, + "counts": {}, + "relationships": {}, + "import_jobs": [], + "cell_status": [], + "geometry_issue_reconciliation": {}, + "geometry_storage": {}, + "issue_summary": {}, + "warnings": [], + } + + conn = None + try: + conn = pymysql.connect( + host=args.host, port=args.port, user=args.user, password=args.password, + database=args.database, charset="utf8mb4", cursorclass=DictCursor, + autocommit=False, connect_timeout=20, read_timeout=600, write_timeout=30, + ) + cur = conn.cursor() + try: + cur.execute("SET SESSION TRANSACTION READ ONLY") + except Exception as e: + report["warnings"].append(f"SET SESSION TRANSACTION READ ONLY failed: {e}") + cur.execute("START TRANSACTION READ ONLY") + + required = [ + "chart_dataset", "chart_cell", "chart_feature", "chart_feature_attribute", + "chart_sounding_point", "chart_point_geometry", "chart_line_geometry", + "chart_area_geometry", "chart_coverage", "chart_import_job", "chart_import_event", + ] + existing = {t: table_exists(cur, args.database, t) for t in required} + report["schema"]["tables"] = existing + report["schema"]["columns"] = {t: columns(cur, args.database, t) for t, ok in existing.items() if ok} + + # ------------------------------------------------------------------ + # Basic counts, scoped correctly through joins when table has no dataset_id. + # ------------------------------------------------------------------ + print("[1/8] Basic counts") + report["counts"]["datasets"] = int(scalar(cur, "SELECT COUNT(*) FROM chart_dataset WHERE id=%s", (args.dataset_id,))) + report["counts"]["cells"] = int(scalar(cur, "SELECT COUNT(*) FROM chart_cell WHERE dataset_id=%s", (args.dataset_id,))) + report["counts"]["features"] = int(scalar(cur, "SELECT COUNT(*) FROM chart_feature WHERE dataset_id=%s", (args.dataset_id,))) + report["counts"]["attributes"] = int(scalar(cur, """ + SELECT COUNT(*) FROM chart_feature_attribute a + JOIN chart_feature f ON f.id=a.feature_id + WHERE f.dataset_id=%s + """, (args.dataset_id,))) + report["counts"]["sounding_points"] = int(scalar(cur, """ + SELECT COUNT(*) FROM chart_sounding_point s + JOIN chart_feature f ON f.id=s.feature_id + WHERE f.dataset_id=%s + """, (args.dataset_id,))) + + # Unknown object/attr dictionary mapping checks. + report["counts"]["feature_object_class_null"] = int(scalar(cur, """ + SELECT COUNT(*) FROM chart_feature + WHERE dataset_id=%s AND (object_class IS NULL OR TRIM(object_class)='') + """, (args.dataset_id,))) + report["counts"]["attribute_name_null"] = int(scalar(cur, """ + SELECT COUNT(*) FROM chart_feature_attribute a + JOIN chart_feature f ON f.id=a.feature_id + WHERE f.dataset_id=%s AND (a.attribute_name IS NULL OR TRIM(a.attribute_name)='') + """, (args.dataset_id,))) + + # ------------------------------------------------------------------ + # Relational integrity. + # ------------------------------------------------------------------ + print("[2/8] Relational integrity") + rel_queries = { + "feature_to_cell_orphan": """ + SELECT COUNT(*) n FROM chart_feature f + LEFT JOIN chart_cell c ON c.id=f.cell_id + WHERE f.dataset_id=%s AND c.id IS NULL + """, + "feature_cell_wrong_dataset": """ + SELECT COUNT(*) n FROM chart_feature f + JOIN chart_cell c ON c.id=f.cell_id + WHERE f.dataset_id=%s AND c.dataset_id<>f.dataset_id + """, + "attribute_to_feature_orphan_global": """ + SELECT COUNT(*) n FROM chart_feature_attribute a + LEFT JOIN chart_feature f ON f.id=a.feature_id + WHERE f.id IS NULL + """, + "sounding_to_feature_orphan_global": """ + SELECT COUNT(*) n FROM chart_sounding_point s + LEFT JOIN chart_feature f ON f.id=s.feature_id + WHERE f.id IS NULL + """, + "point_geometry_orphan_global": """ + SELECT COUNT(*) n FROM chart_point_geometry g + LEFT JOIN chart_feature f ON f.id=g.feature_id + WHERE f.id IS NULL + """, + "line_geometry_orphan_global": """ + SELECT COUNT(*) n FROM chart_line_geometry g + LEFT JOIN chart_feature f ON f.id=g.feature_id + WHERE f.id IS NULL + """, + "area_geometry_orphan_global": """ + SELECT COUNT(*) n FROM chart_area_geometry g + LEFT JOIN chart_feature f ON f.id=g.feature_id + WHERE f.id IS NULL + """, + } + for k, sql in rel_queries.items(): + params = (args.dataset_id,) if "%s" in sql else () + report["relationships"][k] = int(scalar(cur, sql, params)) + + # Duplicate geometry rows by feature_id should not exist. + for table in ["chart_point_geometry", "chart_line_geometry", "chart_area_geometry"]: + report["relationships"][f"{table}_duplicate_feature_ids"] = int(scalar(cur, f""" + SELECT COUNT(*) FROM ( + SELECT g.feature_id FROM {table} g + JOIN chart_feature f ON f.id=g.feature_id + WHERE f.dataset_id=%s + GROUP BY g.feature_id HAVING COUNT(*)>1 + ) x + """, (args.dataset_id,))) + + # ------------------------------------------------------------------ + # Import jobs and cell status: this is likely where 4406 comes from. + # ------------------------------------------------------------------ + print("[3/8] Import job / cell status") + report["import_jobs"] = rows(cur, """ + SELECT id, dataset_id, status, started_at, finished_at, + cells_discovered, cells_done, cells_failed, + features_source, features_imported, + attributes_source, attributes_imported, + unknown_object_count, unknown_attribute_count, + geometry_invalid_count, geometry_repaired_count, summary_json + FROM chart_import_job + WHERE dataset_id=%s + ORDER BY id + """, (args.dataset_id,)) + report["cell_status"] = rows(cur, """ + SELECT import_status, COUNT(*) AS cell_count, + SUM(feature_count) AS feature_count, + SUM(attribute_count) AS attribute_count, + SUM(CASE WHEN error_message IS NOT NULL AND TRIM(error_message)<>'' THEN 1 ELSE 0 END) AS cells_with_error_message + FROM chart_cell + WHERE dataset_id=%s + GROUP BY import_status + ORDER BY cell_count DESC + """, (args.dataset_id,)) + write_csv(out / "import_jobs.csv", report["import_jobs"]) + write_csv(out / "cell_status.csv", report["cell_status"]) + + # ------------------------------------------------------------------ + # Geometry issue storage in chart_feature and chart_coverage. + # ------------------------------------------------------------------ + print("[4/8] Geometry issue reconciliation") + feature_cols = report["schema"]["columns"].get("chart_feature", {}) + coverage_cols = report["schema"]["columns"].get("chart_coverage", {}) + + if "geometry_issue" not in feature_cols: + raise RuntimeError("chart_feature.geometry_issue not found") + + issue_pred = issue_is_present_sql("geometry_issue") + feature_issue_count = int(scalar(cur, f"SELECT COUNT(*) FROM chart_feature WHERE dataset_id=%s AND {issue_pred}", (args.dataset_id,))) + coverage_issue_count = 0 + if "geometry_issue" in coverage_cols: + coverage_issue_count = int(scalar(cur, f"SELECT COUNT(*) FROM chart_coverage WHERE dataset_id=%s AND {issue_pred}", (args.dataset_id,))) + + feature_repaired_count = int(scalar(cur, "SELECT COUNT(*) FROM chart_feature WHERE dataset_id=%s AND COALESCE(geometry_repaired,0)<>0", (args.dataset_id,))) + coverage_repaired_count = 0 + if "geometry_repaired" in coverage_cols: + coverage_repaired_count = int(scalar(cur, "SELECT COUNT(*) FROM chart_coverage WHERE dataset_id=%s AND COALESCE(geometry_repaired,0)<>0", (args.dataset_id,))) + + job_invalid_total = sum(int(r.get("geometry_invalid_count") or 0) for r in report["import_jobs"]) + job_repaired_total = sum(int(r.get("geometry_repaired_count") or 0) for r in report["import_jobs"]) + report["geometry_issue_reconciliation"] = { + "feature_geometry_issue_rows": feature_issue_count, + "coverage_geometry_issue_rows": coverage_issue_count, + "feature_geometry_repaired_rows": feature_repaired_count, + "coverage_geometry_repaired_rows": coverage_repaired_count, + "sum_import_job_geometry_invalid_count": job_invalid_total, + "sum_import_job_geometry_repaired_count": job_repaired_total, + "expected_geometry_issues": EXPECTED["geometry_issues"], + } + + # Distribution of raw feature geometry_issue values. + issue_value_counts = rows(cur, f""" + SELECT CAST(geometry_issue AS CHAR) AS geometry_issue, COUNT(*) AS n + FROM chart_feature + WHERE dataset_id=%s AND {issue_pred} + GROUP BY CAST(geometry_issue AS CHAR) + ORDER BY n DESC, geometry_issue + """, (args.dataset_id,)) + write_csv(out / "geometry_issue_raw_value_counts.csv", issue_value_counts) + + # Import events related to geometry, including raw event_code/message patterns. + geom_events = rows(cur, """ + SELECT e.id, e.job_id, e.cell_id, e.level, e.event_code, e.message, e.details_json, e.created_at, + c.cell_name, c.source_relative_path + FROM chart_import_event e + LEFT JOIN chart_import_job j ON j.id=e.job_id + LEFT JOIN chart_cell c ON c.id=e.cell_id + WHERE j.dataset_id=%s + AND ( + LOWER(COALESCE(e.event_code,'')) LIKE '%%geom%%' + OR LOWER(COALESCE(e.message,'')) LIKE '%%geom%%' + OR LOWER(COALESCE(e.message,'')) LIKE '%%ring%%' + OR LOWER(COALESCE(e.message,'')) LIKE '%%polygon%%' + OR LOWER(COALESCE(e.message,'')) LIKE '%%line%%' + OR LOWER(COALESCE(e.message,'')) LIKE '%%point%%' + OR LOWER(COALESCE(e.message,'')) LIKE '%%topolog%%' + ) + ORDER BY e.id + """, (args.dataset_id,)) + write_csv(out / "geometry_related_import_events.csv", geom_events) + + # ------------------------------------------------------------------ + # Pull exact issue feature rows. Preserve raw geometry_issue and enough context. + # ------------------------------------------------------------------ + print("[5/8] Classifying feature.geometry_issue rows") + issue_rows = rows(cur, f""" + SELECT + f.id AS feature_id, + f.dataset_id, + f.cell_id, + c.cell_name, + c.source_relative_path, + f.coverage_id, + f.source_record_index, + f.object_class_id, + f.object_class, + f.geometry_type, + f.raw_geotype, + f.scale_code, + f.native_scale, + f.geometry_repaired, + CAST(f.geometry_issue AS CHAR) AS geometry_issue, + f.bbox_min_lon, f.bbox_min_lat, f.bbox_max_lon, f.bbox_max_lat, + CASE WHEN pg.feature_id IS NOT NULL THEN 1 ELSE 0 END AS has_point_geometry, + CASE WHEN lg.feature_id IS NOT NULL THEN 1 ELSE 0 END AS has_line_geometry, + CASE WHEN ag.feature_id IS NOT NULL THEN 1 ELSE 0 END AS has_area_geometry, + (SELECT COUNT(*) FROM chart_sounding_point sp WHERE sp.feature_id=f.id) AS sounding_point_count + FROM chart_feature f + JOIN chart_cell c ON c.id=f.cell_id + LEFT JOIN chart_point_geometry pg ON pg.feature_id=f.id + LEFT JOIN chart_line_geometry lg ON lg.feature_id=f.id + LEFT JOIN chart_area_geometry ag ON ag.feature_id=f.id + WHERE f.dataset_id=%s AND {issue_pred.replace(qi('geometry_issue'), 'f.'+qi('geometry_issue'))} + ORDER BY f.id + """, (args.dataset_id,)) + + for r in issue_rows: + raw = normalize_issue_value(r.get("geometry_issue")) + r["issue_normalized"] = raw + r["issue_category"] = classify_category(raw) + r["issue_stage"] = classify_stage(raw) + has_geom = bool(r.get("has_point_geometry") or r.get("has_line_geometry") or r.get("has_area_geometry") or int(r.get("sounding_point_count") or 0) > 0) + r["has_any_geometry_storage"] = 1 if has_geom else 0 + + # Conservative evidence label. Do not claim importer bug from text alone. + if r["issue_category"] == "OTHER": + r["assessment"] = "NEEDS_TEXT_REVIEW" + elif r["issue_category"] in {"TOPOLOGY_MISMATCH", "MULTIPART_ASSEMBLY", "CONVERSION_EXCEPTION"}: + r["assessment"] = "IMPORTER_OR_SOURCE_CANDIDATE" + else: + r["assessment"] = "GEOMETRY_DATA_OR_ASSEMBLY_ISSUE" + + write_csv(out / "geometry_issues_classified_v2.csv", issue_rows) + + # Counts by category/stage/object/cell/raw issue text. + by_cat = Counter(r["issue_category"] for r in issue_rows) + by_stage = Counter(r["issue_stage"] for r in issue_rows) + by_cross = Counter((r["issue_stage"], r["issue_category"]) for r in issue_rows) + by_obj = Counter((text(r.get("object_class_id")), text(r.get("object_class"))) for r in issue_rows) + by_cell = Counter((text(r.get("cell_name")), text(r.get("source_relative_path"))) for r in issue_rows) + by_raw = Counter(r["issue_normalized"] for r in issue_rows) + + issue_summary = { + "feature_issue_rows_loaded": len(issue_rows), + "category_sum": sum(by_cat.values()), + "other_count": by_cat.get("OTHER", 0), + "by_category": [{"category": k, "count": v} for k, v in by_cat.most_common()], + "by_stage": [{"stage": k, "count": v} for k, v in by_stage.most_common()], + "by_stage_category": [{"stage": k[0], "category": k[1], "count": v} for k, v in by_cross.most_common()], + "by_object_class_top50": [{"object_class_id": k[0], "object_class": k[1], "count": v} for k, v in by_obj.most_common(50)], + "by_cell_top50": [{"cell_name": k[0], "source_relative_path": k[1], "count": v} for k, v in by_cell.most_common(50)], + "raw_issue_top100": [{"geometry_issue": k, "count": v} for k, v in by_raw.most_common(100)], + } + report["issue_summary"] = issue_summary + write_csv(out / "geometry_issue_by_category.csv", issue_summary["by_category"]) + write_csv(out / "geometry_issue_by_stage.csv", issue_summary["by_stage"]) + write_csv(out / "geometry_issue_stage_category.csv", issue_summary["by_stage_category"]) + write_csv(out / "geometry_issue_object_class_top50.csv", issue_summary["by_object_class_top50"]) + write_csv(out / "geometry_issue_cell_top50.csv", issue_summary["by_cell_top50"]) + write_csv(out / "geometry_issue_raw_top100.csv", issue_summary["raw_issue_top100"]) + + # Samples: up to 10 per category, prioritizing different object/cell/raw combinations. + samples = [] + seen_per_cat: Dict[str, set] = defaultdict(set) + for r in issue_rows: + cat = r["issue_category"] + sig = (r.get("object_class_id"), r.get("cell_id"), r.get("issue_normalized")) + if len(seen_per_cat[cat]) < 10 and sig not in seen_per_cat[cat]: + samples.append(r) + seen_per_cat[cat].add(sig) + write_csv(out / "geometry_issue_samples_v2.csv", samples) + + # ------------------------------------------------------------------ + # Geometry storage audit: true spatial columns live in separate tables. + # ------------------------------------------------------------------ + print("[6/8] Spatial geometry storage audit") + storage = {} + for name, table in [("POINT", "chart_point_geometry"), ("LINE", "chart_line_geometry"), ("AREA", "chart_area_geometry")]: + item: Dict[str, Any] = {} + item["row_count"] = int(scalar(cur, f""" + SELECT COUNT(*) FROM {table} g + JOIN chart_feature f ON f.id=g.feature_id + WHERE f.dataset_id=%s + """, (args.dataset_id,))) + item["feature_count"] = int(scalar(cur, f""" + SELECT COUNT(DISTINCT g.feature_id) FROM {table} g + JOIN chart_feature f ON f.id=g.feature_id + WHERE f.dataset_id=%s + """, (args.dataset_id,))) + item["null_geom"] = int(scalar(cur, f""" + SELECT COUNT(*) FROM {table} g + JOIN chart_feature f ON f.id=g.feature_id + WHERE f.dataset_id=%s AND g.geom IS NULL + """, (args.dataset_id,))) + + # Spatial functions may fail on a bad row. Capture rather than abort entire audit. + st = safe(cur, f""" + SELECT + ST_GeometryType(g.geom) AS geometry_type, + COUNT(*) AS n, + SUM(CASE WHEN ST_IsEmpty(g.geom) THEN 1 ELSE 0 END) AS empty_count, + SUM(CASE WHEN ST_IsValid(g.geom) THEN 1 ELSE 0 END) AS valid_count, + SUM(CASE WHEN NOT ST_IsValid(g.geom) THEN 1 ELSE 0 END) AS invalid_count + FROM {table} g + JOIN chart_feature f ON f.id=g.feature_id + WHERE f.dataset_id=%s AND g.geom IS NOT NULL + GROUP BY ST_GeometryType(g.geom) + ORDER BY n DESC + """, (args.dataset_id,)) + item["spatial_function_result"] = st + storage[name] = item + + # Sounding points have their own geom. + sounding_item: Dict[str, Any] = { + "row_count": report["counts"]["sounding_points"], + "null_geom": int(scalar(cur, """ + SELECT COUNT(*) FROM chart_sounding_point s + JOIN chart_feature f ON f.id=s.feature_id + WHERE f.dataset_id=%s AND s.geom IS NULL + """, (args.dataset_id,))), + } + sounding_item["spatial_function_result"] = safe(cur, """ + SELECT ST_GeometryType(s.geom) AS geometry_type, COUNT(*) AS n, + SUM(CASE WHEN ST_IsEmpty(s.geom) THEN 1 ELSE 0 END) AS empty_count, + SUM(CASE WHEN ST_IsValid(s.geom) THEN 1 ELSE 0 END) AS valid_count, + SUM(CASE WHEN NOT ST_IsValid(s.geom) THEN 1 ELSE 0 END) AS invalid_count + FROM chart_sounding_point s + JOIN chart_feature f ON f.id=s.feature_id + WHERE f.dataset_id=%s AND s.geom IS NOT NULL + GROUP BY ST_GeometryType(s.geom) + """, (args.dataset_id,)) + storage["SOUNDING"] = sounding_item + report["geometry_storage"] = storage + + # Geometry type distribution from feature semantic field, separate from spatial columns. + geom_type_dist = rows(cur, """ + SELECT geometry_type, raw_geotype, COUNT(*) AS n, + SUM(CASE WHEN geometry_repaired<>0 THEN 1 ELSE 0 END) AS repaired_n, + SUM(CASE WHEN geometry_issue IS NOT NULL AND TRIM(CAST(geometry_issue AS CHAR))<>'' THEN 1 ELSE 0 END) AS issue_n + FROM chart_feature + WHERE dataset_id=%s + GROUP BY geometry_type, raw_geotype + ORDER BY n DESC + """, (args.dataset_id,)) + write_csv(out / "feature_geometry_type_distribution.csv", geom_type_dist) + + # ------------------------------------------------------------------ + # Cell-level issue rates. + # ------------------------------------------------------------------ + print("[7/8] Cell issue rates") + cell_issue_rates = rows(cur, f""" + SELECT c.id AS cell_id, c.cell_name, c.source_relative_path, c.feature_count, + COUNT(f.id) AS feature_rows, + SUM(CASE WHEN {issue_pred.replace(qi('geometry_issue'), 'f.'+qi('geometry_issue'))} THEN 1 ELSE 0 END) AS issue_count, + ROUND(100.0 * SUM(CASE WHEN {issue_pred.replace(qi('geometry_issue'), 'f.'+qi('geometry_issue'))} THEN 1 ELSE 0 END) / NULLIF(COUNT(f.id),0), 6) AS issue_rate_pct + FROM chart_cell c + LEFT JOIN chart_feature f ON f.cell_id=c.id AND f.dataset_id=c.dataset_id + WHERE c.dataset_id=%s + GROUP BY c.id, c.cell_name, c.source_relative_path, c.feature_count + ORDER BY issue_count DESC, c.id + """, (args.dataset_id,)) + write_csv(out / "geometry_issue_cell_rates_all.csv", cell_issue_rates) + write_csv(out / "geometry_issue_cell_top30_count.csv", cell_issue_rates[:30]) + write_csv(out / "geometry_issue_cell_top30_rate.csv", sorted(cell_issue_rates, key=lambda r: float(r.get("issue_rate_pct") or 0), reverse=True)[:30]) + + # ------------------------------------------------------------------ + # Final report and acceptance checks. + # ------------------------------------------------------------------ + print("[8/8] Writing report") + report["acceptance"] = { + "cells_match": report["counts"]["cells"] == EXPECTED["cells"], + "features_match": report["counts"]["features"] == EXPECTED["features"], + "attributes_match": report["counts"]["attributes"] == EXPECTED["attributes"], + "sounding_points_match": report["counts"]["sounding_points"] == EXPECTED["sounding_points"], + "feature_issue_rows_match_4406": len(issue_rows) == EXPECTED["geometry_issues"], + "category_sum_equals_issue_rows": sum(by_cat.values()) == len(issue_rows), + "no_feature_cell_orphans": report["relationships"]["feature_to_cell_orphan"] == 0, + "no_attribute_orphans": report["relationships"]["attribute_to_feature_orphan_global"] == 0, + "no_sounding_orphans": report["relationships"]["sounding_to_feature_orphan_global"] == 0, + } + + (out / "audit_report_v2.json").write_text(json.dumps(report, ensure_ascii=False, indent=2, default=str), encoding="utf-8") + + lines = [] + lines.append("CM93 -> MySQL Dataset ID=1 Audit v2") + lines.append("=" * 78) + lines.append(f"Generated: {report['generated_at']}") + lines.append(f"Database: {args.database}") + lines.append(f"Dataset ID: {args.dataset_id}") + lines.append("Mode: READ ONLY / NO PBF") + lines.append("") + lines.append("COUNTS") + for k, v in report["counts"].items(): + lines.append(f" {k:30s} {v}") + lines.append("") + lines.append("RELATIONSHIPS") + for k, v in report["relationships"].items(): + lines.append(f" {k:40s} {v}") + lines.append("") + lines.append("GEOMETRY ISSUE RECONCILIATION") + for k, v in report["geometry_issue_reconciliation"].items(): + lines.append(f" {k:42s} {v}") + lines.append("") + lines.append("ISSUE CATEGORY") + for r in issue_summary["by_category"]: + pct = (100.0 * r["count"] / len(issue_rows)) if issue_rows else 0 + lines.append(f" {r['category']:30s} {r['count']:7d} {pct:8.3f}%") + lines.append("") + lines.append("ISSUE STAGE") + for r in issue_summary["by_stage"]: + pct = (100.0 * r["count"] / len(issue_rows)) if issue_rows else 0 + lines.append(f" {r['stage']:20s} {r['count']:7d} {pct:8.3f}%") + lines.append("") + lines.append("TOP RAW ISSUE VALUES") + for r in issue_summary["raw_issue_top100"][:30]: + s = r["geometry_issue"].replace("\n", " ") + lines.append(f" {r['count']:7d} {s[:180]}") + lines.append("") + lines.append("ACCEPTANCE") + for k, v in report["acceptance"].items(): + lines.append(f" {k:42s} {'PASS' if v else 'CHECK'}") + lines.append("") + lines.append("IMPORTANT") + lines.append(" - category_sum must equal feature_issue_rows_loaded.") + lines.append(" - OTHER is not considered fully understood; inspect raw top values/samples.") + lines.append(" - No repair or database mutation was performed.") + lines.append(" - PBF remains out of scope.") + (out / "audit_summary_v2.txt").write_text("\n".join(lines), encoding="utf-8") + + conn.rollback() + print("\n".join(lines)) + print(f"\nOutput: {out.resolve()}") + return 0 + + except Exception: + if conn: + try: + conn.rollback() + except Exception: + pass + err = traceback.format_exc() + print(err) + try: + (out / "fatal_error.txt").write_text(err, encoding="utf-8") + except Exception: + pass + return 1 + finally: + if conn: + try: + conn.close() + except Exception: + pass + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/cm93_mysql_fillrule_audit_v8.py b/cm93_mysql_fillrule_audit_v8.py new file mode 100644 index 0000000..a03313c --- /dev/null +++ b/cm93_mysql_fillrule_audit_v8.py @@ -0,0 +1,1162 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +CM93 -> MySQL Invalid Polygon 线性残片与填充规则诊断 +Version: v8.0 + +背景 +==== +前序 V7 已确认： +- invalid total = 55（Area 52 + Coverage 3） +- 41 条 make_valid 后得到 Polygon + LineString +- 14 条得到纯 Polygon/MultiPolygon +- 其中 5 条 make_valid 面选择与“polygonize 全部 faces”不同 +- 31 条 polygonize 有 dangles，10 条有 cuts，恰好合计 41 + +V8 目标 +======= +只做只读、内存诊断，不修复数据库。 + +A. 验证 41 条线性残片的来源 +-------------------------------- +对 make_valid 产生的每一条线性 remainder： +1. 将 remainder 拆成最小线段； +2. 对每个 remainder segment 的中点，反查原始 invalid Polygon 的所有 ring segment； +3. 统计该点被多少条原始 segment 覆盖； +4. 判断覆盖 segment 相对于 remainder segment 的方向： + - 正向 + - 反向 +5. 分类： + - OPPOSITE_DIRECTION_RETRACE + 同一位置至少有两条原始 segment，且存在正反两个方向； + 这是“往返桥接/重走线”的强证据。 + - SAME_DIRECTION_OVERLAP + - SINGLE_TRAVERSAL + - MULTI_COVER_NO_OPPOSITE + - UNMATCHED +6. 对每个对象给出 record-level 分类： + - RETRACED_CONNECTOR_CONFIRMED + - RETRACED_CONNECTOR_MIXED + - NONRETRACED_LINEAR_REMAINDER + - NO_LINEAR_REMAINDER + +B. 验证 polygon face 填充规则 +-------------------------------- +对原始 invalid geometry.boundary： +1. unary_union + polygonize_full，得到所有 planar faces； +2. 对每个 face 取 representative point； +3. 对原始 ring 坐标计算： + - Odd-even parity + - Non-zero winding number +4. 检查该 face 是否被 Shapely make_valid 的 polygonal result 选中； +5. 每条记录判定： + - MATCHES_BOTH_ODD_EVEN_AND_NONZERO + - MATCHES_ODD_EVEN_ONLY + - MATCHES_NONZERO_ONLY + - MATCHES_NEITHER + - NO_FACES + +这一步尤其解释 V7 中 5 条 POLYGON_ONLY_FACE_SELECTION_DIFFERS： +不是坐标变了，而是“全部 polygonized faces 中哪些应该填充”存在语义选择。 + +C. 输出结论候选 +-------------------------------- +仅用于诊断，不自动修改数据库： + +- OGC_AREA_WITH_RETRACED_CONNECTOR + 面部分可有效表达，线性残片基本确认是原 ring 的正反重走桥接线。 + +- OGC_AREA_WITH_NONRETRACED_REMAINDER + 存在线性残片，但无法解释为简单正反重走，需要继续调查。 + +- OGC_POLYGONAL_FILL_RULE_RESOLVED + 无线性残片，且 make_valid face selection 能被 odd-even/nonzero 规则解释。 + +- OGC_POLYGONAL_FILL_RULE_UNRESOLVED + face selection 无法由两种常见规则解释。 + +重要边界 +======== +当前 CM93JAPAN 是日本沿海局部子集，不是完整 CM93： +- 跨未导入海域连续性不作为 FAIL； +- Cell 边界上的共享拓扑保留 boundary context； +- V8 不将“Cell 边界 self-touch”自动判为源数据损坏。 + +禁止 +==== +- 不 UPDATE / INSERT / DELETE +- 不写回 make_valid +- 不 buffer(0) +- 不修改 geometry_issue +- 不开始 PBF + +依赖 +==== + pip install pymysql shapely + +执行 +==== + python cm93_mysql_fillrule_audit_v8.py ^ + --host 127.0.0.1 ^ + --port 3306 ^ + --user root ^ + --database cm93_chart ^ + --dataset-id 1 + +输出 +==== +- fillrule_audit_summary_v8.txt +- fillrule_audit_report_v8.json +- invalid_polygon_fillrule_classification.csv +- linear_remainder_segment_evidence.csv +- polygon_face_fillrule_evidence.csv +- fillrule_by_object_class.csv +- fillrule_by_boundary_context.csv +- retraced_connector_summary.csv +- fillrule_diagnostic.geojson +并自动打 ZIP。 +""" + +from __future__ import annotations + +import argparse +import csv +import getpass +import json +import math +import os +import re +import sys +import traceback +import zipfile +from collections import Counter, defaultdict +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple + +try: + import pymysql + from pymysql.cursors import DictCursor +except ImportError: + print("缺少 pymysql：pip install pymysql shapely") + sys.exit(2) + +try: + import shapely + try: + from shapely import from_wkb as _from_wkb + def load_wkb(data: bytes): + return _from_wkb(data) + except ImportError: + from shapely import wkb as _wkb + def load_wkb(data: bytes): + return _wkb.loads(data) + + try: + from shapely import make_valid as _make_valid + def make_valid_geom(g): + return _make_valid(g) + except ImportError: + from shapely.validation import make_valid as _make_valid + def make_valid_geom(g): + return _make_valid(g) + + from shapely.geometry import Point, LineString, mapping + from shapely.ops import unary_union, polygonize_full + from shapely.validation import explain_validity +except ImportError: + print("缺少 shapely：pip install pymysql shapely") + sys.exit(2) + + +INVALID_POINT_RE = re.compile( + r"\[\s*([-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)\s+" + r"([-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)\s*\]" +) + + +def scalar(cur, sql: str, params: Sequence[Any] = ()) -> Any: + cur.execute(sql, params) + row = cur.fetchone() + return next(iter(row.values())) if row else None + + +def begin_read_only(cur, warnings: List[str]): + try: + cur.execute("SET SESSION TRANSACTION READ ONLY") + except Exception as e: + warnings.append(f"SET SESSION TRANSACTION READ ONLY failed: {e}") + try: + cur.execute("START TRANSACTION READ ONLY") + except Exception as e: + warnings.append(f"START TRANSACTION READ ONLY failed: {e}") + cur.execute("START TRANSACTION") + + +def to_bytes(v: Any) -> Optional[bytes]: + if v is None: + return None + if isinstance(v, bytes): + return v + if isinstance(v, bytearray): + return bytes(v) + if isinstance(v, memoryview): + return v.tobytes() + try: + return bytes(v) + except Exception: + return None + + +def write_csv(path: Path, rows: Iterable[Dict[str, Any]], + fieldnames: Optional[List[str]] = None): + rows = list(rows) + if fieldnames is None: + seen = set() + fieldnames = [] + for r in rows: + for k in r.keys(): + if k not in seen: + seen.add(k) + fieldnames.append(k) + + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8-sig", newline="") as f: + if not fieldnames: + return + w = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore") + w.writeheader() + for r in rows: + out = {} + for k in fieldnames: + v = r.get(k) + if isinstance(v, (dict, list, tuple)): + v = json.dumps(v, ensure_ascii=False, default=str) + out[k] = "" if v is None else v + w.writerow(out) + + +def probe_wkb_fn(cur) -> str: + for fn in ("ST_AsBinary", "ST_AsWKB", "AsBinary", "AsWKB"): + try: + cur.execute(f"SELECT {fn}(geom) AS x FROM chart_area_geometry LIMIT 1") + cur.fetchone() + return fn + except Exception: + pass + raise RuntimeError("找不到 geometry -> WKB 函数") + + +def parse_invalid_point(reason: str) -> Optional[Tuple[float, float]]: + if not reason: + return None + m = INVALID_POINT_RE.search(reason) + if not m: + return None + try: + return float(m.group(1)), float(m.group(2)) + except Exception: + return None + + +def bbox_from_row(row: Dict[str, Any]) -> Optional[Tuple[float, float, float, float]]: + try: + vals = ( + float(row["cell_bbox_min_lon"]), + float(row["cell_bbox_min_lat"]), + float(row["cell_bbox_max_lon"]), + float(row["cell_bbox_max_lat"]), + ) + if all(math.isfinite(v) for v in vals): + return vals + except Exception: + pass + return None + + +def invalid_point_cell_edge( + p: Optional[Tuple[float, float]], + bbox: Optional[Tuple[float, float, float, float]], + tol: float +) -> Tuple[int, str, Optional[float]]: + if p is None or bbox is None: + return 0, "", None + + x, y = p + minx, miny, maxx, maxy = bbox + d = { + "WEST": abs(x - minx), + "EAST": abs(x - maxx), + "SOUTH": abs(y - miny), + "NORTH": abs(y - maxy), + } + edge, dist = min(d.items(), key=lambda kv: kv[1]) + return int(dist <= tol), edge if dist <= tol else "", dist + + +def flatten_components(g) -> Dict[str, List[Any]]: + result = {"polygons": [], "lines": [], "points": [], "others": []} + + def rec(x): + if x is None: + return + t = x.geom_type + if t == "Polygon": + result["polygons"].append(x) + elif t == "MultiPolygon": + for y in x.geoms: + rec(y) + elif t in ("LineString", "LinearRing"): + result["lines"].append(x) + elif t == "MultiLineString": + for y in x.geoms: + rec(y) + elif t == "Point": + result["points"].append(x) + elif t == "MultiPoint": + for y in x.geoms: + rec(y) + elif hasattr(x, "geoms"): + for y in x.geoms: + rec(y) + else: + result["others"].append(x) + + rec(g) + return result + + +def geom_union(items: Sequence[Any]) -> Optional[Any]: + if not items: + return None + try: + return unary_union(list(items)) + except Exception: + return None + + +def original_rings(geom) -> List[List[Tuple[float, float]]]: + rings: List[List[Tuple[float, float]]] = [] + + def add_polygon(poly): + rings.append([(float(c[0]), float(c[1])) for c in poly.exterior.coords]) + for ring in poly.interiors: + rings.append([(float(c[0]), float(c[1])) for c in ring.coords]) + + if geom.geom_type == "Polygon": + add_polygon(geom) + elif geom.geom_type == "MultiPolygon": + for p in geom.geoms: + add_polygon(p) + return rings + + +def original_segments(geom) -> List[Dict[str, Any]]: + segs = [] + for ri, ring in enumerate(original_rings(geom)): + for si in range(len(ring) - 1): + a = ring[si] + b = ring[si + 1] + if a == b: + continue + try: + line = LineString([a, b]) + except Exception: + continue + segs.append({ + "ring_index": ri, + "segment_index": si, + "a": a, + "b": b, + "line": line, + }) + return segs + + +def vector_direction(a: Tuple[float, float], + b: Tuple[float, float]) -> Tuple[float, float]: + return b[0] - a[0], b[1] - a[1] + + +def dot(a: Tuple[float, float], b: Tuple[float, float]) -> float: + return a[0] * b[0] + a[1] * b[1] + + +def cross(a: Tuple[float, float], b: Tuple[float, float]) -> float: + return a[0] * b[1] - a[1] * b[0] + + +def classify_remainder_segment( + ra: Tuple[float, float], + rb: Tuple[float, float], + source_segments: List[Dict[str, Any]], + tol: float, +) -> Dict[str, Any]: + rvec = vector_direction(ra, rb) + rlen = math.hypot(rvec[0], rvec[1]) + if rlen <= 0: + return { + "coverage_count": 0, + "same_direction_count": 0, + "opposite_direction_count": 0, + "segment_class": "ZERO_LENGTH_REMAINDER_SEGMENT", + } + + mid = Point((ra[0] + rb[0]) / 2.0, (ra[1] + rb[1]) / 2.0) + same = 0 + opposite = 0 + covers = 0 + source_hits = [] + + for s in source_segments: + line = s["line"] + # midpoint 必须落在线段上或非常接近 + try: + if line.distance(mid) > tol: + continue + except Exception: + continue + + svec = vector_direction(s["a"], s["b"]) + slen = math.hypot(svec[0], svec[1]) + if slen <= 0: + continue + + # 要求方向基本共线，避免“恰好在交点处”的非同线 segment 被误计。 + sin_angle = abs(cross(rvec, svec)) / (rlen * slen) + if sin_angle > 1e-7: + continue + + covers += 1 + dp = dot(rvec, svec) + if dp >= 0: + same += 1 + direction = "SAME" + else: + opposite += 1 + direction = "OPPOSITE" + + source_hits.append({ + "ring_index": s["ring_index"], + "segment_index": s["segment_index"], + "direction": direction, + }) + + if covers >= 2 and same > 0 and opposite > 0: + cls = "OPPOSITE_DIRECTION_RETRACE" + elif covers >= 2 and opposite == 0: + cls = "SAME_DIRECTION_OVERLAP" + elif covers == 1: + cls = "SINGLE_TRAVERSAL" + elif covers >= 2: + cls = "MULTI_COVER_NO_OPPOSITE" + else: + cls = "UNMATCHED" + + return { + "coverage_count": covers, + "same_direction_count": same, + "opposite_direction_count": opposite, + "segment_class": cls, + "source_hits": source_hits, + } + + +def record_remainder_class(segment_classes: Counter) -> str: + total = sum(segment_classes.values()) + if total == 0: + return "NO_LINEAR_REMAINDER" + + retraced = segment_classes.get("OPPOSITE_DIRECTION_RETRACE", 0) + unmatched = segment_classes.get("UNMATCHED", 0) + single = segment_classes.get("SINGLE_TRAVERSAL", 0) + same = segment_classes.get("SAME_DIRECTION_OVERLAP", 0) + other = total - retraced - unmatched - single - same + + if retraced == total: + return "RETRACED_CONNECTOR_CONFIRMED" + if retraced > 0 and (single > 0 or same > 0 or unmatched > 0 or other > 0): + return "RETRACED_CONNECTOR_MIXED" + return "NONRETRACED_LINEAR_REMAINDER" + + +# --------------------------------------------------------------------------- +# Fill rules +# --------------------------------------------------------------------------- + +def point_on_segment(px, py, ax, ay, bx, by, eps=1e-12) -> bool: + vx, vy = bx - ax, by - ay + wx, wy = px - ax, py - ay + c = abs(vx * wy - vy * wx) + if c > eps * max(1.0, abs(vx), abs(vy)): + return False + d = wx * vx + wy * vy + if d < -eps: + return False + vv = vx * vx + vy * vy + if d - vv > eps: + return False + return True + + +def even_odd_inside(point: Tuple[float, float], + rings: List[List[Tuple[float, float]]]) -> bool: + x, y = point + crossings = 0 + + for ring in rings: + for i in range(len(ring) - 1): + x1, y1 = ring[i] + x2, y2 = ring[i + 1] + + if point_on_segment(x, y, x1, y1, x2, y2): + # representative point 理论上不会落边界；若发生，按 inside 处理。 + return True + + if (y1 > y) != (y2 > y): + xinters = x1 + (y - y1) * (x2 - x1) / (y2 - y1) + if xinters > x: + crossings += 1 + + return (crossings % 2) == 1 + + +def is_left(ax, ay, bx, by, px, py) -> float: + return (bx - ax) * (py - ay) - (px - ax) * (by - ay) + + +def winding_number(point: Tuple[float, float], + rings: List[List[Tuple[float, float]]]) -> int: + x, y = point + wn = 0 + + for ring in rings: + for i in range(len(ring) - 1): + x1, y1 = ring[i] + x2, y2 = ring[i + 1] + + if y1 <= y: + if y2 > y and is_left(x1, y1, x2, y2, x, y) > 0: + wn += 1 + else: + if y2 <= y and is_left(x1, y1, x2, y2, x, y) < 0: + wn -= 1 + return wn + + +def fill_rule_record_class(face_rows: List[Dict[str, Any]]) -> str: + if not face_rows: + return "NO_FACES" + + odd_ok = all( + bool(r["selected_by_make_valid"]) == bool(r["odd_even_inside"]) + for r in face_rows + ) + nonzero_ok = all( + bool(r["selected_by_make_valid"]) == bool(r["nonzero_inside"]) + for r in face_rows + ) + + if odd_ok and nonzero_ok: + return "MATCHES_BOTH_ODD_EVEN_AND_NONZERO" + if odd_ok: + return "MATCHES_ODD_EVEN_ONLY" + if nonzero_ok: + return "MATCHES_NONZERO_ONLY" + return "MATCHES_NEITHER" + + +def final_semantic_class(remainder_class: str, fill_class: str) -> str: + if remainder_class == "RETRACED_CONNECTOR_CONFIRMED": + return "OGC_AREA_WITH_RETRACED_CONNECTOR" + if remainder_class in { + "RETRACED_CONNECTOR_MIXED", + "NONRETRACED_LINEAR_REMAINDER", + }: + return "OGC_AREA_WITH_NONRETRACED_REMAINDER" + + if fill_class != "MATCHES_NEITHER": + return "OGC_POLYGONAL_FILL_RULE_RESOLVED" + return "OGC_POLYGONAL_FILL_RULE_UNRESOLVED" + + +# --------------------------------------------------------------------------- +# DB +# --------------------------------------------------------------------------- + +def load_invalid_records(cur, dataset_id: int, wkb_fn: str) -> List[Dict[str, Any]]: + records = [] + + cur.execute(f""" + SELECT + 'AREA' AS kind, + f.id AS feature_id, + NULL AS coverage_geom_id, + f.object_class, + f.cell_id, + c.cell_name, + f.source_record_index, + f.coverage_id, + f.raw_geotype, + c.bbox_min_lon AS cell_bbox_min_lon, + c.bbox_min_lat AS cell_bbox_min_lat, + c.bbox_max_lon AS cell_bbox_max_lon, + c.bbox_max_lat AS cell_bbox_max_lat, + {wkb_fn}(g.geom) AS geom_wkb + FROM chart_feature f + JOIN chart_area_geometry g ON g.feature_id=f.id + JOIN chart_cell c ON c.id=f.cell_id + WHERE f.dataset_id=%s + AND f.geometry_type='AREA' + ORDER BY f.id + """, (dataset_id,)) + + for r in cur: + try: + geom = load_wkb(to_bytes(r["geom_wkb"])) + if not geom.is_valid: + x = dict(r) + x["_geom"] = geom + records.append(x) + except Exception: + continue + + cur.execute(f""" + SELECT + 'COVERAGE' AS kind, + NULL AS feature_id, + v.id AS coverage_geom_id, + 'COVERAGE' AS object_class, + v.cell_id, + c.cell_name, + v.source_record_index, + NULL AS coverage_id, + NULL AS raw_geotype, + c.bbox_min_lon AS cell_bbox_min_lon, + c.bbox_min_lat AS cell_bbox_min_lat, + c.bbox_max_lon AS cell_bbox_max_lon, + c.bbox_max_lat AS cell_bbox_max_lat, + {wkb_fn}(v.geom) AS geom_wkb + FROM chart_coverage v + JOIN chart_cell c ON c.id=v.cell_id + WHERE v.dataset_id=%s + ORDER BY v.id + """, (dataset_id,)) + + for r in cur: + try: + geom = load_wkb(to_bytes(r["geom_wkb"])) + if not geom.is_valid: + x = dict(r) + x["_geom"] = geom + records.append(x) + except Exception: + continue + + return records + + +def make_geojson(records: List[Dict[str, Any]], path: Path): + features = [] + + def add(g, props): + if g is None: + return + try: + features.append({ + "type": "Feature", + "geometry": mapping(g), + "properties": props, + }) + except Exception: + pass + + for r in records: + rid = r.get("feature_id") if r["kind"] == "AREA" else r.get("coverage_geom_id") + common = { + "kind": r["kind"], + "record_id": rid, + "object_class": r.get("object_class"), + "cell_name": r.get("cell_name"), + "remainder_class": r.get("_remainder_class"), + "fill_rule_class": r.get("_fill_rule_class"), + "final_semantic_class": r.get("_final_semantic_class"), + } + + add(r.get("_geom"), {**common, "layer": "original_invalid"}) + add(r.get("_mv_polygon_union"), {**common, "layer": "make_valid_polygonal"}) + add(r.get("_mv_line_union"), {**common, "layer": "linear_remainder"}) + + p = r.get("_invalid_point") + if p: + add(Point(p), {**common, "layer": "invalidity_point"}) + + path.write_text( + json.dumps( + {"type": "FeatureCollection", "features": features}, + ensure_ascii=False, + separators=(",", ":"), + default=str, + ), + encoding="utf-8" + ) + + +def zip_dir(folder: Path) -> Path: + zpath = folder.with_suffix(".zip") + with zipfile.ZipFile(zpath, "w", compression=zipfile.ZIP_DEFLATED) as z: + for p in sorted(folder.rglob("*")): + if p.is_file(): + z.write(p, p.relative_to(folder.parent)) + return zpath + + +def parse_args(): + p = argparse.ArgumentParser( + description="CM93 invalid Polygon fill rule + retrace audit v8" + ) + p.add_argument("--host", default=os.getenv("MYSQL_HOST", "127.0.0.1")) + p.add_argument("--port", type=int, default=int(os.getenv("MYSQL_PORT", "3306"))) + p.add_argument("--user", default=os.getenv("MYSQL_USER", "root")) + p.add_argument("--password", default=os.getenv("MYSQL_PASSWORD")) + p.add_argument("--database", default=os.getenv("MYSQL_DATABASE", "cm93_chart")) + p.add_argument("--dataset-id", type=int, default=1) + p.add_argument( + "--segment-tolerance", + type=float, + default=2e-8, + help="remainder segment midpoint 与原始 ring segment 匹配容差（degree）" + ) + p.add_argument( + "--cell-edge-tolerance", + type=float, + default=5e-6, + help="invalid point 是否位于 Cell bbox 边界的容差（degree）" + ) + p.add_argument("--output", default=None) + return p.parse_args() + + +def main(): + args = parse_args() + if not args.password: + args.password = getpass.getpass("MySQL password: ") + + stamp = datetime.now().strftime("%Y%m%d_%H%M%S") + outdir = Path( + args.output + or f"cm93_fillrule_audit_v8_dataset_{args.dataset_id}_{stamp}" + ) + outdir.mkdir(parents=True, exist_ok=True) + + report = { + "version": "v8.0", + "generated_at": datetime.now().isoformat(timespec="seconds"), + "database": args.database, + "dataset_id": args.dataset_id, + "read_only": True, + "pbf": "OUT_OF_SCOPE", + "shapely_version": getattr(shapely, "__version__", "unknown"), + "scope_note": ( + "CM93JAPAN 是局部数据集。V8 只验证 invalid Polygon 的线性残片" + "是否为重走桥接线，以及 face selection 对 odd-even/nonzero 填充规则的符合程度。" + ), + "warnings": [], + } + + conn = None + try: + conn = pymysql.connect( + host=args.host, + port=args.port, + user=args.user, + password=args.password, + database=args.database, + charset="utf8mb4", + cursorclass=DictCursor, + autocommit=False, + connect_timeout=20, + read_timeout=600, + write_timeout=30, + ) + + with conn.cursor() as cur: + begin_read_only(cur, report["warnings"]) + report["mysql_version"] = scalar(cur, "SELECT VERSION()") + wkb_fn = probe_wkb_fn(cur) + report["wkb_fn"] = wkb_fn + + print("[1/5] 加载 invalid Area + Coverage ...") + records = load_invalid_records(cur, args.dataset_id, wkb_fn) + print(f" invalid records={len(records)}") + + classification_rows = [] + remainder_evidence = [] + face_evidence = [] + + print("[2/5] 线性残片 retrace 诊断 ...") + for idx, r in enumerate(records, start=1): + geom = r["_geom"] + reason = explain_validity(geom) + invalid_point = parse_invalid_point(reason) + cell_bbox = bbox_from_row(r) + on_edge, edge_name, edge_dist = invalid_point_cell_edge( + invalid_point, cell_bbox, args.cell_edge_tolerance + ) + + mv = make_valid_geom(geom) + comps = flatten_components(mv) + mv_poly_union = geom_union(comps["polygons"]) + mv_line_union = geom_union(comps["lines"]) + + src_segments = original_segments(geom) + seg_class_counts = Counter() + segment_count = 0 + + for li, line in enumerate(comps["lines"]): + coords = [(float(c[0]), float(c[1])) for c in line.coords] + for si in range(len(coords) - 1): + ra, rb = coords[si], coords[si+1] + if ra == rb: + continue + ev = classify_remainder_segment( + ra, rb, src_segments, args.segment_tolerance + ) + segment_count += 1 + seg_class_counts[ev["segment_class"]] += 1 + + remainder_evidence.append({ + "kind": r["kind"], + "feature_id": r.get("feature_id"), + "coverage_geom_id": r.get("coverage_geom_id"), + "object_class": r.get("object_class"), + "cell_name": r.get("cell_name"), + "line_component_index": li, + "remainder_segment_index": si, + "ax": ra[0], + "ay": ra[1], + "bx": rb[0], + "by": rb[1], + "segment_length_deg": LineString([ra, rb]).length, + "coverage_count": ev["coverage_count"], + "same_direction_count": ev["same_direction_count"], + "opposite_direction_count": ev["opposite_direction_count"], + "segment_class": ev["segment_class"], + "source_hits": ev.get("source_hits", []), + }) + + rem_class = record_remainder_class(seg_class_counts) + + # Face analysis + boundary = unary_union(geom.boundary) + polygons_geom, cuts, dangles, invalid_rings = polygonize_full(boundary) + face_comps = flatten_components(polygons_geom)["polygons"] + rings = original_rings(geom) + + this_faces = [] + for fi, face in enumerate(face_comps): + rp = face.representative_point() + pxy = (float(rp.x), float(rp.y)) + odd = even_odd_inside(pxy, rings) + wn = winding_number(pxy, rings) + nonzero = wn != 0 + + selected = False + if mv_poly_union is not None: + try: + selected = bool(mv_poly_union.covers(rp)) + except Exception: + selected = False + + row = { + "kind": r["kind"], + "feature_id": r.get("feature_id"), + "coverage_geom_id": r.get("coverage_geom_id"), + "object_class": r.get("object_class"), + "cell_name": r.get("cell_name"), + "face_index": fi, + "face_area_deg2": float(face.area), + "representative_x": pxy[0], + "representative_y": pxy[1], + "selected_by_make_valid": int(selected), + "odd_even_inside": int(odd), + "winding_number": wn, + "nonzero_inside": int(nonzero), + "odd_even_matches_selection": int(selected == odd), + "nonzero_matches_selection": int(selected == nonzero), + } + face_evidence.append(row) + this_faces.append(row) + + fill_class = fill_rule_record_class(this_faces) + final_class = final_semantic_class(rem_class, fill_class) + + rid = ( + r.get("feature_id") + if r["kind"] == "AREA" + else r.get("coverage_geom_id") + ) + + c = { + "kind": r["kind"], + "feature_id": r.get("feature_id"), + "coverage_geom_id": r.get("coverage_geom_id"), + "object_class": r.get("object_class"), + "cell_id": r.get("cell_id"), + "cell_name": r.get("cell_name"), + "source_record_index": r.get("source_record_index"), + "coverage_id": r.get("coverage_id"), + "validity_reason": reason, + "invalid_point_on_cell_edge": on_edge, + "cell_edge": edge_name, + "cell_edge_distance_deg": edge_dist, + "make_valid_polygon_count": len(comps["polygons"]), + "make_valid_line_count": len(comps["lines"]), + "linear_remainder_segment_count": segment_count, + "remainder_segment_classes": dict(seg_class_counts), + "remainder_class": rem_class, + "polygonize_face_count": len(this_faces), + "faces_selected_by_make_valid": sum( + int(x["selected_by_make_valid"]) for x in this_faces + ), + "faces_odd_even_inside": sum( + int(x["odd_even_inside"]) for x in this_faces + ), + "faces_nonzero_inside": sum( + int(x["nonzero_inside"]) for x in this_faces + ), + "fill_rule_class": fill_class, + "final_semantic_class": final_class, + } + classification_rows.append(c) + + r["_invalid_point"] = invalid_point + r["_mv_polygon_union"] = mv_poly_union + r["_mv_line_union"] = mv_line_union + r["_remainder_class"] = rem_class + r["_fill_rule_class"] = fill_class + r["_final_semantic_class"] = final_class + + print( + f" [{idx:02d}/{len(records)}] {r['kind']} id={rid} " + f"{r.get('object_class')} -> " + f"{rem_class} / {fill_class}" + ) + + print("[3/5] 汇总 ...") + remainder_counts = Counter(r["remainder_class"] for r in classification_rows) + fill_counts = Counter(r["fill_rule_class"] for r in classification_rows) + semantic_counts = Counter(r["final_semantic_class"] for r in classification_rows) + segment_counts = Counter(r["segment_class"] for r in remainder_evidence) + + edge_ctx_counts = Counter( + ( + "CELL_EDGE" if r["invalid_point_on_cell_edge"] else "INTERNAL", + r["final_semantic_class"], + ) + for r in classification_rows + ) + object_counts = Counter( + (r["object_class"], r["final_semantic_class"]) + for r in classification_rows + ) + + report["summary"] = { + "invalid_total": len(classification_rows), + "remainder_class_counts": dict(remainder_counts), + "fill_rule_class_counts": dict(fill_counts), + "final_semantic_class_counts": dict(semantic_counts), + "remainder_segment_class_counts": dict(segment_counts), + "linear_remainder_record_count": sum( + r["remainder_class"] != "NO_LINEAR_REMAINDER" + for r in classification_rows + ), + "retrace_confirmed_record_count": sum( + r["remainder_class"] == "RETRACED_CONNECTOR_CONFIRMED" + for r in classification_rows + ), + "fill_rule_resolved_count": sum( + r["fill_rule_class"] != "MATCHES_NEITHER" + for r in classification_rows + ), + "fill_rule_odd_even_compatible_count": sum( + r["fill_rule_class"] in { + "MATCHES_BOTH_ODD_EVEN_AND_NONZERO", + "MATCHES_ODD_EVEN_ONLY", + } + for r in classification_rows + ), + "fill_rule_nonzero_compatible_count": sum( + r["fill_rule_class"] in { + "MATCHES_BOTH_ODD_EVEN_AND_NONZERO", + "MATCHES_NONZERO_ONLY", + } + for r in classification_rows + ), + } + + print("[4/5] 写文件 ...") + write_csv( + outdir / "invalid_polygon_fillrule_classification.csv", + classification_rows + ) + write_csv( + outdir / "linear_remainder_segment_evidence.csv", + remainder_evidence + ) + write_csv( + outdir / "polygon_face_fillrule_evidence.csv", + face_evidence + ) + + write_csv( + outdir / "retraced_connector_summary.csv", + [ + {"remainder_class": k, "count": v} + for k, v in remainder_counts.most_common() + ], + ["remainder_class", "count"] + ) + write_csv( + outdir / "fillrule_by_object_class.csv", + [ + { + "object_class": obj, + "final_semantic_class": cls, + "count": n, + } + for (obj, cls), n in object_counts.most_common() + ] + ) + write_csv( + outdir / "fillrule_by_boundary_context.csv", + [ + { + "boundary_context": ctx, + "final_semantic_class": cls, + "count": n, + } + for (ctx, cls), n in edge_ctx_counts.most_common() + ] + ) + + make_geojson( + records, + outdir / "fillrule_diagnostic.geojson" + ) + + (outdir / "fillrule_audit_report_v8.json").write_text( + json.dumps(report, ensure_ascii=False, indent=2, default=str), + encoding="utf-8" + ) + + lines = [] + lines.append("CM93 -> MySQL Invalid Polygon 线性残片与填充规则诊断 v8") + lines.append("=" * 92) + lines.append(f"Generated: {report['generated_at']}") + lines.append(f"Dataset ID: {args.dataset_id}") + lines.append(f"MySQL: {report['mysql_version']}") + lines.append(f"Shapely: {report['shapely_version']}") + lines.append("") + lines.append("SCOPE") + lines.append(" CM93JAPAN 为局部数据集；跨未导入海域不连续不作为 FAIL。") + lines.append(" 本轮只诊断 retraced connector 与 polygon fill rule，不写数据库。") + lines.append("") + lines.append("REMAINDER RECORD CLASS") + for k, v in remainder_counts.most_common(): + lines.append(f" {k:42s} {v}") + lines.append("") + lines.append("REMAINDER SEGMENT EVIDENCE") + for k, v in segment_counts.most_common(): + lines.append(f" {k:42s} {v}") + lines.append("") + lines.append("FILL RULE") + for k, v in fill_counts.most_common(): + lines.append(f" {k:42s} {v}") + lines.append("") + lines.append("FINAL SEMANTIC CLASS") + for k, v in semantic_counts.most_common(): + lines.append(f" {k:42s} {v}") + lines.append("") + lines.append("KEY COUNTS") + lines.append( + f" linear remainder records: " + f"{report['summary']['linear_remainder_record_count']}" + ) + lines.append( + f" retraced connector confirmed: " + f"{report['summary']['retrace_confirmed_record_count']}" + ) + lines.append( + f" fill rule resolved: " + f"{report['summary']['fill_rule_resolved_count']}" + ) + lines.append( + f" odd-even compatible records: " + f"{report['summary']['fill_rule_odd_even_compatible_count']}" + ) + lines.append( + f" non-zero compatible records: " + f"{report['summary']['fill_rule_nonzero_compatible_count']}" + ) + lines.append("") + lines.append("INTERPRETATION") + lines.append( + " RETRACED_CONNECTOR_CONFIRMED：make_valid 的线性残片在原始 ring " + "中由相反方向的重走 segment 解释，强烈支持“桥接/connector”而非随机坏线。" + ) + lines.append( + " MATCHES_ODD_EVEN_ONLY / MATCHES_NONZERO_ONLY：" + "说明 polygonize 的全部 planar faces 不能全部填充，必须应用对应 fill rule。" + ) + lines.append( + " MATCHES_NEITHER：不能仅靠常见 odd-even / non-zero 规则解释，需要继续调查。" + ) + lines.append("") + lines.append("NO WRITE / NO REPAIR / NO PBF") + + (outdir / "fillrule_audit_summary_v8.txt").write_text( + "\n".join(lines), encoding="utf-8" + ) + + conn.rollback() + + print("[5/5] 压缩 ...") + zpath = zip_dir(outdir) + print("") + print("完成。") + print(f"Output: {outdir.resolve()}") + print(f"ZIP: {zpath.resolve()}") + print("把 ZIP 整包上传给 ChatGPT。") + + except KeyboardInterrupt: + if conn: + try: + conn.rollback() + except Exception: + pass + print("\n用户中断。") + sys.exit(130) + except Exception: + if conn: + try: + conn.rollback() + except Exception: + pass + tb = traceback.format_exc() + print(tb) + try: + (outdir / "fatal_error.txt").write_text(tb, encoding="utf-8") + except Exception: + pass + sys.exit(1) + finally: + if conn: + try: + conn.close() + except Exception: + pass + + +if __name__ == "__main__": + main() diff --git a/cm93_mysql_final_geometry_audit_v1.py b/cm93_mysql_final_geometry_audit_v1.py new file mode 100644 index 0000000..61edee2 --- /dev/null +++ b/cm93_mysql_final_geometry_audit_v1.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +from __future__ import annotations +import argparse, csv, getpass, json, sys +from datetime import datetime +from pathlib import Path + +try: + import pymysql + from pymysql.cursors import DictCursor +except ImportError: + print("缺少 pymysql：pip install pymysql") + sys.exit(2) + +try: + try: + from shapely import from_wkb + def load_wkb(x): return from_wkb(x) + except ImportError: + from shapely import wkb + def load_wkb(x): return wkb.loads(x) +except ImportError: + print("缺少 shapely：pip install shapely") + sys.exit(2) + +def scalar(cur, sql, params=()): + cur.execute(sql, params) + r = cur.fetchone() + return None if not r else next(iter(r.values())) + +def as_bytes(v): + if isinstance(v, memoryview): return v.tobytes() + if isinstance(v, bytearray): return bytes(v) + return v + +def wkb_fn(cur, table): + for fn in ("ST_AsBinary","ST_AsWKB","AsBinary","AsWKB"): + try: + cur.execute(f"SELECT {fn}(geom) AS g FROM {table} LIMIT 1") + cur.fetchone() + return fn + except Exception: + pass + raise RuntimeError(f"{table}: 无可用 Geometry->WKB 函数") + +def main(): + ap = argparse.ArgumentParser(description="CM93 normalization 最终全库 Geometry 验收") + ap.add_argument("--host", default="127.0.0.1") + ap.add_argument("--port", type=int, default=3306) + ap.add_argument("--user", default="root") + ap.add_argument("--password") + ap.add_argument("--database", default="cm93_chart") + ap.add_argument("--dataset-id", type=int, default=1) + ap.add_argument("--bbox-tolerance", type=float, default=5e-6) + a = ap.parse_args() + if a.password is None: + a.password = getpass.getpass("MySQL password: ") + + stamp = datetime.now().strftime("%Y%m%d_%H%M%S") + outdir = Path(f"cm93_final_geometry_audit_{stamp}") + outdir.mkdir(exist_ok=True) + + conn = pymysql.connect( + host=a.host, port=a.port, user=a.user, password=a.password, + database=a.database, charset="utf8mb4", + cursorclass=DictCursor, autocommit=False, + connect_timeout=20, read_timeout=900 + ) + + failures = [] + report = {"dataset_id": a.dataset_id, "generated_at": datetime.now().isoformat()} + + try: + with conn.cursor() as cur: + try: cur.execute("SET SESSION TRANSACTION READ ONLY") + except Exception: pass + try: cur.execute("START TRANSACTION READ ONLY") + except Exception: cur.execute("START TRANSACTION") + + counts = { + "cells": int(scalar(cur,"SELECT COUNT(*) FROM chart_cell WHERE dataset_id=%s",(a.dataset_id,)) or 0), + "features": int(scalar(cur,"SELECT COUNT(*) FROM chart_feature WHERE dataset_id=%s",(a.dataset_id,)) or 0), + "feature_repaired": int(scalar(cur,"SELECT COUNT(*) FROM chart_feature WHERE dataset_id=%s AND geometry_repaired=1",(a.dataset_id,)) or 0), + "coverage_repaired": int(scalar(cur,"SELECT COUNT(*) FROM chart_coverage WHERE dataset_id=%s AND geometry_repaired=1",(a.dataset_id,)) or 0), + "feature_issues": int(scalar(cur,"SELECT COUNT(*) FROM chart_feature WHERE dataset_id=%s AND geometry_issue IS NOT NULL",(a.dataset_id,)) or 0), + "coverage_issues": int(scalar(cur,"SELECT COUNT(*) FROM chart_coverage WHERE dataset_id=%s AND geometry_issue IS NOT NULL",(a.dataset_id,)) or 0), + "geotype16_total": int(scalar(cur,"SELECT COUNT(*) FROM chart_feature WHERE dataset_id=%s AND raw_geotype=16",(a.dataset_id,)) or 0), + "geotype16_issues": int(scalar(cur,"SELECT COUNT(*) FROM chart_feature WHERE dataset_id=%s AND raw_geotype=16 AND geometry_issue IS NOT NULL",(a.dataset_id,)) or 0), + } + report["counts"] = counts + + expected = { + "cells": 784, "features": 322465, + "feature_repaired": 55, "coverage_repaired": 3, + "feature_issues": 0, "coverage_issues": 0, + "geotype16_total": 4406, "geotype16_issues": 0, + } + for k,v in expected.items(): + if counts[k] != v: + failures.append({"check":k,"expected":v,"actual":counts[k]}) + + afn = wkb_fn(cur, "chart_area_geometry") + cur.execute(f""" + SELECT f.id,f.object_class,f.geometry_repaired, + f.bbox_min_lon,f.bbox_min_lat,f.bbox_max_lon,f.bbox_max_lat, + {afn}(g.geom) AS wkb + FROM chart_feature f + JOIN chart_area_geometry g ON g.feature_id=f.id + WHERE f.dataset_id=%s + """,(a.dataset_id,)) + area_total=area_valid=bbox_bad=0 + for r in cur: + area_total += 1 + try: + g = load_wkb(as_bytes(r["wkb"])) + if g.is_empty or g.geom_type not in ("Polygon","MultiPolygon") or not g.is_valid: + failures.append({"check":"area_invalid","feature_id":r["id"],"object_class":r["object_class"],"geom_type":g.geom_type,"is_valid":bool(g.is_valid)}) + continue + area_valid += 1 + dbb=(float(r["bbox_min_lon"]),float(r["bbox_min_lat"]),float(r["bbox_max_lon"]),float(r["bbox_max_lat"])) + gb=tuple(float(x) for x in g.bounds) + if any(abs(x-y)>a.bbox_tolerance for x,y in zip(dbb,gb)): + bbox_bad += 1 + failures.append({"check":"area_bbox","feature_id":r["id"],"db_bbox":dbb,"geom_bbox":gb}) + except Exception as e: + failures.append({"check":"area_wkb","feature_id":r["id"],"error":repr(e)}) + + cfn = wkb_fn(cur, "chart_coverage") + cur.execute(f""" + SELECT id,cell_id,source_record_index,geometry_repaired,{cfn}(geom) AS wkb + FROM chart_coverage WHERE dataset_id=%s + """,(a.dataset_id,)) + cov_total=cov_valid=0 + for r in cur: + cov_total += 1 + try: + g = load_wkb(as_bytes(r["wkb"])) + if g.is_empty or g.geom_type not in ("Polygon","MultiPolygon") or not g.is_valid: + failures.append({"check":"coverage_invalid","coverage_id":r["id"],"cell_id":r["cell_id"],"geom_type":g.geom_type,"is_valid":bool(g.is_valid)}) + continue + cov_valid += 1 + except Exception as e: + failures.append({"check":"coverage_wkb","coverage_id":r["id"],"error":repr(e)}) + + report["area"]={"total":area_total,"valid":area_valid,"bbox_mismatch":bbox_bad} + report["coverage"]={"total":cov_total,"valid":cov_valid} + conn.rollback() + finally: + conn.close() + + report["failure_count"]=len(failures) + report["overall"]="PASS" if not failures else "FAIL" + + with (outdir/"失败明细_v1.csv").open("w",encoding="utf-8-sig",newline="") as f: + keys=["check","expected","actual","feature_id","coverage_id","cell_id","object_class","geom_type","is_valid","db_bbox","geom_bbox","error"] + w=csv.DictWriter(f,fieldnames=keys,extrasaction="ignore"); w.writeheader(); w.writerows(failures) + + (outdir/"最终验收报告_v1.json").write_text(json.dumps(report,ensure_ascii=False,indent=2),encoding="utf-8") + summary = f"""CM93 normalization 最终全库 Geometry 验收 v1 +Dataset ID : {a.dataset_id} +Cells : {report['counts']['cells']} +Features : {report['counts']['features']} +Feature repaired : {report['counts']['feature_repaired']} +Coverage repaired : {report['counts']['coverage_repaired']} +Feature issues : {report['counts']['feature_issues']} +Coverage issues : {report['counts']['coverage_issues']} +Geotype16 total : {report['counts']['geotype16_total']} +Geotype16 issues : {report['counts']['geotype16_issues']} +Area total : {report['area']['total']} +Area valid : {report['area']['valid']} +Area bbox mismatch : {report['area']['bbox_mismatch']} +Coverage total : {report['coverage']['total']} +Coverage valid : {report['coverage']['valid']} +Failure count : {report['failure_count']} +OVERALL: {report['overall']} +""" + (outdir/"最终验收摘要_v1.txt").write_text(summary,encoding="utf-8") + print(summary) + print("报告目录:", outdir.resolve()) + +if __name__=="__main__": + main() diff --git a/cm93_mysql_invalid_area_audit_v4.py b/cm93_mysql_invalid_area_audit_v4.py new file mode 100644 index 0000000..6263001 --- /dev/null +++ b/cm93_mysql_invalid_area_audit_v4.py @@ -0,0 +1,977 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +CM93 -> MySQL 无效 Area / Coverage 深度分类审计 +Version: v4.0 + +背景： +- Dataset ID=1 为日本沿海局部 CM93 数据子集，不是完整 CM93。 +- V3 已确认： + * Point 113346：全部有效 + * Line 121773：全部有效 + * Sounding 275700：全部有效 + * Area 81080：52 个 Shapely invalid + * Coverage 1156：3 个 Shapely invalid + * feature ↔ geometry table mapping issue = 0 + * raw_geotype=16 / META 的 4406 条为 EXPECTED_PARENT_NO_GEOMETRY + +本脚本目标： +1. 重新找出 invalid Area / Coverage； +2. 对每一个 Shapely 报告的 invalidity point 做局部几何分析； +3. 区分： + - invalid 点本身位于 Cell 边界； + - geometry 只是“整体碰 Cell 边界”，但 invalid 点在内部； + - 重复顶点自接触； + - 线段内部真正交叉； + - shell / hole / ring 结构问题； + - Interior disconnected； +4. 找出多个 Feature 共享同一 invalid point 的簇； +5. 找出 Area 与 Coverage 在同一 Cell、同一 invalid point 的关联； +6. 输出当前数据库中 topology/edge/node/primitive 等相关 schema 线索，供下一阶段追 importer。 +7. 全程只读，不修 geometry，不改 MySQL，不做 PBF。 + +重要： +“当前 Dataset 不是完整 CM93”只影响跨未导入海域的连续性判断。 +一个已经生成的 Polygon 如果在自身内部发生真正 segment crossing， +仍属于需要调查的内在 geometry 问题。 +但如果 invalidity point 正好位于 Cell 裁切边界、并被多个对象/coverage 共用， +更可能是 Cell clipping / shared topology / ring assembly 类问题，不能直接归为原始数据损坏。 + +依赖： + pip install pymysql shapely + +运行： + python cm93_mysql_invalid_area_audit_v4.py ^ + --host 127.0.0.1 ^ + --port 3306 ^ + --user root ^ + --database cm93_chart ^ + --dataset-id 1 + +输出目录会自动压缩成 zip，直接把 zip 发给 ChatGPT。 +""" + +from __future__ import annotations + +import argparse +import csv +import getpass +import json +import math +import os +import re +import sys +import traceback +import zipfile +from collections import Counter, defaultdict +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple + +try: + import pymysql + from pymysql.cursors import DictCursor +except ImportError: + print("缺少 pymysql：pip install pymysql shapely") + sys.exit(2) + +try: + try: + from shapely import from_wkb as _from_wkb + def load_wkb(data: bytes): + return _from_wkb(data) + except ImportError: + from shapely import wkb as _wkb + def load_wkb(data: bytes): + return _wkb.loads(data) + + from shapely.geometry import Point, LineString + from shapely.validation import explain_validity +except ImportError: + print("缺少 shapely：pip install pymysql shapely") + sys.exit(2) + + +INVALID_POINT_RE = re.compile( + r"\[\s*([-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)\s+" + r"([-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)\s*\]" +) + + +def qi(name: str) -> str: + return "`" + name.replace("`", "``") + "`" + + +def scalar(cur, sql: str, params: Sequence[Any] = ()) -> Any: + cur.execute(sql, params) + row = cur.fetchone() + return next(iter(row.values())) if row else None + + +def fetch_all(cur, sql: str, params: Sequence[Any] = ()) -> List[Dict[str, Any]]: + cur.execute(sql, params) + return list(cur.fetchall()) + + +def write_csv(path: Path, rows: Iterable[Dict[str, Any]], fieldnames: Optional[List[str]] = None): + rows = list(rows) + if fieldnames is None: + seen = set() + fieldnames = [] + for row in rows: + for k in row.keys(): + if k not in seen: + seen.add(k) + fieldnames.append(k) + + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8-sig", newline="") as f: + if not fieldnames: + return + w = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore") + w.writeheader() + for row in rows: + out = {} + for k in fieldnames: + v = row.get(k) + if isinstance(v, (tuple, list, dict)): + v = json.dumps(v, ensure_ascii=False) + out[k] = "" if v is None else v + w.writerow(out) + + +def begin_read_only(cur, warnings: List[str]): + try: + cur.execute("SET SESSION TRANSACTION READ ONLY") + except Exception as e: + warnings.append(f"SET SESSION TRANSACTION READ ONLY failed: {e}") + try: + cur.execute("START TRANSACTION READ ONLY") + except Exception as e: + warnings.append(f"START TRANSACTION READ ONLY failed: {e}") + try: + cur.execute("START TRANSACTION") + except Exception: + pass + + +def probe_wkb_fn(cur) -> str: + for fn in ("ST_AsBinary", "ST_AsWKB", "AsBinary", "AsWKB"): + try: + cur.execute(f"SELECT {fn}(geom) AS x FROM chart_area_geometry LIMIT 1") + cur.fetchone() + return fn + except Exception: + pass + raise RuntimeError("找不到可用的 geometry -> WKB 函数") + + +def parse_invalid_point(reason: str) -> Optional[Tuple[float, float]]: + if not reason: + return None + m = INVALID_POINT_RE.search(reason) + if not m: + return None + try: + return float(m.group(1)), float(m.group(2)) + except Exception: + return None + + +def bbox_from_row(row: Dict[str, Any], prefix: str = "") -> Optional[Tuple[float, float, float, float]]: + keys = [ + prefix + "bbox_min_lon", + prefix + "bbox_min_lat", + prefix + "bbox_max_lon", + prefix + "bbox_max_lat", + ] + vals = [] + for k in keys: + try: + v = float(row[k]) + if not math.isfinite(v): + return None + vals.append(v) + except Exception: + return None + return vals[0], vals[1], vals[2], vals[3] + + +def point_cell_edge_context( + p: Optional[Tuple[float, float]], + bbox: Optional[Tuple[float, float, float, float]], + tol: float, +) -> Tuple[bool, str, Optional[float]]: + if not p or not bbox: + return False, "", None + x, y = p + minx, miny, maxx, maxy = bbox + d = { + "WEST": abs(x - minx), + "EAST": abs(x - maxx), + "SOUTH": abs(y - miny), + "NORTH": abs(y - maxy), + } + edge, dist = min(d.items(), key=lambda kv: kv[1]) + return dist <= tol, edge if dist <= tol else "", dist + + +def point_feature_bbox_edge_context( + p: Optional[Tuple[float, float]], + bbox: Optional[Tuple[float, float, float, float]], + tol: float, +) -> Tuple[bool, str, Optional[float]]: + return point_cell_edge_context(p, bbox, tol) + + +def ring_coordinate_lists(geom) -> List[Tuple[str, int, List[Tuple[float, float]]]]: + out = [] + + def add_polygon(poly, poly_index: int): + try: + out.append((f"poly{poly_index}.shell", 0, [(float(x), float(y)) for x, y, *rest in poly.exterior.coords])) + except Exception: + pass + for i, ring in enumerate(poly.interiors): + try: + out.append((f"poly{poly_index}.hole", i, [(float(x), float(y)) for x, y, *rest in ring.coords])) + except Exception: + pass + + if geom.geom_type == "Polygon": + add_polygon(geom, 0) + elif geom.geom_type == "MultiPolygon": + for pi, poly in enumerate(geom.geoms): + add_polygon(poly, pi) + + return out + + +def point_close(a: Tuple[float, float], b: Tuple[float, float], tol: float) -> bool: + return abs(a[0]-b[0]) <= tol and abs(a[1]-b[1]) <= tol + + +def analyze_invalid_point_on_rings( + geom, + invalid_point: Optional[Tuple[float, float]], + tol: float, +) -> Dict[str, Any]: + result = { + "polygon_count": 0, + "ring_count": 0, + "hole_count": 0, + "total_vertices": 0, + "invalid_point_vertex_occurrences": 0, + "invalid_point_distinct_ring_hits": 0, + "invalid_point_segment_hits": 0, + "invalid_point_segment_interior_hits": 0, + "invalid_point_endpoint_hits": 0, + "repeated_nonconsecutive_vertex_count": 0, + "zero_length_segment_count": 0, + "mechanism": "UNRESOLVED", + "ring_hit_labels": "", + } + + if geom.geom_type == "Polygon": + result["polygon_count"] = 1 + elif geom.geom_type == "MultiPolygon": + result["polygon_count"] = len(geom.geoms) + else: + return result + + rings = ring_coordinate_lists(geom) + result["ring_count"] = len(rings) + result["hole_count"] = sum(1 for label, _, _ in rings if ".hole" in label) + + ring_hits = set() + vertex_occurrences = 0 + segment_hits = 0 + segment_interior_hits = 0 + endpoint_hits = 0 + repeated_nonconsecutive = 0 + zero_segments = 0 + + pgeom = Point(invalid_point) if invalid_point else None + + for label, ring_index, coords in rings: + result["total_vertices"] += len(coords) + + # 去掉 ring 最后一个闭合重复点后做“非连续重复顶点”统计。 + core = coords[:-1] if len(coords) > 1 and point_close(coords[0], coords[-1], tol) else coords[:] + seen_positions: Dict[Tuple[int, int], List[int]] = defaultdict(list) + scale = 1.0 / max(tol, 1e-12) + for i, c in enumerate(core): + key = (round(c[0] * scale), round(c[1] * scale)) + seen_positions[key].append(i) + for positions in seen_positions.values(): + if len(positions) > 1: + # 相邻重复点不归到 nonconsecutive；它们通常会形成 zero segment。 + has_nonconsecutive = False + for i in range(len(positions)): + for j in range(i+1, len(positions)): + if abs(positions[j] - positions[i]) > 1: + has_nonconsecutive = True + break + if has_nonconsecutive: + break + if has_nonconsecutive: + repeated_nonconsecutive += 1 + + if invalid_point: + for c in core: + if point_close(c, invalid_point, tol): + vertex_occurrences += 1 + ring_hits.add(f"{label}[{ring_index}]") + + for i in range(len(coords)-1): + a = coords[i] + b = coords[i+1] + + if point_close(a, b, tol): + zero_segments += 1 + + if not invalid_point: + continue + + try: + seg = LineString([a, b]) + if seg.distance(pgeom) <= tol: + segment_hits += 1 + ring_hits.add(f"{label}[{ring_index}]") + at_a = point_close(a, invalid_point, tol) + at_b = point_close(b, invalid_point, tol) + if at_a or at_b: + endpoint_hits += 1 + else: + segment_interior_hits += 1 + except Exception: + pass + + result["invalid_point_vertex_occurrences"] = vertex_occurrences + result["invalid_point_distinct_ring_hits"] = len(ring_hits) + result["invalid_point_segment_hits"] = segment_hits + result["invalid_point_segment_interior_hits"] = segment_interior_hits + result["invalid_point_endpoint_hits"] = endpoint_hits + result["repeated_nonconsecutive_vertex_count"] = repeated_nonconsecutive + result["zero_length_segment_count"] = zero_segments + result["ring_hit_labels"] = "|".join(sorted(ring_hits)) + + # 机制分类：只描述几何表现，不直接断言根因。 + if not invalid_point: + result["mechanism"] = "NO_INVALID_POINT_FROM_VALIDATOR" + elif segment_interior_hits >= 2 and vertex_occurrences == 0: + result["mechanism"] = "TRUE_SEGMENT_CROSSING" + elif vertex_occurrences >= 2: + result["mechanism"] = "REPEATED_VERTEX_SELF_TOUCH" + elif vertex_occurrences >= 1 and segment_interior_hits >= 1: + result["mechanism"] = "VERTEX_TOUCHES_NONADJACENT_SEGMENT" + elif len(ring_hits) >= 2: + result["mechanism"] = "MULTI_RING_TOUCH_OR_CROSS" + elif segment_hits >= 3: + result["mechanism"] = "MULTI_SEGMENT_TOUCH" + elif segment_hits >= 2: + result["mechanism"] = "TWO_SEGMENT_INTERACTION" + else: + result["mechanism"] = "UNRESOLVED_LOCAL_TOPOLOGY" + + return result + + +def validator_reason_category(reason: str) -> str: + s = (reason or "").lower() + if "ring self-intersection" in s: + return "RING_SELF_INTERSECTION" + if "self-intersection" in s: + return "SELF_INTERSECTION" + if "interior is disconnected" in s: + return "INTERIOR_DISCONNECTED" + if "hole lies outside shell" in s: + return "HOLE_OUTSIDE_SHELL" + if "nested shells" in s: + return "NESTED_SHELLS" + if "nested holes" in s: + return "NESTED_HOLES" + if "too few points" in s: + return "TOO_FEW_POINTS" + return "OTHER_INVALID_GEOMETRY" + + +def scan_invalid_geometries( + cur, + *, + table: str, + kind: str, + dataset_id: int, + wkb_fn: str, + chunk_size: int, + point_tol: float, + edge_tol: float, +) -> List[Dict[str, Any]]: + last_id = 0 + invalids = [] + id_col = "feature_id" if kind == "AREA" else "id" + + while True: + if kind == "AREA": + sql = f""" + SELECT + g.feature_id AS row_id, + g.feature_id, + NULL AS coverage_id, + {wkb_fn}(g.geom) AS geom_wkb, + f.object_class, + f.raw_geotype, + f.geometry_type, + f.geometry_issue, + f.geometry_repaired, + f.source_record_index, + f.cell_id, + c.cell_name, + f.bbox_min_lon, f.bbox_min_lat, f.bbox_max_lon, f.bbox_max_lat, + c.bbox_min_lon AS cell_bbox_min_lon, + c.bbox_min_lat AS cell_bbox_min_lat, + c.bbox_max_lon AS cell_bbox_max_lon, + c.bbox_max_lat AS cell_bbox_max_lat + FROM chart_area_geometry g + JOIN chart_feature f ON f.id=g.feature_id + JOIN chart_cell c ON c.id=f.cell_id + WHERE f.dataset_id=%s + AND g.feature_id>%s + ORDER BY g.feature_id + LIMIT %s + """ + else: + sql = f""" + SELECT + v.id AS row_id, + NULL AS feature_id, + v.id AS coverage_id, + {wkb_fn}(v.geom) AS geom_wkb, + 'COVERAGE' AS object_class, + NULL AS raw_geotype, + 'COVERAGE' AS geometry_type, + v.geometry_issue, + v.geometry_repaired, + v.source_record_index, + v.cell_id, + c.cell_name, + NULL AS bbox_min_lon, NULL AS bbox_min_lat, + NULL AS bbox_max_lon, NULL AS bbox_max_lat, + c.bbox_min_lon AS cell_bbox_min_lon, + c.bbox_min_lat AS cell_bbox_min_lat, + c.bbox_max_lon AS cell_bbox_max_lon, + c.bbox_max_lat AS cell_bbox_max_lat + FROM chart_coverage v + JOIN chart_cell c ON c.id=v.cell_id + WHERE v.dataset_id=%s + AND v.id>%s + ORDER BY v.id + LIMIT %s + """ + + cur.execute(sql, (dataset_id, last_id, chunk_size)) + batch = list(cur.fetchall()) + if not batch: + break + + for r in batch: + last_id = int(r["row_id"]) + wkb = r["geom_wkb"] + if isinstance(wkb, memoryview): + wkb = wkb.tobytes() + elif isinstance(wkb, bytearray): + wkb = bytes(wkb) + + try: + geom = load_wkb(wkb) + except Exception: + continue + + try: + valid = bool(geom.is_valid) + except Exception: + continue + if valid: + continue + + reason = explain_validity(geom) + p = parse_invalid_point(reason) + cb = bbox_from_row(r, "cell_") + fb = bbox_from_row(r) + + on_cell_edge, cell_edge, cell_edge_dist = point_cell_edge_context(p, cb, edge_tol) + on_feature_bbox_edge, feature_bbox_edge, feature_bbox_edge_dist = point_feature_bbox_edge_context( + p, fb, edge_tol + ) + + local = analyze_invalid_point_on_rings(geom, p, point_tol) + + geom_bounds = tuple(float(v) for v in geom.bounds) + + row = { + "kind": kind, + "feature_id": r.get("feature_id"), + "coverage_id": r.get("coverage_id"), + "object_class": r.get("object_class"), + "cell_id": r.get("cell_id"), + "cell_name": r.get("cell_name"), + "source_record_index": r.get("source_record_index"), + "raw_geotype": r.get("raw_geotype"), + "geometry_type": r.get("geometry_type"), + "db_geometry_issue": r.get("geometry_issue"), + "db_geometry_repaired": r.get("geometry_repaired"), + "validator_category": validator_reason_category(reason), + "validator_reason": reason, + "invalid_x": p[0] if p else None, + "invalid_y": p[1] if p else None, + "cell_bbox": cb, + "feature_bbox": fb, + "geometry_bounds": geom_bounds, + "invalid_point_on_cell_edge": int(on_cell_edge), + "cell_edge": cell_edge, + "cell_edge_distance_deg": cell_edge_dist, + "invalid_point_on_feature_bbox_edge": int(on_feature_bbox_edge), + "feature_bbox_edge": feature_bbox_edge, + "feature_bbox_edge_distance_deg": feature_bbox_edge_dist, + } + row.update(local) + invalids.append(row) + + print(f" {kind:<8} scanned_to={last_id} invalid_found={len(invalids)}") + + return invalids + + +def rounded_point_key(row: Dict[str, Any], decimals: int = 7) -> Optional[Tuple[float, float]]: + try: + x = float(row["invalid_x"]) + y = float(row["invalid_y"]) + if math.isfinite(x) and math.isfinite(y): + return round(x, decimals), round(y, decimals) + except Exception: + pass + return None + + +def build_shared_point_groups(all_rows: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + groups = defaultdict(list) + for r in all_rows: + k = rounded_point_key(r) + if k is not None: + groups[k].append(r) + + out = [] + for p, rs in groups.items(): + if len(rs) < 2: + continue + kinds = Counter(r["kind"] for r in rs) + cells = sorted({str(r.get("cell_name") or "") for r in rs}) + objs = Counter(str(r.get("object_class") or "") for r in rs) + out.append({ + "invalid_x": p[0], + "invalid_y": p[1], + "record_count": len(rs), + "area_count": kinds.get("AREA", 0), + "coverage_count": kinds.get("COVERAGE", 0), + "cells": "|".join(cells), + "object_classes": "|".join(f"{k}:{v}" for k, v in objs.most_common()), + "cell_edge_records": sum(int(r.get("invalid_point_on_cell_edge") or 0) for r in rs), + "feature_ids": "|".join(str(r["feature_id"]) for r in rs if r.get("feature_id") is not None), + "coverage_ids": "|".join(str(r["coverage_id"]) for r in rs if r.get("coverage_id") is not None), + }) + + out.sort(key=lambda r: (-r["record_count"], r["invalid_x"], r["invalid_y"])) + return out + + +def mark_area_coverage_links( + areas: List[Dict[str, Any]], + coverages: List[Dict[str, Any]], + decimals: int = 7, +): + cov_index = defaultdict(list) + for c in coverages: + p = rounded_point_key(c, decimals) + if p is not None: + cov_index[(str(c.get("cell_name")), p)].append(c) + + for a in areas: + p = rounded_point_key(a, decimals) + matches = cov_index.get((str(a.get("cell_name")), p), []) if p is not None else [] + a["matches_invalid_coverage_same_cell_point"] = int(bool(matches)) + a["matching_coverage_ids"] = "|".join( + str(x.get("coverage_id")) for x in matches + ) + + +def schema_topology_inventory(cur, database: str) -> List[Dict[str, Any]]: + keywords = ( + "edge", "node", "vector", "primitive", "segment", + "geom", "geometry", "coverage", "ring", "topolog" + ) + rs = fetch_all(cur, """ + SELECT TABLE_NAME, COLUMN_NAME, DATA_TYPE, COLUMN_TYPE, IS_NULLABLE + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA=%s + ORDER BY TABLE_NAME, ORDINAL_POSITION + """, (database,)) + out = [] + for r in rs: + blob = (r["TABLE_NAME"] + " " + r["COLUMN_NAME"]).lower() + if any(k in blob for k in keywords): + out.append(r) + return out + + +def table_inventory(cur, database: str) -> List[Dict[str, Any]]: + return fetch_all(cur, """ + SELECT TABLE_NAME, TABLE_ROWS, ENGINE, TABLE_COMMENT + FROM information_schema.TABLES + WHERE TABLE_SCHEMA=%s + ORDER BY TABLE_NAME + """, (database,)) + + +def classify_for_next_step(r: Dict[str, Any]) -> str: + """ + 这是“调查优先级/表现分类”，不是最终根因。 + """ + vc = r.get("validator_category") + mechanism = r.get("mechanism") + on_edge = bool(r.get("invalid_point_on_cell_edge")) + coverage_link = bool(r.get("matches_invalid_coverage_same_cell_point")) + + if vc == "INTERIOR_DISCONNECTED": + return "INTERNAL_RING_STRUCTURE_NEEDS_REVIEW" + + if on_edge and coverage_link: + return "CELL_BOUNDARY_SHARED_WITH_COVERAGE" + if on_edge and mechanism in { + "REPEATED_VERTEX_SELF_TOUCH", + "VERTEX_TOUCHES_NONADJACENT_SEGMENT", + "MULTI_RING_TOUCH_OR_CROSS", + "MULTI_SEGMENT_TOUCH", + "TWO_SEGMENT_INTERACTION", + }: + return "CELL_BOUNDARY_TOPOLOGY_CANDIDATE" + if on_edge and mechanism == "TRUE_SEGMENT_CROSSING": + return "CELL_BOUNDARY_SEGMENT_CROSSING" + if not on_edge and mechanism == "TRUE_SEGMENT_CROSSING": + return "INTERNAL_TRUE_SEGMENT_CROSSING" + if not on_edge and mechanism == "REPEATED_VERTEX_SELF_TOUCH": + return "INTERNAL_REPEATED_VERTEX_SELF_TOUCH" + if not on_edge: + return "INTERNAL_INVALID_GEOMETRY_NEEDS_REVIEW" + return "BOUNDARY_INVALID_GEOMETRY_NEEDS_REVIEW" + + +def zip_dir(folder: Path) -> Path: + zpath = folder.with_suffix(".zip") + with zipfile.ZipFile(zpath, "w", compression=zipfile.ZIP_DEFLATED) as z: + for p in sorted(folder.rglob("*")): + if p.is_file(): + z.write(p, p.relative_to(folder.parent)) + return zpath + + +def parse_args(): + p = argparse.ArgumentParser(description="CM93 invalid Area/Coverage deep audit v4") + p.add_argument("--host", default=os.getenv("MYSQL_HOST", "127.0.0.1")) + p.add_argument("--port", type=int, default=int(os.getenv("MYSQL_PORT", "3306"))) + p.add_argument("--user", default=os.getenv("MYSQL_USER", "root")) + p.add_argument("--password", default=os.getenv("MYSQL_PASSWORD")) + p.add_argument("--database", default=os.getenv("MYSQL_DATABASE", "cm93_chart")) + p.add_argument("--dataset-id", type=int, default=1) + p.add_argument("--chunk-size", type=int, default=2000) + p.add_argument( + "--point-tolerance", + type=float, + default=1e-8, + help="invalid point 与 ring 顶点/segment 的比较容差（degree）" + ) + p.add_argument( + "--cell-edge-tolerance", + type=float, + default=5e-6, + help="invalidity point 是否位于 Cell bbox 边界的容差（degree）" + ) + p.add_argument("--output", default=None) + return p.parse_args() + + +def main(): + args = parse_args() + if not args.password: + args.password = getpass.getpass("MySQL password: ") + + stamp = datetime.now().strftime("%Y%m%d_%H%M%S") + outdir = Path(args.output or f"cm93_invalid_area_audit_v4_dataset_{args.dataset_id}_{stamp}") + outdir.mkdir(parents=True, exist_ok=True) + + report = { + "version": "v4.0", + "generated_at": datetime.now().isoformat(timespec="seconds"), + "database": args.database, + "dataset_id": args.dataset_id, + "read_only": True, + "pbf": "OUT_OF_SCOPE", + "dataset_scope_note": ( + "CM93JAPAN 只是局部子集。跨未导入海域连续性不作为 FAIL；" + "本轮只深挖已经生成的 invalid Polygon/Coverage 自身。" + ), + "warnings": [], + } + + conn = None + try: + conn = pymysql.connect( + host=args.host, + port=args.port, + user=args.user, + password=args.password, + database=args.database, + charset="utf8mb4", + cursorclass=DictCursor, + autocommit=False, + connect_timeout=20, + read_timeout=600, + write_timeout=30, + ) + + with conn.cursor() as cur: + begin_read_only(cur, report["warnings"]) + report["mysql_version"] = scalar(cur, "SELECT VERSION()") + wkb_fn = probe_wkb_fn(cur) + report["wkb_fn"] = wkb_fn + + print("[1/6] 深扫 invalid Area ...") + areas = scan_invalid_geometries( + cur, + table="chart_area_geometry", + kind="AREA", + dataset_id=args.dataset_id, + wkb_fn=wkb_fn, + chunk_size=args.chunk_size, + point_tol=args.point_tolerance, + edge_tol=args.cell_edge_tolerance, + ) + + print("[2/6] 深扫 invalid Coverage ...") + coverages = scan_invalid_geometries( + cur, + table="chart_coverage", + kind="COVERAGE", + dataset_id=args.dataset_id, + wkb_fn=wkb_fn, + chunk_size=args.chunk_size, + point_tol=args.point_tolerance, + edge_tol=args.cell_edge_tolerance, + ) + + print("[3/6] 建立 Area ↔ Coverage 同点关联 ...") + mark_area_coverage_links(areas, coverages) + + for r in areas: + r["next_step_class"] = classify_for_next_step(r) + for r in coverages: + r["matches_invalid_coverage_same_cell_point"] = 0 + r["matching_coverage_ids"] = "" + r["next_step_class"] = classify_for_next_step(r) + + all_invalid = areas + coverages + shared_groups = build_shared_point_groups(all_invalid) + + print("[4/6] 导出 schema/topology 线索 ...") + topology_schema = schema_topology_inventory(cur, args.database) + tables = table_inventory(cur, args.database) + + print("[5/6] 汇总分类 ...") + validator_counts = Counter(r["validator_category"] for r in all_invalid) + mechanism_counts = Counter(r["mechanism"] for r in all_invalid) + next_counts = Counter(r["next_step_class"] for r in all_invalid) + area_next_counts = Counter(r["next_step_class"] for r in areas) + cov_next_counts = Counter(r["next_step_class"] for r in coverages) + + summary = { + "invalid_area_count": len(areas), + "invalid_coverage_count": len(coverages), + "invalid_total": len(all_invalid), + "invalid_point_on_cell_edge_total": sum(int(r["invalid_point_on_cell_edge"]) for r in all_invalid), + "invalid_point_internal_total": sum(1 for r in all_invalid if not r["invalid_point_on_cell_edge"]), + "area_invalid_point_on_cell_edge": sum(int(r["invalid_point_on_cell_edge"]) for r in areas), + "area_invalid_point_internal": sum(1 for r in areas if not r["invalid_point_on_cell_edge"]), + "coverage_invalid_point_on_cell_edge": sum(int(r["invalid_point_on_cell_edge"]) for r in coverages), + "area_matches_invalid_coverage_same_cell_point": sum( + int(r.get("matches_invalid_coverage_same_cell_point") or 0) + for r in areas + ), + "validator_categories": dict(validator_counts), + "mechanisms": dict(mechanism_counts), + "next_step_classes": dict(next_counts), + "area_next_step_classes": dict(area_next_counts), + "coverage_next_step_classes": dict(cov_next_counts), + "shared_invalid_point_group_count": len(shared_groups), + } + report["summary"] = summary + + write_csv(outdir / "invalid_areas_deep_classified.csv", areas) + write_csv(outdir / "invalid_coverages_deep_classified.csv", coverages) + write_csv(outdir / "invalid_all_deep_classified.csv", all_invalid) + write_csv(outdir / "shared_invalid_point_groups.csv", shared_groups) + write_csv(outdir / "topology_schema_inventory.csv", topology_schema) + write_csv(outdir / "database_table_inventory.csv", tables) + + write_csv( + outdir / "validator_category_counts.csv", + [{"category": k, "count": v} for k, v in validator_counts.most_common()], + ["category", "count"] + ) + write_csv( + outdir / "local_mechanism_counts.csv", + [{"mechanism": k, "count": v} for k, v in mechanism_counts.most_common()], + ["mechanism", "count"] + ) + write_csv( + outdir / "next_step_class_counts.csv", + [{"next_step_class": k, "count": v} for k, v in next_counts.most_common()], + ["next_step_class", "count"] + ) + + # 重点样本：每一 next_step_class 最多 10 条 + samples = [] + sc = Counter() + for r in sorted( + all_invalid, + key=lambda x: ( + x.get("next_step_class", ""), + str(x.get("cell_name", "")), + int(x.get("feature_id") or x.get("coverage_id") or 0) + ) + ): + c = r["next_step_class"] + if sc[c] < 10: + samples.append(r) + sc[c] += 1 + write_csv(outdir / "deep_classification_samples.csv", samples) + + (outdir / "invalid_area_audit_report_v4.json").write_text( + json.dumps(report, ensure_ascii=False, indent=2, default=str), + encoding="utf-8" + ) + + lines = [] + lines.append("CM93 -> MySQL Invalid Area/Coverage 深度分类审计 v4") + lines.append("=" * 82) + lines.append(f"Generated: {report['generated_at']}") + lines.append(f"Database: {args.database}") + lines.append(f"Dataset ID: {args.dataset_id}") + lines.append(f"MySQL: {report['mysql_version']}") + lines.append("") + lines.append("DATASET SCOPE") + lines.append(" 当前 CM93JAPAN 是局部数据，不要求与未导入海域连续。") + lines.append(" 但本轮检查的是已生成 Polygon 本身的自交/自接触机制。") + lines.append("") + lines.append("COUNTS") + lines.append(f" invalid Area: {len(areas)}") + lines.append(f" invalid Coverage: {len(coverages)}") + lines.append(f" total: {len(all_invalid)}") + lines.append("") + lines.append("INVALID POINT LOCATION") + lines.append( + f" Area invalid point ON cell edge: " + f"{summary['area_invalid_point_on_cell_edge']}" + ) + lines.append( + f" Area invalid point INTERNAL: " + f"{summary['area_invalid_point_internal']}" + ) + lines.append( + f" Coverage invalid point ON cell edge:" + f" {summary['coverage_invalid_point_on_cell_edge']}" + ) + lines.append("") + lines.append("AREA ↔ COVERAGE") + lines.append( + f" Area same cell + same invalid point as invalid Coverage: " + f"{summary['area_matches_invalid_coverage_same_cell_point']}" + ) + lines.append("") + lines.append("VALIDATOR CATEGORY") + for k, v in validator_counts.most_common(): + lines.append(f" {k:42s} {v}") + lines.append("") + lines.append("LOCAL MECHANISM") + for k, v in mechanism_counts.most_common(): + lines.append(f" {k:42s} {v}") + lines.append("") + lines.append("NEXT-STEP CLASS (表现分类，不是最终根因)") + for k, v in next_counts.most_common(): + lines.append(f" {k:42s} {v}") + lines.append("") + lines.append("SHARED INVALID POINT GROUPS") + lines.append(f" groups with >=2 invalid records: {len(shared_groups)}") + lines.append("") + lines.append("INTERPRETATION RULES") + lines.append( + " CELL_BOUNDARY_SHARED_WITH_COVERAGE / CELL_BOUNDARY_TOPOLOGY_CANDIDATE " + "优先调查 Cell clipping / shared topology / importer ring assembly。" + ) + lines.append( + " INTERNAL_TRUE_SEGMENT_CROSSING 优先调查源拓扑与 importer edge/ring 顺序。" + ) + lines.append( + " INTERNAL_REPEATED_VERTEX_SELF_TOUCH 需要判断 CM93 是否允许 touch-ring 表达，" + "以及 importer 是否应输出 MultiPolygon/拆环。" + ) + lines.append( + " 本脚本不执行 make_valid、buffer(0)、闭环、补点、删点或任何数据库写入。" + ) + lines.append("") + lines.append("PBF: OUT OF SCOPE / NOT STARTED") + + (outdir / "invalid_area_audit_summary_v4.txt").write_text( + "\n".join(lines), + encoding="utf-8" + ) + + conn.rollback() + + print("[6/6] 压缩输出 ...") + zpath = zip_dir(outdir) + print("") + print("完成。") + print(f"Output: {outdir.resolve()}") + print(f"ZIP: {zpath.resolve()}") + print("请把 ZIP 整包发给 ChatGPT。") + + except KeyboardInterrupt: + if conn: + try: + conn.rollback() + except Exception: + pass + print("\n用户中断。") + sys.exit(130) + except Exception: + if conn: + try: + conn.rollback() + except Exception: + pass + tb = traceback.format_exc() + print(tb) + try: + (outdir / "fatal_error.txt").write_text(tb, encoding="utf-8") + except Exception: + pass + sys.exit(1) + finally: + if conn: + try: + conn.close() + except Exception: + pass + + +if __name__ == "__main__": + main() diff --git a/cm93_mysql_invalid_trace_v5.py b/cm93_mysql_invalid_trace_v5.py new file mode 100644 index 0000000..1545653 --- /dev/null +++ b/cm93_mysql_invalid_trace_v5.py @@ -0,0 +1,1041 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +CM93 -> MySQL invalid Area/Coverage 源记录 + 邻接拓扑追踪审计 +Version: v5.0 + +用途： +- 延续 V4，只针对已确认的 52 个 invalid Area + 3 个 invalid Coverage。 +- 不修复 geometry，不写数据库，不做 PBF。 +- 当前 CM93JAPAN 是局部子集；跨未导入海域连续性不作为 FAIL。 + +V5 重点回答： +1. invalid point 是否被同 Cell 的其他 Area（包括有效 Area）共同使用？ +2. invalid point 是邻接对象的 vertex、segment interior，还是仅落在内部？ +3. 同一点是否形成多个 object class 的共享拓扑簇？ +4. chart_feature.coverage_id 与 invalid Coverage 是否有关联？ +5. source_record_index 能否在 chart_raw_record 中找到对应/邻近原始记录？ +6. invalid Feature 的 attributes 是什么？ +7. 当前 DB 是否保留足够的 raw/topology 中间数据，可以继续从 MySQL 追源？ +8. 生成 GeoJSON，供人工用 QGIS/geojson.io 等查看 invalid polygon、invalid point、共享邻居。 + +依赖： + pip install pymysql shapely + +运行： + python cm93_mysql_invalid_trace_v5.py ^ + --host 127.0.0.1 ^ + --port 3306 ^ + --user root ^ + --database cm93_chart ^ + --dataset-id 1 + +输出自动打 zip，直接上传整个 zip。 +""" + +from __future__ import annotations + +import argparse +import csv +import getpass +import json +import math +import os +import re +import sys +import traceback +import zipfile +from collections import Counter, defaultdict +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple + +try: + import pymysql + from pymysql.cursors import DictCursor +except ImportError: + print("缺少 pymysql：pip install pymysql shapely") + sys.exit(2) + +try: + try: + from shapely import from_wkb as _from_wkb + def load_wkb(data: bytes): + return _from_wkb(data) + except ImportError: + from shapely import wkb as _wkb + def load_wkb(data: bytes): + return _wkb.loads(data) + + from shapely.geometry import Point, LineString, box, mapping + from shapely.validation import explain_validity +except ImportError: + print("缺少 shapely：pip install pymysql shapely") + sys.exit(2) + + +INVALID_POINT_RE = re.compile( + r"\[\s*([-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)\s+" + r"([-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)\s*\]" +) + + +def qi(name: str) -> str: + return "`" + name.replace("`", "``") + "`" + + +def scalar(cur, sql: str, params: Sequence[Any] = ()) -> Any: + cur.execute(sql, params) + row = cur.fetchone() + return next(iter(row.values())) if row else None + + +def fetch_all(cur, sql: str, params: Sequence[Any] = ()) -> List[Dict[str, Any]]: + cur.execute(sql, params) + return list(cur.fetchall()) + + +def write_csv(path: Path, rows: Iterable[Dict[str, Any]], fieldnames: Optional[List[str]] = None): + rows = list(rows) + if fieldnames is None: + seen = set() + fieldnames = [] + for row in rows: + for k in row.keys(): + if k not in seen: + seen.add(k) + fieldnames.append(k) + + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8-sig", newline="") as f: + if not fieldnames: + return + w = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore") + w.writeheader() + for row in rows: + out = {} + for k in fieldnames: + v = row.get(k) + if isinstance(v, (tuple, list, dict)): + v = json.dumps(v, ensure_ascii=False, default=str) + elif isinstance(v, (bytes, bytearray, memoryview)): + b = bytes(v) + v = b.hex() + out[k] = "" if v is None else v + w.writerow(out) + + +def begin_read_only(cur, warnings: List[str]): + try: + cur.execute("SET SESSION TRANSACTION READ ONLY") + except Exception as e: + warnings.append(f"SET SESSION TRANSACTION READ ONLY failed: {e}") + try: + cur.execute("START TRANSACTION READ ONLY") + except Exception as e: + warnings.append(f"START TRANSACTION READ ONLY failed: {e}") + cur.execute("START TRANSACTION") + + +def probe_wkb_fn(cur) -> str: + for fn in ("ST_AsBinary", "ST_AsWKB", "AsBinary", "AsWKB"): + try: + cur.execute(f"SELECT {fn}(geom) AS x FROM chart_area_geometry LIMIT 1") + cur.fetchone() + return fn + except Exception: + pass + raise RuntimeError("找不到 geometry -> WKB 函数") + + +def parse_invalid_point(reason: str) -> Optional[Tuple[float, float]]: + if not reason: + return None + m = INVALID_POINT_RE.search(reason) + if not m: + return None + try: + return float(m.group(1)), float(m.group(2)) + except Exception: + return None + + +def to_bytes(v: Any) -> Optional[bytes]: + if v is None: + return None + if isinstance(v, memoryview): + return v.tobytes() + if isinstance(v, bytearray): + return bytes(v) + if isinstance(v, bytes): + return v + return bytes(v) + + +def bbox_from_row(row: Dict[str, Any], prefix: str = "") -> Optional[Tuple[float, float, float, float]]: + keys = ( + prefix + "bbox_min_lon", + prefix + "bbox_min_lat", + prefix + "bbox_max_lon", + prefix + "bbox_max_lat", + ) + vals = [] + for k in keys: + try: + v = float(row[k]) + if not math.isfinite(v): + return None + vals.append(v) + except Exception: + return None + return tuple(vals) # type: ignore + + +def point_close(a: Tuple[float, float], b: Tuple[float, float], tol: float) -> bool: + return abs(a[0] - b[0]) <= tol and abs(a[1] - b[1]) <= tol + + +def ring_lists(geom) -> List[Tuple[str, List[Tuple[float, float]]]]: + out = [] + + def add_poly(poly, pi: int): + try: + out.append((f"p{pi}.shell", [(float(c[0]), float(c[1])) for c in poly.exterior.coords])) + except Exception: + pass + for hi, ring in enumerate(poly.interiors): + try: + out.append((f"p{pi}.hole{hi}", [(float(c[0]), float(c[1])) for c in ring.coords])) + except Exception: + pass + + if geom.geom_type == "Polygon": + add_poly(geom, 0) + elif geom.geom_type == "MultiPolygon": + for pi, poly in enumerate(geom.geoms): + add_poly(poly, pi) + return out + + +def point_relation_to_area_boundary( + geom, + pxy: Tuple[float, float], + tol: float +) -> Dict[str, Any]: + """ + 判断一个目标点与某 Area boundary/ring 的关系。 + """ + p = Point(pxy) + vertex_occ = 0 + segment_hits = 0 + segment_interior_hits = 0 + endpoint_hits = 0 + ring_labels = set() + + for label, coords in ring_lists(geom): + core = coords[:-1] if len(coords) > 1 and point_close(coords[0], coords[-1], tol) else coords + for c in core: + if point_close(c, pxy, tol): + vertex_occ += 1 + ring_labels.add(label) + + for i in range(len(coords) - 1): + a, b = coords[i], coords[i+1] + try: + seg = LineString([a, b]) + if seg.distance(p) <= tol: + segment_hits += 1 + ring_labels.add(label) + at_a = point_close(a, pxy, tol) + at_b = point_close(b, pxy, tol) + if at_a or at_b: + endpoint_hits += 1 + else: + segment_interior_hits += 1 + except Exception: + pass + + try: + boundary_distance = float(geom.boundary.distance(p)) + except Exception: + boundary_distance = None + + try: + contains = bool(geom.contains(p)) + except Exception: + contains = False + try: + covers = bool(geom.covers(p)) + except Exception: + covers = False + + if vertex_occ > 0: + relation = "VERTEX" + elif segment_interior_hits > 0: + relation = "SEGMENT_INTERIOR" + elif segment_hits > 0: + relation = "BOUNDARY" + elif boundary_distance is not None and boundary_distance <= tol: + relation = "BOUNDARY_NEAR" + elif contains: + relation = "INTERIOR" + elif covers: + relation = "COVERS_OTHER" + else: + relation = "NONE" + + return { + "relation": relation, + "vertex_occurrences": vertex_occ, + "segment_hits": segment_hits, + "segment_interior_hits": segment_interior_hits, + "endpoint_hits": endpoint_hits, + "ring_labels": "|".join(sorted(ring_labels)), + "boundary_distance_deg": boundary_distance, + "contains_point": int(contains), + "covers_point": int(covers), + } + + +def get_columns(cur, db: str, table: str) -> List[Dict[str, Any]]: + return fetch_all(cur, """ + SELECT COLUMN_NAME, DATA_TYPE, COLUMN_TYPE, IS_NULLABLE, COLUMN_KEY + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA=%s AND TABLE_NAME=%s + ORDER BY ORDINAL_POSITION + """, (db, table)) + + +def col_names(cols: List[Dict[str, Any]]) -> List[str]: + return [str(x["COLUMN_NAME"]) for x in cols] + + +def find_col(cols: Sequence[str], candidates: Sequence[str]) -> Optional[str]: + lower = {c.lower(): c for c in cols} + for cand in candidates: + if cand.lower() in lower: + return lower[cand.lower()] + for cand in candidates: + for c in cols: + if cand.lower() in c.lower(): + return c + return None + + +def detect_raw_record_link(cur, db: str) -> Dict[str, Any]: + cols_meta = get_columns(cur, db, "chart_raw_record") + cols = col_names(cols_meta) + return { + "columns": cols_meta, + "cell_col": find_col(cols, ["cell_id", "cellid"]), + "record_index_col": find_col(cols, [ + "source_record_index", "record_index", "record_no", + "record_number", "source_index", "index" + ]), + "object_class_col": find_col(cols, ["object_class", "class_name", "object"]), + "payload_cols": [ + c for c in cols + if any(x in c.lower() for x in ( + "raw", "payload", "data", "blob", "json", + "hex", "content", "record" + )) + ], + } + + +def load_invalid_areas(cur, dataset_id: int, wkb_fn: str) -> List[Dict[str, Any]]: + cur.execute(f""" + SELECT + g.feature_id, + {wkb_fn}(g.geom) AS geom_wkb, + f.*, + c.cell_name, + c.bbox_min_lon AS cell_bbox_min_lon, + c.bbox_min_lat AS cell_bbox_min_lat, + c.bbox_max_lon AS cell_bbox_max_lon, + c.bbox_max_lat AS cell_bbox_max_lat + FROM chart_area_geometry g + JOIN chart_feature f ON f.id=g.feature_id + JOIN chart_cell c ON c.id=f.cell_id + WHERE f.dataset_id=%s + ORDER BY g.feature_id + """, (dataset_id,)) + out = [] + for r in cur: + try: + geom = load_wkb(to_bytes(r["geom_wkb"])) + if not geom.is_valid: + reason = explain_validity(geom) + p = parse_invalid_point(reason) + x = dict(r) + x["_geom"] = geom + x["_reason"] = reason + x["_invalid_point"] = p + out.append(x) + except Exception: + continue + return out + + +def load_invalid_coverages(cur, dataset_id: int, wkb_fn: str) -> List[Dict[str, Any]]: + cur.execute(f""" + SELECT + v.*, + {wkb_fn}(v.geom) AS geom_wkb, + c.cell_name, + c.bbox_min_lon AS cell_bbox_min_lon, + c.bbox_min_lat AS cell_bbox_min_lat, + c.bbox_max_lon AS cell_bbox_max_lon, + c.bbox_max_lat AS cell_bbox_max_lat + FROM chart_coverage v + JOIN chart_cell c ON c.id=v.cell_id + WHERE v.dataset_id=%s + ORDER BY v.id + """, (dataset_id,)) + out = [] + for r in cur: + try: + geom = load_wkb(to_bytes(r["geom_wkb"])) + if not geom.is_valid: + reason = explain_validity(geom) + p = parse_invalid_point(reason) + x = dict(r) + x["_geom"] = geom + x["_reason"] = reason + x["_invalid_point"] = p + out.append(x) + except Exception: + continue + return out + + +def load_all_areas_for_cells( + cur, + dataset_id: int, + cell_ids: Sequence[int], + wkb_fn: str, +) -> Dict[int, List[Dict[str, Any]]]: + out = defaultdict(list) + ids = sorted(set(int(x) for x in cell_ids)) + for pos in range(0, len(ids), 100): + batch = ids[pos:pos+100] + ph = ",".join(["%s"] * len(batch)) + sql = f""" + SELECT + g.feature_id, + {wkb_fn}(g.geom) AS geom_wkb, + f.object_class, + f.source_record_index, + f.coverage_id, + f.geometry_issue, + f.geometry_repaired, + f.raw_geotype, + f.cell_id, + c.cell_name + FROM chart_area_geometry g + JOIN chart_feature f ON f.id=g.feature_id + JOIN chart_cell c ON c.id=f.cell_id + WHERE f.dataset_id=%s + AND f.cell_id IN ({ph}) + ORDER BY f.cell_id, f.id + """ + cur.execute(sql, [dataset_id] + batch) + for r in cur.fetchall(): + try: + r["_geom"] = load_wkb(to_bytes(r["geom_wkb"])) + out[int(r["cell_id"])].append(r) + except Exception: + pass + return out + + +def dump_attributes(cur, feature_ids: Sequence[int], db: str) -> List[Dict[str, Any]]: + if not feature_ids: + return [] + + attr_cols = col_names(get_columns(cur, db, "chart_feature_attribute")) + dict_cols = col_names(get_columns(cur, db, "chart_dictionary_attribute")) + + feature_col = find_col(attr_cols, ["feature_id"]) + attr_id_col = find_col(attr_cols, ["attribute_id", "attr_id", "dictionary_attribute_id"]) + if not feature_col: + return [] + + # 为最大兼容性，先原样导出 feature_attribute。 + raw_rows = [] + ids = list(sorted(set(int(x) for x in feature_ids))) + for pos in range(0, len(ids), 500): + b = ids[pos:pos+500] + ph = ",".join(["%s"] * len(b)) + raw_rows.extend(fetch_all( + cur, + f"SELECT * FROM chart_feature_attribute WHERE {qi(feature_col)} IN ({ph}) ORDER BY {qi(feature_col)}", + b + )) + + # 如果能识别字典主键，则补充 dictionary 字段。 + if attr_id_col: + dict_pk = find_col(dict_cols, ["id", "attribute_id", "attr_id"]) + if dict_pk: + dict_ids = sorted({ + int(r[attr_id_col]) + for r in raw_rows + if r.get(attr_id_col) is not None + }) + dictionary = {} + for pos in range(0, len(dict_ids), 500): + b = dict_ids[pos:pos+500] + ph = ",".join(["%s"] * len(b)) + for d in fetch_all( + cur, + f"SELECT * FROM chart_dictionary_attribute WHERE {qi(dict_pk)} IN ({ph})", + b + ): + dictionary[str(d[dict_pk])] = d + + enriched = [] + for r in raw_rows: + x = dict(r) + d = dictionary.get(str(r.get(attr_id_col))) + if d: + for k, v in d.items(): + x["dict__" + k] = v + enriched.append(x) + return enriched + + return raw_rows + + +def dump_raw_records( + cur, + db: str, + invalid_rows: List[Dict[str, Any]], + raw_link: Dict[str, Any], +) -> Tuple[List[Dict[str, Any]], Dict[str, Any]]: + cell_col = raw_link.get("cell_col") + idx_col = raw_link.get("record_index_col") + result_meta = { + "raw_record_total": int(scalar(cur, "SELECT COUNT(*) FROM chart_raw_record") or 0), + "cell_col": cell_col, + "record_index_col": idx_col, + "exact_match_supported": bool(cell_col and idx_col), + "exact_match_count": 0, + "nearby_row_count": 0, + } + + if not cell_col or not idx_col: + # schema 能看，但无法安全猜连接逻辑。 + sample = fetch_all(cur, "SELECT * FROM chart_raw_record LIMIT 50") + result_meta["fallback_sample_count"] = len(sample) + return sample, result_meta + + output = [] + seen = set() + + for r in invalid_rows: + cell_id = r.get("cell_id") + src_idx = r.get("source_record_index") + if cell_id is None or src_idx is None: + continue + + # 精确 + 前后两条，仅用于调查；不假定 record_index 必然等价。 + rs = fetch_all(cur, f""" + SELECT *, + CASE WHEN {qi(idx_col)}=%s THEN 1 ELSE 0 END AS _exact_source_index + FROM chart_raw_record + WHERE {qi(cell_col)}=%s + AND {qi(idx_col)} BETWEEN %s AND %s + ORDER BY {qi(idx_col)} + """, (src_idx, cell_id, int(src_idx)-2, int(src_idx)+2)) + + for x in rs: + key = json.dumps(x, sort_keys=True, default=str) + if key in seen: + continue + seen.add(key) + y = dict(x) + y["_target_feature_id"] = r.get("feature_id") + y["_target_cell_name"] = r.get("cell_name") + y["_target_source_record_index"] = src_idx + output.append(y) + result_meta["nearby_row_count"] += 1 + if int(x.get("_exact_source_index") or 0): + result_meta["exact_match_count"] += 1 + + return output, result_meta + + +def make_geojson( + invalid_areas: List[Dict[str, Any]], + invalid_coverages: List[Dict[str, Any]], + neighbor_rows: List[Dict[str, Any]], + path: Path, +): + features = [] + + def add_geom(geom, props): + try: + features.append({ + "type": "Feature", + "geometry": mapping(geom), + "properties": props, + }) + except Exception: + pass + + # invalid Area + invalid point + for r in invalid_areas: + geom = r["_geom"] + p = r["_invalid_point"] + add_geom(geom, { + "layer": "invalid_area", + "feature_id": r.get("feature_id"), + "object_class": r.get("object_class"), + "cell_name": r.get("cell_name"), + "source_record_index": r.get("source_record_index"), + "coverage_id": r.get("coverage_id"), + "validity": r.get("_reason"), + }) + if p: + add_geom(Point(p), { + "layer": "invalid_point", + "source": "AREA", + "feature_id": r.get("feature_id"), + "cell_name": r.get("cell_name"), + "validity": r.get("_reason"), + }) + + # invalid Coverage + invalid point + for r in invalid_coverages: + geom = r["_geom"] + p = r["_invalid_point"] + add_geom(geom, { + "layer": "invalid_coverage", + "coverage_id": r.get("id"), + "cell_name": r.get("cell_name"), + "source_record_index": r.get("source_record_index"), + "validity": r.get("_reason"), + }) + if p: + add_geom(Point(p), { + "layer": "invalid_point", + "source": "COVERAGE", + "coverage_id": r.get("id"), + "cell_name": r.get("cell_name"), + "validity": r.get("_reason"), + }) + + # 只加入确实共享 invalid point 的 Area 邻居，避免 GeoJSON 爆大。 + added_neighbors = set() + for n in neighbor_rows: + if n.get("relation") not in {"VERTEX", "SEGMENT_INTERIOR", "BOUNDARY", "BOUNDARY_NEAR"}: + continue + fid = n.get("neighbor_feature_id") + geom = n.get("_geom") + if geom is None: + continue + key = (fid, n.get("target_invalid_x"), n.get("target_invalid_y")) + if key in added_neighbors: + continue + added_neighbors.add(key) + add_geom(geom, { + "layer": "shared_neighbor_area", + "feature_id": fid, + "object_class": n.get("neighbor_object_class"), + "cell_name": n.get("cell_name"), + "target_feature_id": n.get("target_feature_id"), + "relation": n.get("relation"), + "neighbor_valid": n.get("neighbor_valid"), + }) + + fc = {"type": "FeatureCollection", "features": features} + path.write_text( + json.dumps(fc, ensure_ascii=False, separators=(",", ":"), default=str), + encoding="utf-8" + ) + + +def zip_dir(folder: Path) -> Path: + zpath = folder.with_suffix(".zip") + with zipfile.ZipFile(zpath, "w", compression=zipfile.ZIP_DEFLATED) as z: + for p in sorted(folder.rglob("*")): + if p.is_file(): + z.write(p, p.relative_to(folder.parent)) + return zpath + + +def parse_args(): + p = argparse.ArgumentParser(description="CM93 invalid geometry source/topology trace v5") + p.add_argument("--host", default=os.getenv("MYSQL_HOST", "127.0.0.1")) + p.add_argument("--port", type=int, default=int(os.getenv("MYSQL_PORT", "3306"))) + p.add_argument("--user", default=os.getenv("MYSQL_USER", "root")) + p.add_argument("--password", default=os.getenv("MYSQL_PASSWORD")) + p.add_argument("--database", default=os.getenv("MYSQL_DATABASE", "cm93_chart")) + p.add_argument("--dataset-id", type=int, default=1) + p.add_argument("--point-tolerance", type=float, default=1e-8) + p.add_argument("--output", default=None) + return p.parse_args() + + +def main(): + args = parse_args() + if not args.password: + args.password = getpass.getpass("MySQL password: ") + + stamp = datetime.now().strftime("%Y%m%d_%H%M%S") + outdir = Path(args.output or f"cm93_invalid_trace_v5_dataset_{args.dataset_id}_{stamp}") + outdir.mkdir(parents=True, exist_ok=True) + + report = { + "version": "v5.0", + "generated_at": datetime.now().isoformat(timespec="seconds"), + "database": args.database, + "dataset_id": args.dataset_id, + "read_only": True, + "pbf": "OUT_OF_SCOPE", + "scope_note": ( + "CM93JAPAN 为局部数据集。V5 不要求跨未导入海域连续，" + "只追 invalid Area/Coverage 的源记录和同 Cell 邻接拓扑。" + ), + "warnings": [], + } + + conn = None + try: + conn = pymysql.connect( + host=args.host, + port=args.port, + user=args.user, + password=args.password, + database=args.database, + charset="utf8mb4", + cursorclass=DictCursor, + autocommit=False, + connect_timeout=20, + read_timeout=600, + write_timeout=30, + ) + + with conn.cursor() as cur: + begin_read_only(cur, report["warnings"]) + report["mysql_version"] = scalar(cur, "SELECT VERSION()") + wkb_fn = probe_wkb_fn(cur) + report["wkb_fn"] = wkb_fn + + print("[1/7] 找出 invalid Area / Coverage ...") + invalid_areas = load_invalid_areas(cur, args.dataset_id, wkb_fn) + invalid_cov = load_invalid_coverages(cur, args.dataset_id, wkb_fn) + print(f" Area={len(invalid_areas)} Coverage={len(invalid_cov)}") + + affected_cells = sorted({ + int(r["cell_id"]) + for r in invalid_areas + invalid_cov + if r.get("cell_id") is not None + }) + report["affected_cell_count"] = len(affected_cells) + + print("[2/7] 加载受影响 Cell 的所有 Area，检查共享 invalid point ...") + all_areas_by_cell = load_all_areas_for_cells( + cur, args.dataset_id, affected_cells, wkb_fn + ) + + neighbor_rows = [] + shared_counter = Counter() + target_summary = [] + + for target in invalid_areas: + p = target["_invalid_point"] + if not p: + continue + fid = int(target["feature_id"]) + cell_id = int(target["cell_id"]) + neighbors_at_point = 0 + valid_neighbors_at_point = 0 + invalid_neighbors_at_point = 0 + relation_counts = Counter() + + for n in all_areas_by_cell.get(cell_id, []): + nf = int(n["feature_id"]) + if nf == fid: + continue + geom = n["_geom"] + + rel = point_relation_to_area_boundary( + geom, p, args.point_tolerance + ) + relation = rel["relation"] + + if relation in {"VERTEX", "SEGMENT_INTERIOR", "BOUNDARY", "BOUNDARY_NEAR"}: + neighbors_at_point += 1 + try: + nvalid = bool(geom.is_valid) + except Exception: + nvalid = False + if nvalid: + valid_neighbors_at_point += 1 + else: + invalid_neighbors_at_point += 1 + relation_counts[relation] += 1 + + row = { + "target_feature_id": fid, + "target_object_class": target.get("object_class"), + "target_source_record_index": target.get("source_record_index"), + "target_invalid_reason": target.get("_reason"), + "target_invalid_x": p[0], + "target_invalid_y": p[1], + "cell_id": cell_id, + "cell_name": target.get("cell_name"), + "neighbor_feature_id": nf, + "neighbor_object_class": n.get("object_class"), + "neighbor_source_record_index": n.get("source_record_index"), + "neighbor_coverage_id": n.get("coverage_id"), + "neighbor_valid": int(nvalid), + "neighbor_validity_reason": "" if nvalid else explain_validity(geom), + } + row.update(rel) + row["_geom"] = geom + neighbor_rows.append(row) + + shared_counter[neighbors_at_point] += 1 + target_summary.append({ + "feature_id": fid, + "object_class": target.get("object_class"), + "cell_id": cell_id, + "cell_name": target.get("cell_name"), + "source_record_index": target.get("source_record_index"), + "coverage_id": target.get("coverage_id"), + "invalid_reason": target.get("_reason"), + "invalid_x": p[0], + "invalid_y": p[1], + "shared_neighbor_count": neighbors_at_point, + "shared_valid_neighbor_count": valid_neighbors_at_point, + "shared_invalid_neighbor_count": invalid_neighbors_at_point, + "neighbor_relations": dict(relation_counts), + }) + + print(f" shared neighbor rows={len(neighbor_rows)}") + + print("[3/7] 导出 invalid Feature attributes ...") + feature_ids = [int(r["feature_id"]) for r in invalid_areas] + attrs = dump_attributes(cur, feature_ids, args.database) + write_csv(outdir / "invalid_feature_attributes.csv", attrs) + + print("[4/7] 追 chart_raw_record / source_record_index ...") + raw_link = detect_raw_record_link(cur, args.database) + raw_rows, raw_meta = dump_raw_records( + cur, args.database, invalid_areas, raw_link + ) + report["raw_record_link"] = raw_meta + write_csv(outdir / "chart_raw_record_schema.csv", raw_link["columns"]) + write_csv(outdir / "invalid_raw_record_trace.csv", raw_rows) + + print("[5/7] 导出 Feature / Coverage 完整元数据 ...") + feature_export = [] + for r in invalid_areas: + x = { + k: v for k, v in r.items() + if not k.startswith("_") and k != "geom_wkb" + } + x["validity_reason"] = r["_reason"] + p = r["_invalid_point"] + x["invalid_x"] = p[0] if p else None + x["invalid_y"] = p[1] if p else None + feature_export.append(x) + + coverage_export = [] + for r in invalid_cov: + x = { + k: v for k, v in r.items() + if not k.startswith("_") and k not in {"geom_wkb", "geom"} + } + x["validity_reason"] = r["_reason"] + p = r["_invalid_point"] + x["invalid_x"] = p[0] if p else None + x["invalid_y"] = p[1] if p else None + coverage_export.append(x) + + write_csv(outdir / "invalid_feature_full_metadata.csv", feature_export) + write_csv(outdir / "invalid_coverage_full_metadata.csv", coverage_export) + write_csv(outdir / "invalid_target_neighbor_summary.csv", target_summary) + + # CSV 不应塞 Shapely object + neighbor_csv = [ + {k: v for k, v in r.items() if k != "_geom"} + for r in neighbor_rows + ] + write_csv(outdir / "invalid_point_shared_area_neighbors.csv", neighbor_csv) + + print("[6/7] 生成 GeoJSON ...") + make_geojson( + invalid_areas, invalid_cov, neighbor_rows, + outdir / "invalid_geometry_trace.geojson" + ) + + # 汇总 + target_counts = Counter() + valid_share = 0 + invalid_share = 0 + no_share = 0 + for t in target_summary: + n = int(t["shared_neighbor_count"]) + if n == 0: + no_share += 1 + target_counts["NO_SHARED_AREA_BOUNDARY_AT_INVALID_POINT"] += 1 + else: + target_counts["HAS_SHARED_AREA_BOUNDARY_AT_INVALID_POINT"] += 1 + if int(t["shared_valid_neighbor_count"]) > 0: + valid_share += 1 + if int(t["shared_invalid_neighbor_count"]) > 0: + invalid_share += 1 + + # coverage_id 分布 + feature_cov = Counter( + "HAS_COVERAGE_ID" if r.get("coverage_id") is not None else "NO_COVERAGE_ID" + for r in invalid_areas + ) + + report["summary"] = { + "invalid_area_count": len(invalid_areas), + "invalid_coverage_count": len(invalid_cov), + "affected_cell_count": len(affected_cells), + "invalid_area_with_shared_area_boundary_at_invalid_point": + target_counts["HAS_SHARED_AREA_BOUNDARY_AT_INVALID_POINT"], + "invalid_area_without_shared_area_boundary_at_invalid_point": + no_share, + "invalid_area_sharing_point_with_at_least_one_valid_area": + valid_share, + "invalid_area_sharing_point_with_at_least_one_invalid_area": + invalid_share, + "shared_neighbor_relation_counts": dict(Counter( + r["relation"] for r in neighbor_rows + )), + "invalid_feature_coverage_id_presence": dict(feature_cov), + "raw_record_trace": raw_meta, + } + + print("[7/7] 写报告并压缩 ...") + (outdir / "invalid_trace_report_v5.json").write_text( + json.dumps(report, ensure_ascii=False, indent=2, default=str), + encoding="utf-8" + ) + + lines = [] + lines.append("CM93 -> MySQL invalid Area/Coverage 源记录 + 邻接拓扑追踪 v5") + lines.append("=" * 88) + lines.append(f"Generated: {report['generated_at']}") + lines.append(f"Dataset ID: {args.dataset_id}") + lines.append(f"MySQL: {report['mysql_version']}") + lines.append("") + lines.append("SCOPE") + lines.append(" CM93JAPAN 为局部数据集；跨未导入海域连续性不作为 FAIL。") + lines.append(" 本轮不修 geometry，只调查 invalid point 的共享拓扑与 source_record。") + lines.append("") + lines.append("INVALID BASELINE") + lines.append(f" invalid Area: {len(invalid_areas)}") + lines.append(f" invalid Coverage: {len(invalid_cov)}") + lines.append(f" affected Cells: {len(affected_cells)}") + lines.append("") + lines.append("SHARED AREA TOPOLOGY AT INVALID POINT") + lines.append( + f" invalid Area with >=1 other Area boundary at same point: " + f"{report['summary']['invalid_area_with_shared_area_boundary_at_invalid_point']}" + ) + lines.append( + f" invalid Area with no other Area boundary at same point: " + f"{report['summary']['invalid_area_without_shared_area_boundary_at_invalid_point']}" + ) + lines.append( + f" invalid Area sharing point with >=1 VALID Area: " + f"{report['summary']['invalid_area_sharing_point_with_at_least_one_valid_area']}" + ) + lines.append( + f" invalid Area sharing point with >=1 INVALID Area: " + f"{report['summary']['invalid_area_sharing_point_with_at_least_one_invalid_area']}" + ) + lines.append("") + lines.append("NEIGHBOR RELATIONS") + for k, v in report["summary"]["shared_neighbor_relation_counts"].items(): + lines.append(f" {k:28s} {v}") + lines.append("") + lines.append("FEATURE coverage_id") + for k, v in feature_cov.items(): + lines.append(f" {k:28s} {v}") + lines.append("") + lines.append("RAW RECORD TRACE") + lines.append(f" chart_raw_record rows: {raw_meta['raw_record_total']}") + lines.append(f" detected cell column: {raw_meta['cell_col']}") + lines.append(f" detected record index column: {raw_meta['record_index_col']}") + lines.append(f" exact source-index match supported:{raw_meta['exact_match_supported']}") + lines.append(f" exact source-index matches: {raw_meta['exact_match_count']}") + lines.append(f" exported nearby raw rows: {raw_meta['nearby_row_count']}") + lines.append("") + lines.append("OUTPUT OF SPECIAL INTEREST") + lines.append(" invalid_target_neighbor_summary.csv") + lines.append(" invalid_point_shared_area_neighbors.csv") + lines.append(" invalid_feature_full_metadata.csv") + lines.append(" invalid_feature_attributes.csv") + lines.append(" chart_raw_record_schema.csv") + lines.append(" invalid_raw_record_trace.csv") + lines.append(" invalid_geometry_trace.geojson") + lines.append("") + lines.append("INTERPRETATION") + lines.append( + " 若 invalid point 被多个正常 Area 共享，优先怀疑共享拓扑/ring assembly 表达，" + "而不是孤立坐标损坏。" + ) + lines.append( + " 若内部 TRUE_SEGMENT_CROSSING 没有任何邻接对象共享该点，" + "则更值得追 source_record / edge ordering。" + ) + lines.append( + " chart_raw_record 若不能按 cell_id + source_record_index 直接关联，" + "说明 DB 当前可能没有保留足够原始 topology，需要回到 CM93 parser/importer 调查。" + ) + lines.append("") + lines.append("NO DATABASE WRITE / NO GEOMETRY REPAIR / NO PBF") + + (outdir / "invalid_trace_summary_v5.txt").write_text( + "\n".join(lines), encoding="utf-8" + ) + + conn.rollback() + + zpath = zip_dir(outdir) + print("") + print("完成。") + print(f"Output: {outdir.resolve()}") + print(f"ZIP: {zpath.resolve()}") + print("把 ZIP 整包上传给 ChatGPT。") + + except KeyboardInterrupt: + if conn: + try: + conn.rollback() + except Exception: + pass + print("\n用户中断。") + sys.exit(130) + except Exception: + if conn: + try: + conn.rollback() + except Exception: + pass + tb = traceback.format_exc() + print(tb) + try: + (outdir / "fatal_error.txt").write_text(tb, encoding="utf-8") + except Exception: + pass + sys.exit(1) + finally: + if conn: + try: + conn.close() + except Exception: + pass + + +if __name__ == "__main__": + main() diff --git a/cm93_mysql_non_geometry_final_audit_v1.py b/cm93_mysql_non_geometry_final_audit_v1.py new file mode 100644 index 0000000..f4ef342 --- /dev/null +++ b/cm93_mysql_non_geometry_final_audit_v1.py @@ -0,0 +1,1272 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +CM93 -> MySQL 非 Geometry 数据完整性全库最终验收 v1 + +目标 +==== +在 Geometry normalization 已通过后，对同一 Dataset 的“非 Geometry”数据链做最终只读验收。 + +检查： +1. Dataset / Cell 完整性 +2. 每个 Cell 的 feature_count / attribute_count 与实际落库一致 +3. Feature 的 dataset/cell/source_record_index 可追溯性 +4. Feature source_record_index 在 Cell 内连续、唯一 +5. Attribute 外键、ordinal、dictionary 映射、类型值通道 +6. Coverage 与 _m_sor Feature 一一对应且同 Cell +7. Sounding 外键、point_index 连续性、depth 公式 +8. raw_feature_blob / raw_attribute_blob 保留 +9. chart_raw_record 分块索引、chunk_count、total_bytes、JSON metadata +10. Object / Attribute dictionary 映射完整性 +11. attributes_json 可解析；_cm93_related_indices 若存在，必须指向同 Cell Feature +12. 可选：源 CM93 文件存在、source_size、SHA-256 与数据库一致 +13. orphan / duplicate 统计必须为 0 + +不检查 Geometry validity（该项已经由独立 Geometry audit 验收）。 +不修改数据库。 +不依赖历史 V 系列测试输出。 +不涉及 PBF。 + +当前 CM93JAPAN 固定基线默认： +- Cells 784 +- Features 322465 +- Attributes 405153 +- Soundings 275700 + +如果以后换 Dataset，可以传 --no-fixed-baseline。 +""" + +from __future__ import annotations + +import argparse +import csv +import getpass +import hashlib +import json +import math +import sys +import zipfile +from collections import defaultdict +from datetime import datetime +from pathlib import Path +from typing import Any, Iterable + +try: + import pymysql + from pymysql.cursors import DictCursor, SSCursor +except ImportError: + print("缺少 pymysql，请执行: pip install pymysql") + sys.exit(2) + + +EXPECTED_COUNTS = { + "cells": 784, + "features": 322465, + "attributes": 405153, + "soundings": 275700, +} + + +def parse_args(): + p = argparse.ArgumentParser( + description="CM93 -> MySQL 非 Geometry 数据完整性全库最终验收 v1" + ) + p.add_argument("--host", default="127.0.0.1") + p.add_argument("--port", type=int, default=3306) + p.add_argument("--user", default="root") + p.add_argument("--password", default=None) + p.add_argument("--database", default="cm93_chart") + p.add_argument("--dataset-id", type=int, default=1) + p.add_argument( + "--source-root", + default=None, + help="可选：CM93 根目录，例如 D:\\Ship\\CM93V2\\CM93JAPAN；提供后校验文件存在/大小/SHA256", + ) + p.add_argument( + "--no-fixed-baseline", + action="store_true", + help="不要求当前 CM93JAPAN 的 784/322465/405153/275700 固定数量", + ) + p.add_argument( + "--output-dir", + default=None, + ) + return p.parse_args() + + +def scalar(cur, sql: str, params=()) -> Any: + cur.execute(sql, params) + row = cur.fetchone() + if row is None: + return None + if isinstance(row, dict): + return next(iter(row.values())) + return row[0] + + +def fetchall(cur, sql: str, params=()): + cur.execute(sql, params) + return list(cur.fetchall()) + + +def add_failure( + failures: list[dict[str, Any]], + check: str, + detail: Any = None, + **kwargs, +): + row = {"check": check} + if detail is not None: + row["detail"] = detail + row.update(kwargs) + failures.append(row) + + +def json_load(value): + if value is None: + return None + if isinstance(value, (dict, list)): + return value + if isinstance(value, (bytes, bytearray, memoryview)): + value = bytes(value).decode("utf-8") + return json.loads(value) + + +def file_sha256(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(1024 * 1024), b""): + h.update(chunk) + return h.hexdigest() + + +def write_csv(path: Path, rows: Iterable[dict[str, Any]]): + rows = list(rows) + if not rows: + path.write_text("", encoding="utf-8-sig") + return + + fields = [] + seen = set() + for row in rows: + for key in row: + if key not in seen: + seen.add(key) + fields.append(key) + + with path.open("w", encoding="utf-8-sig", newline="") as f: + w = csv.DictWriter(f, fieldnames=fields, extrasaction="ignore") + w.writeheader() + for row in rows: + out = {} + for key in fields: + v = row.get(key) + if isinstance(v, (dict, list, tuple)): + v = json.dumps(v, ensure_ascii=False, default=str) + out[key] = v + w.writerow(out) + + +def table_exists(cur, database: str, table: str) -> bool: + return bool(scalar( + cur, + """ + SELECT COUNT(*) + FROM information_schema.tables + WHERE table_schema=%s AND table_name=%s + """, + (database, table), + )) + + +def begin_read_only(cur, warnings: list[str]): + try: + cur.execute("SET SESSION TRANSACTION READ ONLY") + except Exception as e: + warnings.append(f"SET SESSION TRANSACTION READ ONLY: {e}") + + try: + cur.execute("START TRANSACTION READ ONLY") + except Exception as e: + warnings.append(f"START TRANSACTION READ ONLY: {e}") + cur.execute("START TRANSACTION") + + +def main(): + args = parse_args() + + if args.password is None: + args.password = getpass.getpass("MySQL password: ") + + source_root = ( + Path(args.source_root).expanduser().resolve() + if args.source_root + else None + ) + + stamp = datetime.now().strftime("%Y%m%d_%H%M%S") + outdir = Path( + args.output_dir + or f"cm93_non_geometry_final_audit_dataset_{args.dataset_id}_{stamp}" + ).resolve() + outdir.mkdir(parents=True, exist_ok=True) + + failures: list[dict[str, Any]] = [] + warnings: list[str] = [] + + report: dict[str, Any] = { + "version": "v1.0", + "generated_at": datetime.now().isoformat(timespec="seconds"), + "database": args.database, + "dataset_id": args.dataset_id, + "read_only": True, + "pbf": "OUT_OF_SCOPE", + "source_root": str(source_root) if source_root else None, + } + + conn = pymysql.connect( + host=args.host, + port=args.port, + user=args.user, + password=args.password, + database=args.database, + charset="utf8mb4", + cursorclass=DictCursor, + autocommit=False, + connect_timeout=20, + read_timeout=900, + write_timeout=30, + ) + + try: + with conn.cursor() as cur: + begin_read_only(cur, warnings) + + print("[1/10] 基础表 / Dataset / 总数 ...") + + required_tables = [ + "chart_dataset", + "chart_cell", + "chart_feature", + "chart_feature_attribute", + "chart_coverage", + "chart_sounding_point", + "chart_raw_record", + "chart_dictionary_object", + "chart_dictionary_attribute", + ] + missing_tables = [ + t for t in required_tables + if not table_exists(cur, args.database, t) + ] + report["missing_tables"] = missing_tables + if missing_tables: + add_failure( + failures, + "missing_tables", + missing_tables, + ) + raise RuntimeError( + "缺少必要表：" + ", ".join(missing_tables) + ) + + dataset_count = int(scalar( + cur, + "SELECT COUNT(*) FROM chart_dataset WHERE id=%s", + (args.dataset_id,), + ) or 0) + + if dataset_count != 1: + add_failure( + failures, + "dataset_exists", + expected=1, + actual=dataset_count, + ) + + counts = { + "cells": int(scalar( + cur, + "SELECT COUNT(*) FROM chart_cell WHERE dataset_id=%s", + (args.dataset_id,), + ) or 0), + "features": int(scalar( + cur, + "SELECT COUNT(*) FROM chart_feature WHERE dataset_id=%s", + (args.dataset_id,), + ) or 0), + "attributes": int(scalar( + cur, + """ + SELECT COUNT(*) + FROM chart_feature_attribute a + JOIN chart_feature f ON f.id=a.feature_id + WHERE f.dataset_id=%s + """, + (args.dataset_id,), + ) or 0), + "soundings": int(scalar( + cur, + """ + SELECT COUNT(*) + FROM chart_sounding_point s + JOIN chart_feature f ON f.id=s.feature_id + WHERE f.dataset_id=%s + """, + (args.dataset_id,), + ) or 0), + "coverages": int(scalar( + cur, + "SELECT COUNT(*) FROM chart_coverage WHERE dataset_id=%s", + (args.dataset_id,), + ) or 0), + "raw_records": int(scalar( + cur, + """ + SELECT COUNT(*) + FROM chart_raw_record r + JOIN chart_cell c ON c.id=r.cell_id + WHERE c.dataset_id=%s + """, + (args.dataset_id,), + ) or 0), + } + report["counts"] = counts + + if not args.no_fixed_baseline: + for key, expected in EXPECTED_COUNTS.items(): + if counts[key] != expected: + add_failure( + failures, + f"baseline_{key}", + expected=expected, + actual=counts[key], + ) + + done_cells = int(scalar( + cur, + """ + SELECT COUNT(*) + FROM chart_cell + WHERE dataset_id=%s AND import_status='DONE' + """, + (args.dataset_id,), + ) or 0) + non_done_cells = counts["cells"] - done_cells + + report["cell_status"] = { + "done": done_cells, + "non_done": non_done_cells, + } + + if non_done_cells: + add_failure( + failures, + "cell_status_non_done", + actual=non_done_cells, + ) + + print("[2/10] Cell header counts vs 实际落库 ...") + + cell_mismatch_rows = fetchall( + cur, + """ + SELECT + c.id AS cell_id, + c.source_relative_path, + c.feature_count AS expected_features, + COALESCE(f.actual_features,0) AS actual_features, + c.attribute_count AS expected_attributes, + COALESCE(a.actual_attributes,0) AS actual_attributes + FROM chart_cell c + LEFT JOIN ( + SELECT cell_id,COUNT(*) AS actual_features + FROM chart_feature + WHERE dataset_id=%s + GROUP BY cell_id + ) f ON f.cell_id=c.id + LEFT JOIN ( + SELECT f.cell_id,COUNT(*) AS actual_attributes + FROM chart_feature_attribute a + JOIN chart_feature f ON f.id=a.feature_id + WHERE f.dataset_id=%s + GROUP BY f.cell_id + ) a ON a.cell_id=c.id + WHERE c.dataset_id=%s + AND ( + c.feature_count<>COALESCE(f.actual_features,0) + OR c.attribute_count<>COALESCE(a.actual_attributes,0) + ) + ORDER BY c.id + """, + (args.dataset_id, args.dataset_id, args.dataset_id), + ) + + report["cell_count_mismatch"] = len(cell_mismatch_rows) + for row in cell_mismatch_rows: + add_failure( + failures, + "cell_header_count_mismatch", + **row, + ) + + print("[3/10] Feature traceability / source_record_index ...") + + feature_checks = { + "feature_cell_dataset_mismatch": int(scalar( + cur, + """ + SELECT COUNT(*) + FROM chart_feature f + JOIN chart_cell c ON c.id=f.cell_id + WHERE f.dataset_id=%s + AND c.dataset_id<>f.dataset_id + """, + (args.dataset_id,), + ) or 0), + "duplicate_source_record_index": int(scalar( + cur, + """ + SELECT COUNT(*) + FROM ( + SELECT cell_id,source_record_index + FROM chart_feature + WHERE dataset_id=%s + GROUP BY cell_id,source_record_index + HAVING COUNT(*)>1 + ) x + """, + (args.dataset_id,), + ) or 0), + "source_record_index_not_contiguous": int(scalar( + cur, + """ + SELECT COUNT(*) + FROM ( + SELECT + cell_id, + COUNT(*) AS n, + COUNT(DISTINCT source_record_index) AS nd, + MIN(source_record_index) AS min_i, + MAX(source_record_index) AS max_i + FROM chart_feature + WHERE dataset_id=%s + GROUP BY cell_id + HAVING + nd<>n + OR min_i<>0 + OR max_i<>n-1 + ) x + """, + (args.dataset_id,), + ) or 0), + "raw_feature_blob_null": int(scalar( + cur, + """ + SELECT COUNT(*) + FROM chart_feature + WHERE dataset_id=%s + AND raw_feature_blob IS NULL + """, + (args.dataset_id,), + ) or 0), + "raw_attribute_blob_null": int(scalar( + cur, + """ + SELECT COUNT(*) + FROM chart_feature + WHERE dataset_id=%s + AND raw_attribute_blob IS NULL + """, + (args.dataset_id,), + ) or 0), + "attributes_json_null": int(scalar( + cur, + """ + SELECT COUNT(*) + FROM chart_feature + WHERE dataset_id=%s + AND attributes_json IS NULL + """, + (args.dataset_id,), + ) or 0), + } + report["feature_checks"] = feature_checks + + for key, value in feature_checks.items(): + if value: + add_failure( + failures, + key, + expected=0, + actual=value, + ) + + print("[4/10] Object / Attribute dictionary 映射 ...") + + dictionary_checks = { + "feature_unknown_object_id": int(scalar( + cur, + """ + SELECT COUNT(*) + FROM chart_feature f + LEFT JOIN chart_dictionary_object d + ON d.dataset_id=f.dataset_id + AND d.object_class_id=f.object_class_id + WHERE f.dataset_id=%s + AND d.object_class_id IS NULL + """, + (args.dataset_id,), + ) or 0), + "feature_object_name_mismatch": int(scalar( + cur, + """ + SELECT COUNT(*) + FROM chart_feature f + JOIN chart_dictionary_object d + ON d.dataset_id=f.dataset_id + AND d.object_class_id=f.object_class_id + WHERE f.dataset_id=%s + AND f.object_class<>d.object_class + """, + (args.dataset_id,), + ) or 0), + "attribute_unknown_id": int(scalar( + cur, + """ + SELECT COUNT(*) + FROM chart_feature_attribute a + JOIN chart_feature f ON f.id=a.feature_id + LEFT JOIN chart_dictionary_attribute d + ON d.dataset_id=f.dataset_id + AND d.attribute_id=a.attribute_id + WHERE f.dataset_id=%s + AND d.attribute_id IS NULL + """, + (args.dataset_id,), + ) or 0), + "attribute_name_mismatch": int(scalar( + cur, + """ + SELECT COUNT(*) + FROM chart_feature_attribute a + JOIN chart_feature f ON f.id=a.feature_id + JOIN chart_dictionary_attribute d + ON d.dataset_id=f.dataset_id + AND d.attribute_id=a.attribute_id + WHERE f.dataset_id=%s + AND a.attribute_name<>d.attribute_name + """, + (args.dataset_id,), + ) or 0), + } + report["dictionary_checks"] = dictionary_checks + + for key, value in dictionary_checks.items(): + if value: + add_failure( + failures, + key, + expected=0, + actual=value, + ) + + print("[5/10] Attribute orphan / ordinal / value_type ...") + + attribute_checks = { + "attribute_orphan": int(scalar( + cur, + """ + SELECT COUNT(*) + FROM chart_feature_attribute a + LEFT JOIN chart_feature f ON f.id=a.feature_id + WHERE f.id IS NULL + """, + ) or 0), + "attribute_duplicate_ordinal": int(scalar( + cur, + """ + SELECT COUNT(*) + FROM ( + SELECT a.feature_id,a.ordinal_no + FROM chart_feature_attribute a + JOIN chart_feature f ON f.id=a.feature_id + WHERE f.dataset_id=%s + GROUP BY a.feature_id,a.ordinal_no + HAVING COUNT(*)>1 + ) x + """, + (args.dataset_id,), + ) or 0), + "attribute_ordinal_not_contiguous": int(scalar( + cur, + """ + SELECT COUNT(*) + FROM ( + SELECT + a.feature_id, + COUNT(*) AS n, + COUNT(DISTINCT a.ordinal_no) AS nd, + MIN(a.ordinal_no) AS min_o, + MAX(a.ordinal_no) AS max_o + FROM chart_feature_attribute a + JOIN chart_feature f ON f.id=a.feature_id + WHERE f.dataset_id=%s + GROUP BY a.feature_id + HAVING + nd<>n + OR min_o<>0 + OR max_o<>n-1 + ) x + """, + (args.dataset_id,), + ) or 0), + "attribute_unknown_value_type": int(scalar( + cur, + """ + SELECT COUNT(*) + FROM chart_feature_attribute a + JOIN chart_feature f ON f.id=a.feature_id + WHERE f.dataset_id=%s + AND a.value_type NOT IN + ('BYTE','INT','LONG','REAL','WORD10','STRING','LIST','COMPLEX','UNKNOWN') + """, + (args.dataset_id,), + ) or 0), + } + report["attribute_checks"] = attribute_checks + + for key, value in attribute_checks.items(): + if value: + add_failure( + failures, + key, + expected=0, + actual=value, + ) + + print("[6/10] Coverage <-> _m_sor Feature 映射 ...") + + coverage_checks = { + "coverage_duplicate_source_record_index": int(scalar( + cur, + """ + SELECT COUNT(*) + FROM ( + SELECT cell_id,source_record_index + FROM chart_coverage + WHERE dataset_id=%s + GROUP BY cell_id,source_record_index + HAVING COUNT(*)>1 + ) x + """, + (args.dataset_id,), + ) or 0), + "coverage_without_m_sor_feature": int(scalar( + cur, + """ + SELECT COUNT(*) + FROM chart_coverage c + LEFT JOIN chart_feature f + ON f.dataset_id=c.dataset_id + AND f.cell_id=c.cell_id + AND f.source_record_index=c.source_record_index + AND f.object_class='_m_sor' + WHERE c.dataset_id=%s + AND f.id IS NULL + """, + (args.dataset_id,), + ) or 0), + "m_sor_without_coverage": int(scalar( + cur, + """ + SELECT COUNT(*) + FROM chart_feature f + LEFT JOIN chart_coverage c + ON c.dataset_id=f.dataset_id + AND c.cell_id=f.cell_id + AND c.source_record_index=f.source_record_index + WHERE f.dataset_id=%s + AND f.object_class='_m_sor' + AND c.id IS NULL + """, + (args.dataset_id,), + ) or 0), + "feature_coverage_cross_cell": int(scalar( + cur, + """ + SELECT COUNT(*) + FROM chart_feature f + JOIN chart_coverage c ON c.id=f.coverage_id + WHERE f.dataset_id=%s + AND ( + c.dataset_id<>f.dataset_id + OR c.cell_id<>f.cell_id + ) + """, + (args.dataset_id,), + ) or 0), + } + report["coverage_checks"] = coverage_checks + + for key, value in coverage_checks.items(): + if value: + add_failure( + failures, + key, + expected=0, + actual=value, + ) + + print("[7/10] Sounding 数据链 ...") + + sounding_checks = { + "sounding_orphan": int(scalar( + cur, + """ + SELECT COUNT(*) + FROM chart_sounding_point s + LEFT JOIN chart_feature f ON f.id=s.feature_id + WHERE f.id IS NULL + """, + ) or 0), + "sounding_wrong_feature_type": int(scalar( + cur, + """ + SELECT COUNT(*) + FROM chart_sounding_point s + JOIN chart_feature f ON f.id=s.feature_id + WHERE f.dataset_id=%s + AND f.geometry_type<>'SOUNDING' + """, + (args.dataset_id,), + ) or 0), + "sounding_duplicate_point_index": int(scalar( + cur, + """ + SELECT COUNT(*) + FROM ( + SELECT s.feature_id,s.point_index + FROM chart_sounding_point s + JOIN chart_feature f ON f.id=s.feature_id + WHERE f.dataset_id=%s + GROUP BY s.feature_id,s.point_index + HAVING COUNT(*)>1 + ) x + """, + (args.dataset_id,), + ) or 0), + "sounding_index_not_contiguous": int(scalar( + cur, + """ + SELECT COUNT(*) + FROM ( + SELECT + s.feature_id, + COUNT(*) AS n, + COUNT(DISTINCT s.point_index) AS nd, + MIN(s.point_index) AS min_i, + MAX(s.point_index) AS max_i + FROM chart_sounding_point s + JOIN chart_feature f ON f.id=s.feature_id + WHERE f.dataset_id=%s + GROUP BY s.feature_id + HAVING + nd<>n + OR min_i<>0 + OR max_i<>n-1 + ) x + """, + (args.dataset_id,), + ) or 0), + "sounding_depth_formula_mismatch": int(scalar( + cur, + """ + SELECT COUNT(*) + FROM chart_sounding_point s + JOIN chart_feature f ON f.id=s.feature_id + WHERE f.dataset_id=%s + AND ABS( + s.depth - + CASE + WHEN s.raw_z>=12000 THEN s.raw_z-12000 + ELSE s.raw_z/10.0 + END + ) > 0.000001 + """, + (args.dataset_id,), + ) or 0), + } + report["sounding_checks"] = sounding_checks + + for key, value in sounding_checks.items(): + if value: + add_failure( + failures, + key, + expected=0, + actual=value, + ) + + print("[8/10] attributes_json / relation indices 全量解析 ...") + + relation_bad = 0 + attributes_json_bad = 0 + attributes_name_missing = 0 + + cur.execute( + """ + SELECT + id, + cell_id, + source_record_index, + attributes_json + FROM chart_feature + WHERE dataset_id=%s + ORDER BY cell_id,source_record_index + """, + (args.dataset_id,), + ) + feature_json_rows = list(cur.fetchall()) + + indices_by_cell: dict[int, set[int]] = defaultdict(set) + parsed_by_feature: dict[int, Any] = {} + + for row in feature_json_rows: + cell_id = int(row["cell_id"]) + src_idx = int(row["source_record_index"]) + indices_by_cell[cell_id].add(src_idx) + + try: + obj = json_load(row["attributes_json"]) + if not isinstance(obj, dict): + raise ValueError("attributes_json 不是 JSON object") + parsed_by_feature[int(row["id"])] = obj + except Exception as e: + attributes_json_bad += 1 + add_failure( + failures, + "attributes_json_parse", + feature_id=row["id"], + cell_id=cell_id, + source_record_index=src_idx, + detail=f"{type(e).__name__}: {e}", + ) + + for row in feature_json_rows: + fid = int(row["id"]) + obj = parsed_by_feature.get(fid) + if obj is None: + continue + + related = obj.get("_cm93_related_indices") + if related is not None: + if not isinstance(related, list): + relation_bad += 1 + add_failure( + failures, + "related_indices_not_list", + feature_id=fid, + actual_type=type(related).__name__, + ) + else: + cell_indices = indices_by_cell[int(row["cell_id"])] + for target in related: + try: + ti = int(target) + except Exception: + relation_bad += 1 + add_failure( + failures, + "related_index_not_int", + feature_id=fid, + target=repr(target), + ) + continue + + if ti not in cell_indices: + relation_bad += 1 + add_failure( + failures, + "related_index_missing_target", + feature_id=fid, + cell_id=row["cell_id"], + source_record_index=row["source_record_index"], + target_source_record_index=ti, + ) + + # Attribute row 的 attribute_name 应在对应 attributes_json 顶层出现。 + cur.execute( + """ + SELECT + a.feature_id, + a.attribute_name + FROM chart_feature_attribute a + JOIN chart_feature f ON f.id=a.feature_id + WHERE f.dataset_id=%s + """, + (args.dataset_id,), + ) + + for row in cur: + obj = parsed_by_feature.get(int(row["feature_id"])) + if obj is None: + continue + if row["attribute_name"] not in obj: + attributes_name_missing += 1 + if attributes_name_missing <= 1000: + add_failure( + failures, + "attribute_name_missing_from_attributes_json", + feature_id=row["feature_id"], + attribute_name=row["attribute_name"], + ) + + report["json_relation_checks"] = { + "attributes_json_bad": attributes_json_bad, + "related_indices_bad": relation_bad, + "attribute_name_missing_from_attributes_json": attributes_name_missing, + } + + if attributes_name_missing > 1000: + add_failure( + failures, + "attribute_name_missing_from_attributes_json_overflow", + detail=f"总数={attributes_name_missing}，明细只记录前 1000 条", + ) + + print("[9/10] chart_raw_record 分块完整性 ...") + + cur.execute( + """ + SELECT + r.cell_id, + r.record_kind, + r.source_record_index, + OCTET_LENGTH(r.raw_blob) AS raw_bytes, + r.raw_metadata_json + FROM chart_raw_record r + JOIN chart_cell c ON c.id=r.cell_id + WHERE c.dataset_id=%s + ORDER BY r.cell_id,r.record_kind,r.source_record_index + """, + (args.dataset_id,), + ) + + raw_groups: dict[tuple[int, str], list[dict[str, Any]]] = defaultdict(list) + raw_json_bad = 0 + raw_empty_blob = 0 + + for row in cur: + if int(row["raw_bytes"] or 0) <= 0: + raw_empty_blob += 1 + add_failure( + failures, + "raw_record_empty_blob", + cell_id=row["cell_id"], + record_kind=row["record_kind"], + source_record_index=row["source_record_index"], + ) + + try: + meta = json_load(row["raw_metadata_json"]) + if not isinstance(meta, dict): + raise ValueError("raw_metadata_json 不是 JSON object") + except Exception as e: + raw_json_bad += 1 + add_failure( + failures, + "raw_metadata_json_parse", + cell_id=row["cell_id"], + record_kind=row["record_kind"], + source_record_index=row["source_record_index"], + detail=f"{type(e).__name__}: {e}", + ) + meta = None + + raw_groups[ + (int(row["cell_id"]), str(row["record_kind"])) + ].append({ + "index": int(row["source_record_index"]), + "raw_bytes": int(row["raw_bytes"] or 0), + "meta": meta, + }) + + raw_group_bad = 0 + + for (cell_id, kind), rows in raw_groups.items(): + indices = [r["index"] for r in rows] + expected_indices = list(range(len(rows))) + + if indices != expected_indices: + raw_group_bad += 1 + add_failure( + failures, + "raw_chunk_index_not_contiguous", + cell_id=cell_id, + record_kind=kind, + indices=indices[:50], + count=len(indices), + ) + + metas = [r["meta"] for r in rows if isinstance(r["meta"], dict)] + if len(metas) != len(rows): + continue + + chunk_counts = {m.get("chunk_count") for m in metas} + total_bytes_set = {m.get("total_bytes") for m in metas} + chunk_indices_meta = [m.get("chunk_index") for m in metas] + actual_bytes = sum(r["raw_bytes"] for r in rows) + + if chunk_counts != {len(rows)}: + raw_group_bad += 1 + add_failure( + failures, + "raw_chunk_count_mismatch", + cell_id=cell_id, + record_kind=kind, + metadata_chunk_counts=list(chunk_counts), + actual_chunk_count=len(rows), + ) + + if chunk_indices_meta != expected_indices: + raw_group_bad += 1 + add_failure( + failures, + "raw_metadata_chunk_index_mismatch", + cell_id=cell_id, + record_kind=kind, + metadata_indices=chunk_indices_meta[:50], + ) + + if len(total_bytes_set) != 1: + raw_group_bad += 1 + add_failure( + failures, + "raw_total_bytes_inconsistent", + cell_id=cell_id, + record_kind=kind, + values=list(total_bytes_set), + ) + else: + expected_total = next(iter(total_bytes_set)) + try: + expected_total = int(expected_total) + except Exception: + expected_total = -1 + + if expected_total != actual_bytes: + raw_group_bad += 1 + add_failure( + failures, + "raw_total_bytes_mismatch", + cell_id=cell_id, + record_kind=kind, + expected=expected_total, + actual=actual_bytes, + ) + + # 每个 Cell 至少应有 HEADER 和 FEATURE 原始表。 + raw_required_missing = int(scalar( + cur, + """ + SELECT COUNT(*) + FROM ( + SELECT c.id + FROM chart_cell c + LEFT JOIN chart_raw_record h + ON h.cell_id=c.id AND h.record_kind='HEADER' + LEFT JOIN chart_raw_record f + ON f.cell_id=c.id AND f.record_kind='FEATURE' + WHERE c.dataset_id=%s + GROUP BY c.id + HAVING COUNT(DISTINCT h.record_kind)=0 + OR COUNT(DISTINCT f.record_kind)=0 + ) x + """, + (args.dataset_id,), + ) or 0) + + report["raw_checks"] = { + "raw_metadata_json_bad": raw_json_bad, + "raw_empty_blob": raw_empty_blob, + "raw_group_bad": raw_group_bad, + "required_header_or_feature_missing_cells": raw_required_missing, + "raw_group_count": len(raw_groups), + } + + if raw_required_missing: + add_failure( + failures, + "raw_required_kind_missing", + expected=0, + actual=raw_required_missing, + ) + + print("[10/10] 可选源文件 SHA256 + 最终判定 ...") + + source_checks = { + "enabled": source_root is not None, + "files_checked": 0, + "files_missing": 0, + "size_mismatch": 0, + "sha256_mismatch": 0, + } + + if source_root is not None: + if not source_root.is_dir(): + add_failure( + failures, + "source_root_not_found", + path=str(source_root), + ) + else: + cells = fetchall( + cur, + """ + SELECT + id, + source_relative_path, + source_size, + source_sha256 + FROM chart_cell + WHERE dataset_id=%s + ORDER BY id + """, + (args.dataset_id,), + ) + + for cell in cells: + rel = str(cell["source_relative_path"]).replace("\\", "/") + path = source_root / Path(rel) + source_checks["files_checked"] += 1 + + if not path.is_file(): + source_checks["files_missing"] += 1 + add_failure( + failures, + "source_file_missing", + cell_id=cell["id"], + path=str(path), + ) + continue + + actual_size = path.stat().st_size + expected_size = int(cell["source_size"]) + + if actual_size != expected_size: + source_checks["size_mismatch"] += 1 + add_failure( + failures, + "source_size_mismatch", + cell_id=cell["id"], + path=str(path), + expected=expected_size, + actual=actual_size, + ) + + expected_hash = str(cell["source_sha256"] or "").lower() + actual_hash = file_sha256(path).lower() + + if expected_hash != actual_hash: + source_checks["sha256_mismatch"] += 1 + add_failure( + failures, + "source_sha256_mismatch", + cell_id=cell["id"], + path=str(path), + expected=expected_hash, + actual=actual_hash, + ) + + report["source_checks"] = source_checks + + conn.rollback() + + finally: + conn.close() + + report["warnings"] = warnings + report["failure_count"] = len(failures) + report["overall"] = "PASS" if not failures else "FAIL" + + write_csv(outdir / "失败明细_v1.csv", failures) + write_csv(outdir / "Cell计数不一致_v1.csv", cell_mismatch_rows) + + (outdir / "最终验收报告_v1.json").write_text( + json.dumps( + report, + ensure_ascii=False, + indent=2, + default=str, + ), + encoding="utf-8", + ) + + c = report["counts"] + lines = [ + "CM93 -> MySQL 非 Geometry 数据完整性全库最终验收 v1", + "=" * 82, + f"Dataset ID : {args.dataset_id}", + "", + f"Cells : {c['cells']}", + f"Features : {c['features']}", + f"Attributes : {c['attributes']}", + f"Soundings : {c['soundings']}", + f"Coverages : {c['coverages']}", + f"Raw records : {c['raw_records']}", + "", + f"Cell count mismatch : {report['cell_count_mismatch']}", + f"Feature check failures : {sum(report['feature_checks'].values())}", + f"Dictionary failures : {sum(report['dictionary_checks'].values())}", + f"Attribute failures : {sum(report['attribute_checks'].values())}", + f"Coverage failures : {sum(report['coverage_checks'].values())}", + f"Sounding failures : {sum(report['sounding_checks'].values())}", + f"JSON/relation failures : {sum(report['json_relation_checks'].values())}", + f"Raw failures : " + f"{report['raw_checks']['raw_metadata_json_bad'] + report['raw_checks']['raw_empty_blob'] + report['raw_checks']['raw_group_bad'] + report['raw_checks']['required_header_or_feature_missing_cells']}", + "", + ] + + if source_root is not None: + s = report["source_checks"] + lines += [ + f"Source files checked : {s['files_checked']}", + f"Source files missing : {s['files_missing']}", + f"Source size mismatch : {s['size_mismatch']}", + f"Source SHA256 mismatch : {s['sha256_mismatch']}", + "", + ] + + lines += [ + f"Failure count : {report['failure_count']}", + f"OVERALL: {report['overall']}", + ] + + summary = "\n".join(lines) + + (outdir / "最终验收摘要_v1.txt").write_text( + summary, + encoding="utf-8", + ) + + zip_path = outdir.with_suffix(".zip") + with zipfile.ZipFile( + zip_path, + "w", + compression=zipfile.ZIP_DEFLATED, + ) as z: + for p in sorted(outdir.rglob("*")): + if p.is_file(): + z.write(p, p.relative_to(outdir.parent)) + + print("") + print(summary) + print("") + print(f"报告目录: {outdir}") + print(f"报告 ZIP : {zip_path}") + print("把 ZIP 上传给 ChatGPT。") + + +if __name__ == "__main__": + main() diff --git a/cm93_mysql_normalization_canonical_v10.py b/cm93_mysql_normalization_canonical_v10.py new file mode 100644 index 0000000..409d0b6 --- /dev/null +++ b/cm93_mysql_normalization_canonical_v10.py @@ -0,0 +1,1633 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +CM93 -> MySQL OGC Normalization 可重复性 / Canonical WKB 验证 +Version: v10.0 + +背景 +==== +V9 已确认： +- 55/55 invalid Area/Coverage 使用 odd-even face selection 可生成 valid Polygon/MultiPolygon； +- 55/55 与 make_valid polygonal 部分一致； +- 55/55 不引入原始 linework 之外的新边界； +- 55/55 被丢弃的 linework 与已确认 retraced connector 完全一致； +- 55/55 normalization 幂等； +- 82181 条原本有效的 Area/Coverage 全部 PASS-THROUGH，WKB hash 0 变化； +- 27 条 candidate bbox 缩小，其中 27/27 都由被去掉的零面积 retraced connector 解释。 + +V10 目的 +======= +在真正设计 importer/schema 修改前，再验证一件实现层关键要求： + +“相同的 CM93 面语义，在输入 ring 起点、ring 方向、MultiPolygon component 顺序、 +polygonize face 顺序不同的情况下，最终是否能得到稳定的 canonical geometry / WKB hash。” + +原因： +如果 normalized geometry 语义完全相同，但 WKB 因 ring 起点或 component 顺序变化而不同， +以后重新导入、增量同步、缓存或 hash 对账可能出现伪变化。 + +V10 只读、只在内存运行。 + +验证内容 +======== +对 55 个 invalid Area/Coverage： + +1. 基线： + - original invalid geometry + - odd-even normalization candidate + - GEOS/Shapely normalize() canonicalization + - canonical WKB SHA-256 + +2. 输入等价变体： + A. 原始输入 + B. 每个 ring 全部反向 + C. 每个 ring 改变起始 vertex + D. ring 反向 + 改变起点 + E. MultiPolygon component 反序（如适用） + + 以上变体不改变原始线网/odd-even 面语义。 + +3. 对每个变体重新执行 odd-even normalization； + 然后 canonicalize； + 必须满足： + - candidate valid + - candidate polygonal-only + - 与基线 candidate topology equals + - canonical WKB SHA-256 与基线一致 + +4. polygonize selected-face 顺序： + - 原顺序 union + - reverse 顺序 union + - 按面积排序 union + - 按 centroid 排序 union + canonical hash 必须一致。 + +5. 重复执行： + 同一输入连续 normalization 3 次，canonical hash 必须一致。 + +6. Canonicalization 自身不得改变 topology： + - symdiff area = 0 + - Hausdorff distance = 0（数值容差内） + +7. 输出 55 条 golden vectors： + - kind + - feature_id / coverage_id + - object_class + - cell + - source_record_index + - normalized type + - normalized part count + - canonical WKB SHA-256 + - candidate bbox + - candidate area + - discarded connector length + - feature bbox 是否未来需要更新 + 这些 golden vectors 可供后续 importer 实现回归测试。 + +8. 有效 geometry policy： + 仍然只做 PASS-THROUGH，不 canonicalize 后写回； + 统计 valid records，确认 policy 不会改它们。 + +重要： +- CM93JAPAN 是日本沿海局部数据，不是完整 CM93； +- 跨未导入海域边界不连续不作为错误； +- 本脚本不修改数据库； +- 不开始 PBF。 + +依赖 +==== + pip install pymysql shapely + +当前验证环境已知 Shapely 2.0.7。 + +执行 +==== + python cm93_mysql_normalization_canonical_v10.py ^ + --host 127.0.0.1 ^ + --port 3306 ^ + --user root ^ + --database cm93_chart ^ + --dataset-id 1 + +输出 +==== +- normalization_canonical_summary_v10.txt +- normalization_canonical_report_v10.json +- normalization_golden_vectors_v10.csv +- normalization_variant_results.csv +- normalization_face_order_results.csv +- normalization_canonical_failures.csv +- normalization_bbox_metadata_impact.csv +- normalization_canonical_candidates.geojson +- normalization_golden_vectors_v10.json +并自动生成 ZIP。 + +禁止 +==== +- 不 UPDATE / INSERT / DELETE +- 不改 importer +- 不写 normalized geometry +- 不修改 bbox +- 不 PBF +""" + +from __future__ import annotations + +import argparse +import csv +import getpass +import hashlib +import json +import math +import os +import re +import sys +import traceback +import zipfile +from collections import Counter +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple + +try: + import pymysql + from pymysql.cursors import DictCursor +except ImportError: + print("缺少 pymysql：pip install pymysql shapely") + sys.exit(2) + +try: + import shapely + + try: + from shapely import from_wkb as _from_wkb + def load_wkb(data: bytes): + return _from_wkb(data) + except ImportError: + from shapely import wkb as _wkb + def load_wkb(data: bytes): + return _wkb.loads(data) + + try: + from shapely import to_wkb as _to_wkb + def dump_wkb(g) -> bytes: + return bytes(_to_wkb(g, hex=False, byte_order=1)) + except ImportError: + from shapely import wkb as _wkb2 + def dump_wkb(g) -> bytes: + return _wkb2.dumps(g, hex=False, big_endian=False) + + try: + from shapely import normalize as _normalize + def canonicalize(g): + return _normalize(g) + except ImportError: + raise ImportError("V10 需要 Shapely 2.x 的 shapely.normalize()") + + try: + from shapely import make_valid as _make_valid + def make_valid_geom(g): + return _make_valid(g) + except ImportError: + from shapely.validation import make_valid as _make_valid + def make_valid_geom(g): + return _make_valid(g) + + from shapely.geometry import ( + Point, Polygon, MultiPolygon, mapping + ) + from shapely.ops import unary_union, polygonize_full + from shapely.validation import explain_validity + +except ImportError as e: + print(f"Shapely 初始化失败：{e}") + print("请执行：pip install -U shapely") + sys.exit(2) + + +INVALID_POINT_RE = re.compile( + r"\[\s*([-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)\s+" + r"([-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)\s*\]" +) + + +def scalar(cur, sql: str, params: Sequence[Any] = ()) -> Any: + cur.execute(sql, params) + row = cur.fetchone() + return next(iter(row.values())) if row else None + + +def begin_read_only(cur, warnings: List[str]): + try: + cur.execute("SET SESSION TRANSACTION READ ONLY") + except Exception as e: + warnings.append(f"SET SESSION TRANSACTION READ ONLY failed: {e}") + + try: + cur.execute("START TRANSACTION READ ONLY") + except Exception as e: + warnings.append(f"START TRANSACTION READ ONLY failed: {e}") + cur.execute("START TRANSACTION") + + +def to_bytes(v: Any) -> Optional[bytes]: + if v is None: + return None + if isinstance(v, bytes): + return v + if isinstance(v, bytearray): + return bytes(v) + if isinstance(v, memoryview): + return v.tobytes() + try: + return bytes(v) + except Exception: + return None + + +def write_csv( + path: Path, + rows: Iterable[Dict[str, Any]], + fieldnames: Optional[List[str]] = None, +): + rows = list(rows) + + if fieldnames is None: + seen = set() + fieldnames = [] + for row in rows: + for k in row.keys(): + if k not in seen: + seen.add(k) + fieldnames.append(k) + + path.parent.mkdir(parents=True, exist_ok=True) + + with path.open("w", encoding="utf-8-sig", newline="") as f: + if not fieldnames: + return + + writer = csv.DictWriter( + f, + fieldnames=fieldnames, + extrasaction="ignore" + ) + writer.writeheader() + + for row in rows: + out = {} + for k in fieldnames: + v = row.get(k) + if isinstance(v, (dict, list, tuple)): + v = json.dumps(v, ensure_ascii=False, default=str) + out[k] = "" if v is None else v + writer.writerow(out) + + +def probe_wkb_fn(cur) -> str: + for fn in ("ST_AsBinary", "ST_AsWKB", "AsBinary", "AsWKB"): + try: + cur.execute( + f"SELECT {fn}(geom) AS x " + f"FROM chart_area_geometry LIMIT 1" + ) + cur.fetchone() + return fn + except Exception: + pass + + raise RuntimeError("找不到 geometry -> WKB 函数") + + +def geometry_hash(g) -> Optional[str]: + if g is None: + return None + try: + return hashlib.sha256(dump_wkb(g)).hexdigest() + except Exception: + return None + + +def safe_symdiff_area(a, b) -> Optional[float]: + if a is None or b is None: + return None + try: + return float(a.symmetric_difference(b).area) + except Exception: + return None + + +def safe_hausdorff(a, b) -> Optional[float]: + if a is None or b is None: + return None + try: + return float(a.hausdorff_distance(b)) + except Exception: + return None + + +def safe_length(g) -> float: + if g is None: + return 0.0 + try: + return float(g.length) + except Exception: + return 0.0 + + +def flatten_components(g) -> Dict[str, List[Any]]: + result = { + "polygons": [], + "lines": [], + "points": [], + "others": [], + } + + def rec(x): + if x is None: + return + + t = x.geom_type + + if t == "Polygon": + result["polygons"].append(x) + elif t == "MultiPolygon": + for y in x.geoms: + rec(y) + elif t in ("LineString", "LinearRing"): + result["lines"].append(x) + elif t == "MultiLineString": + for y in x.geoms: + rec(y) + elif t == "Point": + result["points"].append(x) + elif t == "MultiPoint": + for y in x.geoms: + rec(y) + elif hasattr(x, "geoms"): + for y in x.geoms: + rec(y) + else: + result["others"].append(x) + + rec(g) + return result + + +def polygon_part_count(g) -> int: + if g is None: + return 0 + if g.geom_type == "Polygon": + return 1 + if g.geom_type == "MultiPolygon": + return len(g.geoms) + return len(flatten_components(g)["polygons"]) + + +# --------------------------------------------------------------------------- +# Odd-even normalization +# --------------------------------------------------------------------------- + +def original_rings(geom) -> List[List[Tuple[float, float]]]: + rings: List[List[Tuple[float, float]]] = [] + + def add_polygon(poly): + rings.append([ + (float(c[0]), float(c[1])) + for c in poly.exterior.coords + ]) + + for ring in poly.interiors: + rings.append([ + (float(c[0]), float(c[1])) + for c in ring.coords + ]) + + if geom.geom_type == "Polygon": + add_polygon(geom) + elif geom.geom_type == "MultiPolygon": + for poly in geom.geoms: + add_polygon(poly) + + return rings + + +def point_on_segment( + px: float, py: float, + ax: float, ay: float, + bx: float, by: float, + eps: float = 1e-12, +) -> bool: + vx = bx - ax + vy = by - ay + wx = px - ax + wy = py - ay + + c = abs(vx * wy - vy * wx) + if c > eps * max(1.0, abs(vx), abs(vy)): + return False + + d = wx * vx + wy * vy + if d < -eps: + return False + + vv = vx * vx + vy * vy + if d - vv > eps: + return False + + return True + + +def even_odd_inside( + point: Tuple[float, float], + rings: List[List[Tuple[float, float]]], +) -> bool: + x, y = point + crossings = 0 + + for ring in rings: + for i in range(len(ring) - 1): + x1, y1 = ring[i] + x2, y2 = ring[i + 1] + + if point_on_segment(x, y, x1, y1, x2, y2): + return True + + if (y1 > y) != (y2 > y): + xinters = x1 + (y - y1) * (x2 - x1) / (y2 - y1) + if xinters > x: + crossings += 1 + + return crossings % 2 == 1 + + +def select_faces_odd_even(geom) -> Tuple[List[Any], Dict[str, Any]]: + rings = original_rings(geom) + noded = unary_union(geom.boundary) + polygons_geom, cuts, dangles, invalid_rings = polygonize_full(noded) + + faces = flatten_components(polygons_geom)["polygons"] + selected = [] + + for face in faces: + rp = face.representative_point() + p = (float(rp.x), float(rp.y)) + if even_odd_inside(p, rings): + selected.append(face) + + return selected, { + "face_count": len(faces), + "selected_face_count": len(selected), + "cuts_length_deg": safe_length(cuts), + "dangles_length_deg": safe_length(dangles), + "invalid_rings_length_deg": safe_length(invalid_rings), + } + + +def candidate_from_selected_faces( + selected: List[Any], + mode: str = "original", +): + if not selected: + return None + + if mode == "original": + items = list(selected) + elif mode == "reverse": + items = list(reversed(selected)) + elif mode == "area": + items = sorted(selected, key=lambda g: (float(g.area), g.bounds)) + elif mode == "centroid": + items = sorted( + selected, + key=lambda g: ( + float(g.centroid.x), + float(g.centroid.y), + float(g.area), + ) + ) + else: + raise ValueError(mode) + + return unary_union(items) + + +def normalize_odd_even(geom): + if geom.is_valid: + return geom + + selected, _ = select_faces_odd_even(geom) + + if not selected: + return None + + return unary_union(selected) + + +# --------------------------------------------------------------------------- +# Equivalent input variants +# --------------------------------------------------------------------------- + +def rotate_closed_ring( + coords: List[Tuple[float, float]], + shift: int +) -> List[Tuple[float, float]]: + if not coords: + return coords + + core = list(coords) + + if len(core) > 1 and core[0] == core[-1]: + core = core[:-1] + + if not core: + return coords + + shift = shift % len(core) + rotated = core[shift:] + core[:shift] + rotated.append(rotated[0]) + + return rotated + + +def reverse_closed_ring( + coords: List[Tuple[float, float]] +) -> List[Tuple[float, float]]: + core = list(coords) + + if len(core) > 1 and core[0] == core[-1]: + core = core[:-1] + + rev = list(reversed(core)) + + if rev: + rev.append(rev[0]) + + return rev + + +def polygon_variant(poly, *, reverse: bool, rotate: bool): + ext = [ + (float(c[0]), float(c[1])) + for c in poly.exterior.coords + ] + + holes = [ + [ + (float(c[0]), float(c[1])) + for c in ring.coords + ] + for ring in poly.interiors + ] + + def tx(coords, index): + out = coords + + if reverse: + out = reverse_closed_ring(out) + + if rotate: + core_len = max(1, len(out) - 1) + shift = max(1, core_len // 3 + index) + out = rotate_closed_ring(out, shift) + + return out + + ext2 = tx(ext, 0) + holes2 = [tx(h, i + 1) for i, h in enumerate(holes)] + + return Polygon(ext2, holes2) + + +def make_input_variant(geom, mode: str): + if mode == "original": + return geom + + reverse = mode in { + "reverse_rings", + "reverse_and_rotate", + "reverse_components_and_rings", + } + + rotate = mode in { + "rotate_rings", + "reverse_and_rotate", + } + + if geom.geom_type == "Polygon": + return polygon_variant( + geom, + reverse=reverse, + rotate=rotate, + ) + + if geom.geom_type == "MultiPolygon": + polys = [ + polygon_variant( + p, + reverse=reverse, + rotate=rotate, + ) + for p in geom.geoms + ] + + if mode in { + "reverse_components", + "reverse_components_and_rings", + }: + polys = list(reversed(polys)) + + return MultiPolygon(polys) + + return geom + + +# --------------------------------------------------------------------------- +# DB loader +# --------------------------------------------------------------------------- + +def load_polygon_records(cur, dataset_id: int, wkb_fn: str): + records = [] + + cur.execute(f""" + SELECT + 'AREA' AS kind, + f.id AS feature_id, + NULL AS coverage_geom_id, + f.object_class, + f.cell_id, + c.cell_name, + f.source_record_index, + f.coverage_id, + f.raw_geotype, + f.bbox_min_lon, + f.bbox_min_lat, + f.bbox_max_lon, + f.bbox_max_lat, + c.bbox_min_lon AS cell_bbox_min_lon, + c.bbox_min_lat AS cell_bbox_min_lat, + c.bbox_max_lon AS cell_bbox_max_lon, + c.bbox_max_lat AS cell_bbox_max_lat, + {wkb_fn}(g.geom) AS geom_wkb + FROM chart_feature f + JOIN chart_area_geometry g + ON g.feature_id=f.id + JOIN chart_cell c + ON c.id=f.cell_id + WHERE f.dataset_id=%s + AND f.geometry_type='AREA' + ORDER BY f.id + """, (dataset_id,)) + + for r in cur: + try: + geom = load_wkb(to_bytes(r["geom_wkb"])) + x = dict(r) + x["_geom"] = geom + records.append(x) + except Exception: + continue + + cur.execute(f""" + SELECT + 'COVERAGE' AS kind, + NULL AS feature_id, + v.id AS coverage_geom_id, + 'COVERAGE' AS object_class, + v.cell_id, + c.cell_name, + v.source_record_index, + NULL AS coverage_id, + NULL AS raw_geotype, + NULL AS bbox_min_lon, + NULL AS bbox_min_lat, + NULL AS bbox_max_lon, + NULL AS bbox_max_lat, + c.bbox_min_lon AS cell_bbox_min_lon, + c.bbox_min_lat AS cell_bbox_min_lat, + c.bbox_max_lon AS cell_bbox_max_lon, + c.bbox_max_lat AS cell_bbox_max_lat, + {wkb_fn}(v.geom) AS geom_wkb + FROM chart_coverage v + JOIN chart_cell c + ON c.id=v.cell_id + WHERE v.dataset_id=%s + ORDER BY v.id + """, (dataset_id,)) + + for r in cur: + try: + geom = load_wkb(to_bytes(r["geom_wkb"])) + x = dict(r) + x["_geom"] = geom + records.append(x) + except Exception: + continue + + return records + + +# --------------------------------------------------------------------------- +# GeoJSON +# --------------------------------------------------------------------------- + +def make_geojson(records, path: Path): + features = [] + + def add(g, props): + if g is None: + return + try: + features.append({ + "type": "Feature", + "geometry": mapping(g), + "properties": props, + }) + except Exception: + pass + + for r in records: + if r["_geom"].is_valid: + continue + + rid = ( + r.get("feature_id") + if r["kind"] == "AREA" + else r.get("coverage_geom_id") + ) + + common = { + "kind": r["kind"], + "record_id": rid, + "object_class": r.get("object_class"), + "cell_name": r.get("cell_name"), + "canonical_hash": r.get("_canonical_hash"), + } + + add( + r["_geom"], + { + **common, + "layer": "original_invalid", + } + ) + + add( + r.get("_canonical_candidate"), + { + **common, + "layer": "canonical_candidate", + } + ) + + path.write_text( + json.dumps( + { + "type": "FeatureCollection", + "features": features, + }, + ensure_ascii=False, + separators=(",", ":"), + default=str, + ), + encoding="utf-8" + ) + + +def parse_args(): + p = argparse.ArgumentParser( + description="CM93 normalization canonical validation v10" + ) + + p.add_argument("--host", default=os.getenv("MYSQL_HOST", "127.0.0.1")) + p.add_argument("--port", type=int, default=int(os.getenv("MYSQL_PORT", "3306"))) + p.add_argument("--user", default=os.getenv("MYSQL_USER", "root")) + p.add_argument("--password", default=os.getenv("MYSQL_PASSWORD")) + p.add_argument("--database", default=os.getenv("MYSQL_DATABASE", "cm93_chart")) + p.add_argument("--dataset-id", type=int, default=1) + p.add_argument( + "--topology-area-tolerance", + type=float, + default=1e-12, + ) + p.add_argument( + "--hausdorff-tolerance", + type=float, + default=1e-12, + ) + p.add_argument( + "--bbox-tolerance", + type=float, + default=5e-6, + ) + p.add_argument("--output", default=None) + + return p.parse_args() + + +def main(): + args = parse_args() + + if not args.password: + args.password = getpass.getpass("MySQL password: ") + + stamp = datetime.now().strftime("%Y%m%d_%H%M%S") + + outdir = Path( + args.output + or f"cm93_normalization_canonical_v10_dataset_{args.dataset_id}_{stamp}" + ) + outdir.mkdir(parents=True, exist_ok=True) + + report = { + "version": "v10.0", + "generated_at": datetime.now().isoformat(timespec="seconds"), + "database": args.database, + "dataset_id": args.dataset_id, + "read_only": True, + "pbf": "OUT_OF_SCOPE", + "shapely_version": getattr(shapely, "__version__", "unknown"), + "warnings": [], + } + + conn = None + + try: + conn = pymysql.connect( + host=args.host, + port=args.port, + user=args.user, + password=args.password, + database=args.database, + charset="utf8mb4", + cursorclass=DictCursor, + autocommit=False, + connect_timeout=20, + read_timeout=600, + write_timeout=30, + ) + + with conn.cursor() as cur: + begin_read_only(cur, report["warnings"]) + + report["mysql_version"] = scalar(cur, "SELECT VERSION()") + wkb_fn = probe_wkb_fn(cur) + + print("[1/6] 加载 Area + Coverage ...") + records = load_polygon_records( + cur, + args.dataset_id, + wkb_fn + ) + + valid = [r for r in records if r["_geom"].is_valid] + invalid = [r for r in records if not r["_geom"].is_valid] + + print( + f" total={len(records):,} " + f"valid={len(valid):,} invalid={len(invalid):,}" + ) + + print("[2/6] Valid PASS-THROUGH policy guard ...") + valid_policy_changed = 0 + valid_hash_changed = 0 + + for i, r in enumerate(valid, start=1): + # policy: valid geometry 不进入 canonical rewrite + out = r["_geom"] + + if out is not r["_geom"]: + valid_policy_changed += 1 + + if geometry_hash(out) != geometry_hash(r["_geom"]): + valid_hash_changed += 1 + + if i % 20000 == 0: + print(f" valid checked={i:,}") + + print("[3/6] 55 invalid canonical / input-variant 验证 ...") + + golden_rows = [] + variant_rows = [] + face_order_rows = [] + failure_rows = [] + bbox_rows = [] + + input_modes = [ + "original", + "reverse_rings", + "rotate_rings", + "reverse_and_rotate", + "reverse_components", + "reverse_components_and_rings", + ] + + for i, r in enumerate(invalid, start=1): + geom = r["_geom"] + + rid = ( + r.get("feature_id") + if r["kind"] == "AREA" + else r.get("coverage_geom_id") + ) + + selected, sel_info = select_faces_odd_even(geom) + baseline = candidate_from_selected_faces( + selected, + "original" + ) + + if baseline is None: + fail = { + "kind": r["kind"], + "record_id": rid, + "object_class": r.get("object_class"), + "cell_name": r.get("cell_name"), + "failure": "BASELINE_CANDIDATE_NONE", + } + failure_rows.append(fail) + continue + + baseline_canonical = canonicalize(baseline) + baseline_hash = geometry_hash(baseline_canonical) + + baseline_valid = bool(baseline_canonical.is_valid) + baseline_polygon_only = ( + baseline_canonical.geom_type + in {"Polygon", "MultiPolygon"} + ) + + # make_valid line remainder length only as metadata + mv = make_valid_geom(geom) + mvcomps = flatten_components(mv) + remainder = geom_union(mvcomps["lines"]) + remainder_length = safe_length(remainder) + + # canonicalization itself topology invariant + canonical_sym = safe_symdiff_area( + baseline, + baseline_canonical + ) + canonical_haus = safe_hausdorff( + baseline, + baseline_canonical + ) + + canonical_topology_ok = ( + canonical_sym is not None + and canonical_sym <= args.topology_area_tolerance + and canonical_haus is not None + and canonical_haus <= args.hausdorff_tolerance + ) + + # face order invariance + face_hashes = {} + for mode in [ + "original", + "reverse", + "area", + "centroid", + ]: + cand = candidate_from_selected_faces( + selected, + mode + ) + canon = canonicalize(cand) + h = geometry_hash(canon) + face_hashes[mode] = h + + face_order_rows.append({ + "kind": r["kind"], + "feature_id": r.get("feature_id"), + "coverage_geom_id": r.get("coverage_geom_id"), + "object_class": r.get("object_class"), + "cell_name": r.get("cell_name"), + "face_order_mode": mode, + "canonical_wkb_sha256": h, + "matches_baseline_hash": + int(h == baseline_hash), + "topology_symdiff_area_deg2": + safe_symdiff_area( + baseline_canonical, + canon + ), + }) + + face_order_stable = ( + len(set(face_hashes.values())) == 1 + ) + + # repeated same input + repeat_hashes = [] + for _ in range(3): + c = normalize_odd_even(geom) + cc = canonicalize(c) + repeat_hashes.append(geometry_hash(cc)) + + repeat_stable = ( + len(set(repeat_hashes)) == 1 + and repeat_hashes[0] == baseline_hash + ) + + # input-equivalent variants + variant_fail = 0 + + for mode in input_modes: + try: + vg = make_input_variant(geom, mode) + + vc = normalize_odd_even(vg) + + if vc is None: + variant_fail += 1 + variant_rows.append({ + "kind": r["kind"], + "feature_id": r.get("feature_id"), + "coverage_geom_id": r.get("coverage_geom_id"), + "object_class": r.get("object_class"), + "cell_name": r.get("cell_name"), + "variant_mode": mode, + "status": "FAIL_NO_CANDIDATE", + }) + continue + + vcanon = canonicalize(vc) + vh = geometry_hash(vcanon) + + sym = safe_symdiff_area( + baseline_canonical, + vcanon + ) + haus = safe_hausdorff( + baseline_canonical, + vcanon + ) + + topology_equal = ( + bool( + baseline_canonical.equals( + vcanon + ) + ) + and sym is not None + and sym <= args.topology_area_tolerance + ) + + hash_equal = (vh == baseline_hash) + + if not ( + vcanon.is_valid + and topology_equal + and hash_equal + ): + variant_fail += 1 + + variant_rows.append({ + "kind": r["kind"], + "feature_id": r.get("feature_id"), + "coverage_geom_id": r.get("coverage_geom_id"), + "object_class": r.get("object_class"), + "cell_name": r.get("cell_name"), + "variant_mode": mode, + "variant_input_valid": + int(bool(vg.is_valid)), + "candidate_valid": + int(bool(vcanon.is_valid)), + "candidate_type": + vcanon.geom_type, + "topology_equal_baseline": + int(topology_equal), + "symdiff_area_deg2": sym, + "hausdorff_distance_deg": haus, + "canonical_wkb_sha256": vh, + "matches_baseline_hash": + int(hash_equal), + "status": + "PASS" + if ( + vcanon.is_valid + and topology_equal + and hash_equal + ) + else "FAIL", + }) + + except Exception as e: + variant_fail += 1 + + variant_rows.append({ + "kind": r["kind"], + "feature_id": r.get("feature_id"), + "coverage_geom_id": r.get("coverage_geom_id"), + "object_class": r.get("object_class"), + "cell_name": r.get("cell_name"), + "variant_mode": mode, + "status": "EXCEPTION", + "detail": + f"{type(e).__name__}: {e}", + }) + + # bbox metadata impact + old_bounds = tuple(float(v) for v in geom.bounds) + new_bounds = tuple( + float(v) for v in baseline_canonical.bounds + ) + + bbox_changed = any( + abs(a - b) > args.bbox_tolerance + for a, b in zip(old_bounds, new_bounds) + ) + + feature_bbox = None + feature_bbox_matches_candidate = None + + if r["kind"] == "AREA": + vals = [ + r.get("bbox_min_lon"), + r.get("bbox_min_lat"), + r.get("bbox_max_lon"), + r.get("bbox_max_lat"), + ] + if all(v is not None for v in vals): + feature_bbox = tuple( + float(v) for v in vals + ) + feature_bbox_matches_candidate = all( + abs(a - b) <= args.bbox_tolerance + for a, b in zip( + feature_bbox, + new_bounds + ) + ) + + bbox_rows.append({ + "kind": r["kind"], + "feature_id": r.get("feature_id"), + "coverage_geom_id": r.get("coverage_geom_id"), + "object_class": r.get("object_class"), + "cell_name": r.get("cell_name"), + "original_geometry_bounds": old_bounds, + "candidate_bounds": new_bounds, + "bbox_changed_over_tolerance": + int(bbox_changed), + "current_feature_bbox": feature_bbox, + "current_feature_bbox_matches_candidate": + ( + int(feature_bbox_matches_candidate) + if feature_bbox_matches_candidate + is not None + else None + ), + "future_feature_bbox_update_needed": + ( + int(not feature_bbox_matches_candidate) + if feature_bbox_matches_candidate + is not None + else None + ), + "discarded_connector_length_deg": + remainder_length, + }) + + record_pass = all([ + baseline_valid, + baseline_polygon_only, + canonical_topology_ok, + face_order_stable, + repeat_stable, + variant_fail == 0, + ]) + + golden = { + "kind": r["kind"], + "feature_id": r.get("feature_id"), + "coverage_geom_id": r.get("coverage_geom_id"), + "object_class": r.get("object_class"), + "cell_id": r.get("cell_id"), + "cell_name": r.get("cell_name"), + "source_record_index": + r.get("source_record_index"), + "coverage_id": r.get("coverage_id"), + "raw_geotype": r.get("raw_geotype"), + + "normalization_rule": + "ODD_EVEN_FACE_SELECTION", + "canonicalization_rule": + "GEOS_NORMALIZE", + + "candidate_type": + baseline_canonical.geom_type, + "candidate_polygon_part_count": + polygon_part_count( + baseline_canonical + ), + "candidate_valid": + int(baseline_valid), + + "polygonize_face_count": + sel_info["face_count"], + "selected_face_count": + sel_info["selected_face_count"], + + "candidate_area_deg2": + float(baseline_canonical.area), + "candidate_bbox": + tuple( + float(v) + for v in baseline_canonical.bounds + ), + + "discarded_connector_length_deg": + remainder_length, + + "canonical_wkb_sha256": + baseline_hash, + + "canonicalization_symdiff_area_deg2": + canonical_sym, + "canonicalization_hausdorff_deg": + canonical_haus, + "canonical_topology_unchanged": + int(canonical_topology_ok), + + "face_order_hash_stable": + int(face_order_stable), + "repeat_hash_stable": + int(repeat_stable), + "input_variant_failure_count": + variant_fail, + + "record_status": + "PASS" + if record_pass + else "FAIL", + } + + golden_rows.append(golden) + + if not record_pass: + failure_rows.append(golden) + + r["_canonical_candidate"] = baseline_canonical + r["_canonical_hash"] = baseline_hash + + print( + f" [{i:02d}/{len(invalid)}] " + f"{r['kind']} id={rid} " + f"{r.get('object_class')} -> " + f"{'PASS' if record_pass else 'FAIL'}" + ) + + print("[4/6] 汇总 canonical stability ...") + + record_status = Counter( + r["record_status"] + for r in golden_rows + ) + + variant_status = Counter( + r.get("status", "UNKNOWN") + for r in variant_rows + ) + + bbox_update_area_count = sum( + int( + r.get( + "future_feature_bbox_update_needed" + ) or 0 + ) + for r in bbox_rows + ) + + report["summary"] = { + "total_polygon_records": len(records), + "valid_polygon_records": len(valid), + "invalid_polygon_records": len(invalid), + + "valid_policy_changed_count": + valid_policy_changed, + "valid_hash_changed_count": + valid_hash_changed, + + "golden_vector_count": + len(golden_rows), + + "golden_record_status_counts": + dict(record_status), + + "input_variant_case_count": + len(variant_rows), + "input_variant_status_counts": + dict(variant_status), + + "face_order_case_count": + len(face_order_rows), + "face_order_hash_mismatch_count": + sum( + not bool( + r["matches_baseline_hash"] + ) + for r in face_order_rows + ), + + "canonical_topology_failure_count": + sum( + not bool( + r["canonical_topology_unchanged"] + ) + for r in golden_rows + ), + + "repeat_hash_failure_count": + sum( + not bool( + r["repeat_hash_stable"] + ) + for r in golden_rows + ), + + "input_variant_failure_record_count": + sum( + int( + r["input_variant_failure_count"] + ) > 0 + for r in golden_rows + ), + + "future_area_feature_bbox_update_needed_count": + bbox_update_area_count, + } + + overall_pass = ( + len(invalid) == 55 + and len(golden_rows) == 55 + and record_status.get("PASS", 0) == 55 + and record_status.get("FAIL", 0) == 0 + and valid_policy_changed == 0 + and valid_hash_changed == 0 + and report["summary"][ + "face_order_hash_mismatch_count" + ] == 0 + and report["summary"][ + "canonical_topology_failure_count" + ] == 0 + and report["summary"][ + "repeat_hash_failure_count" + ] == 0 + and report["summary"][ + "input_variant_failure_record_count" + ] == 0 + ) + + report["summary"]["overall_pass"] = overall_pass + + print("[5/6] 写 golden vectors / 报告 ...") + + write_csv( + outdir / "normalization_golden_vectors_v10.csv", + golden_rows + ) + + ( + outdir / "normalization_golden_vectors_v10.json" + ).write_text( + json.dumps( + golden_rows, + ensure_ascii=False, + indent=2, + default=str, + ), + encoding="utf-8" + ) + + write_csv( + outdir / "normalization_variant_results.csv", + variant_rows + ) + + write_csv( + outdir / "normalization_face_order_results.csv", + face_order_rows + ) + + write_csv( + outdir / "normalization_canonical_failures.csv", + failure_rows + ) + + write_csv( + outdir / "normalization_bbox_metadata_impact.csv", + bbox_rows + ) + + make_geojson( + records, + outdir / "normalization_canonical_candidates.geojson" + ) + + ( + outdir / "normalization_canonical_report_v10.json" + ).write_text( + json.dumps( + report, + ensure_ascii=False, + indent=2, + default=str, + ), + encoding="utf-8" + ) + + lines = [] + + lines.append( + "CM93 -> MySQL OGC Normalization Canonical Validation v10" + ) + lines.append("=" * 96) + lines.append( + f"Generated: {report['generated_at']}" + ) + lines.append( + f"Dataset ID: {args.dataset_id}" + ) + lines.append( + f"MySQL: {report['mysql_version']}" + ) + lines.append( + f"Shapely: {report['shapely_version']}" + ) + lines.append("") + + lines.append("SCOPE") + lines.append( + " CM93JAPAN 是局部 CM93 数据集；" + "跨未导入海域不连续不作为错误。" + ) + lines.append( + " V10 只验证 odd-even candidate 的 canonical WKB " + "可重复性，不修改数据库。" + ) + lines.append("") + + lines.append("BASELINE") + lines.append( + f" polygon records: {len(records)}" + ) + lines.append( + f" valid: {len(valid)}" + ) + lines.append( + f" invalid: {len(invalid)}" + ) + lines.append("") + + lines.append("VALID PASS-THROUGH") + lines.append( + f" policy changed: {valid_policy_changed}" + ) + lines.append( + f" WKB hash changed: {valid_hash_changed}" + ) + lines.append("") + + lines.append("GOLDEN VECTOR STATUS") + for k, v in record_status.most_common(): + lines.append( + f" {k:24s} {v}" + ) + lines.append("") + + lines.append("INPUT VARIANTS") + lines.append( + f" cases: {len(variant_rows)}" + ) + for k, v in variant_status.most_common(): + lines.append( + f" {k:24s} {v}" + ) + lines.append("") + + lines.append("STABILITY") + lines.append( + f" face-order hash mismatch: " + f"{report['summary']['face_order_hash_mismatch_count']}" + ) + lines.append( + f" canonical topology failures: " + f"{report['summary']['canonical_topology_failure_count']}" + ) + lines.append( + f" repeat hash failures: " + f"{report['summary']['repeat_hash_failure_count']}" + ) + lines.append( + f" input-variant failure records: " + f"{report['summary']['input_variant_failure_record_count']}" + ) + lines.append("") + + lines.append("METADATA IMPACT") + lines.append( + f" future Area feature bbox updates needed: " + f"{bbox_update_area_count}" + ) + lines.append( + " bbox 更新仅是未来若真正保存 normalized geometry " + "时的元数据同步要求；V10 不修改任何 bbox。" + ) + lines.append("") + + lines.append("OVERALL") + lines.append( + f" CANONICAL VALIDATION PASS: " + f"{'YES' if overall_pass else 'NO'}" + ) + lines.append("") + + lines.append("NEXT USE") + lines.append( + " normalization_golden_vectors_v10.csv/json " + "可作为未来 importer normalization 实现的 55 条 golden regression vectors。" + ) + lines.append( + " 后续实现应固定：ODD_EVEN_FACE_SELECTION + GEOS_NORMALIZE，" + "并要求输出 canonical_wkb_sha256 与 golden vector 一致。" + ) + lines.append("") + + lines.append( + "NO DATABASE WRITE / NO IMPORTER CHANGE / NO PBF" + ) + + ( + outdir / "normalization_canonical_summary_v10.txt" + ).write_text( + "\n".join(lines), + encoding="utf-8" + ) + + conn.rollback() + + print("[6/6] 压缩 ...") + + zpath = outdir.with_suffix(".zip") + + with zipfile.ZipFile( + zpath, + "w", + compression=zipfile.ZIP_DEFLATED + ) as z: + for p in sorted(outdir.rglob("*")): + if p.is_file(): + z.write( + p, + p.relative_to(outdir.parent) + ) + + print("") + print("完成。") + print(f"Output: {outdir.resolve()}") + print(f"ZIP: {zpath.resolve()}") + print("把 ZIP 整包上传给 ChatGPT。") + + except KeyboardInterrupt: + if conn: + try: + conn.rollback() + except Exception: + pass + + print("\n用户中断。") + sys.exit(130) + + except Exception: + if conn: + try: + conn.rollback() + except Exception: + pass + + tb = traceback.format_exc() + print(tb) + + try: + ( + outdir / "fatal_error.txt" + ).write_text( + tb, + encoding="utf-8" + ) + except Exception: + pass + + sys.exit(1) + + finally: + if conn: + try: + conn.close() + except Exception: + pass + + +if __name__ == "__main__": + main() diff --git a/cm93_mysql_normalization_canonical_v10_1.py b/cm93_mysql_normalization_canonical_v10_1.py new file mode 100644 index 0000000..7f5a63f --- /dev/null +++ b/cm93_mysql_normalization_canonical_v10_1.py @@ -0,0 +1,1651 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +CM93 -> MySQL OGC Normalization 可重复性 / Canonical WKB 验证 +Version: v10.1 + +背景 +==== +V9 已确认： +- 55/55 invalid Area/Coverage 使用 odd-even face selection 可生成 valid Polygon/MultiPolygon； +- 55/55 与 make_valid polygonal 部分一致； +- 55/55 不引入原始 linework 之外的新边界； +- 55/55 被丢弃的 linework 与已确认 retraced connector 完全一致； +- 55/55 normalization 幂等； +- 82181 条原本有效的 Area/Coverage 全部 PASS-THROUGH，WKB hash 0 变化； +- 27 条 candidate bbox 缩小，其中 27/27 都由被去掉的零面积 retraced connector 解释。 + +V10 目的 +======= +在真正设计 importer/schema 修改前，再验证一件实现层关键要求： + +“相同的 CM93 面语义，在输入 ring 起点、ring 方向、MultiPolygon component 顺序、 +polygonize face 顺序不同的情况下，最终是否能得到稳定的 canonical geometry / WKB hash。” + +原因： +如果 normalized geometry 语义完全相同，但 WKB 因 ring 起点或 component 顺序变化而不同， +以后重新导入、增量同步、缓存或 hash 对账可能出现伪变化。 + +V10 只读、只在内存运行。 + +验证内容 +======== +对 55 个 invalid Area/Coverage： + +1. 基线： + - original invalid geometry + - odd-even normalization candidate + - GEOS/Shapely normalize() canonicalization + - canonical WKB SHA-256 + +2. 输入等价变体： + A. 原始输入 + B. 每个 ring 全部反向 + C. 每个 ring 改变起始 vertex + D. ring 反向 + 改变起点 + E. MultiPolygon component 反序（如适用） + + 以上变体不改变原始线网/odd-even 面语义。 + +3. 对每个变体重新执行 odd-even normalization； + 然后 canonicalize； + 必须满足： + - candidate valid + - candidate polygonal-only + - 与基线 candidate topology equals + - canonical WKB SHA-256 与基线一致 + +4. polygonize selected-face 顺序： + - 原顺序 union + - reverse 顺序 union + - 按面积排序 union + - 按 centroid 排序 union + canonical hash 必须一致。 + +5. 重复执行： + 同一输入连续 normalization 3 次，canonical hash 必须一致。 + +6. Canonicalization 自身不得改变 topology： + - symdiff area = 0 + - Hausdorff distance = 0（数值容差内） + +7. 输出 55 条 golden vectors： + - kind + - feature_id / coverage_id + - object_class + - cell + - source_record_index + - normalized type + - normalized part count + - canonical WKB SHA-256 + - candidate bbox + - candidate area + - discarded connector length + - feature bbox 是否未来需要更新 + 这些 golden vectors 可供后续 importer 实现回归测试。 + +8. 有效 geometry policy： + 仍然只做 PASS-THROUGH，不 canonicalize 后写回； + 统计 valid records，确认 policy 不会改它们。 + +重要： +- CM93JAPAN 是日本沿海局部数据，不是完整 CM93； +- 跨未导入海域边界不连续不作为错误； +- 本脚本不修改数据库； +- 不开始 PBF。 + +依赖 +==== + pip install pymysql shapely + +当前验证环境已知 Shapely 2.0.7。 + +执行 +==== + python cm93_mysql_normalization_canonical_v10.py ^ + --host 127.0.0.1 ^ + --port 3306 ^ + --user root ^ + --database cm93_chart ^ + --dataset-id 1 + +输出 +==== +- normalization_canonical_summary_v10_1.txt +- normalization_canonical_report_v10_1.json +- normalization_golden_vectors_v10_1.csv +- normalization_variant_results_v10_1.csv +- normalization_face_order_results_v10_1.csv +- normalization_canonical_failures_v10_1.csv +- normalization_bbox_metadata_impact_v10_1.csv +- normalization_canonical_candidates_v10_1.geojson +- normalization_golden_vectors_v10_1.json +并自动生成 ZIP。 + +禁止 +==== +- 不 UPDATE / INSERT / DELETE +- 不改 importer +- 不写 normalized geometry +- 不修改 bbox +- 不 PBF +""" + +from __future__ import annotations + +import argparse +import csv +import getpass +import hashlib +import json +import math +import os +import re +import sys +import traceback +import zipfile +from collections import Counter +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple + +try: + import pymysql + from pymysql.cursors import DictCursor +except ImportError: + print("缺少 pymysql：pip install pymysql shapely") + sys.exit(2) + +try: + import shapely + + try: + from shapely import from_wkb as _from_wkb + def load_wkb(data: bytes): + return _from_wkb(data) + except ImportError: + from shapely import wkb as _wkb + def load_wkb(data: bytes): + return _wkb.loads(data) + + try: + from shapely import to_wkb as _to_wkb + def dump_wkb(g) -> bytes: + return bytes(_to_wkb(g, hex=False, byte_order=1)) + except ImportError: + from shapely import wkb as _wkb2 + def dump_wkb(g) -> bytes: + return _wkb2.dumps(g, hex=False, big_endian=False) + + try: + from shapely import normalize as _normalize + def canonicalize(g): + return _normalize(g) + except ImportError: + raise ImportError("V10 需要 Shapely 2.x 的 shapely.normalize()") + + try: + from shapely import make_valid as _make_valid + def make_valid_geom(g): + return _make_valid(g) + except ImportError: + from shapely.validation import make_valid as _make_valid + def make_valid_geom(g): + return _make_valid(g) + + from shapely.geometry import ( + Point, Polygon, MultiPolygon, mapping + ) + from shapely.ops import unary_union, polygonize_full + from shapely.validation import explain_validity + +except ImportError as e: + print(f"Shapely 初始化失败：{e}") + print("请执行：pip install -U shapely") + sys.exit(2) + + +INVALID_POINT_RE = re.compile( + r"\[\s*([-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)\s+" + r"([-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)\s*\]" +) + + +def scalar(cur, sql: str, params: Sequence[Any] = ()) -> Any: + cur.execute(sql, params) + row = cur.fetchone() + return next(iter(row.values())) if row else None + + +def begin_read_only(cur, warnings: List[str]): + try: + cur.execute("SET SESSION TRANSACTION READ ONLY") + except Exception as e: + warnings.append(f"SET SESSION TRANSACTION READ ONLY failed: {e}") + + try: + cur.execute("START TRANSACTION READ ONLY") + except Exception as e: + warnings.append(f"START TRANSACTION READ ONLY failed: {e}") + cur.execute("START TRANSACTION") + + +def to_bytes(v: Any) -> Optional[bytes]: + if v is None: + return None + if isinstance(v, bytes): + return v + if isinstance(v, bytearray): + return bytes(v) + if isinstance(v, memoryview): + return v.tobytes() + try: + return bytes(v) + except Exception: + return None + + +def write_csv( + path: Path, + rows: Iterable[Dict[str, Any]], + fieldnames: Optional[List[str]] = None, +): + rows = list(rows) + + if fieldnames is None: + seen = set() + fieldnames = [] + for row in rows: + for k in row.keys(): + if k not in seen: + seen.add(k) + fieldnames.append(k) + + path.parent.mkdir(parents=True, exist_ok=True) + + with path.open("w", encoding="utf-8-sig", newline="") as f: + if not fieldnames: + return + + writer = csv.DictWriter( + f, + fieldnames=fieldnames, + extrasaction="ignore" + ) + writer.writeheader() + + for row in rows: + out = {} + for k in fieldnames: + v = row.get(k) + if isinstance(v, (dict, list, tuple)): + v = json.dumps(v, ensure_ascii=False, default=str) + out[k] = "" if v is None else v + writer.writerow(out) + + +def probe_wkb_fn(cur) -> str: + for fn in ("ST_AsBinary", "ST_AsWKB", "AsBinary", "AsWKB"): + try: + cur.execute( + f"SELECT {fn}(geom) AS x " + f"FROM chart_area_geometry LIMIT 1" + ) + cur.fetchone() + return fn + except Exception: + pass + + raise RuntimeError("找不到 geometry -> WKB 函数") + + +def geometry_hash(g) -> Optional[str]: + if g is None: + return None + try: + return hashlib.sha256(dump_wkb(g)).hexdigest() + except Exception: + return None + + +def safe_symdiff_area(a, b) -> Optional[float]: + if a is None or b is None: + return None + try: + return float(a.symmetric_difference(b).area) + except Exception: + return None + + +def safe_hausdorff(a, b) -> Optional[float]: + if a is None or b is None: + return None + try: + return float(a.hausdorff_distance(b)) + except Exception: + return None + + +def safe_length(g) -> float: + if g is None: + return 0.0 + try: + return float(g.length) + except Exception: + return 0.0 + + +def flatten_components(g) -> Dict[str, List[Any]]: + result = { + "polygons": [], + "lines": [], + "points": [], + "others": [], + } + + def rec(x): + if x is None: + return + + t = x.geom_type + + if t == "Polygon": + result["polygons"].append(x) + elif t == "MultiPolygon": + for y in x.geoms: + rec(y) + elif t in ("LineString", "LinearRing"): + result["lines"].append(x) + elif t == "MultiLineString": + for y in x.geoms: + rec(y) + elif t == "Point": + result["points"].append(x) + elif t == "MultiPoint": + for y in x.geoms: + rec(y) + elif hasattr(x, "geoms"): + for y in x.geoms: + rec(y) + else: + result["others"].append(x) + + rec(g) + return result + + + +def geom_union(items: Sequence[Any]) -> Optional[Any]: + """ + 将一组 geometry 做 unary_union。 + 空集合返回 None；异常时返回 None。 + + V10.1 修正： + V10.0 主流程调用了该辅助函数，但文件中漏掉定义，导致 NameError。 + """ + if not items: + return None + + try: + return unary_union(list(items)) + except Exception: + return None + + +def polygon_part_count(g) -> int: + if g is None: + return 0 + if g.geom_type == "Polygon": + return 1 + if g.geom_type == "MultiPolygon": + return len(g.geoms) + return len(flatten_components(g)["polygons"]) + + +# --------------------------------------------------------------------------- +# Odd-even normalization +# --------------------------------------------------------------------------- + +def original_rings(geom) -> List[List[Tuple[float, float]]]: + rings: List[List[Tuple[float, float]]] = [] + + def add_polygon(poly): + rings.append([ + (float(c[0]), float(c[1])) + for c in poly.exterior.coords + ]) + + for ring in poly.interiors: + rings.append([ + (float(c[0]), float(c[1])) + for c in ring.coords + ]) + + if geom.geom_type == "Polygon": + add_polygon(geom) + elif geom.geom_type == "MultiPolygon": + for poly in geom.geoms: + add_polygon(poly) + + return rings + + +def point_on_segment( + px: float, py: float, + ax: float, ay: float, + bx: float, by: float, + eps: float = 1e-12, +) -> bool: + vx = bx - ax + vy = by - ay + wx = px - ax + wy = py - ay + + c = abs(vx * wy - vy * wx) + if c > eps * max(1.0, abs(vx), abs(vy)): + return False + + d = wx * vx + wy * vy + if d < -eps: + return False + + vv = vx * vx + vy * vy + if d - vv > eps: + return False + + return True + + +def even_odd_inside( + point: Tuple[float, float], + rings: List[List[Tuple[float, float]]], +) -> bool: + x, y = point + crossings = 0 + + for ring in rings: + for i in range(len(ring) - 1): + x1, y1 = ring[i] + x2, y2 = ring[i + 1] + + if point_on_segment(x, y, x1, y1, x2, y2): + return True + + if (y1 > y) != (y2 > y): + xinters = x1 + (y - y1) * (x2 - x1) / (y2 - y1) + if xinters > x: + crossings += 1 + + return crossings % 2 == 1 + + +def select_faces_odd_even(geom) -> Tuple[List[Any], Dict[str, Any]]: + rings = original_rings(geom) + noded = unary_union(geom.boundary) + polygons_geom, cuts, dangles, invalid_rings = polygonize_full(noded) + + faces = flatten_components(polygons_geom)["polygons"] + selected = [] + + for face in faces: + rp = face.representative_point() + p = (float(rp.x), float(rp.y)) + if even_odd_inside(p, rings): + selected.append(face) + + return selected, { + "face_count": len(faces), + "selected_face_count": len(selected), + "cuts_length_deg": safe_length(cuts), + "dangles_length_deg": safe_length(dangles), + "invalid_rings_length_deg": safe_length(invalid_rings), + } + + +def candidate_from_selected_faces( + selected: List[Any], + mode: str = "original", +): + if not selected: + return None + + if mode == "original": + items = list(selected) + elif mode == "reverse": + items = list(reversed(selected)) + elif mode == "area": + items = sorted(selected, key=lambda g: (float(g.area), g.bounds)) + elif mode == "centroid": + items = sorted( + selected, + key=lambda g: ( + float(g.centroid.x), + float(g.centroid.y), + float(g.area), + ) + ) + else: + raise ValueError(mode) + + return unary_union(items) + + +def normalize_odd_even(geom): + if geom.is_valid: + return geom + + selected, _ = select_faces_odd_even(geom) + + if not selected: + return None + + return unary_union(selected) + + +# --------------------------------------------------------------------------- +# Equivalent input variants +# --------------------------------------------------------------------------- + +def rotate_closed_ring( + coords: List[Tuple[float, float]], + shift: int +) -> List[Tuple[float, float]]: + if not coords: + return coords + + core = list(coords) + + if len(core) > 1 and core[0] == core[-1]: + core = core[:-1] + + if not core: + return coords + + shift = shift % len(core) + rotated = core[shift:] + core[:shift] + rotated.append(rotated[0]) + + return rotated + + +def reverse_closed_ring( + coords: List[Tuple[float, float]] +) -> List[Tuple[float, float]]: + core = list(coords) + + if len(core) > 1 and core[0] == core[-1]: + core = core[:-1] + + rev = list(reversed(core)) + + if rev: + rev.append(rev[0]) + + return rev + + +def polygon_variant(poly, *, reverse: bool, rotate: bool): + ext = [ + (float(c[0]), float(c[1])) + for c in poly.exterior.coords + ] + + holes = [ + [ + (float(c[0]), float(c[1])) + for c in ring.coords + ] + for ring in poly.interiors + ] + + def tx(coords, index): + out = coords + + if reverse: + out = reverse_closed_ring(out) + + if rotate: + core_len = max(1, len(out) - 1) + shift = max(1, core_len // 3 + index) + out = rotate_closed_ring(out, shift) + + return out + + ext2 = tx(ext, 0) + holes2 = [tx(h, i + 1) for i, h in enumerate(holes)] + + return Polygon(ext2, holes2) + + +def make_input_variant(geom, mode: str): + if mode == "original": + return geom + + reverse = mode in { + "reverse_rings", + "reverse_and_rotate", + "reverse_components_and_rings", + } + + rotate = mode in { + "rotate_rings", + "reverse_and_rotate", + } + + if geom.geom_type == "Polygon": + return polygon_variant( + geom, + reverse=reverse, + rotate=rotate, + ) + + if geom.geom_type == "MultiPolygon": + polys = [ + polygon_variant( + p, + reverse=reverse, + rotate=rotate, + ) + for p in geom.geoms + ] + + if mode in { + "reverse_components", + "reverse_components_and_rings", + }: + polys = list(reversed(polys)) + + return MultiPolygon(polys) + + return geom + + +# --------------------------------------------------------------------------- +# DB loader +# --------------------------------------------------------------------------- + +def load_polygon_records(cur, dataset_id: int, wkb_fn: str): + records = [] + + cur.execute(f""" + SELECT + 'AREA' AS kind, + f.id AS feature_id, + NULL AS coverage_geom_id, + f.object_class, + f.cell_id, + c.cell_name, + f.source_record_index, + f.coverage_id, + f.raw_geotype, + f.bbox_min_lon, + f.bbox_min_lat, + f.bbox_max_lon, + f.bbox_max_lat, + c.bbox_min_lon AS cell_bbox_min_lon, + c.bbox_min_lat AS cell_bbox_min_lat, + c.bbox_max_lon AS cell_bbox_max_lon, + c.bbox_max_lat AS cell_bbox_max_lat, + {wkb_fn}(g.geom) AS geom_wkb + FROM chart_feature f + JOIN chart_area_geometry g + ON g.feature_id=f.id + JOIN chart_cell c + ON c.id=f.cell_id + WHERE f.dataset_id=%s + AND f.geometry_type='AREA' + ORDER BY f.id + """, (dataset_id,)) + + for r in cur: + try: + geom = load_wkb(to_bytes(r["geom_wkb"])) + x = dict(r) + x["_geom"] = geom + records.append(x) + except Exception: + continue + + cur.execute(f""" + SELECT + 'COVERAGE' AS kind, + NULL AS feature_id, + v.id AS coverage_geom_id, + 'COVERAGE' AS object_class, + v.cell_id, + c.cell_name, + v.source_record_index, + NULL AS coverage_id, + NULL AS raw_geotype, + NULL AS bbox_min_lon, + NULL AS bbox_min_lat, + NULL AS bbox_max_lon, + NULL AS bbox_max_lat, + c.bbox_min_lon AS cell_bbox_min_lon, + c.bbox_min_lat AS cell_bbox_min_lat, + c.bbox_max_lon AS cell_bbox_max_lon, + c.bbox_max_lat AS cell_bbox_max_lat, + {wkb_fn}(v.geom) AS geom_wkb + FROM chart_coverage v + JOIN chart_cell c + ON c.id=v.cell_id + WHERE v.dataset_id=%s + ORDER BY v.id + """, (dataset_id,)) + + for r in cur: + try: + geom = load_wkb(to_bytes(r["geom_wkb"])) + x = dict(r) + x["_geom"] = geom + records.append(x) + except Exception: + continue + + return records + + +# --------------------------------------------------------------------------- +# GeoJSON +# --------------------------------------------------------------------------- + +def make_geojson(records, path: Path): + features = [] + + def add(g, props): + if g is None: + return + try: + features.append({ + "type": "Feature", + "geometry": mapping(g), + "properties": props, + }) + except Exception: + pass + + for r in records: + if r["_geom"].is_valid: + continue + + rid = ( + r.get("feature_id") + if r["kind"] == "AREA" + else r.get("coverage_geom_id") + ) + + common = { + "kind": r["kind"], + "record_id": rid, + "object_class": r.get("object_class"), + "cell_name": r.get("cell_name"), + "canonical_hash": r.get("_canonical_hash"), + } + + add( + r["_geom"], + { + **common, + "layer": "original_invalid", + } + ) + + add( + r.get("_canonical_candidate"), + { + **common, + "layer": "canonical_candidate", + } + ) + + path.write_text( + json.dumps( + { + "type": "FeatureCollection", + "features": features, + }, + ensure_ascii=False, + separators=(",", ":"), + default=str, + ), + encoding="utf-8" + ) + + +def parse_args(): + p = argparse.ArgumentParser( + description="CM93 normalization canonical validation v10" + ) + + p.add_argument("--host", default=os.getenv("MYSQL_HOST", "127.0.0.1")) + p.add_argument("--port", type=int, default=int(os.getenv("MYSQL_PORT", "3306"))) + p.add_argument("--user", default=os.getenv("MYSQL_USER", "root")) + p.add_argument("--password", default=os.getenv("MYSQL_PASSWORD")) + p.add_argument("--database", default=os.getenv("MYSQL_DATABASE", "cm93_chart")) + p.add_argument("--dataset-id", type=int, default=1) + p.add_argument( + "--topology-area-tolerance", + type=float, + default=1e-12, + ) + p.add_argument( + "--hausdorff-tolerance", + type=float, + default=1e-12, + ) + p.add_argument( + "--bbox-tolerance", + type=float, + default=5e-6, + ) + p.add_argument("--output", default=None) + + return p.parse_args() + + +def main(): + args = parse_args() + + if not args.password: + args.password = getpass.getpass("MySQL password: ") + + stamp = datetime.now().strftime("%Y%m%d_%H%M%S") + + outdir = Path( + args.output + or f"cm93_normalization_canonical_v10_1_dataset_{args.dataset_id}_{stamp}" + ) + outdir.mkdir(parents=True, exist_ok=True) + + report = { + "version": "v10.1", + "generated_at": datetime.now().isoformat(timespec="seconds"), + "database": args.database, + "dataset_id": args.dataset_id, + "read_only": True, + "pbf": "OUT_OF_SCOPE", + "shapely_version": getattr(shapely, "__version__", "unknown"), + "warnings": [], + } + + conn = None + + try: + conn = pymysql.connect( + host=args.host, + port=args.port, + user=args.user, + password=args.password, + database=args.database, + charset="utf8mb4", + cursorclass=DictCursor, + autocommit=False, + connect_timeout=20, + read_timeout=600, + write_timeout=30, + ) + + with conn.cursor() as cur: + begin_read_only(cur, report["warnings"]) + + report["mysql_version"] = scalar(cur, "SELECT VERSION()") + wkb_fn = probe_wkb_fn(cur) + + print("[1/6] 加载 Area + Coverage ...") + records = load_polygon_records( + cur, + args.dataset_id, + wkb_fn + ) + + valid = [r for r in records if r["_geom"].is_valid] + invalid = [r for r in records if not r["_geom"].is_valid] + + print( + f" total={len(records):,} " + f"valid={len(valid):,} invalid={len(invalid):,}" + ) + + print("[2/6] Valid PASS-THROUGH policy guard ...") + valid_policy_changed = 0 + valid_hash_changed = 0 + + for i, r in enumerate(valid, start=1): + # policy: valid geometry 不进入 canonical rewrite + out = r["_geom"] + + if out is not r["_geom"]: + valid_policy_changed += 1 + + if geometry_hash(out) != geometry_hash(r["_geom"]): + valid_hash_changed += 1 + + if i % 20000 == 0: + print(f" valid checked={i:,}") + + print("[3/6] 55 invalid canonical / input-variant 验证 ...") + + golden_rows = [] + variant_rows = [] + face_order_rows = [] + failure_rows = [] + bbox_rows = [] + + input_modes = [ + "original", + "reverse_rings", + "rotate_rings", + "reverse_and_rotate", + "reverse_components", + "reverse_components_and_rings", + ] + + for i, r in enumerate(invalid, start=1): + geom = r["_geom"] + + rid = ( + r.get("feature_id") + if r["kind"] == "AREA" + else r.get("coverage_geom_id") + ) + + selected, sel_info = select_faces_odd_even(geom) + baseline = candidate_from_selected_faces( + selected, + "original" + ) + + if baseline is None: + fail = { + "kind": r["kind"], + "record_id": rid, + "object_class": r.get("object_class"), + "cell_name": r.get("cell_name"), + "failure": "BASELINE_CANDIDATE_NONE", + } + failure_rows.append(fail) + continue + + baseline_canonical = canonicalize(baseline) + baseline_hash = geometry_hash(baseline_canonical) + + baseline_valid = bool(baseline_canonical.is_valid) + baseline_polygon_only = ( + baseline_canonical.geom_type + in {"Polygon", "MultiPolygon"} + ) + + # make_valid line remainder length only as metadata + mv = make_valid_geom(geom) + mvcomps = flatten_components(mv) + remainder = geom_union(mvcomps["lines"]) + remainder_length = safe_length(remainder) + + # canonicalization itself topology invariant + canonical_sym = safe_symdiff_area( + baseline, + baseline_canonical + ) + canonical_haus = safe_hausdorff( + baseline, + baseline_canonical + ) + + canonical_topology_ok = ( + canonical_sym is not None + and canonical_sym <= args.topology_area_tolerance + and canonical_haus is not None + and canonical_haus <= args.hausdorff_tolerance + ) + + # face order invariance + face_hashes = {} + for mode in [ + "original", + "reverse", + "area", + "centroid", + ]: + cand = candidate_from_selected_faces( + selected, + mode + ) + canon = canonicalize(cand) + h = geometry_hash(canon) + face_hashes[mode] = h + + face_order_rows.append({ + "kind": r["kind"], + "feature_id": r.get("feature_id"), + "coverage_geom_id": r.get("coverage_geom_id"), + "object_class": r.get("object_class"), + "cell_name": r.get("cell_name"), + "face_order_mode": mode, + "canonical_wkb_sha256": h, + "matches_baseline_hash": + int(h == baseline_hash), + "topology_symdiff_area_deg2": + safe_symdiff_area( + baseline_canonical, + canon + ), + }) + + face_order_stable = ( + len(set(face_hashes.values())) == 1 + ) + + # repeated same input + repeat_hashes = [] + for _ in range(3): + c = normalize_odd_even(geom) + cc = canonicalize(c) + repeat_hashes.append(geometry_hash(cc)) + + repeat_stable = ( + len(set(repeat_hashes)) == 1 + and repeat_hashes[0] == baseline_hash + ) + + # input-equivalent variants + variant_fail = 0 + + for mode in input_modes: + try: + vg = make_input_variant(geom, mode) + + vc = normalize_odd_even(vg) + + if vc is None: + variant_fail += 1 + variant_rows.append({ + "kind": r["kind"], + "feature_id": r.get("feature_id"), + "coverage_geom_id": r.get("coverage_geom_id"), + "object_class": r.get("object_class"), + "cell_name": r.get("cell_name"), + "variant_mode": mode, + "status": "FAIL_NO_CANDIDATE", + }) + continue + + vcanon = canonicalize(vc) + vh = geometry_hash(vcanon) + + sym = safe_symdiff_area( + baseline_canonical, + vcanon + ) + haus = safe_hausdorff( + baseline_canonical, + vcanon + ) + + topology_equal = ( + bool( + baseline_canonical.equals( + vcanon + ) + ) + and sym is not None + and sym <= args.topology_area_tolerance + ) + + hash_equal = (vh == baseline_hash) + + if not ( + vcanon.is_valid + and topology_equal + and hash_equal + ): + variant_fail += 1 + + variant_rows.append({ + "kind": r["kind"], + "feature_id": r.get("feature_id"), + "coverage_geom_id": r.get("coverage_geom_id"), + "object_class": r.get("object_class"), + "cell_name": r.get("cell_name"), + "variant_mode": mode, + "variant_input_valid": + int(bool(vg.is_valid)), + "candidate_valid": + int(bool(vcanon.is_valid)), + "candidate_type": + vcanon.geom_type, + "topology_equal_baseline": + int(topology_equal), + "symdiff_area_deg2": sym, + "hausdorff_distance_deg": haus, + "canonical_wkb_sha256": vh, + "matches_baseline_hash": + int(hash_equal), + "status": + "PASS" + if ( + vcanon.is_valid + and topology_equal + and hash_equal + ) + else "FAIL", + }) + + except Exception as e: + variant_fail += 1 + + variant_rows.append({ + "kind": r["kind"], + "feature_id": r.get("feature_id"), + "coverage_geom_id": r.get("coverage_geom_id"), + "object_class": r.get("object_class"), + "cell_name": r.get("cell_name"), + "variant_mode": mode, + "status": "EXCEPTION", + "detail": + f"{type(e).__name__}: {e}", + }) + + # bbox metadata impact + old_bounds = tuple(float(v) for v in geom.bounds) + new_bounds = tuple( + float(v) for v in baseline_canonical.bounds + ) + + bbox_changed = any( + abs(a - b) > args.bbox_tolerance + for a, b in zip(old_bounds, new_bounds) + ) + + feature_bbox = None + feature_bbox_matches_candidate = None + + if r["kind"] == "AREA": + vals = [ + r.get("bbox_min_lon"), + r.get("bbox_min_lat"), + r.get("bbox_max_lon"), + r.get("bbox_max_lat"), + ] + if all(v is not None for v in vals): + feature_bbox = tuple( + float(v) for v in vals + ) + feature_bbox_matches_candidate = all( + abs(a - b) <= args.bbox_tolerance + for a, b in zip( + feature_bbox, + new_bounds + ) + ) + + bbox_rows.append({ + "kind": r["kind"], + "feature_id": r.get("feature_id"), + "coverage_geom_id": r.get("coverage_geom_id"), + "object_class": r.get("object_class"), + "cell_name": r.get("cell_name"), + "original_geometry_bounds": old_bounds, + "candidate_bounds": new_bounds, + "bbox_changed_over_tolerance": + int(bbox_changed), + "current_feature_bbox": feature_bbox, + "current_feature_bbox_matches_candidate": + ( + int(feature_bbox_matches_candidate) + if feature_bbox_matches_candidate + is not None + else None + ), + "future_feature_bbox_update_needed": + ( + int(not feature_bbox_matches_candidate) + if feature_bbox_matches_candidate + is not None + else None + ), + "discarded_connector_length_deg": + remainder_length, + }) + + record_pass = all([ + baseline_valid, + baseline_polygon_only, + canonical_topology_ok, + face_order_stable, + repeat_stable, + variant_fail == 0, + ]) + + golden = { + "kind": r["kind"], + "feature_id": r.get("feature_id"), + "coverage_geom_id": r.get("coverage_geom_id"), + "object_class": r.get("object_class"), + "cell_id": r.get("cell_id"), + "cell_name": r.get("cell_name"), + "source_record_index": + r.get("source_record_index"), + "coverage_id": r.get("coverage_id"), + "raw_geotype": r.get("raw_geotype"), + + "normalization_rule": + "ODD_EVEN_FACE_SELECTION", + "canonicalization_rule": + "GEOS_NORMALIZE", + + "candidate_type": + baseline_canonical.geom_type, + "candidate_polygon_part_count": + polygon_part_count( + baseline_canonical + ), + "candidate_valid": + int(baseline_valid), + + "polygonize_face_count": + sel_info["face_count"], + "selected_face_count": + sel_info["selected_face_count"], + + "candidate_area_deg2": + float(baseline_canonical.area), + "candidate_bbox": + tuple( + float(v) + for v in baseline_canonical.bounds + ), + + "discarded_connector_length_deg": + remainder_length, + + "canonical_wkb_sha256": + baseline_hash, + + "canonicalization_symdiff_area_deg2": + canonical_sym, + "canonicalization_hausdorff_deg": + canonical_haus, + "canonical_topology_unchanged": + int(canonical_topology_ok), + + "face_order_hash_stable": + int(face_order_stable), + "repeat_hash_stable": + int(repeat_stable), + "input_variant_failure_count": + variant_fail, + + "record_status": + "PASS" + if record_pass + else "FAIL", + } + + golden_rows.append(golden) + + if not record_pass: + failure_rows.append(golden) + + r["_canonical_candidate"] = baseline_canonical + r["_canonical_hash"] = baseline_hash + + print( + f" [{i:02d}/{len(invalid)}] " + f"{r['kind']} id={rid} " + f"{r.get('object_class')} -> " + f"{'PASS' if record_pass else 'FAIL'}" + ) + + print("[4/6] 汇总 canonical stability ...") + + record_status = Counter( + r["record_status"] + for r in golden_rows + ) + + variant_status = Counter( + r.get("status", "UNKNOWN") + for r in variant_rows + ) + + bbox_update_area_count = sum( + int( + r.get( + "future_feature_bbox_update_needed" + ) or 0 + ) + for r in bbox_rows + ) + + report["summary"] = { + "total_polygon_records": len(records), + "valid_polygon_records": len(valid), + "invalid_polygon_records": len(invalid), + + "valid_policy_changed_count": + valid_policy_changed, + "valid_hash_changed_count": + valid_hash_changed, + + "golden_vector_count": + len(golden_rows), + + "golden_record_status_counts": + dict(record_status), + + "input_variant_case_count": + len(variant_rows), + "input_variant_status_counts": + dict(variant_status), + + "face_order_case_count": + len(face_order_rows), + "face_order_hash_mismatch_count": + sum( + not bool( + r["matches_baseline_hash"] + ) + for r in face_order_rows + ), + + "canonical_topology_failure_count": + sum( + not bool( + r["canonical_topology_unchanged"] + ) + for r in golden_rows + ), + + "repeat_hash_failure_count": + sum( + not bool( + r["repeat_hash_stable"] + ) + for r in golden_rows + ), + + "input_variant_failure_record_count": + sum( + int( + r["input_variant_failure_count"] + ) > 0 + for r in golden_rows + ), + + "future_area_feature_bbox_update_needed_count": + bbox_update_area_count, + } + + overall_pass = ( + len(invalid) == 55 + and len(golden_rows) == 55 + and record_status.get("PASS", 0) == 55 + and record_status.get("FAIL", 0) == 0 + and valid_policy_changed == 0 + and valid_hash_changed == 0 + and report["summary"][ + "face_order_hash_mismatch_count" + ] == 0 + and report["summary"][ + "canonical_topology_failure_count" + ] == 0 + and report["summary"][ + "repeat_hash_failure_count" + ] == 0 + and report["summary"][ + "input_variant_failure_record_count" + ] == 0 + ) + + report["summary"]["overall_pass"] = overall_pass + + print("[5/6] 写 golden vectors / 报告 ...") + + write_csv( + outdir / "normalization_golden_vectors_v10_1.csv", + golden_rows + ) + + ( + outdir / "normalization_golden_vectors_v10_1.json" + ).write_text( + json.dumps( + golden_rows, + ensure_ascii=False, + indent=2, + default=str, + ), + encoding="utf-8" + ) + + write_csv( + outdir / "normalization_variant_results_v10_1.csv", + variant_rows + ) + + write_csv( + outdir / "normalization_face_order_results_v10_1.csv", + face_order_rows + ) + + write_csv( + outdir / "normalization_canonical_failures_v10_1.csv", + failure_rows + ) + + write_csv( + outdir / "normalization_bbox_metadata_impact_v10_1.csv", + bbox_rows + ) + + make_geojson( + records, + outdir / "normalization_canonical_candidates_v10_1.geojson" + ) + + ( + outdir / "normalization_canonical_report_v10_1.json" + ).write_text( + json.dumps( + report, + ensure_ascii=False, + indent=2, + default=str, + ), + encoding="utf-8" + ) + + lines = [] + + lines.append( + "CM93 -> MySQL OGC Normalization Canonical Validation v10.1" + ) + lines.append("=" * 96) + lines.append( + f"Generated: {report['generated_at']}" + ) + lines.append( + f"Dataset ID: {args.dataset_id}" + ) + lines.append( + f"MySQL: {report['mysql_version']}" + ) + lines.append( + f"Shapely: {report['shapely_version']}" + ) + lines.append("") + + lines.append("SCOPE") + lines.append( + " CM93JAPAN 是局部 CM93 数据集；" + "跨未导入海域不连续不作为错误。" + ) + lines.append( + " V10 只验证 odd-even candidate 的 canonical WKB " + "可重复性，不修改数据库。" + ) + lines.append("") + + lines.append("BASELINE") + lines.append( + f" polygon records: {len(records)}" + ) + lines.append( + f" valid: {len(valid)}" + ) + lines.append( + f" invalid: {len(invalid)}" + ) + lines.append("") + + lines.append("VALID PASS-THROUGH") + lines.append( + f" policy changed: {valid_policy_changed}" + ) + lines.append( + f" WKB hash changed: {valid_hash_changed}" + ) + lines.append("") + + lines.append("GOLDEN VECTOR STATUS") + for k, v in record_status.most_common(): + lines.append( + f" {k:24s} {v}" + ) + lines.append("") + + lines.append("INPUT VARIANTS") + lines.append( + f" cases: {len(variant_rows)}" + ) + for k, v in variant_status.most_common(): + lines.append( + f" {k:24s} {v}" + ) + lines.append("") + + lines.append("STABILITY") + lines.append( + f" face-order hash mismatch: " + f"{report['summary']['face_order_hash_mismatch_count']}" + ) + lines.append( + f" canonical topology failures: " + f"{report['summary']['canonical_topology_failure_count']}" + ) + lines.append( + f" repeat hash failures: " + f"{report['summary']['repeat_hash_failure_count']}" + ) + lines.append( + f" input-variant failure records: " + f"{report['summary']['input_variant_failure_record_count']}" + ) + lines.append("") + + lines.append("METADATA IMPACT") + lines.append( + f" future Area feature bbox updates needed: " + f"{bbox_update_area_count}" + ) + lines.append( + " bbox 更新仅是未来若真正保存 normalized geometry " + "时的元数据同步要求；V10 不修改任何 bbox。" + ) + lines.append("") + + lines.append("OVERALL") + lines.append( + f" CANONICAL VALIDATION PASS: " + f"{'YES' if overall_pass else 'NO'}" + ) + lines.append("") + + lines.append("NEXT USE") + lines.append( + " normalization_golden_vectors_v10_1.csv/json " + "可作为未来 importer normalization 实现的 55 条 golden regression vectors。" + ) + lines.append( + " 后续实现应固定：ODD_EVEN_FACE_SELECTION + GEOS_NORMALIZE，" + "并要求输出 canonical_wkb_sha256 与 golden vector 一致。" + ) + lines.append("") + + lines.append( + "NO DATABASE WRITE / NO IMPORTER CHANGE / NO PBF" + ) + + ( + outdir / "normalization_canonical_summary_v10_1.txt" + ).write_text( + "\n".join(lines), + encoding="utf-8" + ) + + conn.rollback() + + print("[6/6] 压缩 ...") + + zpath = outdir.with_suffix(".zip") + + with zipfile.ZipFile( + zpath, + "w", + compression=zipfile.ZIP_DEFLATED + ) as z: + for p in sorted(outdir.rglob("*")): + if p.is_file(): + z.write( + p, + p.relative_to(outdir.parent) + ) + + print("") + print("完成。") + print(f"Output: {outdir.resolve()}") + print(f"ZIP: {zpath.resolve()}") + print("把 ZIP 整包上传给 ChatGPT。") + + except KeyboardInterrupt: + if conn: + try: + conn.rollback() + except Exception: + pass + + print("\n用户中断。") + sys.exit(130) + + except Exception: + if conn: + try: + conn.rollback() + except Exception: + pass + + tb = traceback.format_exc() + print(tb) + + try: + ( + outdir / "fatal_error.txt" + ).write_text( + tb, + encoding="utf-8" + ) + except Exception: + pass + + sys.exit(1) + + finally: + if conn: + try: + conn.close() + except Exception: + pass + + +if __name__ == "__main__": + main() diff --git a/cm93_mysql_normalization_dryrun_v9.py b/cm93_mysql_normalization_dryrun_v9.py new file mode 100644 index 0000000..65085b8 --- /dev/null +++ b/cm93_mysql_normalization_dryrun_v9.py @@ -0,0 +1,1713 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +CM93 -> MySQL OGC Polygon Normalization Dry-Run +Version: v9.0 + +背景 +==== +前序审计已经确认： + +1. Dataset ID=1 为日本沿海局部 CM93 数据，不是完整 CM93。 + 因此跨未导入海域的边界断开不作为错误。 + +2. 原来的 4406 条 geometry issue： + 全部是 raw_geotype=16 / META 的正常 parent-no-geometry， + 应分类为 EXPECTED_PARENT_NO_GEOMETRY，而非 geometry corruption。 + +3. 已实际生成的空间数据： + - Point invalid = 0 + - Line invalid = 0 + - Sounding invalid = 0 + - Area OGC-invalid = 52 + - Coverage OGC-invalid = 3 + +4. V6 已证明： + 52/52 invalid Area 的 raw CM93 edge chain 本身就会生成同样的 invalid topology， + importer 没有发现 edge index / edge direction / edge ordering / transform 错误。 + +5. V8 已证明： + - 41 条记录存在 linear remainder； + - 共 109 条 remainder segment； + - 109/109 都是 OPPOSITE_DIRECTION_RETRACE； + - 即同一路径正向走一次、反向走一次的零面积 connector / bridge； + - 55/55 的 face selection 全部兼容 odd-even； + - 54/55 同时兼容 non-zero； + - 1 条只兼容 odd-even。 + 因此 odd-even 是当前证据下更安全的 CM93 Area fill rule。 + +V9 目的 +======= +这是 normalization 的“干跑 / 验证器”，不是数据库修复器。 + +对 52 invalid Area + 3 invalid Coverage： +1. 对原始 boundary 进行 noding + polygonize_full； +2. 对每一个 planar face 取 representative point； +3. 按原始 ring 的 odd-even parity 决定该 face 是否属于 Area； +4. union 所有 odd-even inside faces； +5. 得到 candidate OGC-valid Polygon / MultiPolygon； +6. 只在内存中验证 candidate： + + - candidate 是否 non-empty； + - candidate 是否只包含 Polygon/MultiPolygon； + - candidate 是否 valid； + - candidate 是否与 Shapely make_valid 的 polygonal 部分一致； + - candidate 是否没有产生原始 linework 之外的新边界； + - 原始 boundary 中被 candidate 丢弃的线是否与 make_valid linear remainder 一致； + - candidate 再次 normalization 是否幂等； + - candidate bbox 与原始 bbox 的变化； + - 生成稳定 SHA-256 WKB hash，方便后续重复验证。 + +7. 对全部“本来就 valid”的 Area/Coverage 做 policy guard： + normalization 函数遇到 valid geometry 必须 PASS-THROUGH， + 不得改变任何有效 geometry。 + 本脚本会统计 valid record 数量并确认 policy_changed_count=0。 + +V9 不做什么 +============ +- 不 UPDATE +- 不 INSERT +- 不 DELETE +- 不 ST_MakeValid 写回 +- 不替换 chart_area_geometry +- 不替换 chart_coverage +- 不修改 geometry_issue +- 不重新导入 +- 不开始 PBF + +V9 的 PASS 含义 +=============== +只有当 55 条全部满足： + +- odd-even candidate 成功； +- candidate valid； +- candidate polygonal-only； +- candidate 与 make_valid polygonal 部分面积拓扑一致； +- candidate 没有引入原始 boundary 之外的新线； +- discarded linework 与已确认 retraced remainder 一致（若有）； +- normalization 幂等； + +才允许下一阶段讨论 importer/schema 如何保存 normalized geometry。 + +即使 V9 全 PASS，本脚本也绝不写数据库。 + +依赖 +==== + pip install pymysql shapely + +执行 +==== + python cm93_mysql_normalization_dryrun_v9.py ^ + --host 127.0.0.1 ^ + --port 3306 ^ + --user root ^ + --database cm93_chart ^ + --dataset-id 1 + +输出 +==== +- normalization_dryrun_summary_v9.txt +- normalization_dryrun_report_v9.json +- normalization_candidate_results.csv +- normalization_face_selection.csv +- normalization_boundary_compare.csv +- normalization_failure_detail.csv +- normalization_by_object_class.csv +- normalization_by_boundary_context.csv +- normalization_candidates.geojson +并自动生成 ZIP。 +""" + +from __future__ import annotations + +import argparse +import csv +import getpass +import hashlib +import json +import math +import os +import re +import sys +import traceback +import zipfile +from collections import Counter +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple + +try: + import pymysql + from pymysql.cursors import DictCursor +except ImportError: + print("缺少 pymysql：pip install pymysql shapely") + sys.exit(2) + +try: + import shapely + + try: + from shapely import from_wkb as _from_wkb + def load_wkb(data: bytes): + return _from_wkb(data) + except ImportError: + from shapely import wkb as _wkb + def load_wkb(data: bytes): + return _wkb.loads(data) + + try: + from shapely import to_wkb as _to_wkb + def dump_wkb(g) -> bytes: + return bytes(_to_wkb(g, hex=False, byte_order=1)) + except ImportError: + from shapely import wkb as _wkb2 + def dump_wkb(g) -> bytes: + return _wkb2.dumps(g, hex=False, big_endian=False) + + try: + from shapely import make_valid as _make_valid + def make_valid_geom(g): + return _make_valid(g) + except ImportError: + from shapely.validation import make_valid as _make_valid + def make_valid_geom(g): + return _make_valid(g) + + from shapely.geometry import Point, mapping + from shapely.ops import unary_union, polygonize_full + from shapely.validation import explain_validity + +except ImportError: + print("缺少 shapely：pip install pymysql shapely") + sys.exit(2) + + +INVALID_POINT_RE = re.compile( + r"\[\s*([-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)\s+" + r"([-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)\s*\]" +) + + +# --------------------------------------------------------------------------- +# Basic helpers +# --------------------------------------------------------------------------- + +def scalar(cur, sql: str, params: Sequence[Any] = ()) -> Any: + cur.execute(sql, params) + row = cur.fetchone() + return next(iter(row.values())) if row else None + + +def begin_read_only(cur, warnings: List[str]): + try: + cur.execute("SET SESSION TRANSACTION READ ONLY") + except Exception as e: + warnings.append(f"SET SESSION TRANSACTION READ ONLY failed: {e}") + + try: + cur.execute("START TRANSACTION READ ONLY") + except Exception as e: + warnings.append(f"START TRANSACTION READ ONLY failed: {e}") + cur.execute("START TRANSACTION") + + +def to_bytes(v: Any) -> Optional[bytes]: + if v is None: + return None + if isinstance(v, bytes): + return v + if isinstance(v, bytearray): + return bytes(v) + if isinstance(v, memoryview): + return v.tobytes() + try: + return bytes(v) + except Exception: + return None + + +def write_csv( + path: Path, + rows: Iterable[Dict[str, Any]], + fieldnames: Optional[List[str]] = None, +): + rows = list(rows) + + if fieldnames is None: + seen = set() + fieldnames = [] + for row in rows: + for k in row.keys(): + if k not in seen: + seen.add(k) + fieldnames.append(k) + + path.parent.mkdir(parents=True, exist_ok=True) + + with path.open("w", encoding="utf-8-sig", newline="") as f: + if not fieldnames: + return + + writer = csv.DictWriter( + f, + fieldnames=fieldnames, + extrasaction="ignore" + ) + writer.writeheader() + + for row in rows: + out = {} + for k in fieldnames: + v = row.get(k) + if isinstance(v, (dict, list, tuple)): + v = json.dumps(v, ensure_ascii=False, default=str) + out[k] = "" if v is None else v + writer.writerow(out) + + +def probe_wkb_fn(cur) -> str: + for fn in ("ST_AsBinary", "ST_AsWKB", "AsBinary", "AsWKB"): + try: + cur.execute( + f"SELECT {fn}(geom) AS x " + f"FROM chart_area_geometry LIMIT 1" + ) + cur.fetchone() + return fn + except Exception: + pass + + raise RuntimeError("找不到 geometry -> WKB 函数") + + +def parse_invalid_point(reason: str) -> Optional[Tuple[float, float]]: + if not reason: + return None + + m = INVALID_POINT_RE.search(reason) + if not m: + return None + + try: + return float(m.group(1)), float(m.group(2)) + except Exception: + return None + + +def bbox_from_row(row: Dict[str, Any]) -> Optional[Tuple[float, float, float, float]]: + try: + vals = ( + float(row["cell_bbox_min_lon"]), + float(row["cell_bbox_min_lat"]), + float(row["cell_bbox_max_lon"]), + float(row["cell_bbox_max_lat"]), + ) + if all(math.isfinite(v) for v in vals): + return vals + except Exception: + pass + return None + + +def invalid_point_cell_edge( + p: Optional[Tuple[float, float]], + bbox: Optional[Tuple[float, float, float, float]], + tol: float, +) -> Tuple[int, str, Optional[float]]: + if p is None or bbox is None: + return 0, "", None + + x, y = p + minx, miny, maxx, maxy = bbox + + d = { + "WEST": abs(x - minx), + "EAST": abs(x - maxx), + "SOUTH": abs(y - miny), + "NORTH": abs(y - maxy), + } + + edge, dist = min(d.items(), key=lambda kv: kv[1]) + return int(dist <= tol), edge if dist <= tol else "", dist + + +def geometry_hash(g) -> Optional[str]: + if g is None: + return None + try: + return hashlib.sha256(dump_wkb(g)).hexdigest() + except Exception: + return None + + +def safe_area(g) -> Optional[float]: + if g is None: + return None + try: + return float(g.area) + except Exception: + return None + + +def safe_length(g) -> Optional[float]: + if g is None: + return None + try: + return float(g.length) + except Exception: + return None + + +def safe_symdiff_area(a, b) -> Optional[float]: + if a is None or b is None: + return None + try: + return float(a.symmetric_difference(b).area) + except Exception: + return None + + +def safe_symdiff_length(a, b) -> Optional[float]: + if a is None or b is None: + return None + try: + return float(a.symmetric_difference(b).length) + except Exception: + return None + + +def safe_hausdorff(a, b) -> Optional[float]: + if a is None or b is None: + return None + try: + return float(a.hausdorff_distance(b)) + except Exception: + return None + + +def flatten_components(g) -> Dict[str, List[Any]]: + result = { + "polygons": [], + "lines": [], + "points": [], + "others": [], + } + + def rec(x): + if x is None: + return + + t = x.geom_type + + if t == "Polygon": + result["polygons"].append(x) + elif t == "MultiPolygon": + for y in x.geoms: + rec(y) + elif t in ("LineString", "LinearRing"): + result["lines"].append(x) + elif t == "MultiLineString": + for y in x.geoms: + rec(y) + elif t == "Point": + result["points"].append(x) + elif t == "MultiPoint": + for y in x.geoms: + rec(y) + elif hasattr(x, "geoms"): + for y in x.geoms: + rec(y) + else: + result["others"].append(x) + + rec(g) + return result + + +def geom_union(items: Sequence[Any]) -> Optional[Any]: + if not items: + return None + + try: + return unary_union(list(items)) + except Exception: + return None + + +# --------------------------------------------------------------------------- +# Odd-even fill rule +# --------------------------------------------------------------------------- + +def original_rings(geom) -> List[List[Tuple[float, float]]]: + rings: List[List[Tuple[float, float]]] = [] + + def add_polygon(poly): + rings.append([ + (float(c[0]), float(c[1])) + for c in poly.exterior.coords + ]) + + for ring in poly.interiors: + rings.append([ + (float(c[0]), float(c[1])) + for c in ring.coords + ]) + + if geom.geom_type == "Polygon": + add_polygon(geom) + elif geom.geom_type == "MultiPolygon": + for poly in geom.geoms: + add_polygon(poly) + + return rings + + +def point_on_segment( + px: float, + py: float, + ax: float, + ay: float, + bx: float, + by: float, + eps: float = 1e-12, +) -> bool: + vx = bx - ax + vy = by - ay + wx = px - ax + wy = py - ay + + cross_val = abs(vx * wy - vy * wx) + if cross_val > eps * max(1.0, abs(vx), abs(vy)): + return False + + dot_val = wx * vx + wy * vy + if dot_val < -eps: + return False + + vv = vx * vx + vy * vy + if dot_val - vv > eps: + return False + + return True + + +def even_odd_inside( + point: Tuple[float, float], + rings: List[List[Tuple[float, float]]], +) -> bool: + x, y = point + crossings = 0 + + for ring in rings: + for i in range(len(ring) - 1): + x1, y1 = ring[i] + x2, y2 = ring[i + 1] + + if point_on_segment(x, y, x1, y1, x2, y2): + # representative point 理论上不应落在 face boundary。 + # 若数值误差导致落边界，按 inside 处理。 + return True + + if (y1 > y) != (y2 > y): + xinters = x1 + (y - y1) * (x2 - x1) / (y2 - y1) + if xinters > x: + crossings += 1 + + return (crossings % 2) == 1 + + +# --------------------------------------------------------------------------- +# Candidate normalization +# --------------------------------------------------------------------------- + +def odd_even_normalize_invalid( + geom +) -> Tuple[Optional[Any], Dict[str, Any], List[Dict[str, Any]]]: + """ + 对 invalid Polygon/MultiPolygon： + + 1. noded boundary + 2. polygonize_full + 3. face representative point + 4. odd-even selection + 5. union selected faces + + 不调用 make_valid 生成 candidate。 + make_valid 只在外部作为独立对照参考。 + """ + info: Dict[str, Any] = { + "polygonize_face_count": 0, + "selected_face_count": 0, + "cuts_length_deg": None, + "dangles_length_deg": None, + "invalid_rings_length_deg": None, + "error": None, + } + + face_rows: List[Dict[str, Any]] = [] + + try: + rings = original_rings(geom) + noded_boundary = unary_union(geom.boundary) + + polygons_geom, cuts, dangles, invalid_rings = polygonize_full( + noded_boundary + ) + + face_components = flatten_components(polygons_geom)["polygons"] + + info["polygonize_face_count"] = len(face_components) + info["cuts_length_deg"] = safe_length(cuts) + info["dangles_length_deg"] = safe_length(dangles) + info["invalid_rings_length_deg"] = safe_length(invalid_rings) + + selected = [] + + for idx, face in enumerate(face_components): + rp = face.representative_point() + p = (float(rp.x), float(rp.y)) + + inside = even_odd_inside(p, rings) + + face_rows.append({ + "face_index": idx, + "representative_x": p[0], + "representative_y": p[1], + "face_area_deg2": float(face.area), + "odd_even_inside": int(inside), + }) + + if inside: + selected.append(face) + + info["selected_face_count"] = len(selected) + + if not selected: + return None, info, face_rows + + candidate = unary_union(selected) + + return candidate, info, face_rows + + except Exception as e: + info["error"] = f"{type(e).__name__}: {e}" + return None, info, face_rows + + +def normalize_policy(geom) -> Tuple[Any, str]: + """ + 最终候选 policy 的核心行为： + + - valid geometry: 原样 PASS-THROUGH + - invalid polygonal geometry: odd-even normalization + + 这里只用于 dry-run。 + """ + if geom.is_valid: + return geom, "PASS_THROUGH_VALID" + + candidate, info, _ = odd_even_normalize_invalid(geom) + + if candidate is None: + return geom, "NORMALIZATION_FAILED_RETURN_ORIGINAL" + + return candidate, "ODD_EVEN_NORMALIZED" + + +# --------------------------------------------------------------------------- +# Database loader +# --------------------------------------------------------------------------- + +def load_polygon_records( + cur, + dataset_id: int, + wkb_fn: str +) -> List[Dict[str, Any]]: + records = [] + + cur.execute(f""" + SELECT + 'AREA' AS kind, + f.id AS feature_id, + NULL AS coverage_geom_id, + f.object_class, + f.cell_id, + c.cell_name, + f.source_record_index, + f.coverage_id, + f.raw_geotype, + f.bbox_min_lon, + f.bbox_min_lat, + f.bbox_max_lon, + f.bbox_max_lat, + c.bbox_min_lon AS cell_bbox_min_lon, + c.bbox_min_lat AS cell_bbox_min_lat, + c.bbox_max_lon AS cell_bbox_max_lon, + c.bbox_max_lat AS cell_bbox_max_lat, + {wkb_fn}(g.geom) AS geom_wkb + FROM chart_feature f + JOIN chart_area_geometry g + ON g.feature_id = f.id + JOIN chart_cell c + ON c.id = f.cell_id + WHERE f.dataset_id=%s + AND f.geometry_type='AREA' + ORDER BY f.id + """, (dataset_id,)) + + for r in cur: + try: + geom = load_wkb(to_bytes(r["geom_wkb"])) + x = dict(r) + x["_geom"] = geom + records.append(x) + except Exception: + continue + + cur.execute(f""" + SELECT + 'COVERAGE' AS kind, + NULL AS feature_id, + v.id AS coverage_geom_id, + 'COVERAGE' AS object_class, + v.cell_id, + c.cell_name, + v.source_record_index, + NULL AS coverage_id, + NULL AS raw_geotype, + NULL AS bbox_min_lon, + NULL AS bbox_min_lat, + NULL AS bbox_max_lon, + NULL AS bbox_max_lat, + c.bbox_min_lon AS cell_bbox_min_lon, + c.bbox_min_lat AS cell_bbox_min_lat, + c.bbox_max_lon AS cell_bbox_max_lon, + c.bbox_max_lat AS cell_bbox_max_lat, + {wkb_fn}(v.geom) AS geom_wkb + FROM chart_coverage v + JOIN chart_cell c + ON c.id = v.cell_id + WHERE v.dataset_id=%s + ORDER BY v.id + """, (dataset_id,)) + + for r in cur: + try: + geom = load_wkb(to_bytes(r["geom_wkb"])) + x = dict(r) + x["_geom"] = geom + records.append(x) + except Exception: + continue + + return records + + +# --------------------------------------------------------------------------- +# GeoJSON +# --------------------------------------------------------------------------- + +def make_geojson(records: List[Dict[str, Any]], path: Path): + features = [] + + def add(g, props): + if g is None: + return + + try: + features.append({ + "type": "Feature", + "geometry": mapping(g), + "properties": props, + }) + except Exception: + pass + + for r in records: + if r["_geom"].is_valid: + continue + + rid = ( + r.get("feature_id") + if r["kind"] == "AREA" + else r.get("coverage_geom_id") + ) + + common = { + "kind": r["kind"], + "record_id": rid, + "object_class": r.get("object_class"), + "cell_name": r.get("cell_name"), + "status": r.get("_candidate_status"), + } + + add( + r.get("_geom"), + { + **common, + "layer": "original_invalid", + } + ) + + add( + r.get("_candidate"), + { + **common, + "layer": "odd_even_candidate", + } + ) + + add( + r.get("_discarded_linework"), + { + **common, + "layer": "discarded_linework", + } + ) + + p = r.get("_invalid_point") + if p is not None: + add( + Point(p), + { + **common, + "layer": "invalidity_point", + } + ) + + path.write_text( + json.dumps( + { + "type": "FeatureCollection", + "features": features, + }, + ensure_ascii=False, + separators=(",", ":"), + default=str, + ), + encoding="utf-8" + ) + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +def parse_args(): + parser = argparse.ArgumentParser( + description="CM93 OGC polygon normalization dry-run v9" + ) + + parser.add_argument( + "--host", + default=os.getenv("MYSQL_HOST", "127.0.0.1") + ) + parser.add_argument( + "--port", + type=int, + default=int(os.getenv("MYSQL_PORT", "3306")) + ) + parser.add_argument( + "--user", + default=os.getenv("MYSQL_USER", "root") + ) + parser.add_argument( + "--password", + default=os.getenv("MYSQL_PASSWORD") + ) + parser.add_argument( + "--database", + default=os.getenv("MYSQL_DATABASE", "cm93_chart") + ) + parser.add_argument( + "--dataset-id", + type=int, + default=1 + ) + parser.add_argument( + "--cell-edge-tolerance", + type=float, + default=5e-6 + ) + parser.add_argument( + "--area-abs-tolerance", + type=float, + default=1e-12, + help="candidate vs make_valid polygonal symmetric-difference area absolute tolerance" + ) + parser.add_argument( + "--area-relative-tolerance", + type=float, + default=1e-6 + ) + parser.add_argument( + "--line-length-tolerance", + type=float, + default=1e-10, + help="boundary linework compare tolerance (degree)" + ) + parser.add_argument( + "--output", + default=None + ) + + return parser.parse_args() + + +def main(): + args = parse_args() + + if not args.password: + args.password = getpass.getpass("MySQL password: ") + + stamp = datetime.now().strftime("%Y%m%d_%H%M%S") + + outdir = Path( + args.output + or f"cm93_normalization_dryrun_v9_dataset_{args.dataset_id}_{stamp}" + ) + outdir.mkdir(parents=True, exist_ok=True) + + report: Dict[str, Any] = { + "version": "v9.0", + "generated_at": datetime.now().isoformat(timespec="seconds"), + "database": args.database, + "dataset_id": args.dataset_id, + "read_only": True, + "pbf": "OUT_OF_SCOPE", + "shapely_version": getattr(shapely, "__version__", "unknown"), + "scope_note": ( + "CM93JAPAN 是日本沿海局部 CM93 数据集。" + "V9 只验证 odd-even OGC normalization candidate，" + "不修改数据库。" + ), + "warnings": [], + } + + conn = None + + try: + conn = pymysql.connect( + host=args.host, + port=args.port, + user=args.user, + password=args.password, + database=args.database, + charset="utf8mb4", + cursorclass=DictCursor, + autocommit=False, + connect_timeout=20, + read_timeout=600, + write_timeout=30, + ) + + with conn.cursor() as cur: + begin_read_only(cur, report["warnings"]) + + report["mysql_version"] = scalar( + cur, + "SELECT VERSION()" + ) + + wkb_fn = probe_wkb_fn(cur) + report["wkb_fn"] = wkb_fn + + print("[1/6] 加载全部 Area + Coverage ...") + records = load_polygon_records( + cur, + args.dataset_id, + wkb_fn + ) + + area_count = sum(r["kind"] == "AREA" for r in records) + coverage_count = sum(r["kind"] == "COVERAGE" for r in records) + + valid_records = [r for r in records if r["_geom"].is_valid] + invalid_records = [r for r in records if not r["_geom"].is_valid] + + print( + f" Area={area_count:,} Coverage={coverage_count:,} " + f"valid={len(valid_records):,} invalid={len(invalid_records):,}" + ) + + # ---------------------------------------------------------------- + # Valid geometry policy guard + # ---------------------------------------------------------------- + + print("[2/6] Valid geometry PASS-THROUGH guard ...") + + valid_policy_changed = 0 + valid_policy_hash_changed = 0 + + for idx, r in enumerate(valid_records, start=1): + geom = r["_geom"] + + normalized, mode = normalize_policy(geom) + + if mode != "PASS_THROUGH_VALID": + valid_policy_changed += 1 + + if geometry_hash(normalized) != geometry_hash(geom): + valid_policy_hash_changed += 1 + + if idx % 20000 == 0: + print( + f" valid checked={idx:,}/{len(valid_records):,}" + ) + + print( + f" valid policy changed={valid_policy_changed} " + f"hash_changed={valid_policy_hash_changed}" + ) + + # ---------------------------------------------------------------- + # Invalid dry-run + # ---------------------------------------------------------------- + + print("[3/6] 55 invalid Polygon odd-even normalization dry-run ...") + + candidate_rows: List[Dict[str, Any]] = [] + face_rows: List[Dict[str, Any]] = [] + boundary_rows: List[Dict[str, Any]] = [] + failure_rows: List[Dict[str, Any]] = [] + + for idx, r in enumerate(invalid_records, start=1): + geom = r["_geom"] + + rid = ( + r.get("feature_id") + if r["kind"] == "AREA" + else r.get("coverage_geom_id") + ) + + reason = explain_validity(geom) + invalid_point = parse_invalid_point(reason) + + cell_bbox = bbox_from_row(r) + + on_cell_edge, cell_edge, cell_edge_distance = ( + invalid_point_cell_edge( + invalid_point, + cell_bbox, + args.cell_edge_tolerance + ) + ) + + candidate, norm_info, this_faces = ( + odd_even_normalize_invalid(geom) + ) + + base = { + "kind": r["kind"], + "feature_id": r.get("feature_id"), + "coverage_geom_id": r.get("coverage_geom_id"), + "object_class": r.get("object_class"), + "cell_id": r.get("cell_id"), + "cell_name": r.get("cell_name"), + "source_record_index": r.get("source_record_index"), + "coverage_id": r.get("coverage_id"), + "raw_geotype": r.get("raw_geotype"), + "original_validity_reason": reason, + "invalid_x": ( + invalid_point[0] + if invalid_point else None + ), + "invalid_y": ( + invalid_point[1] + if invalid_point else None + ), + "invalid_point_on_cell_edge": on_cell_edge, + "cell_edge": cell_edge, + "cell_edge_distance_deg": cell_edge_distance, + } + + for face in this_faces: + face_rows.append({ + **base, + **face, + }) + + if candidate is None: + row = { + **base, + **norm_info, + "candidate_status": "FAIL_NO_CANDIDATE", + "candidate_type": None, + "candidate_valid": 0, + } + + candidate_rows.append(row) + failure_rows.append(row) + + r["_candidate"] = None + r["_candidate_status"] = "FAIL_NO_CANDIDATE" + continue + + # candidate must be polygonal-only + candidate_comps = flatten_components(candidate) + + candidate_polygon_only = ( + len(candidate_comps["polygons"]) > 0 + and len(candidate_comps["lines"]) == 0 + and len(candidate_comps["points"]) == 0 + and len(candidate_comps["others"]) == 0 + ) + + candidate_valid = bool(candidate.is_valid) + candidate_empty = bool(candidate.is_empty) + + # independent make_valid reference + mv = make_valid_geom(geom) + mv_comps = flatten_components(mv) + mv_poly_union = geom_union(mv_comps["polygons"]) + mv_line_union = geom_union(mv_comps["lines"]) + + candidate_mv_symarea = safe_symdiff_area( + candidate, + mv_poly_union + ) + + candidate_area = safe_area(candidate) + mv_area = safe_area(mv_poly_union) + + if ( + candidate_mv_symarea is not None + and candidate_area is not None + and mv_area is not None + ): + area_scale = max( + abs(candidate_area), + abs(mv_area), + 1e-30 + ) + + area_allowed = max( + args.area_abs_tolerance, + args.area_relative_tolerance * area_scale + ) + + area_match = ( + candidate_mv_symarea <= area_allowed + ) + else: + area_allowed = None + area_match = False + + # Boundary evidence: + # original linework is noded before comparison. + try: + original_noded_boundary = unary_union( + geom.boundary + ) + except Exception: + original_noded_boundary = geom.boundary + + try: + candidate_boundary = unary_union( + candidate.boundary + ) + except Exception: + candidate_boundary = candidate.boundary + + # candidate must not invent boundary outside original linework + try: + candidate_extra_boundary = ( + candidate_boundary.difference( + original_noded_boundary + ) + ) + except Exception: + candidate_extra_boundary = None + + candidate_extra_length = safe_length( + candidate_extra_boundary + ) + + no_new_boundary = ( + candidate_extra_length is not None + and candidate_extra_length <= args.line_length_tolerance + ) + + # linework removed from polygon boundary + try: + discarded_linework = ( + original_noded_boundary.difference( + candidate_boundary + ) + ) + except Exception: + discarded_linework = None + + discarded_length = safe_length(discarded_linework) + mv_remainder_length = safe_length(mv_line_union) + + discarded_vs_mv_sym_length = safe_symdiff_length( + discarded_linework, + mv_line_union + ) + + if mv_line_union is None: + discarded_matches_mv = ( + (discarded_length or 0.0) + <= args.line_length_tolerance + ) + else: + discarded_matches_mv = ( + discarded_vs_mv_sym_length is not None + and discarded_vs_mv_sym_length + <= args.line_length_tolerance + ) + + discarded_mv_hausdorff = safe_hausdorff( + discarded_linework, + mv_line_union + ) + + # Idempotence: + candidate2, mode2 = normalize_policy(candidate) + + candidate_idempotent_symarea = safe_symdiff_area( + candidate, + candidate2 + ) + + idempotent = ( + candidate2.is_valid + and candidate_idempotent_symarea is not None + and candidate_idempotent_symarea + <= args.area_abs_tolerance + and mode2 == "PASS_THROUGH_VALID" + ) + + # bbox + original_bounds = tuple( + float(v) for v in geom.bounds + ) + candidate_bounds = tuple( + float(v) for v in candidate.bounds + ) + + bbox_changed = ( + any( + abs(a - b) > args.cell_edge_tolerance + for a, b in zip( + original_bounds, + candidate_bounds + ) + ) + ) + + # Final PASS gate + pass_gate = all([ + not candidate_empty, + candidate_polygon_only, + candidate_valid, + area_match, + no_new_boundary, + discarded_matches_mv, + idempotent, + ]) + + candidate_status = ( + "PASS_NORMALIZATION_CANDIDATE" + if pass_gate + else "CHECK_NORMALIZATION_CANDIDATE" + ) + + row = { + **base, + **norm_info, + + "candidate_status": candidate_status, + "candidate_type": candidate.geom_type, + "candidate_empty": int(candidate_empty), + "candidate_valid": int(candidate_valid), + "candidate_validity_reason": ( + "Valid Geometry" + if candidate_valid + else explain_validity(candidate) + ), + "candidate_polygon_only": int(candidate_polygon_only), + + "candidate_polygon_part_count": + len(candidate_comps["polygons"]), + "candidate_line_part_count": + len(candidate_comps["lines"]), + "candidate_point_part_count": + len(candidate_comps["points"]), + "candidate_other_part_count": + len(candidate_comps["others"]), + + "candidate_area_deg2": candidate_area, + "make_valid_polygonal_area_deg2": mv_area, + "candidate_vs_make_valid_symdiff_area_deg2": + candidate_mv_symarea, + "candidate_make_valid_area_allowed_deg2": + area_allowed, + "candidate_matches_make_valid_polygonal": + int(area_match), + + "candidate_extra_boundary_length_deg": + candidate_extra_length, + "candidate_introduces_no_new_boundary": + int(no_new_boundary), + + "discarded_linework_length_deg": + discarded_length, + "make_valid_linear_remainder_length_deg": + mv_remainder_length, + "discarded_vs_make_valid_symdiff_length_deg": + discarded_vs_mv_sym_length, + "discarded_vs_make_valid_hausdorff_deg": + discarded_mv_hausdorff, + "discarded_matches_make_valid_remainder": + int(discarded_matches_mv), + + "normalization_idempotent": int(idempotent), + "idempotent_symdiff_area_deg2": + candidate_idempotent_symarea, + + "original_bounds": original_bounds, + "candidate_bounds": candidate_bounds, + "bbox_changed_over_tolerance": + int(bbox_changed), + + "original_wkb_sha256": + geometry_hash(geom), + "candidate_wkb_sha256": + geometry_hash(candidate), + } + + candidate_rows.append(row) + + boundary_rows.append({ + **base, + "candidate_status": candidate_status, + "original_boundary_length_deg": + safe_length(original_noded_boundary), + "candidate_boundary_length_deg": + safe_length(candidate_boundary), + "candidate_extra_boundary_length_deg": + candidate_extra_length, + "discarded_linework_length_deg": + discarded_length, + "make_valid_linear_remainder_length_deg": + mv_remainder_length, + "discarded_vs_make_valid_symdiff_length_deg": + discarded_vs_mv_sym_length, + "discarded_vs_make_valid_hausdorff_deg": + discarded_mv_hausdorff, + }) + + if not pass_gate: + failure_rows.append(row) + + r["_candidate"] = candidate + r["_discarded_linework"] = discarded_linework + r["_candidate_status"] = candidate_status + r["_invalid_point"] = invalid_point + + print( + f" [{idx:02d}/{len(invalid_records)}] " + f"{r['kind']} id={rid} " + f"{r.get('object_class')} -> " + f"{candidate_status}" + ) + + # ---------------------------------------------------------------- + # Summary + # ---------------------------------------------------------------- + + print("[4/6] 汇总 PASS gate ...") + + status_counts = Counter( + r["candidate_status"] + for r in candidate_rows + ) + + object_counts = Counter( + ( + r["object_class"], + r["candidate_status"], + ) + for r in candidate_rows + ) + + boundary_context_counts = Counter( + ( + "CELL_EDGE" + if r["invalid_point_on_cell_edge"] + else "INTERNAL", + r["candidate_status"], + ) + for r in candidate_rows + ) + + report["summary"] = { + "area_count": area_count, + "coverage_count": coverage_count, + + "valid_polygon_records": len(valid_records), + "invalid_polygon_records": len(invalid_records), + + "valid_policy_changed_count": + valid_policy_changed, + "valid_policy_hash_changed_count": + valid_policy_hash_changed, + + "candidate_status_counts": + dict(status_counts), + + "candidate_pass_count": + status_counts.get( + "PASS_NORMALIZATION_CANDIDATE", 0 + ), + + "candidate_check_count": + status_counts.get( + "CHECK_NORMALIZATION_CANDIDATE", 0 + ) + + status_counts.get( + "FAIL_NO_CANDIDATE", 0 + ), + + "candidate_valid_count": + sum( + int(r.get("candidate_valid") or 0) + for r in candidate_rows + ), + + "candidate_polygon_only_count": + sum( + int(r.get("candidate_polygon_only") or 0) + for r in candidate_rows + ), + + "candidate_matches_make_valid_polygonal_count": + sum( + int( + r.get( + "candidate_matches_make_valid_polygonal" + ) or 0 + ) + for r in candidate_rows + ), + + "candidate_no_new_boundary_count": + sum( + int( + r.get( + "candidate_introduces_no_new_boundary" + ) or 0 + ) + for r in candidate_rows + ), + + "discarded_matches_make_valid_remainder_count": + sum( + int( + r.get( + "discarded_matches_make_valid_remainder" + ) or 0 + ) + for r in candidate_rows + ), + + "idempotent_count": + sum( + int( + r.get( + "normalization_idempotent" + ) or 0 + ) + for r in candidate_rows + ), + + "bbox_changed_count": + sum( + int( + r.get( + "bbox_changed_over_tolerance" + ) or 0 + ) + for r in candidate_rows + ), + } + + full_pass = ( + len(invalid_records) == 55 + and report["summary"]["candidate_pass_count"] == 55 + and report["summary"]["candidate_check_count"] == 0 + and valid_policy_changed == 0 + and valid_policy_hash_changed == 0 + ) + + report["summary"]["overall_dryrun_pass"] = full_pass + + # ---------------------------------------------------------------- + # Outputs + # ---------------------------------------------------------------- + + print("[5/6] 写 CSV / JSON / GeoJSON ...") + + write_csv( + outdir / "normalization_candidate_results.csv", + candidate_rows + ) + + write_csv( + outdir / "normalization_face_selection.csv", + face_rows + ) + + write_csv( + outdir / "normalization_boundary_compare.csv", + boundary_rows + ) + + write_csv( + outdir / "normalization_failure_detail.csv", + failure_rows + ) + + write_csv( + outdir / "normalization_by_object_class.csv", + [ + { + "object_class": obj, + "candidate_status": status, + "count": count, + } + for (obj, status), count + in object_counts.most_common() + ] + ) + + write_csv( + outdir / "normalization_by_boundary_context.csv", + [ + { + "boundary_context": ctx, + "candidate_status": status, + "count": count, + } + for (ctx, status), count + in boundary_context_counts.most_common() + ] + ) + + make_geojson( + records, + outdir / "normalization_candidates.geojson" + ) + + (outdir / "normalization_dryrun_report_v9.json").write_text( + json.dumps( + report, + ensure_ascii=False, + indent=2, + default=str + ), + encoding="utf-8" + ) + + lines = [] + + lines.append( + "CM93 -> MySQL OGC Polygon Normalization Dry-Run v9" + ) + lines.append("=" * 92) + lines.append( + f"Generated: {report['generated_at']}" + ) + lines.append( + f"Dataset ID: {args.dataset_id}" + ) + lines.append( + f"MySQL: {report['mysql_version']}" + ) + lines.append( + f"Shapely: {report['shapely_version']}" + ) + lines.append("") + + lines.append("SCOPE") + lines.append( + " CM93JAPAN 是日本沿海局部 CM93 数据集；" + "跨未导入海域边界不连续不作为错误。" + ) + lines.append( + " V9 只在内存构造 odd-even OGC normalization candidate，" + "不修改数据库。" + ) + lines.append("") + + lines.append("BASELINE") + lines.append( + f" Area records: {area_count}" + ) + lines.append( + f" Coverage records: {coverage_count}" + ) + lines.append( + f" valid polygon records: {len(valid_records)}" + ) + lines.append( + f" invalid polygon records: {len(invalid_records)}" + ) + lines.append("") + + lines.append("VALID PASS-THROUGH GUARD") + lines.append( + f" policy changed valid geometry: " + f"{valid_policy_changed}" + ) + lines.append( + f" WKB hash changed valid geometry:" + f" {valid_policy_hash_changed}" + ) + lines.append("") + + lines.append("CANDIDATE STATUS") + for k, v in status_counts.most_common(): + lines.append( + f" {k:44s} {v}" + ) + lines.append("") + + lines.append("PASS GATES") + lines.append( + f" candidate valid: " + f"{report['summary']['candidate_valid_count']}" + f" / {len(invalid_records)}" + ) + lines.append( + f" polygonal-only: " + f"{report['summary']['candidate_polygon_only_count']}" + f" / {len(invalid_records)}" + ) + lines.append( + f" matches make_valid polygonal: " + f"{report['summary']['candidate_matches_make_valid_polygonal_count']}" + f" / {len(invalid_records)}" + ) + lines.append( + f" introduces no new boundary: " + f"{report['summary']['candidate_no_new_boundary_count']}" + f" / {len(invalid_records)}" + ) + lines.append( + f" discarded linework matches remainder: " + f"{report['summary']['discarded_matches_make_valid_remainder_count']}" + f" / {len(invalid_records)}" + ) + lines.append( + f" normalization idempotent: " + f"{report['summary']['idempotent_count']}" + f" / {len(invalid_records)}" + ) + lines.append( + f" bbox changed over tolerance: " + f"{report['summary']['bbox_changed_count']}" + ) + lines.append("") + + lines.append("OVERALL") + lines.append( + f" DRY-RUN PASS: " + f"{'YES' if full_pass else 'NO'}" + ) + lines.append("") + + lines.append("INTERPRETATION") + lines.append( + " 若 55/55 全部 PASS，说明 odd-even face selection 可以稳定生成" + " OGC-valid Polygon/MultiPolygon candidate，且不会给已有有效 geometry " + "引入变化。" + ) + lines.append( + " discarded linework 只是已确认的 retraced connector 时，" + "可把它理解为零面积 CM93 topology bridge，而不是 Area 面积。" + ) + lines.append( + " 即使 DRY-RUN PASS，本阶段仍不允许写回 Dataset ID=1。" + ) + lines.append("") + + lines.append( + "NO DATABASE WRITE / NO IMPORTER CHANGE / NO PBF" + ) + + ( + outdir / "normalization_dryrun_summary_v9.txt" + ).write_text( + "\n".join(lines), + encoding="utf-8" + ) + + conn.rollback() + + print("[6/6] 压缩输出 ...") + + zip_path = outdir.with_suffix(".zip") + + with zipfile.ZipFile( + zip_path, + "w", + compression=zipfile.ZIP_DEFLATED + ) as z: + for p in sorted(outdir.rglob("*")): + if p.is_file(): + z.write( + p, + p.relative_to(outdir.parent) + ) + + print("") + print("完成。") + print(f"Output: {outdir.resolve()}") + print(f"ZIP: {zip_path.resolve()}") + print("把 ZIP 整包上传给 ChatGPT。") + + except KeyboardInterrupt: + if conn: + try: + conn.rollback() + except Exception: + pass + + print("\n用户中断。") + sys.exit(130) + + except Exception: + if conn: + try: + conn.rollback() + except Exception: + pass + + tb = traceback.format_exc() + print(tb) + + try: + ( + outdir / "fatal_error.txt" + ).write_text( + tb, + encoding="utf-8" + ) + except Exception: + pass + + sys.exit(1) + + finally: + if conn: + try: + conn.close() + except Exception: + pass + + +if __name__ == "__main__": + main() diff --git a/cm93_mysql_ogc_validity_audit_v7.py b/cm93_mysql_ogc_validity_audit_v7.py new file mode 100644 index 0000000..305900c --- /dev/null +++ b/cm93_mysql_ogc_validity_audit_v7.py @@ -0,0 +1,1179 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +CM93 -> MySQL Invalid Polygon OGC Validity 表达诊断 +Version: v7.0 + +背景 +==== +前序审计已经确认： +- Dataset ID=1 是日本沿海局部 CM93 子集，不是完整 CM93； +- raw_geotype=16 / META 的 4406 条不是 geometry corruption； +- Point / Line / Sounding 已通过； +- Area 共有 52 个 Shapely invalid； +- Coverage 共有 3 个 Shapely invalid； +- V6 从 raw_feature_blob + VECTOR 原始 edge chain 按 OpenCPN 规则重建： + * 52/52 raw chain 仍然 invalid； + * 52/52 与 DB Polygon 几乎完全重合； + * 0 edge index 越界； + * 0 edge junction 断裂； + * 0 Feature 重复引用同一 edge； + * importer edge 顺序/方向/坐标转换造成 geometry 损坏的假设已基本排除。 + +V7 目的 +======= +只做“OGC 有效面表达诊断”，不做任何数据库修复。 + +对 52 invalid Area + 3 invalid Coverage： +1. 调用 Shapely make_valid()，但只在内存中。 +2. 分解 make_valid 输出： + - Polygon / MultiPolygon + - LineString / MultiLineString + - Point / MultiPoint +3. 对原始 invalid geometry.boundary 做： + unary_union(boundary) -> polygonize_full() +4. 统计： + - 可 polygonize 的 face 数； + - cuts / dangles / invalid rings； + - make_valid polygon parts 数； + - make_valid linear remainder 数和总长度； + - make_valid point remainder 数； + - 原始 invalid polygon area； + - make_valid polygonal area； + - polygonize faces union area； + - make_valid polygonal 与 polygonize union 的 symmetric difference； + - 原始 boundary 与 make_valid polygon boundary 的 Hausdorff distance； + - boundary symmetric-difference length； +5. 精确标记 invalid point 是否在 Cell bbox 边界。 +6. 给出“诊断分类”，注意这不是自动修复建议： + - POLYGON_ONLY_CONSISTENT + make_valid 只产生 polygonal geometry，且与 polygonize faces 基本一致。 + - POLYGON_ONLY_FACE_SELECTION_DIFFERS + make_valid 只产生面，但与 polygonize 全部 faces 的结果明显不同， + 说明不能简单假设“所有 face 都应该填充”。 + - POLYGON_PLUS_LINEAR_REMAINDER + make_valid 同时产生面 + 线，说明原始 linework 中有不能直接归入有效面内部的线段。 + - POLYGON_PLUS_POINT_REMAINDER + - POLYGON_PLUS_MIXED_REMAINDER + - NO_POLYGONAL_RESULT + - MAKE_VALID_FAILED +7. 单独按： + - Area / Coverage + - Cell 边界 / Cell 内部 + - Object Class + - validity reason + 统计。 +8. 生成 GeoJSON： + - original invalid + - invalidity point + - make_valid polygonal result + - make_valid linear remainder + - polygonize faces +9. 输出是否存在“可以统一无损转成 Polygon/MultiPolygon”的证据。 + 注意：即使所有记录都能变成有效面，也不能在本任务写回 DB。 + +非常重要 +======== +当前 Dataset 只覆盖日本沿海一部分，因此： +- 数据集外海域不连续不算错误； +- Cell 边界上的 self-touch / shared topology 要保留边界上下文； +- V7 不把 Cell 边界异常自动判成“坏数据”； +- V7 只回答“OGC Polygon/MultiPolygon 能否干净表达”。 + +禁止 +==== +- 不 UPDATE / INSERT / DELETE +- 不 ST_MakeValid 写回 +- 不 buffer(0) +- 不替换 chart_area_geometry +- 不替换 chart_coverage +- 不修改 geometry_issue +- 不重新导入 +- 不开始 PBF + +依赖 +==== + pip install pymysql shapely + +执行 +==== + python cm93_mysql_ogc_validity_audit_v7.py ^ + --host 127.0.0.1 ^ + --port 3306 ^ + --user root ^ + --database cm93_chart ^ + --dataset-id 1 + +输出 +==== +- ogc_validity_summary_v7.txt +- ogc_validity_report_v7.json +- invalid_polygon_ogc_diagnostic.csv +- make_valid_component_detail.csv +- polygonize_face_detail.csv +- ogc_diagnostic_by_class.csv +- ogc_diagnostic_by_object_class.csv +- ogc_diagnostic_by_cell.csv +- ogc_diagnostic.geojson +并自动打 zip。 +""" + +from __future__ import annotations + +import argparse +import csv +import getpass +import json +import math +import os +import re +import sys +import traceback +import zipfile +from collections import Counter, defaultdict +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple + +try: + import pymysql + from pymysql.cursors import DictCursor +except ImportError: + print("缺少 pymysql：pip install pymysql shapely") + sys.exit(2) + +try: + import shapely + try: + from shapely import from_wkb as _from_wkb + def load_wkb(data: bytes): + return _from_wkb(data) + except ImportError: + from shapely import wkb as _wkb + def load_wkb(data: bytes): + return _wkb.loads(data) + + try: + from shapely import make_valid as _make_valid + def make_valid_geom(g): + return _make_valid(g) + except ImportError: + from shapely.validation import make_valid as _make_valid + def make_valid_geom(g): + return _make_valid(g) + + from shapely.geometry import Point, mapping + from shapely.ops import unary_union, polygonize_full + from shapely.validation import explain_validity +except ImportError: + print("缺少 shapely：pip install pymysql shapely") + sys.exit(2) + + +INVALID_POINT_RE = re.compile( + r"\[\s*([-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)\s+" + r"([-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)\s*\]" +) + + +def scalar(cur, sql: str, params: Sequence[Any] = ()) -> Any: + cur.execute(sql, params) + row = cur.fetchone() + return next(iter(row.values())) if row else None + + +def fetch_all(cur, sql: str, params: Sequence[Any] = ()) -> List[Dict[str, Any]]: + cur.execute(sql, params) + return list(cur.fetchall()) + + +def begin_read_only(cur, warnings: List[str]): + try: + cur.execute("SET SESSION TRANSACTION READ ONLY") + except Exception as e: + warnings.append(f"SET SESSION TRANSACTION READ ONLY failed: {e}") + try: + cur.execute("START TRANSACTION READ ONLY") + except Exception as e: + warnings.append(f"START TRANSACTION READ ONLY failed: {e}") + cur.execute("START TRANSACTION") + + +def to_bytes(v: Any) -> Optional[bytes]: + if v is None: + return None + if isinstance(v, bytes): + return v + if isinstance(v, bytearray): + return bytes(v) + if isinstance(v, memoryview): + return v.tobytes() + try: + return bytes(v) + except Exception: + return None + + +def write_csv(path: Path, rows: Iterable[Dict[str, Any]], + fieldnames: Optional[List[str]] = None): + rows = list(rows) + if fieldnames is None: + seen = set() + fieldnames = [] + for r in rows: + for k in r.keys(): + if k not in seen: + seen.add(k) + fieldnames.append(k) + + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8-sig", newline="") as f: + if not fieldnames: + return + w = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore") + w.writeheader() + for r in rows: + out = {} + for k in fieldnames: + v = r.get(k) + if isinstance(v, (dict, tuple, list)): + v = json.dumps(v, ensure_ascii=False, default=str) + out[k] = "" if v is None else v + w.writerow(out) + + +def probe_wkb_fn(cur) -> str: + for fn in ("ST_AsBinary", "ST_AsWKB", "AsBinary", "AsWKB"): + try: + cur.execute( + f"SELECT {fn}(geom) AS x FROM chart_area_geometry LIMIT 1" + ) + cur.fetchone() + return fn + except Exception: + pass + raise RuntimeError("找不到 geometry -> WKB 函数") + + +def parse_invalid_point(reason: str) -> Optional[Tuple[float, float]]: + if not reason: + return None + m = INVALID_POINT_RE.search(reason) + if not m: + return None + try: + return float(m.group(1)), float(m.group(2)) + except Exception: + return None + + +def bbox_from_row(row: Dict[str, Any]) -> Optional[Tuple[float, float, float, float]]: + try: + vals = ( + float(row["cell_bbox_min_lon"]), + float(row["cell_bbox_min_lat"]), + float(row["cell_bbox_max_lon"]), + float(row["cell_bbox_max_lat"]), + ) + if all(math.isfinite(v) for v in vals): + return vals + except Exception: + pass + return None + + +def invalid_point_cell_edge( + p: Optional[Tuple[float, float]], + bbox: Optional[Tuple[float, float, float, float]], + tol: float, +) -> Tuple[int, str, Optional[float]]: + if p is None or bbox is None: + return 0, "", None + + x, y = p + minx, miny, maxx, maxy = bbox + ds = { + "WEST": abs(x - minx), + "EAST": abs(x - maxx), + "SOUTH": abs(y - miny), + "NORTH": abs(y - maxy), + } + edge, dist = min(ds.items(), key=lambda kv: kv[1]) + return int(dist <= tol), edge if dist <= tol else "", dist + + +def validity_reason_class(reason: str) -> str: + s = (reason or "").lower() + if "ring self-intersection" in s: + return "RING_SELF_INTERSECTION" + if "self-intersection" in s: + return "SELF_INTERSECTION" + if "interior is disconnected" in s: + return "INTERIOR_DISCONNECTED" + if "hole lies outside shell" in s: + return "HOLE_OUTSIDE_SHELL" + if "nested shells" in s: + return "NESTED_SHELLS" + if "nested holes" in s: + return "NESTED_HOLES" + if "too few points" in s: + return "TOO_FEW_POINTS" + return "OTHER_INVALID" + + +def flatten_components(g) -> Dict[str, List[Any]]: + result = { + "polygons": [], + "lines": [], + "points": [], + "others": [], + } + + def rec(x): + if x is None: + return + t = x.geom_type + if t == "Polygon": + result["polygons"].append(x) + elif t == "MultiPolygon": + for y in x.geoms: + rec(y) + elif t in ("LineString", "LinearRing"): + result["lines"].append(x) + elif t == "MultiLineString": + for y in x.geoms: + rec(y) + elif t == "Point": + result["points"].append(x) + elif t == "MultiPoint": + for y in x.geoms: + rec(y) + elif hasattr(x, "geoms"): + for y in x.geoms: + rec(y) + else: + result["others"].append(x) + + rec(g) + return result + + +def geom_union(items: Sequence[Any]) -> Optional[Any]: + if not items: + return None + try: + return unary_union(list(items)) + except Exception: + return None + + +def safe_area(g) -> Optional[float]: + if g is None: + return None + try: + return float(g.area) + except Exception: + return None + + +def safe_length(g) -> Optional[float]: + if g is None: + return None + try: + return float(g.length) + except Exception: + return None + + +def safe_hausdorff(a, b) -> Optional[float]: + if a is None or b is None: + return None + try: + return float(a.hausdorff_distance(b)) + except Exception: + return None + + +def safe_symdiff_area(a, b) -> Optional[float]: + if a is None or b is None: + return None + try: + return float(a.symmetric_difference(b).area) + except Exception: + return None + + +def safe_symdiff_length(a, b) -> Optional[float]: + if a is None or b is None: + return None + try: + return float(a.symmetric_difference(b).length) + except Exception: + return None + + +def count_collection_parts(g) -> int: + if g is None: + return 0 + try: + if hasattr(g, "geoms"): + return len(g.geoms) + return 1 + except Exception: + return 0 + + +def diagnostic_class( + *, + polygon_count: int, + line_count: int, + point_count: int, + other_count: int, + polygonize_agrees: Optional[bool], +) -> str: + if polygon_count == 0: + return "NO_POLYGONAL_RESULT" + + if line_count == 0 and point_count == 0 and other_count == 0: + if polygonize_agrees is True: + return "POLYGON_ONLY_CONSISTENT" + if polygonize_agrees is False: + return "POLYGON_ONLY_FACE_SELECTION_DIFFERS" + return "POLYGON_ONLY_UNRESOLVED_COMPARISON" + + if line_count > 0 and point_count == 0 and other_count == 0: + return "POLYGON_PLUS_LINEAR_REMAINDER" + + if line_count == 0 and point_count > 0 and other_count == 0: + return "POLYGON_PLUS_POINT_REMAINDER" + + return "POLYGON_PLUS_MIXED_REMAINDER" + + +def load_invalid_records(cur, dataset_id: int, wkb_fn: str) -> List[Dict[str, Any]]: + records = [] + + cur.execute(f""" + SELECT + 'AREA' AS kind, + f.id AS feature_id, + NULL AS coverage_geom_id, + f.object_class, + f.cell_id, + c.cell_name, + f.source_record_index, + f.coverage_id, + f.raw_geotype, + f.geometry_issue, + f.geometry_repaired, + c.bbox_min_lon AS cell_bbox_min_lon, + c.bbox_min_lat AS cell_bbox_min_lat, + c.bbox_max_lon AS cell_bbox_max_lon, + c.bbox_max_lat AS cell_bbox_max_lat, + {wkb_fn}(g.geom) AS geom_wkb + FROM chart_feature f + JOIN chart_area_geometry g ON g.feature_id=f.id + JOIN chart_cell c ON c.id=f.cell_id + WHERE f.dataset_id=%s + AND f.geometry_type='AREA' + ORDER BY f.id + """, (dataset_id,)) + + for r in cur: + try: + geom = load_wkb(to_bytes(r["geom_wkb"])) + if not geom.is_valid: + x = dict(r) + x["_geom"] = geom + records.append(x) + except Exception: + continue + + cur.execute(f""" + SELECT + 'COVERAGE' AS kind, + NULL AS feature_id, + v.id AS coverage_geom_id, + 'COVERAGE' AS object_class, + v.cell_id, + c.cell_name, + v.source_record_index, + NULL AS coverage_id, + NULL AS raw_geotype, + v.geometry_issue, + v.geometry_repaired, + c.bbox_min_lon AS cell_bbox_min_lon, + c.bbox_min_lat AS cell_bbox_min_lat, + c.bbox_max_lon AS cell_bbox_max_lon, + c.bbox_max_lat AS cell_bbox_max_lat, + {wkb_fn}(v.geom) AS geom_wkb + FROM chart_coverage v + JOIN chart_cell c ON c.id=v.cell_id + WHERE v.dataset_id=%s + ORDER BY v.id + """, (dataset_id,)) + + for r in cur: + try: + geom = load_wkb(to_bytes(r["geom_wkb"])) + if not geom.is_valid: + x = dict(r) + x["_geom"] = geom + records.append(x) + except Exception: + continue + + return records + + +def make_geojson(records: List[Dict[str, Any]], path: Path): + features = [] + + def add(g, props): + if g is None: + return + try: + features.append({ + "type": "Feature", + "geometry": mapping(g), + "properties": props, + }) + except Exception: + pass + + for r in records: + rid = ( + r.get("feature_id") + if r["kind"] == "AREA" + else r.get("coverage_geom_id") + ) + common = { + "kind": r["kind"], + "record_id": rid, + "object_class": r.get("object_class"), + "cell_name": r.get("cell_name"), + "diagnostic_class": r.get("_diagnostic_class"), + "validity_reason": r.get("_validity_reason"), + "cell_edge": r.get("_invalid_point_on_cell_edge"), + } + + add(r.get("_geom"), { + **common, + "layer": "original_invalid", + }) + + p = r.get("_invalid_point") + if p is not None: + add(Point(p), { + **common, + "layer": "invalidity_point", + }) + + add(r.get("_mv_polygon_union"), { + **common, + "layer": "make_valid_polygonal", + }) + + add(r.get("_mv_line_union"), { + **common, + "layer": "make_valid_linear_remainder", + }) + + add(r.get("_polygonize_union"), { + **common, + "layer": "polygonize_faces_union", + }) + + path.write_text( + json.dumps( + {"type": "FeatureCollection", "features": features}, + ensure_ascii=False, + separators=(",", ":"), + default=str, + ), + encoding="utf-8" + ) + + +def zip_dir(folder: Path) -> Path: + zpath = folder.with_suffix(".zip") + with zipfile.ZipFile(zpath, "w", compression=zipfile.ZIP_DEFLATED) as z: + for p in sorted(folder.rglob("*")): + if p.is_file(): + z.write(p, p.relative_to(folder.parent)) + return zpath + + +def parse_args(): + p = argparse.ArgumentParser( + description="CM93 invalid Polygon OGC validity audit v7" + ) + p.add_argument("--host", default=os.getenv("MYSQL_HOST", "127.0.0.1")) + p.add_argument("--port", type=int, default=int(os.getenv("MYSQL_PORT", "3306"))) + p.add_argument("--user", default=os.getenv("MYSQL_USER", "root")) + p.add_argument("--password", default=os.getenv("MYSQL_PASSWORD")) + p.add_argument("--database", default=os.getenv("MYSQL_DATABASE", "cm93_chart")) + p.add_argument("--dataset-id", type=int, default=1) + p.add_argument( + "--cell-edge-tolerance", + type=float, + default=5e-6, + help="invalid point 距 Cell bbox 边界判定容差（degree）" + ) + p.add_argument( + "--face-symdiff-abs-tolerance", + type=float, + default=1e-12, + help="make_valid polygon 与 polygonize union 面积差绝对容差（degree^2）" + ) + p.add_argument( + "--face-symdiff-relative-tolerance", + type=float, + default=1e-6, + help="make_valid polygon 与 polygonize union 面积差相对容差" + ) + p.add_argument("--output", default=None) + return p.parse_args() + + +def main(): + args = parse_args() + if not args.password: + args.password = getpass.getpass("MySQL password: ") + + stamp = datetime.now().strftime("%Y%m%d_%H%M%S") + outdir = Path( + args.output + or f"cm93_ogc_validity_audit_v7_dataset_{args.dataset_id}_{stamp}" + ) + outdir.mkdir(parents=True, exist_ok=True) + + report = { + "version": "v7.0", + "generated_at": datetime.now().isoformat(timespec="seconds"), + "database": args.database, + "dataset_id": args.dataset_id, + "read_only": True, + "pbf": "OUT_OF_SCOPE", + "scope_note": ( + "CM93JAPAN 是日本沿海局部数据集。" + "V7 只诊断 invalid Polygon 如何映射到 OGC valid polygonal geometry，" + "不做任何数据库写回。" + ), + "warnings": [], + "shapely_version": getattr(shapely, "__version__", "unknown"), + } + + conn = None + try: + conn = pymysql.connect( + host=args.host, + port=args.port, + user=args.user, + password=args.password, + database=args.database, + charset="utf8mb4", + cursorclass=DictCursor, + autocommit=False, + connect_timeout=20, + read_timeout=600, + write_timeout=30, + ) + + with conn.cursor() as cur: + begin_read_only(cur, report["warnings"]) + report["mysql_version"] = scalar(cur, "SELECT VERSION()") + wkb_fn = probe_wkb_fn(cur) + report["wkb_fn"] = wkb_fn + + print("[1/5] 加载 invalid Area + Coverage ...") + records = load_invalid_records(cur, args.dataset_id, wkb_fn) + print( + f" total={len(records)} " + f"Area={sum(r['kind']=='AREA' for r in records)} " + f"Coverage={sum(r['kind']=='COVERAGE' for r in records)}" + ) + + print("[2/5] make_valid + polygonize_full 诊断 ...") + diag_rows = [] + component_rows = [] + face_rows = [] + + for i, r in enumerate(records, start=1): + geom = r["_geom"] + reason = explain_validity(geom) + reason_class = validity_reason_class(reason) + invalid_point = parse_invalid_point(reason) + cell_bbox = bbox_from_row(r) + on_edge, edge_name, edge_dist = invalid_point_cell_edge( + invalid_point, cell_bbox, args.cell_edge_tolerance + ) + + base = { + "kind": r["kind"], + "feature_id": r.get("feature_id"), + "coverage_geom_id": r.get("coverage_geom_id"), + "object_class": r.get("object_class"), + "cell_id": r.get("cell_id"), + "cell_name": r.get("cell_name"), + "source_record_index": r.get("source_record_index"), + "coverage_id": r.get("coverage_id"), + "raw_geotype": r.get("raw_geotype"), + "validity_reason": reason, + "validity_reason_class": reason_class, + "invalid_x": invalid_point[0] if invalid_point else None, + "invalid_y": invalid_point[1] if invalid_point else None, + "invalid_point_on_cell_edge": on_edge, + "cell_edge": edge_name, + "cell_edge_distance_deg": edge_dist, + } + + try: + mv = make_valid_geom(geom) + except Exception as e: + diag_rows.append({ + **base, + "make_valid_type": None, + "diagnostic_class": "MAKE_VALID_FAILED", + "detail": f"{type(e).__name__}: {e}", + }) + continue + + comps = flatten_components(mv) + mv_poly_union = geom_union(comps["polygons"]) + mv_line_union = geom_union(comps["lines"]) + mv_point_union = geom_union(comps["points"]) + + for idx, p in enumerate(comps["polygons"]): + component_rows.append({ + **base, + "component_dimension": "POLYGON", + "component_index": idx, + "geom_type": p.geom_type, + "area_deg2": safe_area(p), + "length_deg": safe_length(p.boundary), + }) + for idx, l in enumerate(comps["lines"]): + component_rows.append({ + **base, + "component_dimension": "LINE", + "component_index": idx, + "geom_type": l.geom_type, + "area_deg2": 0.0, + "length_deg": safe_length(l), + }) + for idx, p in enumerate(comps["points"]): + component_rows.append({ + **base, + "component_dimension": "POINT", + "component_index": idx, + "geom_type": p.geom_type, + "area_deg2": 0.0, + "length_deg": 0.0, + }) + for idx, o in enumerate(comps["others"]): + component_rows.append({ + **base, + "component_dimension": "OTHER", + "component_index": idx, + "geom_type": getattr(o, "geom_type", type(o).__name__), + "area_deg2": safe_area(o), + "length_deg": safe_length(o), + }) + + # polygonize_full on noded linework + polygonize_error = "" + polygonize_union = None + polygons_geom = cuts = dangles = invalid_rings = None + poly_faces: List[Any] = [] + try: + noded_boundary = unary_union(geom.boundary) + polygons_geom, cuts, dangles, invalid_rings = polygonize_full( + noded_boundary + ) + pcomps = flatten_components(polygons_geom) + poly_faces = pcomps["polygons"] + polygonize_union = geom_union(poly_faces) + except Exception as e: + polygonize_error = f"{type(e).__name__}: {e}" + + for idx, face in enumerate(poly_faces): + face_rows.append({ + **base, + "face_index": idx, + "area_deg2": safe_area(face), + "boundary_length_deg": safe_length(face.boundary), + }) + + orig_area = safe_area(geom) + mv_poly_area = safe_area(mv_poly_union) + polyz_area = safe_area(polygonize_union) + + mv_vs_polyz_symarea = safe_symdiff_area( + mv_poly_union, polygonize_union + ) + + if ( + mv_vs_polyz_symarea is not None + and mv_poly_area is not None + and polyz_area is not None + ): + scale = max(abs(mv_poly_area), abs(polyz_area), 1e-30) + allowed = max( + args.face_symdiff_abs_tolerance, + args.face_symdiff_relative_tolerance * scale + ) + polygonize_agrees = mv_vs_polyz_symarea <= allowed + else: + allowed = None + polygonize_agrees = None + + cls = diagnostic_class( + polygon_count=len(comps["polygons"]), + line_count=len(comps["lines"]), + point_count=len(comps["points"]), + other_count=len(comps["others"]), + polygonize_agrees=polygonize_agrees, + ) + + original_boundary = None + mv_boundary = None + try: + original_boundary = unary_union(geom.boundary) + except Exception: + pass + try: + if mv_poly_union is not None: + mv_boundary = unary_union(mv_poly_union.boundary) + except Exception: + pass + + boundary_haus = safe_hausdorff( + original_boundary, mv_boundary + ) + boundary_sym_len = safe_symdiff_length( + original_boundary, mv_boundary + ) + + line_remainder_len = sum( + safe_length(x) or 0.0 for x in comps["lines"] + ) + point_remainder_count = len(comps["points"]) + + diag = { + **base, + "original_geom_type": geom.geom_type, + "original_area_deg2": orig_area, + "make_valid_type": mv.geom_type, + "make_valid_is_valid": int(bool(mv.is_valid)), + "make_valid_polygon_count": len(comps["polygons"]), + "make_valid_line_count": len(comps["lines"]), + "make_valid_point_count": len(comps["points"]), + "make_valid_other_count": len(comps["others"]), + "make_valid_polygon_area_deg2": mv_poly_area, + "make_valid_linear_remainder_length_deg": line_remainder_len, + "make_valid_point_remainder_count": point_remainder_count, + "polygonize_face_count": len(poly_faces), + "polygonize_union_area_deg2": polyz_area, + "polygonize_cuts_length_deg": safe_length(cuts), + "polygonize_dangles_length_deg": safe_length(dangles), + "polygonize_invalid_rings_length_deg": safe_length(invalid_rings), + "polygonize_error": polygonize_error, + "make_valid_vs_polygonize_symdiff_area_deg2": + mv_vs_polyz_symarea, + "face_agreement_allowed_area_deg2": allowed, + "make_valid_polygonize_agree": ( + int(polygonize_agrees) + if polygonize_agrees is not None else None + ), + "original_vs_make_valid_boundary_hausdorff_deg": + boundary_haus, + "original_vs_make_valid_boundary_symdiff_length_deg": + boundary_sym_len, + "diagnostic_class": cls, + } + diag_rows.append(diag) + + r["_validity_reason"] = reason + r["_invalid_point"] = invalid_point + r["_invalid_point_on_cell_edge"] = on_edge + r["_mv_polygon_union"] = mv_poly_union + r["_mv_line_union"] = mv_line_union + r["_mv_point_union"] = mv_point_union + r["_polygonize_union"] = polygonize_union + r["_diagnostic_class"] = cls + + rid = ( + r.get("feature_id") + if r["kind"] == "AREA" + else r.get("coverage_geom_id") + ) + print( + f" [{i:02d}/{len(records)}] " + f"{r['kind']} id={rid} {r.get('object_class')} " + f"{reason_class} -> {cls} " + f"poly={len(comps['polygons'])} " + f"line={len(comps['lines'])}" + ) + + print("[3/5] 汇总分类 ...") + class_counts = Counter( + r.get("diagnostic_class", "UNKNOWN") for r in diag_rows + ) + kind_class_counts = Counter( + (r.get("kind"), r.get("diagnostic_class")) + for r in diag_rows + ) + edge_class_counts = Counter( + ( + "CELL_EDGE" if r.get("invalid_point_on_cell_edge") else "INTERNAL", + r.get("diagnostic_class"), + ) + for r in diag_rows + ) + object_counts = Counter( + (r.get("object_class"), r.get("diagnostic_class")) + for r in diag_rows + ) + cell_counts = Counter( + (r.get("cell_name"), r.get("diagnostic_class")) + for r in diag_rows + ) + + report["summary"] = { + "invalid_total": len(diag_rows), + "invalid_area_count": sum( + r.get("kind") == "AREA" for r in diag_rows + ), + "invalid_coverage_count": sum( + r.get("kind") == "COVERAGE" for r in diag_rows + ), + "diagnostic_class_counts": dict(class_counts), + "cell_edge_count": sum( + int(r.get("invalid_point_on_cell_edge") or 0) + for r in diag_rows + ), + "internal_count": sum( + 1 for r in diag_rows + if not r.get("invalid_point_on_cell_edge") + ), + "make_valid_with_linear_remainder": sum( + int(r.get("make_valid_line_count") or 0) > 0 + for r in diag_rows + ), + "make_valid_polygon_only": sum( + int(r.get("make_valid_polygon_count") or 0) > 0 + and int(r.get("make_valid_line_count") or 0) == 0 + and int(r.get("make_valid_point_count") or 0) == 0 + and int(r.get("make_valid_other_count") or 0) == 0 + for r in diag_rows + ), + "make_valid_vs_polygonize_agree": sum( + int(r.get("make_valid_polygonize_agree") or 0) + for r in diag_rows + ), + "records_with_polygonize_dangles": sum( + (r.get("polygonize_dangles_length_deg") or 0.0) > 0 + for r in diag_rows + ), + "records_with_polygonize_cuts": sum( + (r.get("polygonize_cuts_length_deg") or 0.0) > 0 + for r in diag_rows + ), + } + + by_class_rows = [] + for (kind, cls), n in kind_class_counts.most_common(): + by_class_rows.append({ + "group": "KIND", + "key": kind, + "diagnostic_class": cls, + "count": n, + }) + for (ctx, cls), n in edge_class_counts.most_common(): + by_class_rows.append({ + "group": "BOUNDARY_CONTEXT", + "key": ctx, + "diagnostic_class": cls, + "count": n, + }) + + obj_rows = [ + { + "object_class": obj, + "diagnostic_class": cls, + "count": n, + } + for (obj, cls), n in object_counts.most_common() + ] + cell_rows = [ + { + "cell_name": cell, + "diagnostic_class": cls, + "count": n, + } + for (cell, cls), n in cell_counts.most_common() + ] + + print("[4/5] 写输出文件 ...") + write_csv( + outdir / "invalid_polygon_ogc_diagnostic.csv", + diag_rows + ) + write_csv( + outdir / "make_valid_component_detail.csv", + component_rows + ) + write_csv( + outdir / "polygonize_face_detail.csv", + face_rows + ) + write_csv( + outdir / "ogc_diagnostic_by_class.csv", + by_class_rows + ) + write_csv( + outdir / "ogc_diagnostic_by_object_class.csv", + obj_rows + ) + write_csv( + outdir / "ogc_diagnostic_by_cell.csv", + cell_rows + ) + + make_geojson( + [r for r in records if "_diagnostic_class" in r], + outdir / "ogc_diagnostic.geojson" + ) + + (outdir / "ogc_validity_report_v7.json").write_text( + json.dumps(report, ensure_ascii=False, indent=2, default=str), + encoding="utf-8" + ) + + lines = [] + lines.append("CM93 -> MySQL Invalid Polygon OGC Validity 表达诊断 v7") + lines.append("=" * 90) + lines.append(f"Generated: {report['generated_at']}") + lines.append(f"Dataset ID: {args.dataset_id}") + lines.append(f"MySQL: {report['mysql_version']}") + lines.append(f"Shapely: {report['shapely_version']}") + lines.append("") + lines.append("SCOPE") + lines.append( + " CM93JAPAN 是局部数据集；Cell/Dataset 外边界不连续不作为错误。" + ) + lines.append( + " 本轮仅在内存中运行 make_valid / polygonize_full 作为诊断，" + "绝不写回数据库。" + ) + lines.append("") + lines.append("BASELINE") + lines.append( + f" invalid total: {report['summary']['invalid_total']}" + ) + lines.append( + f" Area: {report['summary']['invalid_area_count']}" + ) + lines.append( + f" Coverage: {report['summary']['invalid_coverage_count']}" + ) + lines.append( + f" invalid point Cell edge: {report['summary']['cell_edge_count']}" + ) + lines.append( + f" invalid point internal: {report['summary']['internal_count']}" + ) + lines.append("") + lines.append("DIAGNOSTIC CLASS") + for k, v in class_counts.most_common(): + lines.append(f" {k:46s} {v}") + lines.append("") + lines.append("MAKE_VALID COMPOSITION") + lines.append( + f" polygon-only records: " + f"{report['summary']['make_valid_polygon_only']}" + ) + lines.append( + f" records with linear remainder: " + f"{report['summary']['make_valid_with_linear_remainder']}" + ) + lines.append("") + lines.append("POLYGONIZE") + lines.append( + f" make_valid polygon agrees with polygonize faces: " + f"{report['summary']['make_valid_vs_polygonize_agree']}" + ) + lines.append( + f" records with polygonize dangles: " + f"{report['summary']['records_with_polygonize_dangles']}" + ) + lines.append( + f" records with polygonize cuts: " + f"{report['summary']['records_with_polygonize_cuts']}" + ) + lines.append("") + lines.append("HOW TO READ") + lines.append( + " POLYGON_ONLY_CONSISTENT：存在比较干净的 OGC Polygon/MultiPolygon 表达候选，" + "但本轮仍不允许写回。" + ) + lines.append( + " POLYGON_PLUS_LINEAR_REMAINDER：make_valid 为了得到有效面，" + "还必须留下线性几何；这说明不能粗暴把结果只取 Polygon 后当成无损修复。" + ) + lines.append( + " POLYGON_ONLY_FACE_SELECTION_DIFFERS：make_valid 与把所有 noded faces " + "全部 polygonize 的结果不同，说明 fill/face 选择存在语义问题。" + ) + lines.append( + " 当前目标只是确定表达性质，不决定 importer 是否要规范化。" + ) + lines.append("") + lines.append("NO WRITE / NO REPAIR / NO PBF") + + (outdir / "ogc_validity_summary_v7.txt").write_text( + "\n".join(lines), encoding="utf-8" + ) + + conn.rollback() + + print("[5/5] 压缩输出 ...") + zpath = zip_dir(outdir) + print("") + print("完成。") + print(f"Output: {outdir.resolve()}") + print(f"ZIP: {zpath.resolve()}") + print("把 ZIP 整包上传给 ChatGPT。") + + except KeyboardInterrupt: + if conn: + try: + conn.rollback() + except Exception: + pass + print("\n用户中断。") + sys.exit(130) + except Exception: + if conn: + try: + conn.rollback() + except Exception: + pass + tb = traceback.format_exc() + print(tb) + try: + (outdir / "fatal_error.txt").write_text(tb, encoding="utf-8") + except Exception: + pass + sys.exit(1) + finally: + if conn: + try: + conn.close() + except Exception: + pass + + +if __name__ == "__main__": + main() diff --git a/cm93_mysql_raw_edge_audit_v6.py b/cm93_mysql_raw_edge_audit_v6.py new file mode 100644 index 0000000..77b8ff3 --- /dev/null +++ b/cm93_mysql_raw_edge_audit_v6.py @@ -0,0 +1,1461 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +CM93 -> MySQL invalid Area 原始 Edge Chain 重建与 Importer 对账 +Version: v6.0 + +背景 +==== +Dataset ID=1 为日本沿海局部 CM93 数据，并非完整 CM93。 +V3/V4/V5 已确认： +- 4406 条 raw_geotype=16 / META 是正常 parent-no-geometry，不是真 geometry 损坏； +- Point / Line / Sounding 均通过； +- Area 81080 中仅 52 个 Shapely invalid； +- Coverage 1156 中仅 3 个 invalid； +- 52 个 invalid Area 中 50 个的 invalid point 还被至少一个“有效 Area”边界共享； +- chart_feature.raw_feature_blob 已逐 Feature 保存原始（已解码）Feature record； +- chart_raw_record 中保存每个 Cell 的 HEADER / VECTOR 等整表 blob。 + +本脚本目的 +========== +只针对 invalid Area： +1. 从 chart_feature.raw_feature_blob 解码 CM93 Area 的 edge references： + - object_type + - geotype + - obj_desc_bytes + - n_elements + - 每个 geometry word + - edge_index = word & 0x1fff + - segment_usage = word >> 13 + - usage bit 4 表示该 edge 反向使用 +2. 从 chart_raw_record(record_kind='VECTOR') 解析该 Cell 的全部原始 edge： + - ushort npoints + - npoints * (ushort x, ushort y) +3. 按 OpenCPN CM93 BuildGeom 的规则重建 ring： + - 根据 segment_usage 选择正向/反向； + - 每段拼接时跳过前一段共享的最后点； + - 当前 edge 终点回到 ring 起点时结束一个 ring。 +4. 检查： + - Feature edge index 是否越界； + - 相邻 edge 是否连续； + - ring 是否闭合； + - 同一 edge 是否重复使用； + - ring 是否有非连续重复顶点； + - raw ring 在原始 0..65535 坐标空间是否自交/无效； + - raw invalidity 与数据库 WGS84 Polygon invalidity 是否拓扑一致。 +5. 使用 HEADER 中 easting/northing transform + chart_coverage.wgsox/wgsoy， + 将 raw CM93 点转换到 WGS84，重建参考 Polygon。 +6. 将“参考重建 Polygon”与数据库 chart_area_geometry.geom 对账： + - geometry type + - validity + - validity reason + - ring 数 + - Hausdorff distance + - symmetric difference area（仅辅助，不作为面积物理量） +7. 对 52 条给出表现分类： + A. RAW_EDGE_CHAIN_ITSELF_INVALID + 原始 edge chain 按 OpenCPN 规则组出来已经 invalid； + 更接近 CM93 拓扑表达 / 数据本身 / OGC Polygon 表达差异。 + B. RAW_CHAIN_VALID_DB_INVALID + 原始 edge chain 重建有效，但 DB Polygon 无效； + 强烈指向 importer geometry assembly / conversion bug。 + C. EDGE_CHAIN_DISCONTINUITY + Feature 引用的相邻 edge 不连续或 ring 不闭合； + 需查 edge ordering / source data。 + D. RECONSTRUCTION_MISMATCH + 参考重建与 DB geometry 明显不同； + 指向 importer transform / ring assembly / coverage offset 差异。 + E. RECONSTRUCTION_MATCHES_DB + 参考重建和 DB topology/shape 基本一致。 +8. 输出最值得进一步人工看的孤立 Feature（V5 中无共享邻居的 2 条）详细 edge chain。 +9. 全程只读；不 make_valid、不 buffer(0)、不修改 DB、不做 PBF。 + +参考 +==== +算法按 OpenCPN 当前 cm93.cpp 的公开实现复核： +- read_vector_record_table() +- read_feature_record_table() +- cm93chart::BuildGeom() + +注意： +本脚本是“对账/取证工具”，不是修复器。 +即使 raw chain 本身 invalid，也不能直接断言 CM93 原始数据“坏了”。 +CM93 的共享拓扑表达转换为 OGC Polygon 后，可能出现 self-touch 等 validity 差异。 + +依赖 +==== + pip install pymysql shapely + +执行 +==== + python cm93_mysql_raw_edge_audit_v6.py ^ + --host 127.0.0.1 ^ + --port 3306 ^ + --user root ^ + --database cm93_chart ^ + --dataset-id 1 + +输出 +==== +自动生成目录和 ZIP： +- raw_edge_audit_summary_v6.txt +- raw_edge_audit_report_v6.json +- invalid_area_raw_edge_classification.csv +- invalid_area_edge_references.csv +- invalid_area_ring_summary.csv +- invalid_area_edge_junction_issues.csv +- raw_vs_db_geometry_compare.csv +- isolated_invalid_area_edge_detail.csv +- affected_cell_vector_table_audit.csv +- invalid_area_raw_reconstruction.geojson + +安全 +==== +只执行 SELECT / read-only transaction。 +""" + +from __future__ import annotations + +import argparse +import csv +import getpass +import json +import math +import os +import struct +import sys +import traceback +import zipfile +from collections import Counter, defaultdict +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple + +try: + import pymysql + from pymysql.cursors import DictCursor +except ImportError: + print("缺少 pymysql：pip install pymysql shapely") + sys.exit(2) + +try: + try: + from shapely import from_wkb as _from_wkb + def load_wkb(data: bytes): + return _from_wkb(data) + except ImportError: + from shapely import wkb as _wkb + def load_wkb(data: bytes): + return _wkb.loads(data) + + from shapely.geometry import ( + Point, LineString, LinearRing, Polygon, MultiPolygon, mapping + ) + from shapely.validation import explain_validity +except ImportError: + print("缺少 shapely：pip install pymysql shapely") + sys.exit(2) + + +CM93_SEMIMAJOR_AXIS_METERS = 6378388.0 +DEGREE = math.pi / 180.0 + + +def qi(name: str) -> str: + return "`" + name.replace("`", "``") + "`" + + +def scalar(cur, sql: str, params: Sequence[Any] = ()) -> Any: + cur.execute(sql, params) + row = cur.fetchone() + return next(iter(row.values())) if row else None + + +def fetch_all(cur, sql: str, params: Sequence[Any] = ()) -> List[Dict[str, Any]]: + cur.execute(sql, params) + return list(cur.fetchall()) + + +def begin_read_only(cur, warnings: List[str]): + try: + cur.execute("SET SESSION TRANSACTION READ ONLY") + except Exception as e: + warnings.append(f"SET SESSION TRANSACTION READ ONLY failed: {e}") + try: + cur.execute("START TRANSACTION READ ONLY") + except Exception as e: + warnings.append(f"START TRANSACTION READ ONLY failed: {e}") + cur.execute("START TRANSACTION") + + +def to_bytes(v: Any) -> Optional[bytes]: + if v is None: + return None + if isinstance(v, bytes): + return v + if isinstance(v, bytearray): + return bytes(v) + if isinstance(v, memoryview): + return v.tobytes() + try: + return bytes(v) + except Exception: + return None + + +def write_csv(path: Path, rows: Iterable[Dict[str, Any]], + fieldnames: Optional[List[str]] = None): + rows = list(rows) + if fieldnames is None: + seen = set() + fieldnames = [] + for r in rows: + for k in r.keys(): + if k not in seen: + seen.add(k) + fieldnames.append(k) + + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8-sig", newline="") as f: + if not fieldnames: + return + w = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore") + w.writeheader() + for r in rows: + out = {} + for k in fieldnames: + v = r.get(k) + if isinstance(v, (dict, tuple, list)): + v = json.dumps(v, ensure_ascii=False, default=str) + elif isinstance(v, (bytes, bytearray, memoryview)): + v = bytes(v).hex() + out[k] = "" if v is None else v + w.writerow(out) + + +def probe_wkb_fn(cur) -> str: + for fn in ("ST_AsBinary", "ST_AsWKB", "AsBinary", "AsWKB"): + try: + cur.execute( + f"SELECT {fn}(geom) AS x FROM chart_area_geometry LIMIT 1" + ) + cur.fetchone() + return fn + except Exception: + pass + raise RuntimeError("找不到 geometry -> WKB 函数") + + +# --------------------------------------------------------------------------- +# CM93 raw structures +# --------------------------------------------------------------------------- + +def u16le(buf: bytes, offset: int) -> int: + if offset + 2 > len(buf): + raise ValueError(f"ushort 越界 offset={offset}, len={len(buf)}") + return struct.unpack_from(" Dict[str, Any]: + """ + OpenCPN read_feature_record_table() 对 Area 的前部格式： + uint8 object_type + uint8 geom_prim + uint16 obj_desc_bytes + uint16 n_elements + uint16 edge_ref[n_elements] + ... related/attrs follow according to geotype flags + + edge_ref: + edge_index = edge_ref & 0x1fff + segment_usage = edge_ref >> 13 + reversed when segment_usage & 4 + """ + if raw is None or len(raw) < 6: + raise ValueError("raw_feature_blob 太短") + + object_type = raw[0] + geotype = raw[1] + obj_desc_bytes = u16le(raw, 2) + primitive = geotype & 0x0F + if primitive != 4: + raise ValueError( + f"不是 Area primitive: geotype={geotype}, low_nibble={primitive}" + ) + + n_elements = u16le(raw, 4) + need = 6 + n_elements * 2 + if len(raw) < need: + raise ValueError( + f"raw_feature_blob edge refs 不完整: " + f"n_elements={n_elements}, need={need}, len={len(raw)}" + ) + + refs = [] + pos = 6 + for i in range(n_elements): + word = u16le(raw, pos) + pos += 2 + usage = word >> 13 + edge_index = word & 0x1FFF + refs.append({ + "sequence": i, + "raw_word": word, + "raw_word_hex": f"0x{word:04x}", + "edge_index": edge_index, + "segment_usage": usage, + "reverse": int(bool(usage & 4)), + "usage_bit0": int(bool(usage & 1)), + "usage_bit1": int(bool(usage & 2)), + "usage_bit2_reverse": int(bool(usage & 4)), + }) + + return { + "object_type": object_type, + "geotype": geotype, + "obj_desc_bytes": obj_desc_bytes, + "primitive": primitive, + "has_children_related": int(bool(geotype & 0x10)), + "has_related_count_word": int(bool(geotype & 0x20)), + "has_bit_0x40": int(bool(geotype & 0x40)), + "has_attributes": int(bool(geotype & 0x80)), + "n_elements": n_elements, + "geometry_refs_end_offset": pos, + "raw_length": len(raw), + "refs": refs, + } + + +def parse_vector_table(raw: bytes) -> List[List[Tuple[int, int]]]: + """ + OpenCPN read_vector_record_table(): + for each edge: + uint16 npoints + repeat npoints: + uint16 x + uint16 y + """ + if raw is None: + raise ValueError("VECTOR raw_blob is NULL") + + edges: List[List[Tuple[int, int]]] = [] + pos = 0 + while pos < len(raw): + if pos + 2 > len(raw): + raise ValueError( + f"VECTOR 尾部不足 2 bytes: pos={pos}, len={len(raw)}" + ) + npoints = u16le(raw, pos) + pos += 2 + + need = npoints * 4 + if pos + need > len(raw): + raise ValueError( + f"VECTOR edge#{len(edges)} 数据越界: " + f"npoints={npoints}, pos={pos}, len={len(raw)}" + ) + + pts = [] + for _ in range(npoints): + x = u16le(raw, pos) + y = u16le(raw, pos + 2) + pos += 4 + pts.append((x, y)) + + edges.append(pts) + + if pos != len(raw): + raise ValueError(f"VECTOR 未精确消费: pos={pos}, len={len(raw)}") + return edges + + +def parse_header_metadata(raw_metadata_json: Any) -> Dict[str, Any]: + if raw_metadata_json is None: + raise ValueError("HEADER raw_metadata_json is NULL") + if isinstance(raw_metadata_json, (bytes, bytearray, memoryview)): + s = bytes(raw_metadata_json).decode("utf-8", errors="replace") + obj = json.loads(s) + elif isinstance(raw_metadata_json, str): + obj = json.loads(raw_metadata_json) + elif isinstance(raw_metadata_json, dict): + obj = raw_metadata_json + else: + obj = json.loads(str(raw_metadata_json)) + + h = obj.get("header", obj) + required = [ + "easting_min", "easting_max", + "northing_min", "northing_max", + "usn_vector_records" + ] + missing = [k for k in required if k not in h] + if missing: + raise ValueError(f"HEADER metadata 缺字段: {missing}") + return h + + +def transform_cm93_point_to_wgs84( + p: Tuple[int, int], + header: Dict[str, Any], + wgsox: float = 0.0, + wgsoy: float = 0.0, +) -> Tuple[float, float]: + """ + 对齐 OpenCPN： + transform_x_rate = delta_x / 65535 + transform_y_rate = delta_y / 65535 + valx = x * rate + origin + valy = y * rate + origin + valx -= wgsox + valy -= wgsoy + lat = inverse mercator + lon = valx / (DEGREE * CM93_semimajor_axis_meters) + + 返回 (lon, lat) + """ + x, y = p + + e_min = float(header["easting_min"]) + e_max = float(header["easting_max"]) + n_min = float(header["northing_min"]) + n_max = float(header["northing_max"]) + + dx = e_max - e_min + if dx < 0: + dx += CM93_SEMIMAJOR_AXIS_METERS * 2.0 * math.pi + + x_rate = dx / 65535.0 + y_rate = (n_max - n_min) / 65535.0 + + valx = x * x_rate + e_min + valy = y * y_rate + n_min + + valx -= float(wgsox or 0.0) + valy -= float(wgsoy or 0.0) + + lat = ( + 2.0 * math.atan(math.exp(valy / CM93_SEMIMAJOR_AXIS_METERS)) + - math.pi / 2.0 + ) / DEGREE + lon = valx / (DEGREE * CM93_SEMIMAJOR_AXIS_METERS) + return lon, lat + + +def orient_edge(points: List[Tuple[int, int]], reverse: bool) -> List[Tuple[int, int]]: + return list(reversed(points)) if reverse else list(points) + + +def build_rings_like_opencpn( + refs: List[Dict[str, Any]], + edges: List[List[Tuple[int, int]]], +) -> Tuple[List[List[Tuple[int, int]]], List[Dict[str, Any]], List[Dict[str, Any]]]: + """ + 按 OpenCPN BuildGeom Area 的 ring 判定方式重建。 + + OpenCPN 对每个 segment 把全部点写入，然后 ip-- 跳掉该段最后点； + 当 cur_end_point == start_point 时，把 closure point 留下并结束 ring。 + + 这里用更直观的列表实现，但保留相同语义。 + """ + rings: List[List[Tuple[int, int]]] = [] + edge_rows: List[Dict[str, Any]] = [] + junction_issues: List[Dict[str, Any]] = [] + + current: List[Tuple[int, int]] = [] + ring_start: Optional[Tuple[int, int]] = None + previous_end: Optional[Tuple[int, int]] = None + ring_index = 0 + + for ref in refs: + ei = int(ref["edge_index"]) + if ei < 0 or ei >= len(edges): + edge_rows.append({ + **ref, + "edge_exists": 0, + "edge_point_count": None, + "ring_index": ring_index, + }) + junction_issues.append({ + "ring_index": ring_index, + "sequence": ref["sequence"], + "issue_category": "EDGE_INDEX_OUT_OF_RANGE", + "detail": f"edge_index={ei}, vector_edge_count={len(edges)}", + }) + continue + + raw_pts = edges[ei] + pts = orient_edge(raw_pts, bool(ref["reverse"])) + if not pts: + edge_rows.append({ + **ref, + "edge_exists": 1, + "edge_point_count": 0, + "ring_index": ring_index, + }) + junction_issues.append({ + "ring_index": ring_index, + "sequence": ref["sequence"], + "issue_category": "EMPTY_EDGE", + "detail": f"edge_index={ei}", + }) + continue + + start = pts[0] + end = pts[-1] + + if ring_start is None: + ring_start = start + previous_end = None + + continuous_from_previous = ( + previous_end is None or previous_end == start + ) + + if previous_end is not None and previous_end != start: + junction_issues.append({ + "ring_index": ring_index, + "sequence": ref["sequence"], + "issue_category": "EDGE_JUNCTION_DISCONTINUITY", + "detail": ( + f"previous_end={previous_end}, current_start={start}, " + f"edge_index={ei}" + ), + "previous_end_x": previous_end[0], + "previous_end_y": previous_end[1], + "current_start_x": start[0], + "current_start_y": start[1], + }) + + # OpenCPN 在 segment 之间跳过共享 end point。 + # 若连续，当前 ring 已包含 previous_end；所以追加本 edge 除首点外的点。 + # 第一段则全部加入。 + if not current: + current.extend(pts) + else: + if current[-1] == pts[0]: + current.extend(pts[1:]) + else: + # 为保持 source 顺序证据，不自动补连；直接追加完整 edge。 + current.extend(pts) + + closed_here = (end == ring_start) + + edge_rows.append({ + **ref, + "edge_exists": 1, + "edge_point_count": len(raw_pts), + "oriented_start_x": start[0], + "oriented_start_y": start[1], + "oriented_end_x": end[0], + "oriented_end_y": end[1], + "continuous_from_previous": int(continuous_from_previous), + "ring_index": ring_index, + "closes_ring": int(closed_here), + }) + + previous_end = end + + if closed_here: + if current and current[0] != current[-1]: + current.append(current[0]) + rings.append(current) + current = [] + ring_start = None + previous_end = None + ring_index += 1 + + if current: + junction_issues.append({ + "ring_index": ring_index, + "sequence": None, + "issue_category": "RING_NOT_CLOSED_AT_END_OF_REFS", + "detail": ( + f"unclosed ring vertices={len(current)}, " + f"start={ring_start}, end={current[-1] if current else None}" + ), + }) + rings.append(current) + + return rings, edge_rows, junction_issues + + +# --------------------------------------------------------------------------- +# Geometry construction and comparison +# --------------------------------------------------------------------------- + +def repeated_nonconsecutive_vertices( + ring: List[Tuple[int, int]] +) -> List[Tuple[int, int]]: + if not ring: + return [] + core = ring[:-1] if len(ring) > 1 and ring[0] == ring[-1] else ring + positions = defaultdict(list) + for i, p in enumerate(core): + positions[p].append(i) + + out = [] + n = len(core) + for p, pos in positions.items(): + if len(pos) < 2: + continue + nonconsecutive = False + for i in range(len(pos)): + for j in range(i + 1, len(pos)): + d = abs(pos[j] - pos[i]) + if d > 1 and d < n - 1: + nonconsecutive = True + break + if nonconsecutive: + break + if nonconsecutive: + out.append(p) + return out + + +def signed_area(coords: List[Tuple[float, float]]) -> float: + if len(coords) < 3: + return 0.0 + s = 0.0 + for i in range(len(coords) - 1): + x1, y1 = coords[i] + x2, y2 = coords[i + 1] + s += x1 * y2 - x2 * y1 + return s / 2.0 + + +def ring_to_polygon_candidate( + rings: List[List[Tuple[float, float]]] +): + """ + 为“对账”构造 Polygon/MultiPolygon。 + 不尝试 make_valid。 + + CM93/OpenCPN BuildGeom 只记录 contour 顺序，不在这里做复杂 shell/hole 修复。 + 优先策略： + - 1 ring -> Polygon(shell) + - 多 ring -> 先把第一个 ring 作为 shell，其余作为 holes； + 若失败，再以每个 ring 独立 Polygon 组成 MultiPolygon，仅用于比较。 + + 返回 (geom, construction_mode) + """ + clean = [] + for ring in rings: + if len(ring) >= 4: + rr = list(ring) + if rr[0] != rr[-1]: + rr.append(rr[0]) + clean.append(rr) + + if not clean: + return None, "NO_USABLE_RING" + + if len(clean) == 1: + try: + return Polygon(clean[0]), "SINGLE_RING_POLYGON" + except Exception: + return None, "SINGLE_RING_CONSTRUCTION_FAILED" + + try: + return Polygon(clean[0], clean[1:]), "FIRST_SHELL_REST_HOLES" + except Exception: + pass + + try: + polys = [Polygon(r) for r in clean] + return MultiPolygon(polys), "MULTIPOLYGON_PER_RING_FALLBACK" + except Exception: + return None, "MULTIRING_CONSTRUCTION_FAILED" + + +def safe_validity(g) -> Tuple[Optional[bool], str]: + if g is None: + return None, "NULL_GEOMETRY" + try: + valid = bool(g.is_valid) + return valid, "Valid Geometry" if valid else explain_validity(g) + except Exception as e: + return None, f"{type(e).__name__}: {e}" + + +def safe_hausdorff(a, b) -> Optional[float]: + try: + return float(a.hausdorff_distance(b)) + except Exception: + return None + + +def safe_symdiff_area(a, b) -> Optional[float]: + try: + return float(a.symmetric_difference(b).area) + except Exception: + return None + + +def topology_reason_class(reason: str) -> str: + s = (reason or "").lower() + if "ring self-intersection" in s: + return "RING_SELF_INTERSECTION" + if "self-intersection" in s: + return "SELF_INTERSECTION" + if "interior is disconnected" in s: + return "INTERIOR_DISCONNECTED" + if "hole lies outside shell" in s: + return "HOLE_OUTSIDE_SHELL" + if "nested" in s: + return "NESTED_RING" + if "valid geometry" in s: + return "VALID" + return "OTHER" + + +# --------------------------------------------------------------------------- +# DB loaders +# --------------------------------------------------------------------------- + +def load_invalid_areas(cur, dataset_id: int, wkb_fn: str) -> List[Dict[str, Any]]: + cur.execute(f""" + SELECT + f.id AS feature_id, + f.dataset_id, + f.cell_id, + f.coverage_id, + f.source_record_index, + f.object_class, + f.geometry_type, + f.raw_geotype, + f.normalized_srid, + f.raw_feature_blob, + f.geometry_issue, + f.geometry_repaired, + c.cell_name, + {wkb_fn}(g.geom) AS geom_wkb, + cv.wgsox AS coverage_wgsox, + cv.wgsoy AS coverage_wgsoy, + cv.source_record_index AS coverage_source_record_index + FROM chart_feature f + JOIN chart_area_geometry g ON g.feature_id=f.id + JOIN chart_cell c ON c.id=f.cell_id + LEFT JOIN chart_coverage cv ON cv.id=f.coverage_id + WHERE f.dataset_id=%s + AND f.geometry_type='AREA' + ORDER BY f.id + """, (dataset_id,)) + + out = [] + for r in cur: + try: + db_geom = load_wkb(to_bytes(r["geom_wkb"])) + if not db_geom.is_valid: + x = dict(r) + x["_db_geom"] = db_geom + x["_db_reason"] = explain_validity(db_geom) + out.append(x) + except Exception: + continue + return out + + +def load_cell_raw_tables( + cur, cell_id: int +) -> Dict[str, Dict[str, Any]]: + rs = fetch_all(cur, """ + SELECT id, record_kind, source_record_index, raw_blob, raw_metadata_json + FROM chart_raw_record + WHERE cell_id=%s + ORDER BY id + """, (cell_id,)) + d = {} + for r in rs: + kind = str(r["record_kind"]).upper() + # 当前 import 设计中每种 table 通常一个 chunk。 + # 如未来出现 chunk_count>1，先保留全部并在调用端报错。 + if kind in d: + # 用 list 暂存重复。 + existing = d[kind] + if not isinstance(existing, dict) or "_chunks" not in existing: + d[kind] = {"_chunks": [existing, r]} + else: + existing["_chunks"].append(r) + else: + d[kind] = r + return d + + +def normalize_chunk_record(rec: Dict[str, Any], kind: str) -> Dict[str, Any]: + if "_chunks" not in rec: + return rec + + chunks = rec["_chunks"] + parsed = [] + for c in chunks: + meta = c.get("raw_metadata_json") + try: + if isinstance(meta, str): + mo = json.loads(meta) + elif isinstance(meta, dict): + mo = meta + else: + mo = json.loads(bytes(meta).decode("utf-8")) + except Exception: + mo = {} + parsed.append((int(mo.get("chunk_index", 0)), c)) + parsed.sort(key=lambda t: t[0]) + + raw = b"".join(to_bytes(c.get("raw_blob")) or b"" for _, c in parsed) + first = dict(parsed[0][1]) + first["raw_blob"] = raw + first["_joined_chunks"] = len(parsed) + return first + + +# --------------------------------------------------------------------------- +# GeoJSON +# --------------------------------------------------------------------------- + +def make_geojson( + records: List[Dict[str, Any]], + path: Path, +): + features = [] + + def add(g, props): + if g is None: + return + try: + features.append({ + "type": "Feature", + "geometry": mapping(g), + "properties": props, + }) + except Exception: + pass + + for r in records: + fid = r["feature_id"] + add(r.get("_db_geom"), { + "layer": "db_invalid_area", + "feature_id": fid, + "object_class": r.get("object_class"), + "cell_name": r.get("cell_name"), + "db_reason": r.get("_db_reason"), + "final_class": r.get("_final_class"), + }) + add(r.get("_raw_wgs_geom"), { + "layer": "raw_reconstructed_area", + "feature_id": fid, + "object_class": r.get("object_class"), + "cell_name": r.get("cell_name"), + "raw_reason": r.get("_raw_wgs_reason"), + "construction_mode": r.get("_construction_mode"), + }) + + path.write_text( + json.dumps( + {"type": "FeatureCollection", "features": features}, + ensure_ascii=False, + separators=(",", ":"), + default=str, + ), + encoding="utf-8" + ) + + +def classify_record( + *, + junction_issues: List[Dict[str, Any]], + raw_valid: Optional[bool], + db_valid: bool, + hausdorff: Optional[float], + symdiff: Optional[float], + compare_tolerance: float, +) -> Tuple[str, List[str]]: + flags = [] + + if junction_issues: + flags.extend(sorted(set(x["issue_category"] for x in junction_issues))) + if any( + x["issue_category"] in { + "EDGE_INDEX_OUT_OF_RANGE", + "EDGE_JUNCTION_DISCONTINUITY", + "RING_NOT_CLOSED_AT_END_OF_REFS", + "EMPTY_EDGE", + } + for x in junction_issues + ): + return "EDGE_CHAIN_DISCONTINUITY", flags + + matches = ( + hausdorff is not None + and hausdorff <= compare_tolerance + ) + + if raw_valid is True and db_valid is False: + if matches: + # 极少见：几何几乎一致但 validity 不同，可能构造策略/holes 差异。 + return "RAW_CHAIN_VALID_DB_INVALID_NEAR_MATCH", flags + return "RAW_CHAIN_VALID_DB_INVALID", flags + + if raw_valid is False: + if matches: + return "RAW_EDGE_CHAIN_ITSELF_INVALID_AND_MATCHES_DB", flags + return "RAW_EDGE_CHAIN_ITSELF_INVALID_RECONSTRUCTION_DIFFERS", flags + + if matches: + return "RECONSTRUCTION_MATCHES_DB", flags + + return "RECONSTRUCTION_MISMATCH", flags + + +def zip_dir(folder: Path) -> Path: + zpath = folder.with_suffix(".zip") + with zipfile.ZipFile(zpath, "w", compression=zipfile.ZIP_DEFLATED) as z: + for p in sorted(folder.rglob("*")): + if p.is_file(): + z.write(p, p.relative_to(folder.parent)) + return zpath + + +def parse_args(): + p = argparse.ArgumentParser( + description="CM93 invalid Area raw edge chain audit v6" + ) + p.add_argument("--host", default=os.getenv("MYSQL_HOST", "127.0.0.1")) + p.add_argument("--port", type=int, default=int(os.getenv("MYSQL_PORT", "3306"))) + p.add_argument("--user", default=os.getenv("MYSQL_USER", "root")) + p.add_argument("--password", default=os.getenv("MYSQL_PASSWORD")) + p.add_argument("--database", default=os.getenv("MYSQL_DATABASE", "cm93_chart")) + p.add_argument("--dataset-id", type=int, default=1) + p.add_argument( + "--compare-tolerance", + type=float, + default=2e-6, + help="参考重建与 DB geometry Hausdorff distance 容差（degree）" + ) + p.add_argument("--output", default=None) + return p.parse_args() + + +def main(): + args = parse_args() + if not args.password: + args.password = getpass.getpass("MySQL password: ") + + stamp = datetime.now().strftime("%Y%m%d_%H%M%S") + outdir = Path( + args.output + or f"cm93_raw_edge_audit_v6_dataset_{args.dataset_id}_{stamp}" + ) + outdir.mkdir(parents=True, exist_ok=True) + + report = { + "version": "v6.0", + "generated_at": datetime.now().isoformat(timespec="seconds"), + "database": args.database, + "dataset_id": args.dataset_id, + "read_only": True, + "pbf": "OUT_OF_SCOPE", + "scope_note": ( + "CM93JAPAN 是局部 CM93 数据集；本轮只重建已导入 invalid Area " + "自身的 raw edge chain，不要求跨未导入海域连续。" + ), + "warnings": [], + } + + conn = None + try: + conn = pymysql.connect( + host=args.host, + port=args.port, + user=args.user, + password=args.password, + database=args.database, + charset="utf8mb4", + cursorclass=DictCursor, + autocommit=False, + connect_timeout=20, + read_timeout=600, + write_timeout=30, + ) + + with conn.cursor() as cur: + begin_read_only(cur, report["warnings"]) + report["mysql_version"] = scalar(cur, "SELECT VERSION()") + wkb_fn = probe_wkb_fn(cur) + report["wkb_fn"] = wkb_fn + + print("[1/6] 找出 52 个 invalid Area ...") + invalids = load_invalid_areas(cur, args.dataset_id, wkb_fn) + print(f" invalid Area = {len(invalids)}") + + affected_cells = sorted(set(int(r["cell_id"]) for r in invalids)) + print(f" affected Cells = {len(affected_cells)}") + + print("[2/6] 解析受影响 Cell 的 HEADER / VECTOR ...") + cell_cache = {} + cell_audit = [] + + for cell_id in affected_cells: + rawtables = load_cell_raw_tables(cur, cell_id) + if "HEADER" not in rawtables or "VECTOR" not in rawtables: + cell_cache[cell_id] = { + "error": "HEADER_OR_VECTOR_MISSING", + "rawtables": rawtables, + } + cell_audit.append({ + "cell_id": cell_id, + "status": "FAIL", + "issue": "HEADER_OR_VECTOR_MISSING", + }) + continue + + hrec = normalize_chunk_record(rawtables["HEADER"], "HEADER") + vrec = normalize_chunk_record(rawtables["VECTOR"], "VECTOR") + + try: + header = parse_header_metadata(hrec.get("raw_metadata_json")) + vector_blob = to_bytes(vrec.get("raw_blob")) + edges = parse_vector_table(vector_blob) + expected_edges = int(header["usn_vector_records"]) + exact = len(edges) == expected_edges + + cell_cache[cell_id] = { + "header": header, + "edges": edges, + "rawtables": rawtables, + } + cell_audit.append({ + "cell_id": cell_id, + "status": "PASS" if exact else "CHECK", + "parsed_vector_edges": len(edges), + "header_usn_vector_records": expected_edges, + "vector_count_match": int(exact), + "vector_blob_bytes": len(vector_blob or b""), + }) + except Exception as e: + cell_cache[cell_id] = { + "error": f"{type(e).__name__}: {e}", + "rawtables": rawtables, + } + cell_audit.append({ + "cell_id": cell_id, + "status": "FAIL", + "issue": f"{type(e).__name__}: {e}", + }) + + print("[3/6] 解码 invalid Feature raw_feature_blob 并重建 edge rings ...") + classification_rows = [] + edge_ref_rows = [] + ring_rows = [] + junction_rows = [] + compare_rows = [] + isolated_detail = [] + + # V5 已确认的两条无共享邻接对象 feature，用于特别详细导出。 + # 不硬编码成“结论”，只是当前已知调查重点；若 ID 不存在也不影响主流程。 + isolated_ids = {24057, 227278} + + for idx, r in enumerate(invalids, start=1): + fid = int(r["feature_id"]) + cell_id = int(r["cell_id"]) + db_geom = r["_db_geom"] + db_valid, db_reason = safe_validity(db_geom) + + cache = cell_cache.get(cell_id, {}) + base = { + "feature_id": fid, + "object_class": r.get("object_class"), + "cell_id": cell_id, + "cell_name": r.get("cell_name"), + "source_record_index": r.get("source_record_index"), + "coverage_id": r.get("coverage_id"), + "raw_geotype_db": r.get("raw_geotype"), + "db_valid": int(bool(db_valid)) if db_valid is not None else None, + "db_validity_reason": db_reason, + } + + if "error" in cache: + classification_rows.append({ + **base, + "final_class": "CELL_RAW_TABLE_UNAVAILABLE", + "detail": cache["error"], + }) + continue + + raw_feature = to_bytes(r.get("raw_feature_blob")) + if not raw_feature: + classification_rows.append({ + **base, + "final_class": "RAW_FEATURE_BLOB_MISSING", + }) + continue + + try: + parsed = parse_area_feature_blob(raw_feature) + except Exception as e: + classification_rows.append({ + **base, + "final_class": "RAW_FEATURE_PARSE_ERROR", + "detail": f"{type(e).__name__}: {e}", + "raw_feature_hex": raw_feature.hex(), + }) + continue + + refs = parsed["refs"] + edges = cache["edges"] + header = cache["header"] + + rings_raw, erows, jissues = build_rings_like_opencpn( + refs, edges + ) + + # Edge refs export + for er in erows: + rr = { + **base, + "feature_geotype_parsed": parsed["geotype"], + "feature_object_type": parsed["object_type"], + "feature_n_elements": parsed["n_elements"], + **er, + } + edge_ref_rows.append(rr) + if fid in isolated_ids: + isolated_detail.append(rr) + + for ji in jissues: + junction_rows.append({**base, **ji}) + + # raw coordinate ring stats + repeated_total = 0 + for ri, ring in enumerate(rings_raw): + reps = repeated_nonconsecutive_vertices(ring) + repeated_total += len(reps) + + raw_line = None + try: + raw_line = LinearRing(ring) if len(ring) >= 4 else None + except Exception: + raw_line = None + + ring_rows.append({ + **base, + "ring_index": ri, + "raw_vertex_count": len(ring), + "raw_closed": int(bool(ring) and ring[0] == ring[-1]), + "repeated_nonconsecutive_vertex_count": len(reps), + "repeated_vertices": reps[:50], + "raw_ring_is_simple": ( + int(bool(raw_line.is_simple)) + if raw_line is not None else None + ), + "signed_area_raw_units": ( + signed_area([(float(x), float(y)) for x, y in ring]) + if ring else None + ), + }) + + # WGS84 transform + wgsox = float(r.get("coverage_wgsox") or 0.0) + wgsoy = float(r.get("coverage_wgsoy") or 0.0) + + rings_wgs = [] + for ring in rings_raw: + w = [ + transform_cm93_point_to_wgs84( + p, header, wgsox, wgsoy + ) + for p in ring + ] + rings_wgs.append(w) + + raw_wgs_geom, construction_mode = ring_to_polygon_candidate( + rings_wgs + ) + raw_valid, raw_reason = safe_validity(raw_wgs_geom) + + haus = safe_hausdorff(raw_wgs_geom, db_geom) if raw_wgs_geom else None + symdiff = safe_symdiff_area(raw_wgs_geom, db_geom) if raw_wgs_geom else None + + final_class, flags = classify_record( + junction_issues=jissues, + raw_valid=raw_valid, + db_valid=bool(db_valid), + hausdorff=haus, + symdiff=symdiff, + compare_tolerance=args.compare_tolerance, + ) + + raw_reason_class = topology_reason_class(raw_reason) + db_reason_class = topology_reason_class(db_reason) + + row = { + **base, + "parsed_object_type": parsed["object_type"], + "parsed_geotype": parsed["geotype"], + "parsed_obj_desc_bytes": parsed["obj_desc_bytes"], + "parsed_n_elements": parsed["n_elements"], + "parsed_has_attributes": parsed["has_attributes"], + "vector_edge_count_cell": len(edges), + "ring_count_reconstructed": len(rings_raw), + "edge_junction_issue_count": len(jissues), + "edge_junction_issue_flags": flags, + "repeated_nonconsecutive_vertex_total": repeated_total, + "coverage_wgsox": wgsox, + "coverage_wgsoy": wgsoy, + "construction_mode": construction_mode, + "raw_reconstructed_valid": ( + int(bool(raw_valid)) if raw_valid is not None else None + ), + "raw_validity_reason": raw_reason, + "raw_validity_reason_class": raw_reason_class, + "db_validity_reason_class": db_reason_class, + "raw_db_same_validity_reason_class": int( + raw_reason_class == db_reason_class + ), + "hausdorff_distance_deg": haus, + "symmetric_difference_area_deg2": symdiff, + "within_compare_tolerance": ( + int(haus is not None and haus <= args.compare_tolerance) + ), + "final_class": final_class, + } + classification_rows.append(row) + + compare_rows.append({ + **base, + "construction_mode": construction_mode, + "raw_geom_type": ( + raw_wgs_geom.geom_type if raw_wgs_geom is not None else None + ), + "db_geom_type": db_geom.geom_type, + "raw_valid": ( + int(bool(raw_valid)) if raw_valid is not None else None + ), + "db_valid": int(bool(db_valid)), + "raw_reason": raw_reason, + "db_reason": db_reason, + "hausdorff_distance_deg": haus, + "symmetric_difference_area_deg2": symdiff, + "within_compare_tolerance": ( + int(haus is not None and haus <= args.compare_tolerance) + ), + }) + + # 挂到 GeoJSON 使用。 + r["_raw_wgs_geom"] = raw_wgs_geom + r["_raw_wgs_reason"] = raw_reason + r["_construction_mode"] = construction_mode + r["_final_class"] = final_class + + print( + f" [{idx:02d}/{len(invalids)}] " + f"feature={fid} {r.get('object_class')} " + f"edges={len(refs)} rings={len(rings_raw)} " + f"class={final_class}" + ) + + print("[4/6] 汇总 raw vs DB 证据 ...") + class_counts = Counter( + r.get("final_class", "UNKNOWN") for r in classification_rows + ) + raw_reason_counts = Counter( + r.get("raw_validity_reason_class", "UNKNOWN") + for r in classification_rows + if r.get("raw_validity_reason_class") + ) + db_reason_counts = Counter( + r.get("db_validity_reason_class", "UNKNOWN") + for r in classification_rows + if r.get("db_validity_reason_class") + ) + + vector_bad_cells = sum( + 1 for r in cell_audit + if r.get("vector_count_match") == 0 + or r.get("status") == "FAIL" + ) + + report["summary"] = { + "invalid_area_count": len(invalids), + "affected_cell_count": len(affected_cells), + "cell_vector_audit_bad_count": vector_bad_cells, + "classification_counts": dict(class_counts), + "raw_validity_reason_counts": dict(raw_reason_counts), + "db_validity_reason_counts": dict(db_reason_counts), + "edge_junction_issue_count": len(junction_rows), + "features_with_edge_junction_issue": len(set( + int(r["feature_id"]) for r in junction_rows + if r.get("feature_id") is not None + )), + "reference_reconstruction_within_tolerance": sum( + int(r.get("within_compare_tolerance") or 0) + for r in classification_rows + ), + "same_validity_reason_class_count": sum( + int(r.get("raw_db_same_validity_reason_class") or 0) + for r in classification_rows + ), + } + + print("[5/6] 写 CSV / JSON / GeoJSON ...") + write_csv( + outdir / "invalid_area_raw_edge_classification.csv", + classification_rows + ) + write_csv( + outdir / "invalid_area_edge_references.csv", + edge_ref_rows + ) + write_csv( + outdir / "invalid_area_ring_summary.csv", + ring_rows + ) + write_csv( + outdir / "invalid_area_edge_junction_issues.csv", + junction_rows + ) + write_csv( + outdir / "raw_vs_db_geometry_compare.csv", + compare_rows + ) + write_csv( + outdir / "isolated_invalid_area_edge_detail.csv", + isolated_detail + ) + write_csv( + outdir / "affected_cell_vector_table_audit.csv", + cell_audit + ) + + make_geojson( + [ + r for r in invalids + if "_raw_wgs_geom" in r + ], + outdir / "invalid_area_raw_reconstruction.geojson" + ) + + (outdir / "raw_edge_audit_report_v6.json").write_text( + json.dumps(report, ensure_ascii=False, indent=2, default=str), + encoding="utf-8" + ) + + lines = [] + lines.append("CM93 -> MySQL invalid Area 原始 Edge Chain 重建审计 v6") + lines.append("=" * 88) + lines.append(f"Generated: {report['generated_at']}") + lines.append(f"Dataset ID: {args.dataset_id}") + lines.append(f"MySQL: {report['mysql_version']}") + lines.append("") + lines.append("SCOPE") + lines.append( + " 当前 CM93JAPAN 是局部数据集；不要求与未导入海域跨边界连续。" + ) + lines.append( + " 本轮直接从 DB 保存的 raw_feature_blob + VECTOR raw_blob " + "重建 invalid Area 的 CM93 edge chain。" + ) + lines.append("") + lines.append("BASELINE") + lines.append(f" invalid Area: {len(invalids)}") + lines.append(f" affected Cells: {len(affected_cells)}") + lines.append(f" bad VECTOR-table audits: {vector_bad_cells}") + lines.append("") + lines.append("FINAL CLASSIFICATION") + for k, v in class_counts.most_common(): + lines.append(f" {k:56s} {v}") + lines.append("") + lines.append("RAW RECONSTRUCTED VALIDITY") + for k, v in raw_reason_counts.most_common(): + lines.append(f" {k:40s} {v}") + lines.append("") + lines.append("DB VALIDITY") + for k, v in db_reason_counts.most_common(): + lines.append(f" {k:40s} {v}") + lines.append("") + lines.append("EDGE CHAIN") + lines.append( + f" junction issue rows: {len(junction_rows)}" + ) + lines.append( + f" features with junction issue:" + f" {report['summary']['features_with_edge_junction_issue']}" + ) + lines.append("") + lines.append("RAW ↔ DB") + lines.append( + f" within Hausdorff tolerance: " + f"{report['summary']['reference_reconstruction_within_tolerance']}" + f" / {len(classification_rows)}" + ) + lines.append( + f" same validity reason class: " + f"{report['summary']['same_validity_reason_class_count']}" + f" / {len(classification_rows)}" + ) + lines.append("") + lines.append("INTERPRETATION") + lines.append( + " RAW_CHAIN_VALID_DB_INVALID：优先判为 importer assembly/conversion 候选。" + ) + lines.append( + " RAW_EDGE_CHAIN_ITSELF_INVALID_AND_MATCHES_DB：" + "说明 DB 基本忠实反映了 CM93 edge chain 组装结果；" + "后续重点研究 CM93 shared topology / OGC Polygon validity 语义，" + "而不是先修改坐标。" + ) + lines.append( + " EDGE_CHAIN_DISCONTINUITY：检查 edge index / usage / ordering；" + "不得自动闭环或补边。" + ) + lines.append( + " RECONSTRUCTION_MISMATCH：检查 coverage WGS offset、ring shell/hole " + "解释以及 importer 的坐标转换。" + ) + lines.append("") + lines.append("NO WRITE / NO REPAIR / NO PBF") + + (outdir / "raw_edge_audit_summary_v6.txt").write_text( + "\n".join(lines), encoding="utf-8" + ) + + conn.rollback() + + print("[6/6] 压缩输出 ...") + zpath = zip_dir(outdir) + print("") + print("完成。") + print(f"Output: {outdir.resolve()}") + print(f"ZIP: {zpath.resolve()}") + print("请把 ZIP 整包上传给 ChatGPT。") + + except KeyboardInterrupt: + if conn: + try: + conn.rollback() + except Exception: + pass + print("\n用户中断。") + sys.exit(130) + except Exception: + if conn: + try: + conn.rollback() + except Exception: + pass + tb = traceback.format_exc() + print(tb) + try: + (outdir / "fatal_error.txt").write_text(tb, encoding="utf-8") + except Exception: + pass + sys.exit(1) + finally: + if conn: + try: + conn.close() + except Exception: + pass + + +if __name__ == "__main__": + main() diff --git a/cm93_mysql_spatial_audit_v3.py b/cm93_mysql_spatial_audit_v3.py new file mode 100644 index 0000000..3c346e9 --- /dev/null +++ b/cm93_mysql_spatial_audit_v3.py @@ -0,0 +1,1570 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +CM93 -> MySQL Dataset ID=1 实体空间 Geometry 只读审计 +Version: v3.0 + +目的： +1. 审计已经真正写入 MySQL 的 Point / Line / Area / Sounding / Coverage geometry。 +2. 不再把 raw_geotype=16 的正常 META parent 当作 geometry 错误。 +3. 检查单个 geometry 的“内在有效性”，不要求当前日本局部 Dataset 与未导入海域连续。 +4. 不修改数据库，不修 geometry，不做 PBF。 + +特别边界规则： +当前 CM93JAPAN 只是 CM93 的日本沿海局部子集，并非完整 CM93。 +因此： +- 不做跨 Dataset 外边界连续性 FAIL 判定； +- 不因 Line 在 Cell/Dataset 边缘终止而判坏； +- 不因缺少未导入邻区对象而判坏； +- 对靠近 Cell/Dataset 边界的异常只增加 boundary_context 提示，不自动归因于 importer。 + +依赖： + pip install pymysql shapely + +示例： + python cm93_mysql_spatial_audit_v3.py ^ + --host 127.0.0.1 ^ + --port 3306 ^ + --user root ^ + --database cm93_chart ^ + --dataset-id 1 + +密码缺省时交互输入。 + +输出： +- spatial_audit_summary.txt +- spatial_audit_report.json +- feature_geometry_mapping_issues.csv +- intrinsic_geometry_issues.csv +- intrinsic_issue_category_counts.csv +- invalid_geometry_reason_counts.csv +- geometry_stats_by_kind.csv +- issue_object_class_counts.csv +- issue_cell_counts.csv +- issue_samples.csv +- sounding_feature_bbox_issues.csv +- coverage_geometry_issues.csv +并自动生成同名 zip。 + +安全： +- SQL 仅包含 SELECT / SHOW / SET TRANSACTION READ ONLY +- 无 INSERT / UPDATE / DELETE / DDL +""" + +from __future__ import annotations + +import argparse +import csv +import getpass +import json +import math +import os +import sys +import traceback +import zipfile +from collections import Counter, defaultdict +from dataclasses import dataclass, asdict +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, Iterable, Iterator, List, Optional, Sequence, Tuple + +try: + import pymysql + from pymysql.cursors import DictCursor +except ImportError: + print("缺少 pymysql。请执行：pip install pymysql shapely") + sys.exit(2) + +try: + import shapely + try: + from shapely import from_wkb as _from_wkb + def load_wkb(data: bytes): + return _from_wkb(data) + except ImportError: + from shapely import wkb as _wkb + def load_wkb(data: bytes): + return _wkb.loads(data) + try: + from shapely.validation import explain_validity + except Exception: + def explain_validity(g): + return "invalid" +except ImportError: + print("缺少 shapely。请执行：pip install pymysql shapely") + sys.exit(2) + + +EXPECTED_DATASET1 = { + "cells": 784, + "features": 322465, + "attributes": 405153, + "sounding_points": 275700, +} + +EXPECTED_GEOMETRY_ISSUE_INTERPRETATION = { + "raw_geotype_16_expected_parent_no_geometry": 4406, +} + + +def qi(name: str) -> str: + return "`" + name.replace("`", "``") + "`" + + +def finite(v: Optional[float]) -> bool: + return v is not None and math.isfinite(float(v)) + + +def safe_float(v: Any) -> Optional[float]: + if v is None: + return None + try: + x = float(v) + return x if math.isfinite(x) else None + except Exception: + return None + + +def bbox_tuple(row: Dict[str, Any], prefix: str = "") -> Optional[Tuple[float, float, float, float]]: + vals = [ + safe_float(row.get(prefix + "bbox_min_lon")), + safe_float(row.get(prefix + "bbox_min_lat")), + safe_float(row.get(prefix + "bbox_max_lon")), + safe_float(row.get(prefix + "bbox_max_lat")), + ] + if any(v is None for v in vals): + return None + return vals[0], vals[1], vals[2], vals[3] + + +def bbox_close(a: Tuple[float, float, float, float], + b: Tuple[float, float, float, float], + tol: float) -> bool: + return all(abs(float(x) - float(y)) <= tol for x, y in zip(a, b)) + + +def touches_bbox_edge(inner: Optional[Tuple[float, float, float, float]], + outer: Optional[Tuple[float, float, float, float]], + tol: float) -> bool: + if not inner or not outer: + return False + return ( + abs(inner[0] - outer[0]) <= tol or + abs(inner[1] - outer[1]) <= tol or + abs(inner[2] - outer[2]) <= tol or + abs(inner[3] - outer[3]) <= tol + ) + + +def bbox_in_lonlat_range(b: Optional[Tuple[float, float, float, float]]) -> bool: + if not b: + return False + minx, miny, maxx, maxy = b + return ( + -180.0 <= minx <= 180.0 and + -180.0 <= maxx <= 180.0 and + -90.0 <= miny <= 90.0 and + -90.0 <= maxy <= 90.0 and + minx <= maxx and miny <= maxy + ) + + +def write_csv(path: Path, rows: Iterable[Dict[str, Any]], fieldnames: Optional[List[str]] = None): + rows = list(rows) + path.parent.mkdir(parents=True, exist_ok=True) + if fieldnames is None: + seen = set() + fieldnames = [] + for r in rows: + for k in r.keys(): + if k not in seen: + seen.add(k) + fieldnames.append(k) + with path.open("w", encoding="utf-8-sig", newline="") as f: + w = csv.DictWriter(f, fieldnames=fieldnames or [], extrasaction="ignore") + if fieldnames: + w.writeheader() + for r in rows: + w.writerow({k: "" if r.get(k) is None else r.get(k) for k in fieldnames}) + + +def scalar(cur, sql: str, params: Sequence[Any] = ()) -> Any: + cur.execute(sql, params) + row = cur.fetchone() + if not row: + return None + return next(iter(row.values())) + + +def rows(cur, sql: str, params: Sequence[Any] = ()) -> List[Dict[str, Any]]: + cur.execute(sql, params) + return list(cur.fetchall()) + + +def begin_read_only(cur, warnings: List[str]): + try: + cur.execute("SET SESSION TRANSACTION READ ONLY") + except Exception as e: + warnings.append(f"SET SESSION TRANSACTION READ ONLY 不支持/失败：{e}") + try: + cur.execute("START TRANSACTION READ ONLY") + except Exception as e: + warnings.append(f"START TRANSACTION READ ONLY 不支持/失败：{e}") + try: + cur.execute("START TRANSACTION") + except Exception: + pass + + +def probe_spatial_functions(cur) -> Dict[str, Optional[str]]: + """ + 探测把 MySQL/MariaDB geometry 转成标准 WKB 的函数和 SRID 函数。 + 不使用 ST_IsValid，实际 validity 统一交给 Shapely，避免 DB 方言差异。 + """ + result = {"wkb_fn": None, "srid_fn": None} + + for fn in ["ST_AsBinary", "ST_AsWKB", "AsBinary", "AsWKB"]: + try: + cur.execute( + f"SELECT {fn}(geom) AS x " + f"FROM chart_point_geometry LIMIT 1" + ) + cur.fetchone() + result["wkb_fn"] = fn + break + except Exception: + continue + + for fn in ["ST_SRID", "SRID"]: + try: + cur.execute( + f"SELECT {fn}(geom) AS x " + f"FROM chart_point_geometry LIMIT 1" + ) + cur.fetchone() + result["srid_fn"] = fn + break + except Exception: + continue + + return result + + +def classify_invalid_reason(reason: str) -> str: + s = (reason or "").lower() + if "self-intersection" in s: + if "ring self-intersection" in s: + return "RING_SELF_INTERSECTION" + return "SELF_INTERSECTION" + if "too few points" in s: + return "TOO_FEW_POINTS" + if "hole lies outside shell" in s: + return "HOLE_OUTSIDE_SHELL" + if "nested shells" in s: + return "NESTED_SHELLS" + if "nested holes" in s: + return "NESTED_HOLES" + if "disconnected interior" in s: + return "DISCONNECTED_INTERIOR" + if "invalid coordinate" in s or "nan" in s or "inf" in s: + return "INVALID_COORDINATE" + if "ring" in s and "closed" in s: + return "RING_NOT_CLOSED" + return "OTHER_INVALID_GEOMETRY" + + +def actual_type_ok(kind: str, geom_type: str) -> bool: + if kind == "POINT": + return geom_type == "Point" + if kind == "LINE": + return geom_type in {"LineString", "MultiLineString"} + if kind == "AREA": + return geom_type in {"Polygon", "MultiPolygon"} + if kind == "SOUNDING": + return geom_type == "Point" + if kind == "COVERAGE": + return geom_type in {"Polygon", "MultiPolygon"} + return True + + +def intrinsic_check( + *, + kind: str, + feature_id: Optional[int], + object_class: str, + cell_id: Optional[int], + cell_name: str, + source_record_index: Optional[int], + wkb_data: Any, + srid: Any, + normalized_srid: Any, + feature_bbox: Optional[Tuple[float, float, float, float]], + cell_bbox: Optional[Tuple[float, float, float, float]], + dataset_bbox: Optional[Tuple[float, float, float, float]], + bbox_tolerance: float, + boundary_tolerance: float, + extra_id: Optional[int] = None, +) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]: + """ + 返回： + stat: 单 geometry 统计 + issues: 0..N 条 issue + """ + base = { + "kind": kind, + "feature_id": feature_id, + "extra_id": extra_id, + "object_class": object_class, + "cell_id": cell_id, + "cell_name": cell_name, + "source_record_index": source_record_index, + "srid": srid, + "normalized_srid": normalized_srid, + } + + stat = dict(base) + stat.update({ + "parsed": 0, + "empty": 0, + "valid": None, + "geom_type": None, + "simple": None, + "bounds": None, + "touches_cell_boundary": 0, + "touches_dataset_bbox": 0, + }) + issues: List[Dict[str, Any]] = [] + + if wkb_data is None: + x = dict(base) + x.update({ + "issue_category": "WKB_MISSING", + "issue_detail": "DB geometry exists but WKB conversion returned NULL", + "invalid_reason": "", + "boundary_context": "", + }) + issues.append(x) + return stat, issues + + if isinstance(wkb_data, memoryview): + wkb_data = wkb_data.tobytes() + if isinstance(wkb_data, bytearray): + wkb_data = bytes(wkb_data) + + try: + geom = load_wkb(wkb_data) + stat["parsed"] = 1 + except Exception as e: + x = dict(base) + x.update({ + "issue_category": "WKB_PARSE_ERROR", + "issue_detail": f"{type(e).__name__}: {e}", + "invalid_reason": "", + "boundary_context": "", + }) + issues.append(x) + return stat, issues + + geom_type = getattr(geom, "geom_type", None) + stat["geom_type"] = geom_type + + try: + is_empty = bool(geom.is_empty) + except Exception: + is_empty = False + stat["empty"] = int(is_empty) + + if is_empty: + x = dict(base) + x.update({ + "issue_category": "EMPTY_GEOMETRY", + "issue_detail": f"{kind} geometry is EMPTY", + "invalid_reason": "", + "actual_geom_type": geom_type, + "boundary_context": "", + }) + issues.append(x) + return stat, issues + + try: + b = tuple(float(v) for v in geom.bounds) + if len(b) == 4 and all(math.isfinite(v) for v in b): + stat["bounds"] = b + else: + b = None + except Exception: + b = None + + stat["touches_cell_boundary"] = int(touches_bbox_edge(b, cell_bbox, boundary_tolerance)) + stat["touches_dataset_bbox"] = int(touches_bbox_edge(b, dataset_bbox, boundary_tolerance)) + + boundary_bits = [] + if stat["touches_cell_boundary"]: + boundary_bits.append("CELL_EDGE") + if stat["touches_dataset_bbox"]: + boundary_bits.append("DATASET_BBOX_EDGE") + boundary_context = "|".join(boundary_bits) + + if not actual_type_ok(kind, geom_type or ""): + x = dict(base) + x.update({ + "issue_category": "GEOMETRY_TYPE_MISMATCH", + "issue_detail": f"expected={kind}, actual={geom_type}", + "invalid_reason": "", + "actual_geom_type": geom_type, + "boundary_context": boundary_context, + }) + issues.append(x) + + # 坐标范围：CM93 normalized SRID 当前应为经纬度数据。 + if b is None: + x = dict(base) + x.update({ + "issue_category": "NON_FINITE_OR_UNREADABLE_BOUNDS", + "issue_detail": "geometry bounds unavailable/non-finite", + "invalid_reason": "", + "actual_geom_type": geom_type, + "boundary_context": boundary_context, + }) + issues.append(x) + elif not bbox_in_lonlat_range(b): + x = dict(base) + x.update({ + "issue_category": "OUT_OF_RANGE_COORDINATE", + "issue_detail": f"bounds={b}", + "invalid_reason": "", + "actual_geom_type": geom_type, + "boundary_context": boundary_context, + }) + issues.append(x) + + # SRID 一致性。无法探测 SRID 函数时 srid=None，不判 FAIL。 + if srid is not None and normalized_srid is not None: + try: + if int(srid) != int(normalized_srid): + x = dict(base) + x.update({ + "issue_category": "SRID_MISMATCH", + "issue_detail": f"geom_srid={srid}, feature_normalized_srid={normalized_srid}", + "invalid_reason": "", + "actual_geom_type": geom_type, + "boundary_context": boundary_context, + }) + issues.append(x) + except Exception: + pass + + # Feature bbox 对账。Sounding 在后面按 feature 聚合单独核对。 + if kind in {"POINT", "LINE", "AREA"}: + if feature_bbox is None: + x = dict(base) + x.update({ + "issue_category": "FEATURE_BBOX_MISSING", + "issue_detail": f"{kind} has geometry but feature bbox is NULL/incomplete", + "invalid_reason": "", + "actual_geom_type": geom_type, + "geometry_bounds": b, + "boundary_context": boundary_context, + }) + issues.append(x) + elif b is not None and not bbox_close(b, feature_bbox, bbox_tolerance): + x = dict(base) + x.update({ + "issue_category": "FEATURE_BBOX_MISMATCH", + "issue_detail": f"feature_bbox={feature_bbox}, geometry_bounds={b}, tol={bbox_tolerance}", + "invalid_reason": "", + "actual_geom_type": geom_type, + "geometry_bounds": b, + "feature_bbox": feature_bbox, + "boundary_context": boundary_context, + }) + issues.append(x) + + # Shapely validity： + # - Area/Coverage 的 invalid 是核心检查； + # - Point/Line 也记录 validity； + # - Line 非 simple 不等同 invalid，只作为统计，不生成 issue。 + try: + valid = bool(geom.is_valid) + stat["valid"] = int(valid) + except Exception: + valid = True + stat["valid"] = None + + try: + stat["simple"] = int(bool(geom.is_simple)) + except Exception: + stat["simple"] = None + + if not valid: + try: + reason = explain_validity(geom) + except Exception as e: + reason = f"validity explanation unavailable: {e}" + reason_class = classify_invalid_reason(reason) + x = dict(base) + x.update({ + "issue_category": reason_class, + "issue_detail": reason, + "invalid_reason": reason, + "actual_geom_type": geom_type, + "geometry_bounds": b, + "boundary_context": boundary_context, + }) + issues.append(x) + + return stat, issues + + +def get_dataset_bbox(cur, dataset_id: int) -> Optional[Tuple[float, float, float, float]]: + cur.execute(""" + SELECT + MIN(bbox_min_lon) AS minx, + MIN(bbox_min_lat) AS miny, + MAX(bbox_max_lon) AS maxx, + MAX(bbox_max_lat) AS maxy + FROM chart_cell + WHERE dataset_id=%s + AND bbox_min_lon IS NOT NULL + AND bbox_min_lat IS NOT NULL + AND bbox_max_lon IS NOT NULL + AND bbox_max_lat IS NOT NULL + """, (dataset_id,)) + r = cur.fetchone() or {} + vals = [safe_float(r.get("minx")), safe_float(r.get("miny")), + safe_float(r.get("maxx")), safe_float(r.get("maxy"))] + if any(v is None for v in vals): + return None + return vals[0], vals[1], vals[2], vals[3] + + +def audit_mapping(cur, dataset_id: int) -> Tuple[List[Dict[str, Any]], Dict[str, Any]]: + """ + 检查 feature.geometry_type 与实际存储表是否一致。 + """ + mapping_issues: List[Dict[str, Any]] = [] + stats: Dict[str, Any] = {} + + # Feature type counts + ft = rows(cur, """ + SELECT geometry_type, COUNT(*) AS n + FROM chart_feature + WHERE dataset_id=%s + GROUP BY geometry_type + ORDER BY geometry_type + """, (dataset_id,)) + stats["feature_type_counts"] = ft + + # 实体表 counts（只统计 Dataset 1） + for name, table in [ + ("point_rows", "chart_point_geometry"), + ("line_rows", "chart_line_geometry"), + ("area_rows", "chart_area_geometry"), + ]: + stats[name] = int(scalar(cur, f""" + SELECT COUNT(*) + FROM {table} g + JOIN chart_feature f ON f.id=g.feature_id + WHERE f.dataset_id=%s + """, (dataset_id,)) or 0) + + stats["sounding_rows"] = int(scalar(cur, """ + SELECT COUNT(*) + FROM chart_sounding_point s + JOIN chart_feature f ON f.id=s.feature_id + WHERE f.dataset_id=%s + """, (dataset_id,)) or 0) + + # 缺 expected geometry + checks = [ + ("POINT_MISSING_GEOMETRY", "POINT", "chart_point_geometry"), + ("LINE_MISSING_GEOMETRY", "LINE", "chart_line_geometry"), + ("AREA_MISSING_GEOMETRY", "AREA", "chart_area_geometry"), + ] + for issue_cat, gtype, table in checks: + rs = rows(cur, f""" + SELECT + f.id AS feature_id, f.object_class, f.cell_id, c.cell_name, + f.source_record_index, f.raw_geotype, f.geometry_type + FROM chart_feature f + JOIN chart_cell c ON c.id=f.cell_id + LEFT JOIN {table} g ON g.feature_id=f.id + WHERE f.dataset_id=%s + AND f.geometry_type=%s + AND g.feature_id IS NULL + ORDER BY f.id + """, (dataset_id, gtype)) + for r in rs: + r["issue_category"] = issue_cat + mapping_issues.append(r) + + # SOUNDING feature 没有任何 sounding point + rs = rows(cur, """ + SELECT + f.id AS feature_id, f.object_class, f.cell_id, c.cell_name, + f.source_record_index, f.raw_geotype, f.geometry_type + FROM chart_feature f + JOIN chart_cell c ON c.id=f.cell_id + LEFT JOIN chart_sounding_point s ON s.feature_id=f.id + WHERE f.dataset_id=%s + AND f.geometry_type='SOUNDING' + GROUP BY f.id, f.object_class, f.cell_id, c.cell_name, + f.source_record_index, f.raw_geotype, f.geometry_type + HAVING COUNT(s.id)=0 + ORDER BY f.id + """, (dataset_id,)) + for r in rs: + r["issue_category"] = "SOUNDING_MISSING_POINTS" + mapping_issues.append(r) + + # geometry 表串类型 + for issue_cat, table, expected in [ + ("POINT_TABLE_WRONG_FEATURE_TYPE", "chart_point_geometry", "POINT"), + ("LINE_TABLE_WRONG_FEATURE_TYPE", "chart_line_geometry", "LINE"), + ("AREA_TABLE_WRONG_FEATURE_TYPE", "chart_area_geometry", "AREA"), + ]: + rs = rows(cur, f""" + SELECT + f.id AS feature_id, f.object_class, f.cell_id, c.cell_name, + f.source_record_index, f.raw_geotype, f.geometry_type + FROM {table} g + JOIN chart_feature f ON f.id=g.feature_id + JOIN chart_cell c ON c.id=f.cell_id + WHERE f.dataset_id=%s + AND f.geometry_type<>%s + ORDER BY f.id + """, (dataset_id, expected)) + for r in rs: + r["issue_category"] = issue_cat + mapping_issues.append(r) + + # sounding point 串到非 SOUNDING feature + rs = rows(cur, """ + SELECT DISTINCT + f.id AS feature_id, f.object_class, f.cell_id, c.cell_name, + f.source_record_index, f.raw_geotype, f.geometry_type + FROM chart_sounding_point s + JOIN chart_feature f ON f.id=s.feature_id + JOIN chart_cell c ON c.id=f.cell_id + WHERE f.dataset_id=%s + AND f.geometry_type<>'SOUNDING' + ORDER BY f.id + """, (dataset_id,)) + for r in rs: + r["issue_category"] = "SOUNDING_POINT_WRONG_FEATURE_TYPE" + mapping_issues.append(r) + + # META/UNKNOWN 不应偷偷挂实际 geometry + rs = rows(cur, """ + SELECT + f.id AS feature_id, f.object_class, f.cell_id, c.cell_name, + f.source_record_index, f.raw_geotype, f.geometry_type, + CASE WHEN pg.feature_id IS NOT NULL THEN 1 ELSE 0 END AS has_point, + CASE WHEN lg.feature_id IS NOT NULL THEN 1 ELSE 0 END AS has_line, + CASE WHEN ag.feature_id IS NOT NULL THEN 1 ELSE 0 END AS has_area, + COUNT(sp.id) AS sounding_point_count + FROM chart_feature f + JOIN chart_cell c ON c.id=f.cell_id + LEFT JOIN chart_point_geometry pg ON pg.feature_id=f.id + LEFT JOIN chart_line_geometry lg ON lg.feature_id=f.id + LEFT JOIN chart_area_geometry ag ON ag.feature_id=f.id + LEFT JOIN chart_sounding_point sp ON sp.feature_id=f.id + WHERE f.dataset_id=%s + AND f.geometry_type IN ('META','UNKNOWN') + GROUP BY f.id, f.object_class, f.cell_id, c.cell_name, + f.source_record_index, f.raw_geotype, f.geometry_type, + pg.feature_id, lg.feature_id, ag.feature_id + HAVING has_point=1 OR has_line=1 OR has_area=1 OR sounding_point_count>0 + ORDER BY f.id + """, (dataset_id,)) + for r in rs: + r["issue_category"] = "META_OR_UNKNOWN_HAS_UNEXPECTED_GEOMETRY" + mapping_issues.append(r) + + stats["mapping_issue_count"] = len(mapping_issues) + stats["mapping_issue_categories"] = dict(Counter(r["issue_category"] for r in mapping_issues)) + return mapping_issues, stats + + +def scan_feature_geometry_table( + cur, + *, + table: str, + kind: str, + dataset_id: int, + wkb_fn: str, + srid_fn: Optional[str], + dataset_bbox: Optional[Tuple[float, float, float, float]], + chunk_size: int, + bbox_tolerance: float, + boundary_tolerance: float, +) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]: + last_id = 0 + stats = Counter() + geom_types = Counter() + invalid_reasons = Counter() + issues: List[Dict[str, Any]] = [] + + srid_expr = f"{srid_fn}(g.geom)" if srid_fn else "NULL" + + while True: + sql = f""" + SELECT + g.feature_id, + {wkb_fn}(g.geom) AS geom_wkb, + {srid_expr} AS geom_srid, + f.object_class, + f.cell_id, + c.cell_name, + f.source_record_index, + f.normalized_srid, + f.bbox_min_lon, f.bbox_min_lat, f.bbox_max_lon, f.bbox_max_lat, + c.bbox_min_lon AS cell_bbox_min_lon, + c.bbox_min_lat AS cell_bbox_min_lat, + c.bbox_max_lon AS cell_bbox_max_lon, + c.bbox_max_lat AS cell_bbox_max_lat + FROM {table} g + JOIN chart_feature f ON f.id=g.feature_id + JOIN chart_cell c ON c.id=f.cell_id + WHERE f.dataset_id=%s + AND g.feature_id>%s + ORDER BY g.feature_id + LIMIT %s + """ + cur.execute(sql, (dataset_id, last_id, chunk_size)) + batch = list(cur.fetchall()) + if not batch: + break + + for r in batch: + last_id = int(r["feature_id"]) + fb = bbox_tuple(r) + cb = bbox_tuple(r, "cell_") + + stat, found = intrinsic_check( + kind=kind, + feature_id=r["feature_id"], + object_class=r.get("object_class") or "", + cell_id=r.get("cell_id"), + cell_name=r.get("cell_name") or "", + source_record_index=r.get("source_record_index"), + wkb_data=r.get("geom_wkb"), + srid=r.get("geom_srid"), + normalized_srid=r.get("normalized_srid"), + feature_bbox=fb, + cell_bbox=cb, + dataset_bbox=dataset_bbox, + bbox_tolerance=bbox_tolerance, + boundary_tolerance=boundary_tolerance, + ) + + stats["rows"] += 1 + stats["parsed"] += stat["parsed"] + stats["empty"] += stat["empty"] + if stat["valid"] == 1: + stats["valid"] += 1 + elif stat["valid"] == 0: + stats["invalid"] += 1 + if kind == "LINE" and stat["simple"] == 0: + stats["non_simple_line_info"] += 1 + if stat["touches_cell_boundary"]: + stats["touch_cell_boundary_info"] += 1 + if stat["touches_dataset_bbox"]: + stats["touch_dataset_bbox_info"] += 1 + geom_types[stat.get("geom_type") or "UNPARSED"] += 1 + + for x in found: + issues.append(x) + invalid_reasons[x["issue_category"]] += 1 + + print( + f" {kind:<8} scanned={stats['rows']:,} " + f"issues={len(issues):,} last_feature_id={last_id}" + ) + + return { + "kind": kind, + "table": table, + "rows": stats["rows"], + "parsed": stats["parsed"], + "empty": stats["empty"], + "valid": stats["valid"], + "invalid": stats["invalid"], + "non_simple_line_info": stats["non_simple_line_info"], + "touch_cell_boundary_info": stats["touch_cell_boundary_info"], + "touch_dataset_bbox_info": stats["touch_dataset_bbox_info"], + "actual_geom_types": dict(geom_types), + "issue_count": len(issues), + "issue_categories": dict(invalid_reasons), + }, issues + + +def scan_soundings( + cur, + *, + dataset_id: int, + wkb_fn: str, + srid_fn: Optional[str], + dataset_bbox: Optional[Tuple[float, float, float, float]], + chunk_size: int, + bbox_tolerance: float, + boundary_tolerance: float, +) -> Tuple[Dict[str, Any], List[Dict[str, Any]], List[Dict[str, Any]]]: + last_id = 0 + stats = Counter() + geom_types = Counter() + issues: List[Dict[str, Any]] = [] + + # feature_id -> [minx,miny,maxx,maxy,count] + feature_bounds: Dict[int, List[float]] = {} + + srid_expr = f"{srid_fn}(s.geom)" if srid_fn else "NULL" + + while True: + sql = f""" + SELECT + s.id AS sounding_id, + s.feature_id, + s.depth, + {wkb_fn}(s.geom) AS geom_wkb, + {srid_expr} AS geom_srid, + f.object_class, + f.cell_id, + c.cell_name, + f.source_record_index, + f.normalized_srid, + f.bbox_min_lon, f.bbox_min_lat, f.bbox_max_lon, f.bbox_max_lat, + c.bbox_min_lon AS cell_bbox_min_lon, + c.bbox_min_lat AS cell_bbox_min_lat, + c.bbox_max_lon AS cell_bbox_max_lon, + c.bbox_max_lat AS cell_bbox_max_lat + FROM chart_sounding_point s + JOIN chart_feature f ON f.id=s.feature_id + JOIN chart_cell c ON c.id=f.cell_id + WHERE f.dataset_id=%s + AND s.id>%s + ORDER BY s.id + LIMIT %s + """ + cur.execute(sql, (dataset_id, last_id, chunk_size)) + batch = list(cur.fetchall()) + if not batch: + break + + for r in batch: + last_id = int(r["sounding_id"]) + fb = bbox_tuple(r) + cb = bbox_tuple(r, "cell_") + + stat, found = intrinsic_check( + kind="SOUNDING", + feature_id=r["feature_id"], + extra_id=r["sounding_id"], + object_class=r.get("object_class") or "", + cell_id=r.get("cell_id"), + cell_name=r.get("cell_name") or "", + source_record_index=r.get("source_record_index"), + wkb_data=r.get("geom_wkb"), + srid=r.get("geom_srid"), + normalized_srid=r.get("normalized_srid"), + feature_bbox=fb, + cell_bbox=cb, + dataset_bbox=dataset_bbox, + bbox_tolerance=bbox_tolerance, + boundary_tolerance=boundary_tolerance, + ) + + stats["rows"] += 1 + stats["parsed"] += stat["parsed"] + stats["empty"] += stat["empty"] + if stat["valid"] == 1: + stats["valid"] += 1 + elif stat["valid"] == 0: + stats["invalid"] += 1 + geom_types[stat.get("geom_type") or "UNPARSED"] += 1 + + # depth 必须 finite + depth = safe_float(r.get("depth")) + if depth is None: + issues.append({ + "kind": "SOUNDING", + "feature_id": r["feature_id"], + "extra_id": r["sounding_id"], + "object_class": r.get("object_class") or "", + "cell_id": r.get("cell_id"), + "cell_name": r.get("cell_name") or "", + "source_record_index": r.get("source_record_index"), + "issue_category": "INVALID_SOUNDING_DEPTH", + "issue_detail": f"depth={r.get('depth')}", + "boundary_context": "", + }) + + issues.extend(found) + + b = stat.get("bounds") + if b and len(b) == 4: + fid = int(r["feature_id"]) + if fid not in feature_bounds: + feature_bounds[fid] = [b[0], b[1], b[2], b[3], 1] + else: + a = feature_bounds[fid] + a[0] = min(a[0], b[0]) + a[1] = min(a[1], b[1]) + a[2] = max(a[2], b[2]) + a[3] = max(a[3], b[3]) + a[4] += 1 + + print( + f" SOUNDING scanned={stats['rows']:,} " + f"issues={len(issues):,} last_sounding_id={last_id}" + ) + + # 按 feature 聚合 sounding bbox 与 chart_feature bbox 对账。 + bbox_issues: List[Dict[str, Any]] = [] + feature_ids = list(feature_bounds.keys()) + # 704 个 feature 左右，可直接分批 IN + for pos in range(0, len(feature_ids), 500): + ids = feature_ids[pos:pos+500] + placeholders = ",".join(["%s"] * len(ids)) + rs = rows(cur, f""" + SELECT + f.id AS feature_id, f.object_class, f.cell_id, c.cell_name, + f.source_record_index, + f.bbox_min_lon, f.bbox_min_lat, f.bbox_max_lon, f.bbox_max_lat, + c.bbox_min_lon AS cell_bbox_min_lon, + c.bbox_min_lat AS cell_bbox_min_lat, + c.bbox_max_lon AS cell_bbox_max_lon, + c.bbox_max_lat AS cell_bbox_max_lat + FROM chart_feature f + JOIN chart_cell c ON c.id=f.cell_id + WHERE f.id IN ({placeholders}) + """, ids) + for r in rs: + fid = int(r["feature_id"]) + a = feature_bounds[fid] + actual = (a[0], a[1], a[2], a[3]) + fb = bbox_tuple(r) + cb = bbox_tuple(r, "cell_") + context = [] + if touches_bbox_edge(actual, cb, boundary_tolerance): + context.append("CELL_EDGE") + if touches_bbox_edge(actual, dataset_bbox, boundary_tolerance): + context.append("DATASET_BBOX_EDGE") + if fb is None: + bbox_issues.append({ + "feature_id": fid, + "object_class": r.get("object_class"), + "cell_id": r.get("cell_id"), + "cell_name": r.get("cell_name"), + "source_record_index": r.get("source_record_index"), + "issue_category": "SOUNDING_FEATURE_BBOX_MISSING", + "point_count": int(a[4]), + "actual_bbox": actual, + "feature_bbox": None, + "boundary_context": "|".join(context), + }) + elif not bbox_close(actual, fb, bbox_tolerance): + bbox_issues.append({ + "feature_id": fid, + "object_class": r.get("object_class"), + "cell_id": r.get("cell_id"), + "cell_name": r.get("cell_name"), + "source_record_index": r.get("source_record_index"), + "issue_category": "SOUNDING_FEATURE_BBOX_MISMATCH", + "point_count": int(a[4]), + "actual_bbox": actual, + "feature_bbox": fb, + "boundary_context": "|".join(context), + }) + + return { + "kind": "SOUNDING", + "table": "chart_sounding_point", + "rows": stats["rows"], + "parsed": stats["parsed"], + "empty": stats["empty"], + "valid": stats["valid"], + "invalid": stats["invalid"], + "actual_geom_types": dict(geom_types), + "issue_count_excluding_feature_bbox": len(issues), + "feature_bbox_issue_count": len(bbox_issues), + }, issues, bbox_issues + + +def scan_coverage( + cur, + *, + dataset_id: int, + wkb_fn: str, + srid_fn: Optional[str], + dataset_bbox: Optional[Tuple[float, float, float, float]], + bbox_tolerance: float, + boundary_tolerance: float, +) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]: + srid_expr = f"{srid_fn}(v.geom)" if srid_fn else "NULL" + cur.execute(f""" + SELECT + v.id AS coverage_id, + v.cell_id, + c.cell_name, + v.source_record_index, + {wkb_fn}(v.geom) AS geom_wkb, + {srid_expr} AS geom_srid, + c.bbox_min_lon AS cell_bbox_min_lon, + c.bbox_min_lat AS cell_bbox_min_lat, + c.bbox_max_lon AS cell_bbox_max_lon, + c.bbox_max_lat AS cell_bbox_max_lat + FROM chart_coverage v + JOIN chart_cell c ON c.id=v.cell_id + WHERE v.dataset_id=%s + ORDER BY v.id + """, (dataset_id,)) + rs = list(cur.fetchall()) + + stats = Counter() + geom_types = Counter() + issues: List[Dict[str, Any]] = [] + for r in rs: + cb = bbox_tuple(r, "cell_") + stat, found = intrinsic_check( + kind="COVERAGE", + feature_id=None, + extra_id=r["coverage_id"], + object_class="COVERAGE", + cell_id=r.get("cell_id"), + cell_name=r.get("cell_name") or "", + source_record_index=r.get("source_record_index"), + wkb_data=r.get("geom_wkb"), + srid=r.get("geom_srid"), + normalized_srid=None, + feature_bbox=None, + cell_bbox=cb, + dataset_bbox=dataset_bbox, + bbox_tolerance=bbox_tolerance, + boundary_tolerance=boundary_tolerance, + ) + stats["rows"] += 1 + stats["parsed"] += stat["parsed"] + stats["empty"] += stat["empty"] + if stat["valid"] == 1: + stats["valid"] += 1 + elif stat["valid"] == 0: + stats["invalid"] += 1 + geom_types[stat.get("geom_type") or "UNPARSED"] += 1 + issues.extend(found) + + return { + "kind": "COVERAGE", + "table": "chart_coverage", + "rows": stats["rows"], + "parsed": stats["parsed"], + "empty": stats["empty"], + "valid": stats["valid"], + "invalid": stats["invalid"], + "actual_geom_types": dict(geom_types), + "issue_count": len(issues), + }, issues + + +def zip_directory(folder: Path) -> Path: + zpath = folder.with_suffix(".zip") + with zipfile.ZipFile(zpath, "w", compression=zipfile.ZIP_DEFLATED) as z: + for p in sorted(folder.rglob("*")): + if p.is_file(): + z.write(p, p.relative_to(folder.parent)) + return zpath + + +def parse_args(): + p = argparse.ArgumentParser( + description="CM93 MySQL 实体空间 geometry 只读审计 v3" + ) + p.add_argument("--host", default=os.getenv("MYSQL_HOST", "127.0.0.1")) + p.add_argument("--port", type=int, default=int(os.getenv("MYSQL_PORT", "3306"))) + p.add_argument("--user", default=os.getenv("MYSQL_USER", "root")) + p.add_argument("--password", default=os.getenv("MYSQL_PASSWORD")) + p.add_argument("--database", default=os.getenv("MYSQL_DATABASE", "cm93_chart")) + p.add_argument("--dataset-id", type=int, default=1) + p.add_argument("--chunk-size", type=int, default=2000) + p.add_argument( + "--bbox-tolerance", + type=float, + default=1e-6, + help="Feature bbox 与实际 geometry bounds 对账容差（degree），默认 1e-6" + ) + p.add_argument( + "--boundary-tolerance", + type=float, + default=1e-5, + help="仅用于标注 geometry 是否碰 Cell/Dataset bbox 边缘，不用于 PASS/FAIL" + ) + p.add_argument("--output", default=None) + p.add_argument( + "--skip-coverage", + action="store_true", + help="跳过 chart_coverage geometry 审计" + ) + return p.parse_args() + + +def main(): + args = parse_args() + if not args.password: + args.password = getpass.getpass("MySQL password: ") + + stamp = datetime.now().strftime("%Y%m%d_%H%M%S") + outdir = Path( + args.output or + f"cm93_spatial_audit_v3_dataset_{args.dataset_id}_{stamp}" + ) + outdir.mkdir(parents=True, exist_ok=True) + + report: Dict[str, Any] = { + "version": "v3.0", + "generated_at": datetime.now().isoformat(timespec="seconds"), + "dataset_id": args.dataset_id, + "database": args.database, + "read_only": True, + "pbf": "OUT_OF_SCOPE", + "dataset_scope_note": ( + "当前 CM93JAPAN 只是日本沿海局部子集，并非完整 CM93。" + "不把跨未导入海域/邻区的不连续性作为 geometry corruption。" + ), + "warnings": [], + "server": {}, + "dataset_bbox": None, + "counts": {}, + "mapping": {}, + "geometry_stats": [], + "issue_summary": {}, + } + + conn = None + try: + conn = pymysql.connect( + host=args.host, + port=args.port, + user=args.user, + password=args.password, + database=args.database, + charset="utf8mb4", + cursorclass=DictCursor, + autocommit=False, + connect_timeout=20, + read_timeout=600, + write_timeout=30, + ) + with conn.cursor() as cur: + begin_read_only(cur, report["warnings"]) + + print("[1/8] 读取服务器与 Dataset 基线...") + report["server"]["version"] = scalar(cur, "SELECT VERSION()") + try: + report["server"]["version_comment"] = scalar(cur, "SELECT @@version_comment") + except Exception: + report["server"]["version_comment"] = None + + funcs = probe_spatial_functions(cur) + report["server"]["spatial_functions"] = funcs + if not funcs["wkb_fn"]: + raise RuntimeError( + "无法找到可用的 geometry -> WKB 函数 " + "(ST_AsBinary/ST_AsWKB/AsBinary/AsWKB)" + ) + print(f" DB: {report['server']['version']}") + print(f" WKB function: {funcs['wkb_fn']}") + print(f" SRID function: {funcs['srid_fn']}") + + dataset_bbox = get_dataset_bbox(cur, args.dataset_id) + report["dataset_bbox"] = dataset_bbox + print(f" Dataset bbox: {dataset_bbox}") + + # 基线 counts + report["counts"]["cells"] = int(scalar(cur, + "SELECT COUNT(*) FROM chart_cell WHERE dataset_id=%s", + (args.dataset_id,)) or 0) + report["counts"]["features"] = int(scalar(cur, + "SELECT COUNT(*) FROM chart_feature WHERE dataset_id=%s", + (args.dataset_id,)) or 0) + report["counts"]["attributes"] = int(scalar(cur, """ + SELECT COUNT(*) + FROM chart_feature_attribute a + JOIN chart_feature f ON f.id=a.feature_id + WHERE f.dataset_id=%s + """, (args.dataset_id,)) or 0) + report["counts"]["sounding_points"] = int(scalar(cur, """ + SELECT COUNT(*) + FROM chart_sounding_point s + JOIN chart_feature f ON f.id=s.feature_id + WHERE f.dataset_id=%s + """, (args.dataset_id,)) or 0) + + report["counts"]["raw_geotype16_meta"] = int(scalar(cur, """ + SELECT COUNT(*) + FROM chart_feature + WHERE dataset_id=%s + AND raw_geotype=16 + AND geometry_type='META' + """, (args.dataset_id,)) or 0) + + report["counts"]["raw_geotype16_geometry_issue"] = int(scalar(cur, """ + SELECT COUNT(*) + FROM chart_feature + WHERE dataset_id=%s + AND raw_geotype=16 + AND geometry_type='META' + AND geometry_issue IS NOT NULL + """, (args.dataset_id,)) or 0) + + print("[2/8] 检查 Feature geometry_type ↔ 实际 geometry 表映射...") + mapping_issues, mapping_stats = audit_mapping(cur, args.dataset_id) + report["mapping"] = mapping_stats + write_csv(outdir / "feature_geometry_mapping_issues.csv", mapping_issues) + print(f" mapping issues: {len(mapping_issues):,}") + + all_intrinsic_issues: List[Dict[str, Any]] = [] + geometry_stats: List[Dict[str, Any]] = [] + + print("[3/8] 扫描 Point geometry...") + st, iss = scan_feature_geometry_table( + cur, + table="chart_point_geometry", + kind="POINT", + dataset_id=args.dataset_id, + wkb_fn=funcs["wkb_fn"], + srid_fn=funcs["srid_fn"], + dataset_bbox=dataset_bbox, + chunk_size=args.chunk_size, + bbox_tolerance=args.bbox_tolerance, + boundary_tolerance=args.boundary_tolerance, + ) + geometry_stats.append(st) + all_intrinsic_issues.extend(iss) + + print("[4/8] 扫描 Line geometry...") + st, iss = scan_feature_geometry_table( + cur, + table="chart_line_geometry", + kind="LINE", + dataset_id=args.dataset_id, + wkb_fn=funcs["wkb_fn"], + srid_fn=funcs["srid_fn"], + dataset_bbox=dataset_bbox, + chunk_size=args.chunk_size, + bbox_tolerance=args.bbox_tolerance, + boundary_tolerance=args.boundary_tolerance, + ) + geometry_stats.append(st) + all_intrinsic_issues.extend(iss) + + print("[5/8] 扫描 Area geometry...") + st, iss = scan_feature_geometry_table( + cur, + table="chart_area_geometry", + kind="AREA", + dataset_id=args.dataset_id, + wkb_fn=funcs["wkb_fn"], + srid_fn=funcs["srid_fn"], + dataset_bbox=dataset_bbox, + chunk_size=args.chunk_size, + bbox_tolerance=args.bbox_tolerance, + boundary_tolerance=args.boundary_tolerance, + ) + geometry_stats.append(st) + all_intrinsic_issues.extend(iss) + + print("[6/8] 扫描 Sounding points，并按 Feature 对账 bbox...") + st, iss, sounding_bbox_issues = scan_soundings( + cur, + dataset_id=args.dataset_id, + wkb_fn=funcs["wkb_fn"], + srid_fn=funcs["srid_fn"], + dataset_bbox=dataset_bbox, + chunk_size=max(args.chunk_size, 5000), + bbox_tolerance=args.bbox_tolerance, + boundary_tolerance=args.boundary_tolerance, + ) + geometry_stats.append(st) + all_intrinsic_issues.extend(iss) + write_csv(outdir / "sounding_feature_bbox_issues.csv", sounding_bbox_issues) + + coverage_issues: List[Dict[str, Any]] = [] + if not args.skip_coverage: + print("[7/8] 扫描 Coverage geometry...") + st, coverage_issues = scan_coverage( + cur, + dataset_id=args.dataset_id, + wkb_fn=funcs["wkb_fn"], + srid_fn=funcs["srid_fn"], + dataset_bbox=dataset_bbox, + bbox_tolerance=args.bbox_tolerance, + boundary_tolerance=args.boundary_tolerance, + ) + geometry_stats.append(st) + write_csv(outdir / "coverage_geometry_issues.csv", coverage_issues) + else: + print("[7/8] Coverage skipped by option.") + + print("[8/8] 汇总输出...") + + report["geometry_stats"] = geometry_stats + + # 主 intrinsic issue 输出（Feature Point/Line/Area + sounding point） + write_csv(outdir / "intrinsic_geometry_issues.csv", all_intrinsic_issues) + write_csv(outdir / "geometry_stats_by_kind.csv", geometry_stats) + + cat_counts = Counter( + x.get("issue_category", "UNKNOWN") + for x in all_intrinsic_issues + ) + cat_rows = [ + {"issue_category": k, "count": v} + for k, v in cat_counts.most_common() + ] + write_csv( + outdir / "intrinsic_issue_category_counts.csv", + cat_rows, + ["issue_category", "count"] + ) + + invalid_reason_counts = Counter( + x.get("invalid_reason") + for x in all_intrinsic_issues + if x.get("invalid_reason") + ) + write_csv( + outdir / "invalid_geometry_reason_counts.csv", + [ + {"reason": k, "count": v} + for k, v in invalid_reason_counts.most_common() + ], + ["reason", "count"] + ) + + obj_counts = Counter( + (x.get("object_class") or "", x.get("issue_category") or "") + for x in all_intrinsic_issues + ) + write_csv( + outdir / "issue_object_class_counts.csv", + [ + {"object_class": k[0], "issue_category": k[1], "count": v} + for k, v in obj_counts.most_common() + ], + ["object_class", "issue_category", "count"] + ) + + cell_counts = Counter( + (x.get("cell_name") or "", x.get("issue_category") or "") + for x in all_intrinsic_issues + ) + write_csv( + outdir / "issue_cell_counts.csv", + [ + {"cell_name": k[0], "issue_category": k[1], "count": v} + for k, v in cell_counts.most_common() + ], + ["cell_name", "issue_category", "count"] + ) + + # 每类最多 10 条样本 + samples = [] + sample_seen = Counter() + for x in all_intrinsic_issues: + cat = x.get("issue_category", "UNKNOWN") + if sample_seen[cat] < 10: + samples.append(x) + sample_seen[cat] += 1 + write_csv(outdir / "issue_samples.csv", samples) + + report["issue_summary"] = { + "mapping_issue_count": len(mapping_issues), + "intrinsic_issue_count": len(all_intrinsic_issues), + "sounding_feature_bbox_issue_count": len(sounding_bbox_issues), + "coverage_issue_count": len(coverage_issues), + "intrinsic_by_category": dict(cat_counts), + "issues_with_cell_boundary_context": sum( + 1 for x in all_intrinsic_issues + if "CELL_EDGE" in (x.get("boundary_context") or "") + ), + "issues_with_dataset_bbox_context": sum( + 1 for x in all_intrinsic_issues + if "DATASET_BBOX_EDGE" in (x.get("boundary_context") or "") + ), + } + + # 基线匹配 + acceptance = {} + for k, exp in EXPECTED_DATASET1.items(): + actual = report["counts"].get(k) + acceptance[k] = { + "actual": actual, + "expected": exp, + "pass": actual == exp, + } + + acceptance["raw_geotype16_expected_parent_count"] = { + "actual": report["counts"]["raw_geotype16_meta"], + "expected": 4406, + "pass": report["counts"]["raw_geotype16_meta"] == 4406, + "interpretation": "EXPECTED_PARENT_NO_GEOMETRY, not intrinsic geometry corruption", + } + + acceptance["mapping"] = { + "issue_count": len(mapping_issues), + "pass": len(mapping_issues) == 0, + } + + # 注意：intrinsic issue 非 0 时不自动称“Importer FAIL”。 + # 需要结合 category、对象类型和 boundary_context 分析。 + acceptance["intrinsic_geometry"] = { + "issue_count": len(all_intrinsic_issues), + "pass_if_zero": len(all_intrinsic_issues) == 0, + "note": ( + "非零时需要人工分类；局部 Dataset 边界不连续本身不属于 FAIL。" + ), + } + report["acceptance"] = acceptance + + # JSON + (outdir / "spatial_audit_report.json").write_text( + json.dumps(report, ensure_ascii=False, indent=2, default=str), + encoding="utf-8" + ) + + # TXT summary + lines = [] + lines.append("CM93 -> MySQL 实体空间 Geometry 只读审计 v3") + lines.append("=" * 78) + lines.append(f"Generated: {report['generated_at']}") + lines.append(f"Database: {args.database}") + lines.append(f"Dataset ID: {args.dataset_id}") + lines.append(f"DB version: {report['server']['version']}") + lines.append(f"WKB fn: {funcs['wkb_fn']}") + lines.append(f"SRID fn: {funcs['srid_fn']}") + lines.append(f"Dataset bbox: {dataset_bbox}") + lines.append("") + lines.append("DATASET SCOPE") + lines.append( + " 当前 CM93JAPAN 是日本沿海局部子集，不是完整 CM93。" + ) + lines.append( + " 跨未导入海域/邻区的几何或对象连续性不作为本轮 FAIL 条件。" + ) + lines.append( + " boundary_context 仅用于后续解释，不会自动把记录判成 importer bug。" + ) + lines.append("") + lines.append("BASELINE COUNTS") + for k in ["cells", "features", "attributes", "sounding_points", + "raw_geotype16_meta", "raw_geotype16_geometry_issue"]: + lines.append(f" {k:36s} {report['counts'].get(k)}") + lines.append("") + lines.append("FEATURE ↔ GEOMETRY TABLE MAPPING") + lines.append(f" mapping issues: {len(mapping_issues)}") + for k, v in sorted(mapping_stats.get("mapping_issue_categories", {}).items()): + lines.append(f" {k:40s} {v}") + lines.append("") + lines.append("INTRINSIC GEOMETRY STATS") + for s in geometry_stats: + lines.append( + f" {s['kind']:<9} rows={s.get('rows',0):>9} " + f"parsed={s.get('parsed',0):>9} " + f"empty={s.get('empty',0):>6} " + f"valid={s.get('valid',0):>9} " + f"invalid={s.get('invalid',0):>6} " + f"issues={s.get('issue_count', s.get('issue_count_excluding_feature_bbox',0)):>6}" + ) + lines.append( + f" actual types={s.get('actual_geom_types')}" + ) + if s["kind"] == "LINE": + lines.append( + f" non_simple_line_info={s.get('non_simple_line_info',0)} " + f"(informational; not automatically invalid)" + ) + lines.append("") + lines.append("INTRINSIC ISSUE CATEGORIES") + if cat_counts: + for k, v in cat_counts.most_common(): + lines.append(f" {k:36s} {v}") + else: + lines.append(" none") + lines.append("") + lines.append("SOUNDING FEATURE BBOX") + lines.append(f" bbox issues: {len(sounding_bbox_issues)}") + lines.append("") + lines.append("COVERAGE") + lines.append(f" coverage issues: {len(coverage_issues)}") + lines.append("") + lines.append("BOUNDARY CONTEXT (INFORMATIONAL)") + lines.append( + f" intrinsic issues touching cell bbox edge: " + f"{report['issue_summary']['issues_with_cell_boundary_context']}" + ) + lines.append( + f" intrinsic issues touching dataset bbox edge: " + f"{report['issue_summary']['issues_with_dataset_bbox_context']}" + ) + lines.append("") + lines.append("IMPORTANT INTERPRETATION") + lines.append( + " raw_geotype=16 / META 的 4406 条正常 parent-no-geometry " + "不进入本轮 intrinsic geometry issue。" + ) + lines.append( + " Line 在 Cell/Dataset 边缘结束、缺少未导入邻区对象，不属于本轮 geometry corruption。" + ) + lines.append( + " Polygon 自相交、WKB 无法解析、坐标越界、类型串表等属于单 geometry 内在问题，" + "需要单独分析。" + ) + lines.append("") + lines.append("PBF: OUT OF SCOPE / NOT STARTED") + + (outdir / "spatial_audit_summary.txt").write_text( + "\n".join(lines), + encoding="utf-8" + ) + + conn.rollback() + + zpath = zip_directory(outdir) + print("") + print("审计完成。") + print(f"输出目录: {outdir.resolve()}") + print(f"ZIP: {zpath.resolve()}") + print("") + print("请把 ZIP 整包发给 ChatGPT 分析。") + + except KeyboardInterrupt: + if conn: + try: + conn.rollback() + except Exception: + pass + print("\n用户中断。") + sys.exit(130) + except Exception: + if conn: + try: + conn.rollback() + except Exception: + pass + tb = traceback.format_exc() + print(tb) + try: + (outdir / "fatal_error.txt").write_text(tb, encoding="utf-8") + except Exception: + pass + sys.exit(1) + finally: + if conn: + try: + conn.close() + except Exception: + pass + + +if __name__ == "__main__": + main() diff --git a/cm93_normalization_implementation_precheck_v11.py b/cm93_normalization_implementation_precheck_v11.py new file mode 100644 index 0000000..67e0131 --- /dev/null +++ b/cm93_normalization_implementation_precheck_v11.py @@ -0,0 +1,1270 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +CM93 -> MySQL Normalization 实施前只读预检 +Version: v11.0 + +目标 +==== +V11 不再依赖把 V10.1 大包上传给 ChatGPT。 + +它在用户本机完成两个门控： + +Gate A：V10.1 canonical validation 必须 PASS +------------------------------------------------ +从以下任一输入读取： +- V10.1 完整输出目录 +- V10.1 完整 ZIP +- v10.2 digest ZIP + +要求至少能确认： +- overall_pass = true +- invalid_polygon_records = 55 +- golden_vector_count = 55 +- 55 条 golden 全 PASS +- valid_policy_changed_count = 0 +- valid_hash_changed_count = 0 +- input variant / face order / repeat / topology failures = 0 + +若不满足，V11 立即停止，不继续。 + +Gate B：当前 MySQL 数据必须仍与 V10.1 golden vectors 一致 +----------------------------------------------------------- +V11 会重新从数据库读取 52 invalid Area + 3 invalid Coverage， +使用已经冻结的规则： + + ODD_EVEN_FACE_SELECTION + + + GEOS_NORMALIZE + +重新生成 candidate，然后逐条比较： +- canonical WKB SHA-256 +- geometry type +- polygon part count +- candidate bbox +- candidate area + +必须与 V10.1 golden vector 一致。 + +之后做实施前 schema 检查： +- chart_feature +- chart_area_geometry +- chart_coverage +- chart_import_job + +检查： +- geometry 字段真实 SQL 类型 +- 是否能存 MultiPolygon +- SRID / SRS 约束 +- 主键、唯一键、外键、索引 +- bbox 字段 +- geometry_issue / geometry_repaired 字段 +- import job geometry counters +- 4406 raw_geotype=16 当前状态 +- 55 OGC-invalid 当前状态 +- 未来若 normalized geometry 替换原 geometry，需要同步修改哪些 metadata + +输出： +- implementation_precheck_summary_v11.txt +- implementation_precheck_report_v11.json +- golden_revalidation_v11.csv +- schema_columns_v11.csv +- schema_indexes_v11.csv +- schema_foreign_keys_v11.csv +- show_create_tables_v11.txt +- normalization_metadata_impact_v11.csv +- proposed_implementation_contract_v11.md +- proposed_migration_plan_v11.sql.txt + +注意： +proposed_migration_plan_v11.sql.txt 只是文本计划。 +V11 绝不会执行这些 SQL。 + +禁止： +- 不 UPDATE +- 不 INSERT +- 不 DELETE +- 不 ALTER TABLE +- 不修改 importer +- 不写 normalized geometry +- 不修 4406 +- 不 PBF + +依赖： + pip install pymysql shapely + +执行例 1：直接使用 V10.1 输出目录 + python cm93_normalization_implementation_precheck_v11.py ^ + --v10-input "D:\\React\\cm93-mysql-importer-starter\\cm93_normalization_canonical_v10_1_dataset_1_20260831_215602" ^ + --host 127.0.0.1 ^ + --port 3306 ^ + --user root ^ + --database cm93_chart ^ + --dataset-id 1 + +执行例 2：使用 v10.2 digest ZIP + python cm93_normalization_implementation_precheck_v11.py ^ + --v10-input "D:\\...\\cm93_v10_1_result_digest_v10_2.zip" ^ + --host 127.0.0.1 ^ + --port 3306 ^ + --user root ^ + --database cm93_chart ^ + --dataset-id 1 +""" + +from __future__ import annotations + +import argparse +import csv +import getpass +import hashlib +import json +import math +import os +import shutil +import sys +import tempfile +import zipfile +from collections import Counter +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple + +try: + import pymysql + from pymysql.cursors import DictCursor +except ImportError: + print("缺少 pymysql：pip install pymysql shapely") + sys.exit(2) + +try: + import shapely + + try: + from shapely import from_wkb as _from_wkb + def load_wkb(data: bytes): + return _from_wkb(data) + except ImportError: + from shapely import wkb as _wkb + def load_wkb(data: bytes): + return _wkb.loads(data) + + try: + from shapely import to_wkb as _to_wkb + def dump_wkb(g) -> bytes: + return bytes(_to_wkb(g, hex=False, byte_order=1)) + except ImportError: + from shapely import wkb as _wkb2 + def dump_wkb(g) -> bytes: + return _wkb2.dumps(g, hex=False, big_endian=False) + + try: + from shapely import normalize as _normalize + def canonicalize(g): + return _normalize(g) + except ImportError: + raise ImportError("V11 需要 Shapely 2.x 的 shapely.normalize()") + + from shapely.ops import unary_union, polygonize_full + +except ImportError as e: + print(f"Shapely 初始化失败：{e}") + print("请执行：pip install -U shapely") + sys.exit(2) + + +# --------------------------------------------------------------------------- +# General helpers +# --------------------------------------------------------------------------- + +def scalar(cur, sql: str, params: Sequence[Any] = ()) -> Any: + cur.execute(sql, params) + row = cur.fetchone() + return next(iter(row.values())) if row else None + + +def fetch_all(cur, sql: str, params: Sequence[Any] = ()) -> List[Dict[str, Any]]: + cur.execute(sql, params) + return list(cur.fetchall()) + + +def begin_read_only(cur, warnings: List[str]): + try: + cur.execute("SET SESSION TRANSACTION READ ONLY") + except Exception as e: + warnings.append(f"SET SESSION TRANSACTION READ ONLY failed: {e}") + + try: + cur.execute("START TRANSACTION READ ONLY") + except Exception as e: + warnings.append(f"START TRANSACTION READ ONLY failed: {e}") + cur.execute("START TRANSACTION") + + +def to_bytes(v: Any) -> Optional[bytes]: + if v is None: + return None + if isinstance(v, bytes): + return v + if isinstance(v, bytearray): + return bytes(v) + if isinstance(v, memoryview): + return v.tobytes() + try: + return bytes(v) + except Exception: + return None + + +def write_csv( + path: Path, + rows: Iterable[Dict[str, Any]], + fieldnames: Optional[List[str]] = None, +): + rows = list(rows) + + if fieldnames is None: + seen = set() + fieldnames = [] + for row in rows: + for k in row.keys(): + if k not in seen: + seen.add(k) + fieldnames.append(k) + + path.parent.mkdir(parents=True, exist_ok=True) + + with path.open("w", encoding="utf-8-sig", newline="") as f: + if not fieldnames: + return + w = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore") + w.writeheader() + for row in rows: + out = {} + for k in fieldnames: + v = row.get(k) + if isinstance(v, (dict, list, tuple)): + v = json.dumps(v, ensure_ascii=False, default=str) + out[k] = "" if v is None else v + w.writerow(out) + + +def intv(v: Any, default: int = 0) -> int: + try: + return int(v) + except Exception: + return default + + +def boolv(v: Any) -> bool: + if isinstance(v, bool): + return v + if v is None: + return False + return str(v).strip().lower() in {"1", "true", "yes", "y", "pass"} + + +def floatv(v: Any, default: Optional[float] = None) -> Optional[float]: + try: + x = float(v) + return x if math.isfinite(x) else default + except Exception: + return default + + +def geometry_hash(g) -> Optional[str]: + if g is None: + return None + try: + return hashlib.sha256(dump_wkb(g)).hexdigest() + except Exception: + return None + + +def flatten_components(g) -> Dict[str, List[Any]]: + result = {"polygons": [], "lines": [], "points": [], "others": []} + + def rec(x): + if x is None: + return + t = x.geom_type + if t == "Polygon": + result["polygons"].append(x) + elif t == "MultiPolygon": + for y in x.geoms: + rec(y) + elif t in ("LineString", "LinearRing"): + result["lines"].append(x) + elif t == "MultiLineString": + for y in x.geoms: + rec(y) + elif t == "Point": + result["points"].append(x) + elif t == "MultiPoint": + for y in x.geoms: + rec(y) + elif hasattr(x, "geoms"): + for y in x.geoms: + rec(y) + else: + result["others"].append(x) + + rec(g) + return result + + +def polygon_part_count(g) -> int: + if g is None: + return 0 + if g.geom_type == "Polygon": + return 1 + if g.geom_type == "MultiPolygon": + return len(g.geoms) + return len(flatten_components(g)["polygons"]) + + +def probe_wkb_fn(cur) -> str: + for fn in ("ST_AsBinary", "ST_AsWKB", "AsBinary", "AsWKB"): + try: + cur.execute(f"SELECT {fn}(geom) AS x FROM chart_area_geometry LIMIT 1") + cur.fetchone() + return fn + except Exception: + pass + raise RuntimeError("找不到 geometry -> WKB 函数") + + +# --------------------------------------------------------------------------- +# V10 input / gate +# --------------------------------------------------------------------------- + +def find_file(root: Path, filename: str) -> Optional[Path]: + direct = root / filename + if direct.exists(): + return direct + hits = list(root.rglob(filename)) + return hits[0] if hits else None + + +def read_csv(path: Path) -> List[Dict[str, str]]: + with path.open("r", encoding="utf-8-sig", newline="") as f: + return list(csv.DictReader(f)) + + +def open_v10_input(src: Path) -> Tuple[Path, Optional[Path]]: + if src.is_dir(): + return src, None + + if src.is_file() and src.suffix.lower() == ".zip": + temp = Path(tempfile.mkdtemp(prefix="cm93_v11_v10_input_")) + with zipfile.ZipFile(src, "r") as z: + z.extractall(temp) + return temp, temp + + raise ValueError("v10-input 必须是目录或 ZIP") + + +def load_v10_gate(root: Path) -> Tuple[Dict[str, Any], Dict[str, Dict[str, Any]]]: + digest_json = find_file(root, "V10_1_结果摘要_v10_2.json") + report_json = find_file(root, "normalization_canonical_report_v10_1.json") + golden_csv = find_file(root, "normalization_golden_vectors_v10_1.csv") + + gate: Dict[str, Any] = { + "source": None, + "overall_pass": False, + "invalid_polygon_records": None, + "valid_policy_changed_count": None, + "valid_hash_changed_count": None, + "golden_vector_count": None, + "golden_pass_count": None, + "golden_fail_count": None, + "face_order_hash_mismatch_count": None, + "canonical_topology_failure_count": None, + "repeat_hash_failure_count": None, + "input_variant_failure_record_count": None, + "errors": [], + } + + golden_map: Dict[str, Dict[str, Any]] = {} + + if digest_json: + digest = json.loads(digest_json.read_text(encoding="utf-8")) + gate["source"] = "v10.2-digest" + gate["digest_overall"] = digest.get("overall") + if digest.get("overall") == "PASS": + gate["overall_pass"] = True + checks = digest.get("checks", {}) + rc = checks.get("report", {}) + gc = checks.get("golden", {}) + gate["invalid_polygon_records"] = rc.get("invalid_polygon_records") + gate["valid_policy_changed_count"] = rc.get("valid_policy_changed_count") + gate["valid_hash_changed_count"] = rc.get("valid_hash_changed_count") + gate["golden_vector_count"] = rc.get("golden_vector_count") + gate["golden_pass_count"] = (gc.get("status_counts") or {}).get("PASS") + gate["golden_fail_count"] = (gc.get("status_counts") or {}).get("FAIL", 0) + gate["face_order_hash_mismatch_count"] = rc.get("face_order_hash_mismatch_count") + gate["canonical_topology_failure_count"] = rc.get("canonical_topology_failure_count") + gate["repeat_hash_failure_count"] = rc.get("repeat_hash_failure_count") + gate["input_variant_failure_record_count"] = rc.get("input_variant_failure_record_count") + + if report_json: + report = json.loads(report_json.read_text(encoding="utf-8")) + s = report.get("summary", {}) + gate["source"] = ( + gate["source"] + "+v10.1-report" + if gate["source"] + else "v10.1-report" + ) + gate["overall_pass"] = boolv(s.get("overall_pass")) + gate["invalid_polygon_records"] = s.get("invalid_polygon_records") + gate["valid_policy_changed_count"] = s.get("valid_policy_changed_count") + gate["valid_hash_changed_count"] = s.get("valid_hash_changed_count") + gate["golden_vector_count"] = s.get("golden_vector_count") + gate["golden_pass_count"] = (s.get("golden_record_status_counts") or {}).get("PASS") + gate["golden_fail_count"] = (s.get("golden_record_status_counts") or {}).get("FAIL", 0) + gate["face_order_hash_mismatch_count"] = s.get("face_order_hash_mismatch_count") + gate["canonical_topology_failure_count"] = s.get("canonical_topology_failure_count") + gate["repeat_hash_failure_count"] = s.get("repeat_hash_failure_count") + gate["input_variant_failure_record_count"] = s.get("input_variant_failure_record_count") + + if golden_csv: + rows = read_csv(golden_csv) + for r in rows: + kind = r.get("kind") + if kind == "AREA": + key = f"AREA:{r.get('feature_id')}" + else: + key = f"COVERAGE:{r.get('coverage_geom_id')}" + golden_map[key] = r + + statuses = Counter(str(r.get("record_status", "")).strip() for r in rows) + gate["golden_vector_count"] = len(rows) + gate["golden_pass_count"] = statuses.get("PASS", 0) + gate["golden_fail_count"] = statuses.get("FAIL", 0) + + pass_conditions = [ + gate["overall_pass"] is True, + intv(gate["invalid_polygon_records"]) == 55, + intv(gate["valid_policy_changed_count"]) == 0, + intv(gate["valid_hash_changed_count"]) == 0, + intv(gate["golden_vector_count"]) == 55, + intv(gate["golden_pass_count"]) == 55, + intv(gate["golden_fail_count"]) == 0, + intv(gate["face_order_hash_mismatch_count"]) == 0, + intv(gate["canonical_topology_failure_count"]) == 0, + intv(gate["repeat_hash_failure_count"]) == 0, + intv(gate["input_variant_failure_record_count"]) == 0, + ] + + gate["pass_conditions"] = pass_conditions + gate["gate_pass"] = all(pass_conditions) + + if len(golden_map) != 55: + gate["errors"].append( + f"Golden vector 明细不足：{len(golden_map)} / 55。" + ) + gate["gate_pass"] = False + + return gate, golden_map + + +# --------------------------------------------------------------------------- +# Odd-even normalization +# --------------------------------------------------------------------------- + +def original_rings(geom) -> List[List[Tuple[float, float]]]: + rings = [] + + def add_poly(poly): + rings.append([(float(c[0]), float(c[1])) for c in poly.exterior.coords]) + for ring in poly.interiors: + rings.append([(float(c[0]), float(c[1])) for c in ring.coords]) + + if geom.geom_type == "Polygon": + add_poly(geom) + elif geom.geom_type == "MultiPolygon": + for p in geom.geoms: + add_poly(p) + + return rings + + +def point_on_segment(px, py, ax, ay, bx, by, eps=1e-12) -> bool: + vx, vy = bx - ax, by - ay + wx, wy = px - ax, py - ay + c = abs(vx * wy - vy * wx) + if c > eps * max(1.0, abs(vx), abs(vy)): + return False + d = wx * vx + wy * vy + if d < -eps: + return False + vv = vx * vx + vy * vy + if d - vv > eps: + return False + return True + + +def even_odd_inside( + point: Tuple[float, float], + rings: List[List[Tuple[float, float]]] +) -> bool: + x, y = point + crossings = 0 + + for ring in rings: + for i in range(len(ring) - 1): + x1, y1 = ring[i] + x2, y2 = ring[i + 1] + + if point_on_segment(x, y, x1, y1, x2, y2): + return True + + if (y1 > y) != (y2 > y): + xinters = x1 + (y - y1) * (x2 - x1) / (y2 - y1) + if xinters > x: + crossings += 1 + + return crossings % 2 == 1 + + +def normalize_invalid(geom): + noded = unary_union(geom.boundary) + polygons_geom, cuts, dangles, invalid_rings = polygonize_full(noded) + faces = flatten_components(polygons_geom)["polygons"] + rings = original_rings(geom) + + selected = [] + for face in faces: + rp = face.representative_point() + if even_odd_inside((float(rp.x), float(rp.y)), rings): + selected.append(face) + + if not selected: + return None + + return canonicalize(unary_union(selected)) + + +# --------------------------------------------------------------------------- +# DB loaders / schema +# --------------------------------------------------------------------------- + +def load_invalid_db_records(cur, dataset_id: int, wkb_fn: str) -> List[Dict[str, Any]]: + out = [] + + cur.execute(f""" + SELECT + 'AREA' AS kind, + f.id AS feature_id, + NULL AS coverage_geom_id, + f.object_class, + f.cell_id, + c.cell_name, + f.source_record_index, + f.coverage_id, + f.raw_geotype, + f.bbox_min_lon, + f.bbox_min_lat, + f.bbox_max_lon, + f.bbox_max_lat, + {wkb_fn}(g.geom) AS geom_wkb + FROM chart_feature f + JOIN chart_area_geometry g ON g.feature_id=f.id + JOIN chart_cell c ON c.id=f.cell_id + WHERE f.dataset_id=%s AND f.geometry_type='AREA' + ORDER BY f.id + """, (dataset_id,)) + + for r in cur: + try: + geom = load_wkb(to_bytes(r["geom_wkb"])) + if not geom.is_valid: + x = dict(r) + x["_geom"] = geom + out.append(x) + except Exception: + continue + + cur.execute(f""" + SELECT + 'COVERAGE' AS kind, + NULL AS feature_id, + v.id AS coverage_geom_id, + 'COVERAGE' AS object_class, + v.cell_id, + c.cell_name, + v.source_record_index, + NULL AS coverage_id, + NULL AS raw_geotype, + NULL AS bbox_min_lon, + NULL AS bbox_min_lat, + NULL AS bbox_max_lon, + NULL AS bbox_max_lat, + {wkb_fn}(v.geom) AS geom_wkb + FROM chart_coverage v + JOIN chart_cell c ON c.id=v.cell_id + WHERE v.dataset_id=%s + ORDER BY v.id + """, (dataset_id,)) + + for r in cur: + try: + geom = load_wkb(to_bytes(r["geom_wkb"])) + if not geom.is_valid: + x = dict(r) + x["_geom"] = geom + out.append(x) + except Exception: + continue + + return out + + +def schema_columns(cur, database: str, tables: Sequence[str]) -> List[Dict[str, Any]]: + ph = ",".join(["%s"] * len(tables)) + sql = f""" + SELECT + TABLE_NAME, + ORDINAL_POSITION, + COLUMN_NAME, + DATA_TYPE, + COLUMN_TYPE, + IS_NULLABLE, + COLUMN_DEFAULT, + COLUMN_KEY, + EXTRA, + COLUMN_COMMENT + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA=%s + AND TABLE_NAME IN ({ph}) + ORDER BY TABLE_NAME, ORDINAL_POSITION + """ + return fetch_all(cur, sql, [database] + list(tables)) + + +def schema_indexes(cur, database: str, tables: Sequence[str]) -> List[Dict[str, Any]]: + ph = ",".join(["%s"] * len(tables)) + sql = f""" + SELECT + TABLE_NAME, + INDEX_NAME, + NON_UNIQUE, + SEQ_IN_INDEX, + COLUMN_NAME, + INDEX_TYPE + FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA=%s + AND TABLE_NAME IN ({ph}) + ORDER BY TABLE_NAME, INDEX_NAME, SEQ_IN_INDEX + """ + return fetch_all(cur, sql, [database] + list(tables)) + + +def schema_foreign_keys(cur, database: str, tables: Sequence[str]) -> List[Dict[str, Any]]: + ph = ",".join(["%s"] * len(tables)) + sql = f""" + SELECT + TABLE_NAME, + COLUMN_NAME, + CONSTRAINT_NAME, + REFERENCED_TABLE_NAME, + REFERENCED_COLUMN_NAME + FROM information_schema.KEY_COLUMN_USAGE + WHERE TABLE_SCHEMA=%s + AND TABLE_NAME IN ({ph}) + AND REFERENCED_TABLE_NAME IS NOT NULL + ORDER BY TABLE_NAME, CONSTRAINT_NAME, ORDINAL_POSITION + """ + return fetch_all(cur, sql, [database] + list(tables)) + + +def show_create(cur, tables: Sequence[str]) -> str: + chunks = [] + for table in tables: + try: + cur.execute(f"SHOW CREATE TABLE `{table}`") + row = cur.fetchone() + chunks.append(f"-- {table}\n{list(row.values())[-1]};\n") + except Exception as e: + chunks.append(f"-- {table}: ERROR {e}\n") + return "\n".join(chunks) + + +def get_column_type(columns: List[Dict[str, Any]], table: str, col: str) -> Optional[str]: + for r in columns: + if r["TABLE_NAME"] == table and r["COLUMN_NAME"] == col: + return str(r["COLUMN_TYPE"]) + return None + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +def parse_args(): + p = argparse.ArgumentParser( + description="CM93 normalization implementation precheck v11" + ) + p.add_argument("--v10-input", required=True) + p.add_argument("--host", default=os.getenv("MYSQL_HOST", "127.0.0.1")) + p.add_argument("--port", type=int, default=int(os.getenv("MYSQL_PORT", "3306"))) + p.add_argument("--user", default=os.getenv("MYSQL_USER", "root")) + p.add_argument("--password", default=os.getenv("MYSQL_PASSWORD")) + p.add_argument("--database", default=os.getenv("MYSQL_DATABASE", "cm93_chart")) + p.add_argument("--dataset-id", type=int, default=1) + p.add_argument("--bbox-tolerance", type=float, default=5e-6) + p.add_argument("--area-relative-tolerance", type=float, default=1e-9) + p.add_argument("--output", default=None) + return p.parse_args() + + +def main(): + args = parse_args() + + v10_src = Path(args.v10_input).expanduser().resolve() + if not v10_src.exists(): + print(f"V10 输入不存在：{v10_src}") + sys.exit(2) + + print("[1/7] V10.1 PASS gate ...") + v10_root = None + temp_v10 = None + + try: + v10_root, temp_v10 = open_v10_input(v10_src) + gate, golden_map = load_v10_gate(v10_root) + + print(json.dumps(gate, ensure_ascii=False, indent=2)) + + if not gate["gate_pass"]: + print("") + print("STOP：V10.1 gate 未通过。") + print("V11 不连接数据库、不继续实施预检。") + sys.exit(3) + + print(" V10.1 gate: PASS") + + if not args.password: + args.password = getpass.getpass("MySQL password: ") + + stamp = datetime.now().strftime("%Y%m%d_%H%M%S") + outdir = Path( + args.output + or f"cm93_normalization_implementation_precheck_v11_dataset_{args.dataset_id}_{stamp}" + ) + outdir.mkdir(parents=True, exist_ok=True) + + report: Dict[str, Any] = { + "version": "v11.0", + "generated_at": datetime.now().isoformat(timespec="seconds"), + "database": args.database, + "dataset_id": args.dataset_id, + "read_only": True, + "pbf": "OUT_OF_SCOPE", + "v10_gate": gate, + "warnings": [], + } + + conn = pymysql.connect( + host=args.host, + port=args.port, + user=args.user, + password=args.password, + database=args.database, + charset="utf8mb4", + cursorclass=DictCursor, + autocommit=False, + connect_timeout=20, + read_timeout=600, + write_timeout=30, + ) + + try: + with conn.cursor() as cur: + begin_read_only(cur, report["warnings"]) + report["mysql_version"] = scalar(cur, "SELECT VERSION()") + wkb_fn = probe_wkb_fn(cur) + + print("[2/7] 当前 55 条 invalid 与 golden vectors 重新对账 ...") + db_invalid = load_invalid_db_records(cur, args.dataset_id, wkb_fn) + + revalidation = [] + metadata_impact = [] + + for i, r in enumerate(db_invalid, start=1): + key = ( + f"AREA:{r['feature_id']}" + if r["kind"] == "AREA" + else f"COVERAGE:{r['coverage_geom_id']}" + ) + golden = golden_map.get(key) + + if golden is None: + row = { + "key": key, + "status": "FAIL_MISSING_GOLDEN", + } + revalidation.append(row) + continue + + candidate = normalize_invalid(r["_geom"]) + + if candidate is None: + revalidation.append({ + "key": key, + "status": "FAIL_NO_CANDIDATE", + }) + continue + + h = geometry_hash(candidate) + expected_hash = golden.get("canonical_wkb_sha256") + type_match = candidate.geom_type == golden.get("candidate_type") + part_match = ( + polygon_part_count(candidate) + == intv(golden.get("candidate_polygon_part_count")) + ) + + expected_area = floatv(golden.get("candidate_area_deg2")) + area = float(candidate.area) + if expected_area is None: + area_match = False + else: + scale = max(abs(area), abs(expected_area), 1e-30) + area_match = ( + abs(area - expected_area) + <= args.area_relative_tolerance * scale + ) + + try: + expected_bbox_raw = golden.get("candidate_bbox") + expected_bbox = ( + json.loads(expected_bbox_raw) + if isinstance(expected_bbox_raw, str) + else expected_bbox_raw + ) + if isinstance(expected_bbox, str): + # CSV 里 tuple 可能不是合法 JSON + import ast + expected_bbox = ast.literal_eval(expected_bbox) + expected_bbox = tuple(float(v) for v in expected_bbox) + except Exception: + expected_bbox = None + + bbox = tuple(float(v) for v in candidate.bounds) + bbox_match = ( + expected_bbox is not None + and len(expected_bbox) == 4 + and all( + abs(a - b) <= args.bbox_tolerance + for a, b in zip(bbox, expected_bbox) + ) + ) + + status = ( + "PASS" + if ( + candidate.is_valid + and candidate.geom_type in {"Polygon", "MultiPolygon"} + and h == expected_hash + and type_match + and part_match + and area_match + and bbox_match + ) + else "FAIL" + ) + + revalidation.append({ + "key": key, + "kind": r["kind"], + "feature_id": r.get("feature_id"), + "coverage_geom_id": r.get("coverage_geom_id"), + "object_class": r.get("object_class"), + "cell_name": r.get("cell_name"), + "status": status, + "candidate_valid": int(bool(candidate.is_valid)), + "candidate_type": candidate.geom_type, + "expected_type": golden.get("candidate_type"), + "type_match": int(type_match), + "candidate_part_count": polygon_part_count(candidate), + "expected_part_count": intv(golden.get("candidate_polygon_part_count")), + "part_match": int(part_match), + "candidate_hash": h, + "expected_hash": expected_hash, + "hash_match": int(h == expected_hash), + "candidate_area_deg2": area, + "expected_area_deg2": expected_area, + "area_match": int(area_match), + "candidate_bbox": bbox, + "expected_bbox": expected_bbox, + "bbox_match": int(bbox_match), + }) + + if r["kind"] == "AREA": + old_bbox = ( + floatv(r.get("bbox_min_lon")), + floatv(r.get("bbox_min_lat")), + floatv(r.get("bbox_max_lon")), + floatv(r.get("bbox_max_lat")), + ) + old_bbox_valid = all(v is not None for v in old_bbox) + feature_bbox_matches = ( + old_bbox_valid + and all( + abs(float(a) - float(b)) <= args.bbox_tolerance + for a, b in zip(old_bbox, bbox) + ) + ) + else: + old_bbox = None + feature_bbox_matches = None + + metadata_impact.append({ + "key": key, + "kind": r["kind"], + "feature_id": r.get("feature_id"), + "coverage_geom_id": r.get("coverage_geom_id"), + "object_class": r.get("object_class"), + "cell_name": r.get("cell_name"), + "old_feature_bbox": old_bbox, + "candidate_bbox": bbox, + "feature_bbox_matches_candidate": + ( + int(feature_bbox_matches) + if feature_bbox_matches is not None + else None + ), + "future_bbox_update_needed": + ( + int(not feature_bbox_matches) + if feature_bbox_matches is not None + else None + ), + }) + + print( + f" [{i:02d}/{len(db_invalid)}] {key} -> {status}" + ) + + rv_status = Counter(r.get("status") for r in revalidation) + + print("[3/7] Schema / index / FK 只读调查 ...") + tables = [ + "chart_feature", + "chart_area_geometry", + "chart_coverage", + "chart_import_job", + ] + columns = schema_columns(cur, args.database, tables) + indexes = schema_indexes(cur, args.database, tables) + fks = schema_foreign_keys(cur, args.database, tables) + creates = show_create(cur, tables) + + area_geom_type = get_column_type( + columns, "chart_area_geometry", "geom" + ) + coverage_geom_type = get_column_type( + columns, "chart_coverage", "geom" + ) + + print("[4/7] 4406 parent-no-geometry 与 counters 当前状态 ...") + parent_4406 = int(scalar(cur, """ + SELECT COUNT(*) + FROM chart_feature + WHERE dataset_id=%s + AND geometry_type='META' + AND raw_geotype=16 + """, (args.dataset_id,)) or 0) + + parent_issues = int(scalar(cur, """ + SELECT COUNT(*) + FROM chart_feature + WHERE dataset_id=%s + AND geometry_type='META' + AND raw_geotype=16 + AND geometry_issue IS NOT NULL + """, (args.dataset_id,)) or 0) + + total_feature_issues = int(scalar(cur, """ + SELECT COUNT(*) + FROM chart_feature + WHERE dataset_id=%s + AND geometry_issue IS NOT NULL + """, (args.dataset_id,)) or 0) + + import_invalid_sum = int(scalar(cur, """ + SELECT COALESCE(SUM(geometry_invalid_count),0) + FROM chart_import_job + WHERE dataset_id=%s + """, (args.dataset_id,)) or 0) + + bbox_update_count = sum( + int(r.get("future_bbox_update_needed") or 0) + for r in metadata_impact + ) + + schema_can_store_multipolygon = ( + area_geom_type is not None + and area_geom_type.lower().startswith("geometry") + ) + coverage_can_store_multipolygon = ( + coverage_geom_type is not None + and coverage_geom_type.lower().startswith("geometry") + ) + + report["summary"] = { + "v10_gate_pass": True, + "db_invalid_count": len(db_invalid), + "golden_revalidation_status_counts": dict(rv_status), + "golden_revalidation_pass_count": rv_status.get("PASS", 0), + "golden_revalidation_fail_count": + len(revalidation) - rv_status.get("PASS", 0), + "area_geometry_column_type": area_geom_type, + "coverage_geometry_column_type": coverage_geom_type, + "area_column_can_store_multipolygon": + schema_can_store_multipolygon, + "coverage_column_can_store_multipolygon": + coverage_can_store_multipolygon, + "future_area_bbox_update_needed_count": bbox_update_count, + "raw_geotype16_meta_count": parent_4406, + "raw_geotype16_with_issue_count": parent_issues, + "total_feature_geometry_issue_count": total_feature_issues, + "import_job_geometry_invalid_sum": import_invalid_sum, + } + + implementation_ready = ( + len(db_invalid) == 55 + and rv_status.get("PASS", 0) == 55 + and len(revalidation) == 55 + and schema_can_store_multipolygon + and coverage_can_store_multipolygon + ) + + report["summary"]["implementation_precheck_pass"] = ( + implementation_ready + ) + + print("[5/7] 生成实施契约与 SQL 计划（仅文本） ...") + + contract = f"""# CM93 OGC Normalization 实施契约 v11 + +## 当前门控 + +- V10.1 canonical gate：PASS +- 当前数据库 55 条 golden revalidation：{rv_status.get("PASS", 0)} / 55 PASS +- Area geometry SQL type：`{area_geom_type}` +- Coverage geometry SQL type：`{coverage_geom_type}` +- Area 可直接容纳 MultiPolygon：{schema_can_store_multipolygon} +- Coverage 可直接容纳 MultiPolygon：{coverage_can_store_multipolygon} +- 未来需要同步更新 Area feature bbox：{bbox_update_count} + +## 冻结 normalization 规则 + +仅当 Polygon/Area geometry 在 OGC/GEOS 意义下 invalid 时执行： + +1. 读取原始 polygon boundary。 +2. `unary_union(boundary)` 完成 noding。 +3. `polygonize_full()` 生成 planar faces。 +4. 对每个 face 的 representative point 使用原始 ring 的 **odd-even parity**。 +5. 只保留 odd-even inside faces。 +6. union selected faces。 +7. 使用 GEOS/Shapely `normalize()` canonicalize。 +8. 最终结果必须是 `Polygon` 或 `MultiPolygon`。 +9. 必须 `is_valid == true`。 +10. 现有 valid geometry 必须原样 PASS-THROUGH，不做 canonical rewrite。 + +## retraced connector + +已确认的 retraced connector 是零面积拓扑桥接线： +- 不作为 normalized Area 面积保存； +- raw source / raw feature blob 继续保留； +- 不从原始 CM93 记录中删除。 + +## bbox + +如果 normalized Area bbox 与当前 `chart_feature.bbox_*` 不同： +- normalized geometry 保存时必须同步更新 feature bbox； +- 不得继续保留 connector 扩大的旧 bbox 作为 normalized geometry bbox。 + +## raw_geotype=16 + +`raw_geotype=16 && geometry_type=META`： +- 不要求 spatial geometry； +- 不设置 `geometry_issue='geometry missing'`； +- 不增加 `geometry_invalid_count`； +- 不制造 EMPTY geometry。 + +## 原始数据保留 + +实施 normalization 时不得删除： +- `raw_feature_blob` +- `chart_raw_record` +- raw geotype +- source_record_index +- object attributes + +## PBF + +本实施契约仍不启动 PBF。 +""" + + sql_plan = f"""-- CM93 normalization migration plan v11 +-- !!! 仅计划文本，V11 没有执行任何 SQL !!! +-- +-- 当前 Area geom 类型: {area_geom_type} +-- 当前 Coverage geom 类型: {coverage_geom_type} +-- 未来 Area bbox 更新数量（当前 Dataset 1）: {bbox_update_count} +-- +-- 实施建议顺序： +-- +-- STEP 1 +-- 先修改 importer： +-- raw_geotype=16 / META 不再写 geometry missing / invalid counter。 +-- +-- STEP 2 +-- importer 在 Area/Coverage geometry 写入前： +-- valid -> 原样写入 +-- invalid -> odd-even normalization -> GEOS normalize -> valid Polygon/MultiPolygon +-- +-- STEP 3 +-- normalized Area 写入时重新计算： +-- bbox_min_lon +-- bbox_min_lat +-- bbox_max_lon +-- bbox_max_lat +-- +-- STEP 4 +-- 对现有 Dataset ID={args.dataset_id} 的历史回填应单独执行， +-- 不应与 importer 代码修改混在一次不可回退操作中。 +-- +-- STEP 5 +-- 历史回填前必须备份/快照，并再次用 V10 golden vectors 校验。 +-- +-- 本文件故意不生成可直接执行的 UPDATE SQL。 +-- 原因：当前阶段只冻结规则和 schema 影响，不执行数据修改。 +""" + + (outdir / "proposed_implementation_contract_v11.md").write_text( + contract, encoding="utf-8" + ) + (outdir / "proposed_migration_plan_v11.sql.txt").write_text( + sql_plan, encoding="utf-8" + ) + + print("[6/7] 写报告 ...") + write_csv(outdir / "golden_revalidation_v11.csv", revalidation) + write_csv(outdir / "schema_columns_v11.csv", columns) + write_csv(outdir / "schema_indexes_v11.csv", indexes) + write_csv(outdir / "schema_foreign_keys_v11.csv", fks) + write_csv( + outdir / "normalization_metadata_impact_v11.csv", + metadata_impact + ) + (outdir / "show_create_tables_v11.txt").write_text( + creates, encoding="utf-8" + ) + (outdir / "implementation_precheck_report_v11.json").write_text( + json.dumps(report, ensure_ascii=False, indent=2, default=str), + encoding="utf-8" + ) + + lines = [] + lines.append("CM93 Normalization 实施前只读预检 v11") + lines.append("=" * 88) + lines.append(f"Generated: {report['generated_at']}") + lines.append("") + lines.append("V10 GATE") + lines.append(" PASS") + lines.append("") + lines.append("GOLDEN REVALIDATION") + lines.append( + f" DB invalid: {len(db_invalid)}" + ) + lines.append( + f" PASS: {rv_status.get('PASS', 0)}" + ) + lines.append( + f" FAIL: {len(revalidation) - rv_status.get('PASS', 0)}" + ) + lines.append("") + lines.append("SCHEMA") + lines.append( + f" chart_area_geometry.geom: {area_geom_type}" + ) + lines.append( + f" chart_coverage.geom: {coverage_geom_type}" + ) + lines.append( + f" Area supports MultiPolygon directly: " + f"{schema_can_store_multipolygon}" + ) + lines.append( + f" Coverage supports MultiPolygon directly: " + f"{coverage_can_store_multipolygon}" + ) + lines.append("") + lines.append("METADATA") + lines.append( + f" future Area bbox updates needed: {bbox_update_count}" + ) + lines.append("") + lines.append("GEOTYPE16") + lines.append( + f" META raw_geotype=16: {parent_4406}" + ) + lines.append( + f" still has geometry_issue: {parent_issues}" + ) + lines.append( + f" total feature issues: {total_feature_issues}" + ) + lines.append( + f" import invalid counter sum:{import_invalid_sum}" + ) + lines.append("") + lines.append("OVERALL") + lines.append( + f" IMPLEMENTATION PRECHECK PASS: " + f"{'YES' if implementation_ready else 'NO'}" + ) + lines.append("") + lines.append( + "NO DATABASE WRITE / NO ALTER TABLE / NO PBF" + ) + + (outdir / "implementation_precheck_summary_v11.txt").write_text( + "\n".join(lines), encoding="utf-8" + ) + + conn.rollback() + + print("[7/7] 压缩 ...") + zpath = outdir.with_suffix(".zip") + with zipfile.ZipFile( + zpath, "w", compression=zipfile.ZIP_DEFLATED + ) as z: + for p in sorted(outdir.rglob("*")): + if p.is_file(): + z.write(p, p.relative_to(outdir.parent)) + + print("") + print( + "IMPLEMENTATION PRECHECK PASS:", + "YES" if report["summary"]["implementation_precheck_pass"] else "NO" + ) + print(f"ZIP: {zpath.resolve()}") + print("把这个 V11 ZIP 上传给 ChatGPT。") + + finally: + try: + conn.close() + except Exception: + pass + + finally: + if temp_v10 and temp_v10.exists(): + shutil.rmtree(temp_v10, ignore_errors=True) + + +if __name__ == "__main__": + main() diff --git a/cm93_v10_1_result_digest_v10_2.py b/cm93_v10_1_result_digest_v10_2.py new file mode 100644 index 0000000..39fb7ab --- /dev/null +++ b/cm93_v10_1_result_digest_v10_2.py @@ -0,0 +1,475 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +CM93 V10.1 结果汇总与完整性验证器 +Version: v10.2-result-digest + +用途： +- 不连接 MySQL。 +- 不重新运行 normalization。 +- 直接读取 V10.1 的输出目录或 ZIP。 +- 自动判断 V10.1 是否整体 PASS。 +- 生成一个很小的结果摘要 ZIP，便于上传给 ChatGPT。 + +支持输入： +1. V10.1 完整输出目录，例如： + cm93_normalization_canonical_v10_1_dataset_1_20260831_215602 + +2. V10.1 完整 ZIP，例如： + cm93_normalization_canonical_v10_1_dataset_1_20260831_215602.zip + +3. 只有部分 CSV 的 ZIP。 + 此时脚本会明确显示“INCOMPLETE”，不会误判整体 PASS。 + +执行： + python cm93_v10_1_result_digest_v10_2.py ^ + --input "D:\...\cm93_normalization_canonical_v10_1_dataset_1_20260831_215602" + +或： + + python cm93_v10_1_result_digest_v10_2.py ^ + --input "D:\...\cm93_normalization_canonical_v10_1_dataset_1_20260831_215602.zip" + +输出： +- V10_1_结果摘要_v10_2.txt +- V10_1_结果摘要_v10_2.json +- cm93_v10_1_result_digest_v10_2.zip + +只读，不修改原始结果。 +""" + +from __future__ import annotations + +import argparse +import csv +import json +import shutil +import sys +import tempfile +import zipfile +from collections import Counter +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, List, Optional + + +EXPECTED_FILES = { + "summary": "normalization_canonical_summary_v10_1.txt", + "report": "normalization_canonical_report_v10_1.json", + "golden_csv": "normalization_golden_vectors_v10_1.csv", + "golden_json": "normalization_golden_vectors_v10_1.json", + "variant": "normalization_variant_results_v10_1.csv", + "face_order": "normalization_face_order_results_v10_1.csv", + "failures": "normalization_canonical_failures_v10_1.csv", + "bbox": "normalization_bbox_metadata_impact_v10_1.csv", +} + + +def parse_args(): + p = argparse.ArgumentParser( + description="CM93 V10.1 result digest verifier v10.2" + ) + p.add_argument("--input", required=True, help="V10.1 输出目录或 ZIP") + p.add_argument("--output-dir", default=None) + return p.parse_args() + + +def find_file(root: Path, filename: str) -> Optional[Path]: + direct = root / filename + if direct.exists(): + return direct + + hits = list(root.rglob(filename)) + if hits: + return hits[0] + + return None + + +def read_csv(path: Path) -> List[Dict[str, str]]: + with path.open("r", encoding="utf-8-sig", newline="") as f: + return list(csv.DictReader(f)) + + +def intv(v: Any, default: int = 0) -> int: + try: + return int(v) + except Exception: + return default + + +def boolv(v: Any) -> bool: + if isinstance(v, bool): + return v + if v is None: + return False + s = str(v).strip().lower() + return s in {"1", "true", "yes", "y", "pass"} + + +def main(): + args = parse_args() + src = Path(args.input).expanduser().resolve() + + if not src.exists(): + print(f"输入不存在：{src}") + sys.exit(2) + + temp_dir = None + + try: + if src.is_file() and src.suffix.lower() == ".zip": + temp_dir = Path(tempfile.mkdtemp(prefix="cm93_v10_1_digest_")) + with zipfile.ZipFile(src, "r") as z: + z.extractall(temp_dir) + root = temp_dir + elif src.is_dir(): + root = src + else: + print("输入必须是 V10.1 输出目录或 ZIP。") + sys.exit(2) + + found: Dict[str, Optional[Path]] = { + key: find_file(root, filename) + for key, filename in EXPECTED_FILES.items() + } + + present = {k: bool(v) for k, v in found.items()} + + digest: Dict[str, Any] = { + "version": "v10.2-result-digest", + "generated_at": datetime.now().isoformat(timespec="seconds"), + "input": str(src), + "files_present": present, + "complete_core_set": all( + present[k] + for k in ( + "report", + "golden_csv", + "variant", + "face_order", + "failures", + "bbox", + ) + ), + "checks": {}, + "overall": "INCOMPLETE", + } + + checks = digest["checks"] + + # -------------------------------------------------------------- + # Report JSON + # -------------------------------------------------------------- + if found["report"]: + report = json.loads( + found["report"].read_text(encoding="utf-8") + ) + summary = report.get("summary", {}) + + checks["report"] = { + "overall_pass": summary.get("overall_pass"), + "total_polygon_records": summary.get("total_polygon_records"), + "valid_polygon_records": summary.get("valid_polygon_records"), + "invalid_polygon_records": summary.get("invalid_polygon_records"), + "valid_policy_changed_count": + summary.get("valid_policy_changed_count"), + "valid_hash_changed_count": + summary.get("valid_hash_changed_count"), + "golden_vector_count": + summary.get("golden_vector_count"), + "golden_record_status_counts": + summary.get("golden_record_status_counts"), + "input_variant_case_count": + summary.get("input_variant_case_count"), + "input_variant_status_counts": + summary.get("input_variant_status_counts"), + "face_order_case_count": + summary.get("face_order_case_count"), + "face_order_hash_mismatch_count": + summary.get("face_order_hash_mismatch_count"), + "canonical_topology_failure_count": + summary.get("canonical_topology_failure_count"), + "repeat_hash_failure_count": + summary.get("repeat_hash_failure_count"), + "input_variant_failure_record_count": + summary.get("input_variant_failure_record_count"), + "future_area_feature_bbox_update_needed_count": + summary.get("future_area_feature_bbox_update_needed_count"), + } + + # -------------------------------------------------------------- + # Golden vectors + # -------------------------------------------------------------- + if found["golden_csv"]: + rows = read_csv(found["golden_csv"]) + statuses = Counter( + str(r.get("record_status", "")).strip() + for r in rows + ) + hashes = [ + r.get("canonical_wkb_sha256", "") + for r in rows + if r.get("canonical_wkb_sha256") + ] + + checks["golden"] = { + "rows": len(rows), + "status_counts": dict(statuses), + "unique_hash_count": len(set(hashes)), + "missing_hash_count": + sum(not bool(r.get("canonical_wkb_sha256")) for r in rows), + "variant_failure_records": + sum( + intv(r.get("input_variant_failure_count")) > 0 + for r in rows + ), + "face_order_unstable_records": + sum( + not boolv(r.get("face_order_hash_stable")) + for r in rows + ), + "repeat_unstable_records": + sum( + not boolv(r.get("repeat_hash_stable")) + for r in rows + ), + "canonical_topology_changed_records": + sum( + not boolv(r.get("canonical_topology_unchanged")) + for r in rows + ), + } + + # -------------------------------------------------------------- + # Variant + # -------------------------------------------------------------- + if found["variant"]: + rows = read_csv(found["variant"]) + statuses = Counter( + str(r.get("status", "")).strip() + for r in rows + ) + + checks["variant"] = { + "rows": len(rows), + "status_counts": dict(statuses), + "fail_rows": + sum(str(r.get("status", "")).strip() != "PASS" for r in rows), + "hash_mismatch_rows": + sum( + not boolv(r.get("matches_baseline_hash")) + for r in rows + if str(r.get("status", "")).strip() != "EXCEPTION" + ), + "topology_mismatch_rows": + sum( + not boolv(r.get("topology_equal_baseline")) + for r in rows + if str(r.get("status", "")).strip() != "EXCEPTION" + ), + } + + # -------------------------------------------------------------- + # Face order + # -------------------------------------------------------------- + if found["face_order"]: + rows = read_csv(found["face_order"]) + checks["face_order"] = { + "rows": len(rows), + "hash_mismatch_rows": + sum( + not boolv(r.get("matches_baseline_hash")) + for r in rows + ), + } + + # -------------------------------------------------------------- + # Failure CSV + # -------------------------------------------------------------- + if found["failures"]: + rows = read_csv(found["failures"]) + # 只有表头时 DictReader 返回 0 rows，正是期望。 + checks["failures"] = { + "rows": len(rows), + } + + # -------------------------------------------------------------- + # BBox metadata impact + # -------------------------------------------------------------- + if found["bbox"]: + rows = read_csv(found["bbox"]) + checks["bbox"] = { + "rows": len(rows), + "bbox_changed_rows": + sum( + boolv(r.get("bbox_changed_over_tolerance")) + for r in rows + ), + "future_feature_bbox_update_needed_rows": + sum( + boolv(r.get("future_feature_bbox_update_needed")) + for r in rows + ), + "rows_with_connector": + sum( + float(r.get("discarded_connector_length_deg") or 0) > 0 + for r in rows + ), + } + + # -------------------------------------------------------------- + # Independent overall verdict + # -------------------------------------------------------------- + required_for_pass = all( + k in checks + for k in ( + "report", + "golden", + "variant", + "face_order", + "failures", + "bbox", + ) + ) + + if required_for_pass: + report_check = checks["report"] + golden = checks["golden"] + variant = checks["variant"] + face = checks["face_order"] + failures = checks["failures"] + + pass_conditions = [ + boolv(report_check.get("overall_pass")), + intv(report_check.get("invalid_polygon_records")) == 55, + intv(report_check.get("valid_policy_changed_count")) == 0, + intv(report_check.get("valid_hash_changed_count")) == 0, + + golden["rows"] == 55, + golden["status_counts"].get("PASS", 0) == 55, + golden["status_counts"].get("FAIL", 0) == 0, + golden["missing_hash_count"] == 0, + golden["variant_failure_records"] == 0, + golden["face_order_unstable_records"] == 0, + golden["repeat_unstable_records"] == 0, + golden["canonical_topology_changed_records"] == 0, + + variant["rows"] == 55 * 6, + variant["fail_rows"] == 0, + variant["hash_mismatch_rows"] == 0, + variant["topology_mismatch_rows"] == 0, + + face["rows"] == 55 * 4, + face["hash_mismatch_rows"] == 0, + + failures["rows"] == 0, + ] + + digest["overall"] = ( + "PASS" + if all(pass_conditions) + else "FAIL" + ) + digest["pass_conditions"] = pass_conditions + else: + digest["overall"] = "INCOMPLETE" + + # -------------------------------------------------------------- + # Output + # -------------------------------------------------------------- + outdir = ( + Path(args.output_dir).expanduser().resolve() + if args.output_dir + else src.parent + ) + outdir.mkdir(parents=True, exist_ok=True) + + txt_path = outdir / "V10_1_结果摘要_v10_2.txt" + json_path = outdir / "V10_1_结果摘要_v10_2.json" + zip_path = outdir / "cm93_v10_1_result_digest_v10_2.zip" + + json_path.write_text( + json.dumps(digest, ensure_ascii=False, indent=2), + encoding="utf-8" + ) + + lines = [] + lines.append("CM93 V10.1 结果汇总与完整性验证 v10.2") + lines.append("=" * 78) + lines.append(f"Input: {src}") + lines.append(f"Overall: {digest['overall']}") + lines.append("") + + lines.append("FILES") + for key, filename in EXPECTED_FILES.items(): + lines.append( + f" {key:12s} " + f"{'YES' if present[key] else 'NO ':3s} " + f"{filename}" + ) + + for section, value in checks.items(): + lines.append("") + lines.append(section.upper()) + for k, v in value.items(): + lines.append(f" {k}: {v}") + + if digest["overall"] == "INCOMPLETE": + lines.append("") + lines.append( + "INCOMPLETE：当前输入不足以判定 V10.1 整体 PASS。" + ) + elif digest["overall"] == "PASS": + lines.append("") + lines.append( + "PASS：V10.1 canonical normalization 全部门槛通过。" + ) + else: + lines.append("") + lines.append( + "FAIL：至少一个 V10.1 canonical validation 门槛未通过。" + ) + + txt_path.write_text( + "\n".join(lines), + encoding="utf-8" + ) + + with zipfile.ZipFile( + zip_path, + "w", + compression=zipfile.ZIP_DEFLATED + ) as z: + z.write(txt_path, txt_path.name) + z.write(json_path, json_path.name) + + # 把关键小文件一起放进 digest ZIP。 + for key in ( + "summary", + "report", + "golden_csv", + "variant", + "face_order", + "failures", + "bbox", + ): + p = found.get(key) + if p and p.exists(): + z.write(p, p.name) + + print("") + print(f"Overall: {digest['overall']}") + print(f"TXT: {txt_path}") + print(f"JSON: {json_path}") + print(f"ZIP: {zip_path}") + print("") + print("把这个很小的 digest ZIP 上传给 ChatGPT。") + + finally: + if temp_dir and temp_dir.exists(): + shutil.rmtree(temp_dir, ignore_errors=True) + + +if __name__ == "__main__": + main() diff --git a/codex/10_第一阶段接手任务.md b/codex/10_第一阶段接手任务.md new file mode 100644 index 0000000..2a55bc3 --- /dev/null +++ b/codex/10_第一阶段接手任务.md @@ -0,0 +1,87 @@ +# 已归档：该任务已由 v0.2.0 实现 + +当前请执行 `codex/20_导入与审计接手.md`。 + +--- + +# Codex：CM93 → MySQL 第一阶段接手任务 + +## 目标 + +在现有 starter 基线上完成可实际导入大目录 CM93 的 importer。 + +最终标准不是“部分对象能显示”，而是： +- cell 可完整解析； +- feature/attribute 计数可核对； +- geometry 可追踪； +- 数据可断点续导； +- 精度满足 `docs/精度规范.md`。 + +## 开始前 + +1. 完整阅读： + - `AGENTS.md` + - `README.md` + - `docs/精度规范.md` + - `docs/数据库设计.md` +2. 运行： + - `python --version` + - `cm93db env-check` + - `cm93db db-check` +3. 不修改第二阶段 PBF 范围。 + +## 必做清单 + +- [ ] 完成 CM93 256-byte substitution decoder 的独立实现 +- [ ] 完成 prolog + 128-byte header +- [ ] 校验 cell 文件长度公式 +- [ ] 完成 vector record table +- [ ] 完成 2D point table +- [ ] 完成 3D sounding table +- [ ] 完成 feature record table +- [ ] 按 `CM93OBJ.DIC` 映射 object class +- [ ] 按 `CM93ATTR.DIC/ATTRLUT.DIC` typed decode attribute +- [ ] 支持 B/S/R/C/L/W/G 属性类型 +- [ ] 保留 raw feature / raw attribute +- [ ] 完成 Point +- [ ] 完成 Line edge chain + reverse flag +- [ ] 完成 Area ring 组装 +- [ ] 完成 Sounding 深度 +- [ ] 完成 `_m_sor` coverage +- [ ] 保存 `_wgsox/_wgsoy` +- [ ] 形成并文档化 WGS offset policy +- [ ] 标准化 geometry 为 WGS84 double +- [ ] 完成 MySQL bulk insert +- [ ] 每 cell 独立事务 +- [ ] DONE 跳过 / FAILED 重试 +- [ ] 未知 ID 不静默丢弃 +- [ ] 完成 import summary +- [ ] 增加精度审计命令 +- [ ] 用实际 CM93 小样本全链路验证 + +## 禁止 + +- 不做 PBF +- 不 simplify +- 不 quantize +- 不为了通过 geometry validity 静默改数据 +- 不使用 MySQL 8 专用 SQL +- 不复制 OpenCPN GPL 源码 + +## 完成报告 + +写入： +`docs/报告/CM93_MySQL导入器完成报告_v1.md` + +至少包括： +- 实现范围 +- 实际样本 +- cell 数 +- source/imported feature 数 +- attribute 数 +- unknown 数 +- geometry invalid/repaired 数 +- 精度审计结果 +- MySQL 版本 +- Python 版本 +- 已知限制 diff --git a/codex/20_导入与审计接手.md b/codex/20_导入与审计接手.md new file mode 100644 index 0000000..9467a18 --- /dev/null +++ b/codex/20_导入与审计接手.md @@ -0,0 +1,82 @@ +# Codex：CM93JAPAN 导入与审计接手任务 + +## 当前状态 + +`cm93db import` 和 `cm93db audit` 已实现。 + +不要重新设计 parser，也不要开始 PBF。 + +## 固定环境 + +- Windows 11 +- Python 3.12.10 +- MySQL 5.7.34-log +- DB: cm93_chart +- CM93: `D:\Ship\CM93V2\CM93JAPAN` + +## 接手顺序 + +- [ ] 阅读 AGENTS.md +- [ ] 阅读 README.md +- [ ] 阅读 docs/精度规范.md +- [ ] 阅读 docs/CM93解析与审计设计.md +- [ ] `cm93db env-check` +- [ ] `cm93db db-check` +- [ ] `cm93db scan` +- [ ] `cm93db inspect-dicts` +- [ ] G 层 10-cell pilot import +- [ ] 同批 audit +- [ ] 若失败，只修真实失败原因 +- [ ] Pilot PASS 后全量 import +- [ ] 全量 audit +- [ ] 记录 FAILED cell，不得静默跳过 + +## Pilot + +```powershell +& ".\.venv\Scripts\cm93db.exe" import --scale G --limit 10 +& ".\.venv\Scripts\cm93db.exe" audit --scale G --limit 10 --precision-samples 32 +``` + +## 全量 + +```powershell +& ".\.venv\Scripts\cm93db.exe" import +& ".\.venv\Scripts\cm93db.exe" audit --precision-samples 256 +``` + +## 修复边界 + +允许修改： + +- src/cm93db/* +- SQL（仅确认 MySQL 5.7.34 实际兼容问题时） +- docs +- scripts + +禁止为了“跑完”而： + +- 跳过无法解析 attribute +- 降低坐标精度 +- round/simplify geometry +- 删除 raw preservation +- 对 Polygon 静默 make_valid +- 把失败 cell 标记 DONE +- 改为 MySQL 8 语法 +- 开始 PBF + +## 完成报告 + +写入：`docs/报告/CM93JAPAN_导入审计完成报告_v1.md` + +记录： + +- source cell 数及 Z/A～G 分布 +- DONE / FAILED / SKIP +- features/attributes +- unknown IDs +- raw preservation +- precision audit +- geometry validity informational issues +- MySQL/Python 版本 +- 失败 cell 清单 diff --git a/codex/ARCHITECTURE.md b/codex/ARCHITECTURE.md new file mode 100644 index 0000000..8446b6b --- /dev/null +++ b/codex/ARCHITECTURE.md @@ -0,0 +1,55 @@ +# 架构设计 + +## 数据流 + +CM93 原始目录 +→ SourceScanner +→ DictionaryLoader +→ CellBinaryReader +→ FeatureDecoder +→ CoverageResolver +→ PrecisionProjection +→ ChartFeature Model +→ MySQLBatchWriter +→ MySQL 母数据库 + +## 分层 + +### `cm93` +只负责格式解析和坐标/几何语义，不知道 MySQL。 + +### `model` +定义稳定中间模型。 + +### `database` +负责 MySQL 5.7 写入、事务、批量提交和恢复。 + +### `cli` +只做命令组织。 + +## 数据库与 CM93 边界 + +写入数据库以后： +- geometry 已标准化为 WGS84 经纬度； +- 原始 CM93 元数据仍保留； +- 后续 PBF builder 不需要理解 CM93 二进制。 + +## Cell 事务 + +每个 cell 为独立事务边界： + +DISCOVERED → IMPORTING → DONE + +失败： + +IMPORTING → FAILED + +再次运行时： +- DONE 默认跳过； +- FAILED 可重试； +- 强制参数可重新导入指定 cell。 + +## Geometry + +母数据库保留标准化 geometry，但不依赖 geometry 作为唯一事实来源。 +原始记录和 source index 是审计入口。 diff --git a/docs/CM93解析与审计设计.md b/docs/CM93解析与审计设计.md new file mode 100644 index 0000000..83cd3ff --- /dev/null +++ b/docs/CM93解析与审计设计.md @@ -0,0 +1,150 @@ +# CM93 解析与审计设计 v1 + +## 1. 原始格式解析 + +独立 Python 实现依据已公开的 OpenCPN CM93 行为进行互操作核对，但不复制 OpenCPN 的类结构或 GUI/S52 代码。 + +核心数据顺序： + +1. 10-byte prolog +2. 128-byte header +3. vector records +4. 3D point descriptors +5. 2D points +6. feature records + +完整性条件： + +`word0 + table1_length + table2_length == file_size` + +且 `word0 == 138`。 + +进一步核对： + +- vector point 总数 = header `n_vector_record_points` +- 3D point 总数 = header `m_50` +- feature 总数 = header `usn_feature_records` +- attribute payload bytes = header `m_78` +- feature geometry refs = header `m_46 + m_4a` +- related object pointers = header `m_nrelated_object_pointers` +- 最终 cursor = EOF + +任何一项不成立，该 cell 解析失败，不允许“尽量导入”。 + +## 2. 属性 + +按 `CM93ATTR.DIC/ATTRLUT.DIC`： + +- B: 1-byte integer +- W: uint16 / 10.0 +- G: int32 +- R: float32，标准化后以 Python double/MySQL DOUBLE 保存 +- S: NUL-terminated string +- C: 3-byte prefix + NUL-terminated string，prefix 与 text 均保留 +- L: count byte + count 个 byte values + +raw value bytes 始终另存。 + +## 3. Geometry + +Feature geotype 低 4 bit： + +- 1 Point +- 2 Line +- 4 Area +- 8 3D Sounding + +Line/Area edge index：低 13 bit 是 edge index，高位是 usage；`usage & 4` 表示反向遍历 edge。 + +Area 按 edge chain 闭合形成 contour/ring。系统不会调用 `make_valid` 或改变原始顶点坐标。 + +## 4. 坐标 + +CM93 raw X/Y：0..65535。 + +Cell 变换： + +`easting = raw_x * x_rate + easting_min` + +`northing = raw_y * y_rate + northing_min` + +CM93 半长轴：`6378388.0 m`。 + +若 coverage 给出 `_wgsox/_wgsoy`，标准化母数据采用： + +`easting -= wgsox` + +`northing -= wgsoy` + +然后执行 CM93 Mercator → lon/lat。 + +Coverage polygon 自身保留未偏移坐标，用于判断 feature 属于哪个 coverage；offset 元数据独立保存。 + +## 5. Sounding + +CM93 Z： + +- `z >= 12000` → `depth = z - 12000` +- 否则 → `depth = z / 10.0` + +每个 sounding point 同时保存 raw_x/raw_y/raw_z、depth 和 WGS84 Point。 + +## 6. Raw preservation + +每个 cell 保存 decoded： + +- prolog +- header +- vector table +- point3d table +- point2d table +- feature table + +较大的 table 按 512 KiB 分块写入 `chart_raw_record`，避免 MySQL `max_allowed_packet` 造成单大包问题。 + +审计要求所有 raw chunk `OCTET_LENGTH` 总和 = 原 cell 文件字节数。 + +## 7. 审计分层 + +### Source parser audit + +重新读取原 CM93，验证所有结构计数和 EOF。 + +### Database completeness audit + +逐 cell 对比源数据与： + +- chart_feature +- chart_feature_attribute +- point/line/area geometry +- chart_coverage +- chart_sounding_point + +### Raw preservation audit + +比较 raw chunk 总字节数与原文件长度。 + +### Hash audit + +比较数据库 `source_sha256` 与当前原文件。 + +### Precision audit + +从 Z/A～G 分层抽样，重新从 CM93 raw geometry 生成 WKB，然后从 MySQL `ST_AsWKB()` 回读，使用 `equals_exact` 比较。 + +默认 tolerance：`1e-12 degree`。 + +该标准比最初讨论的 `1e-9 degree` 更严格。 + +## 8. 不做的事情 + +第一阶段不做： + +- PBF +- zoom selection +- simplify +- topology repair +- scale 去重 +- cartographic style 映射 + +这些都属于第二阶段。 diff --git a/docs/Win11环境安装.md b/docs/Win11环境安装.md new file mode 100644 index 0000000..c5236e4 --- /dev/null +++ b/docs/Win11环境安装.md @@ -0,0 +1,50 @@ +# Windows 11 + pyenv 环境安装 + +已知机器： +- pyenv 已安装 +- Python 3.9.2 当前全局默认 +- Python 3.12.10 已安装 + +本项目不要修改全局默认版本，推荐只设置项目 local： + +```powershell +cd <项目目录> +pyenv local 3.12.10 +python --version +``` + +必须为： + +```text +Python 3.12.10 +``` + +建立虚拟环境： + +```powershell +python -m venv .venv +.\.venv\Scripts\Activate.ps1 +python -m pip install --upgrade pip +pip install -e . +``` + +或者直接： + +```powershell +.\scripts\setup_env.ps1 +``` + +数据库配置： + +```powershell +Copy-Item .env.example .env +notepad .env +``` + +然后： + +```powershell +cm93db env-check +cm93db db-init --create-database +cm93db db-check +``` diff --git a/docs/外部参考.md b/docs/外部参考.md new file mode 100644 index 0000000..1fa63ae --- /dev/null +++ b/docs/外部参考.md @@ -0,0 +1,25 @@ +# 外部参考 + +本项目独立实现 CM93 importer，不复制 OpenCPN GPL 源码。 + +用于核对行为和 MySQL 兼容性的公开参考： + +- OpenCPN CM93 implementation: + https://github.com/OpenCPN/OpenCPN/blob/master/gui/src/cm93.cpp + +- MySQL 5.7 Spatial Data Types: + https://dev.mysql.com/doc/refman/5.7/en/spatial-type-overview.html + +- MySQL 5.7 Geometry Class: + https://dev.mysql.com/doc/refman/5.7/en/gis-class-geometry.html + +- MySQL 5.7 Spatial Index: + https://dev.mysql.com/doc/refman/5.7/en/creating-spatial-indexes.html + +注意：MySQL 5.7 Geometry 坐标本身为 double precision，但对实际 SRID 的地理计算能力不能按 MySQL 8 的行为假设。 + +## v0.2.0 互操作说明 + +CM93 binary 行为用于互操作核对的核心公开实现是 OpenCPN `cm93.cpp` 的 decode/header/table/feature/attribute/Transform 逻辑。 + +本工程采用独立 Python 数据模型、数据库模型和实现结构。CM93 substitution mapping 属于解码互操作所需的固定映射；如未来公开分发或商业分发本工具，应自行进行许可证/数据授权审查。 diff --git a/docs/导入与审计.md b/docs/导入与审计.md new file mode 100644 index 0000000..dc3d37c --- /dev/null +++ b/docs/导入与审计.md @@ -0,0 +1,151 @@ +# CM93 导入与审计操作说明 + +## 1. 适用环境 + +- Windows 11 +- Python 3.12.10 +- MySQL 5.7.34 +- 已完成 `cm93db db-init --create-database` + +## 2. 正式导入 + +如果 `.env` 已配置： + +```text +CM93_SOURCE_DIR=D:\Ship\CM93V2\CM93_2014 +``` + +可直接运行： + +```powershell +& ".\.venv\Scripts\cm93db.exe" import +``` + +也可显式给路径： + +```powershell +& ".\.venv\Scripts\cm93db.exe" import D:\Ship\CM93V2\CM93_2014 +``` + +默认行为： + +- 递归扫描 Z/A/B/C/D/E/F/G； +- 每个 cell 独立事务； +- DONE + 文件 SHA256 相同自动跳过； +- FAILED cell 下次自动重试； +- 保存 typed attributes； +- 保存标准化 WGS84 geometry； +- 保存 decoded CM93 原始 table block； +- `_wgsox/_wgsoy` 默认按 `embedded` policy 应用； +- 不 simplify / snap / quantize / round； +- 不自动 `make_valid`。 + +### 先小批验证 + +推荐正式 30,097 cell 前先跑： + +```powershell +& ".\.venv\Scripts\cm93db.exe" import --scale G --limit-cells 5 +``` + +然后： + +```powershell +& ".\.venv\Scripts\cm93db.exe" audit --sample-cells 5 +``` + +通过后再跑完整导入。 + +### 只导某些尺度 + +```powershell +& ".\.venv\Scripts\cm93db.exe" import --scale G --scale F +``` + +### 强制重导 + +```powershell +& ".\.venv\Scripts\cm93db.exe" import --force +``` + +## 3. 审计 + +### 快速审计 + +```powershell +& ".\.venv\Scripts\cm93db.exe" audit --sample-cells 20 +``` + +包括： + +1. 数据库 cell / feature / attribute 计数； +2. 每个 DONE cell 的 source count 与数据库 count 对账； +3. Point / Line / Area geometry presence； +4. 随机重新解析源 CM93 cell； +5. 源 feature / attribute count 对比； +6. 从 MySQL `ST_AsWKB` 读回 geometry； +7. 与重新计算的 double precision geometry 做逐顶点比较； +8. sounding raw X/Y/Z、depth、lon/lat 比较。 + +默认坐标容差： + +```text +1e-12 degree +``` + +这个容差用于发现数据库序列化/写入造成的精度损失，不是让 importer 人为 round 到该精度。 + +### 全源最终审计 + +完整导入后： + +```powershell +& ".\.venv\Scripts\cm93db.exe" audit --all-source +``` + +这会重新解析所有 DONE cell，耗时明显高于随机抽样，但适合作为母数据库最终验收。 + +报告自动写入： + +```text +logs/audit-YYYYMMDD-HHMMSS.json +``` + +## 4. offset policy + +正式母库默认： + +```text +--offset-policy embedded +``` + +含义： + +1. `_m_sor` coverage 自身使用未偏移 cell 坐标建立； +2. 保存 `_wgsox/_wgsoy` 原始值； +3. 对能够匹配 coverage 的普通几何，在 CM93 Mercator 阶段减去对应 offset； +4. 然后使用 CM93 半长轴 `6378388.0` 转换 WGS84 lon/lat； +5. feature JSON 中记录实际应用的 offset 和 coverage match method。 + +为研究/对照可运行： + +```powershell +--offset-policy none +``` + +但不要把同一个 dataset 混用两种 policy。改变 policy 时应 `--force` 全部重导，并重新执行全源审计。 + +## 5. Geometry issue + +源 CM93 中可能存在自交 polygon 等问题。 + +规则： + +- raw record 永久保留； +- `geometry_issue` 记录 Shapely validity / ring assembly 问题； +- `geometry_repaired=0`； +- 当前版本不自动修复； +- 可以保存结构上可编码的 invalid WKB； +- 无法形成 geometry 时不伪造 geometry row。 + +第二阶段 PBF 如需要显示修复，应在发布层制定独立策略，不能修改母数据库事实。 diff --git a/docs/报告/.gitkeep b/docs/报告/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/docs/报告/CM93_Importer_v0.2.0_离线验证报告.md b/docs/报告/CM93_Importer_v0.2.0_离线验证报告.md new file mode 100644 index 0000000..4a2380a --- /dev/null +++ b/docs/报告/CM93_Importer_v0.2.0_离线验证报告.md @@ -0,0 +1,69 @@ +# CM93 Importer v0.2.0 离线验证报告 + +## 范围 + +本报告验证 CM93 parser、geometry 标准化和 MySQL 写入批次结构。 +由于 ChatGPT 执行环境无法访问用户本机 MySQL，本报告不声称已经替用户执行 MySQL 实际插入；真实数据库验收由 `import --limit-cells 5` + `audit` 完成。 + +## 固定真实样本 + +用户提供的 147 个 A~G CM93 cell。 + +结果： + +```text +cells 147 +parse failures 0 +features 100621 +attributes 112746 +``` + +与前期独立解析计数一致。 + +## Geometry + 写入结构回归 + +对 147 个 cell 全部执行： + +```text +parse +→ coverage build +→ Point/Line/Area/Sounding normalization +→ embedded offset matching +→ WKB generation +→ 模拟 MySQL insert tuple / dependent-id mapping +``` + +结果： + +```text +cells 147 +features 100621 +attributes 112746 +geometry issues 36 +unmatched coverage 4 +``` + +36 个 geometry issue 主要为源 polygon self-intersection；v0.2.0 不自动 make-valid。 +4 个 unmatched coverage 为仍有可保存几何、但 representative point 未落入 `_m_sor` coverage 的 cartographic/text 类对象；按 policy 保存 0 offset 并在 `_cm93.coverage_match_method` 中记录 `unmatched`，不静默猜测最近 coverage。 + +## 精度规则 + +- Python float / C double +- MySQL Geometry double +- CM93 Mercator `6378388.0` +- 不 round +- 不 simplify +- 不 quantize +- sounding 原始 X/Y/Z 同时保存 +- raw feature / raw attribute / decoded CM93 table block 保存 + +## 下一验证门槛 + +用户机器： + +```powershell +cm93db import --scale G --limit-cells 5 +cm93db audit --sample-cells 5 +``` + +只有数据库实际写入和 WKB 读回精度审计 PASS 后，再启动 30,097 cell 全量导入。 diff --git a/docs/报告/CM93导入器开发验证报告_v0.2.0.md b/docs/报告/CM93导入器开发验证报告_v0.2.0.md new file mode 100644 index 0000000..5e68bd9 --- /dev/null +++ b/docs/报告/CM93导入器开发验证报告_v0.2.0.md @@ -0,0 +1,69 @@ +# CM93 导入器开发验证报告 v0.2.0 + +## 结论 + +v0.2.0 已完成 CM93 binary parser、标准化 geometry、MySQL 5.7 writer、断点续导和 audit 代码。 + +## 实际样本回归 + +输入：用户上传 `04501020.zip` + `CM93OBJ.DIC` + `CM93ATTR.DIC`。 + +结果： + +- Cell: 147 +- Parser PASS: 147 +- Parser FAIL: 0 +- Feature: 100,621 +- Attribute: 112,746 +- 未知 object ID: 0 +- 未知 attribute ID: 0 +- Sounding points: 76,126 +- `_m_sor` coverage: 258 +- 非零 `_wgsox/_wgsoy` coverage: 34 + +结构级验证： + +- prolog/header/file-size formula: PASS +- vector point header count: PASS +- 3D point header count: PASS +- feature header count: PASS +- attribute payload byte header count: PASS +- geometry-reference header count: PASS +- related-pointer header count: PASS +- EOF exact: PASS +- decoded raw table 总字节 == 原文件字节: PASS + +## Geometry + +样本中所有应产生 Point/Line/Area 的 source feature 均生成了 geometry；没有 geometry missing。 + +部分 CM93 Area 在严格拓扑检查下存在自相交/边界问题。v0.2.0 不自动修复，只在 `--validate-geometry` 时报告；这符合母数据库不改变原数据顶点的原则。 + +## MySQL + +目标：MySQL 5.7.34-log。 + +用户已在前一版本实际验证： + +- 数据库创建成功 +- 14 张表初始化成功 +- Schema contract PASS + +v0.2.0 未修改表数量或需要迁移的核心 Schema，因此现有数据库可继续使用。 + +v0.2.0 新增的实际 feature/geometry bulk write 尚需在用户 Windows/MySQL 上执行 10-cell pilot；README 和 Codex 接手文档已固定 pilot → audit → full import 顺序。 + +## 当前正式数据目录 + +`D:\Ship\CM93V2\CM93JAPAN` + +不再使用旧的 `CM93_2014` 全目录作为当前正式导入目标。 + +## 下一验收点 + +```powershell +cm93db import --scale G --limit 10 +cm93db audit --scale G --limit 10 --precision-samples 32 +``` + +只有审计 `RESULT: PASS` 后才继续全量日本数据。 diff --git a/docs/数据库设计.md b/docs/数据库设计.md new file mode 100644 index 0000000..feaab8d --- /dev/null +++ b/docs/数据库设计.md @@ -0,0 +1,50 @@ +# MySQL 5.7.34 数据库设计 + +## 核心表 + +- `chart_dataset`：一个 CM93 数据集根目录 +- `chart_dictionary_object`：对象字典快照 +- `chart_dictionary_attribute`：属性字典快照 +- `chart_cell`：CM93 cell 元数据 +- `chart_coverage`：`_m_sor` coverage +- `chart_feature`：公共 feature 元数据 +- `chart_feature_attribute`：规范化属性 +- `chart_point_geometry` +- `chart_line_geometry` +- `chart_area_geometry` +- `chart_sounding_point` +- `chart_raw_record` +- `chart_import_job` +- `chart_import_event` + +## Geometry + +MySQL 5.7 不使用 MySQL 8 的 SRID 列限制。 +所有标准化 geometry 写入时显式： + +`ST_GeomFromWKB(%s, 4326)` + +并在 `chart_feature.normalized_srid` 记录 4326。 + +## 原始值 + +CM93 的： +- header bounds +- source scale +- raw geotype +- raw feature bytes +- raw attribute bytes +- coverage offset + +必须保存。 + +## 数值属性 + +`chart_feature_attribute` 同时提供： +- `value_int` +- `value_real` +- `value_text` +- `value_json` +- `raw_value_blob` + +这样既保留 typed value，也允许审计原始数据。 diff --git a/docs/架构设计.md b/docs/架构设计.md new file mode 100644 index 0000000..8446b6b --- /dev/null +++ b/docs/架构设计.md @@ -0,0 +1,55 @@ +# 架构设计 + +## 数据流 + +CM93 原始目录 +→ SourceScanner +→ DictionaryLoader +→ CellBinaryReader +→ FeatureDecoder +→ CoverageResolver +→ PrecisionProjection +→ ChartFeature Model +→ MySQLBatchWriter +→ MySQL 母数据库 + +## 分层 + +### `cm93` +只负责格式解析和坐标/几何语义，不知道 MySQL。 + +### `model` +定义稳定中间模型。 + +### `database` +负责 MySQL 5.7 写入、事务、批量提交和恢复。 + +### `cli` +只做命令组织。 + +## 数据库与 CM93 边界 + +写入数据库以后： +- geometry 已标准化为 WGS84 经纬度； +- 原始 CM93 元数据仍保留； +- 后续 PBF builder 不需要理解 CM93 二进制。 + +## Cell 事务 + +每个 cell 为独立事务边界： + +DISCOVERED → IMPORTING → DONE + +失败： + +IMPORTING → FAILED + +再次运行时： +- DONE 默认跳过； +- FAILED 可重试； +- 强制参数可重新导入指定 cell。 + +## Geometry + +母数据库保留标准化 geometry，但不依赖 geometry 作为唯一事实来源。 +原始记录和 source index 是审计入口。 diff --git a/docs/精度规范.md b/docs/精度规范.md new file mode 100644 index 0000000..03840bb --- /dev/null +++ b/docs/精度规范.md @@ -0,0 +1,55 @@ +# CM93 母数据库精度规范 v2 + +## 固定常量 + +- CM93 raw grid: 0..65535 +- CM93 Mercator semi-major axis: **6378388.0 m** +- 标准化母库：WGS84 经纬度语义 / SRID 4326 +- Python 坐标计算：CPython float（C double） +- WKB X/Y：IEEE-754 8-byte double + +## 禁止 + +- numpy.float32 作为坐标 +- MySQL FLOAT 保存最终经纬度 +- importer 阶段 round +- simplify +- snap +- quantize +- 自动 make_valid 后替换原 geometry + +## 转换 + +1. 读取 raw uint16 X/Y。 +2. 使用 header 的 easting/northing bounds 恢复 CM93 Mercator。 +3. 若 feature 所属 `_m_sor` coverage 有 `_wgsox/_wgsoy`：在 Mercator 米坐标中减去 offset。 +4. 使用 6378388.0 执行 CM93 Mercator → lon/lat。 +5. WKB 写 MySQL，第二参数显式 SRID=4326。 + +## 数据可追踪 + +必须同时保存： + +- source cell path +- SHA-256 +- header +- raw decoded tables +- raw feature record +- raw attribute payload/value +- `_wgsox/_wgsoy` +- normalized geometry + +因此标准化公式未来如需修订，不必重新取得原 CM93 光盘/目录即可重新解释数据库 raw 数据。 + +## 审计门槛 + +默认： + +- parser 结构计数全部一致 +- EOF 精确一致 +- source SHA-256 一致 +- raw chunk 总字节 = source file bytes +- source/database Feature/Attribute/Geometry/Sounding 计数一致 +- WKB round-trip precision tolerance = **1e-12 degree** + +`--validate-geometry` 的拓扑 validity 是额外信息，不允许通过自动修复掩盖原数据结构。 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..65f57e6 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,24 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "cm93-mysql-importer" +version = "0.2.0" +description = "CM93 complete high-precision importer and auditor for MySQL 5.7" +readme = "README.md" +requires-python = ">=3.12,<3.13" +dependencies = [ + "PyMySQL>=1.1,<2", + "python-dotenv>=1.0,<2", + "Shapely>=2.0,<3", +] + +[project.scripts] +cm93db = "cm93db.cli:main" + +[tool.setuptools] +package-dir = {"" = "src"} + +[tool.setuptools.packages.find] +where = ["src"] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..43fd8cd --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +PyMySQL>=1.1,<2 +python-dotenv>=1.0,<2 +Shapely>=2.0,<3 diff --git a/samples/README.md b/samples/README.md new file mode 100644 index 0000000..aa53462 --- /dev/null +++ b/samples/README.md @@ -0,0 +1,17 @@ +# 样本数据 + +本 ZIP 不重新分发用户的 CM93 数据。 + +建议将已经验证的小样本放在项目外，例如： + +`D:\Ship\CM93V2\CM93JAPAN_TEST\04501020\` + +然后运行： + +`cm93db scan D:\Ship\CM93V2\CM93JAPAN_TEST\04501020` + +和： + +`cm93db inspect-dicts D:\Ship\CM93V2\CM93JAPAN_TEST` + +正式 Codex 开发时，以真实样本作为格式回归基线。 diff --git a/scripts/audit_cm93.ps1 b/scripts/audit_cm93.ps1 new file mode 100644 index 0000000..ac9c88e --- /dev/null +++ b/scripts/audit_cm93.ps1 @@ -0,0 +1,7 @@ +$ErrorActionPreference = "Stop" + +if (-not (Test-Path ".\.venv\Scripts\cm93db.exe")) { + throw "未发现 cm93db，请先运行 scripts\setup_env.ps1" +} + +& ".\.venv\Scripts\cm93db.exe" audit @args diff --git a/scripts/import_cm93.ps1 b/scripts/import_cm93.ps1 new file mode 100644 index 0000000..bb383e0 --- /dev/null +++ b/scripts/import_cm93.ps1 @@ -0,0 +1,10 @@ +$ErrorActionPreference = "Stop" + +if (-not (Test-Path ".\.venv\Scripts\cm93db.exe")) { + throw "未发现 cm93db，请先运行 scripts\setup_env.ps1" +} +if (-not (Test-Path ".env")) { + throw "未发现 .env，请从 .env.example 复制并配置 CM93_SOURCE_DIR / MySQL" +} + +& ".\.venv\Scripts\cm93db.exe" import @args diff --git a/scripts/init_db.ps1 b/scripts/init_db.ps1 new file mode 100644 index 0000000..1c1fced --- /dev/null +++ b/scripts/init_db.ps1 @@ -0,0 +1,11 @@ +$ErrorActionPreference = "Stop" + +if (-not (Test-Path ".venv")) { + throw "未发现 .venv，请先运行 .\scripts\setup_env.ps1" +} +if (-not (Test-Path ".env")) { + throw "未发现 .env，请从 .env.example 复制并填写数据库信息" +} + +& ".\.venv\Scripts\cm93db.exe" db-init --create-database +& ".\.venv\Scripts\cm93db.exe" db-check diff --git a/scripts/repair_env.ps1 b/scripts/repair_env.ps1 new file mode 100644 index 0000000..4d90f83 --- /dev/null +++ b/scripts/repair_env.ps1 @@ -0,0 +1,23 @@ +$ErrorActionPreference = "Stop" + +$Python312 = (pyenv which python | Select-Object -First 1).Trim() +if (-not (Test-Path $Python312)) { + throw "找不到 pyenv Python: $Python312" +} + +$Version = (& $Python312 --version 2>&1).Trim() +if ($Version -ne "Python 3.12.10") { + throw "pyenv 当前不是 Python 3.12.10: $Version" +} + +if (Test-Path ".venv") { + Remove-Item -Recurse -Force ".venv" +} + +& $Python312 -m venv .venv +$VenvPython = (Resolve-Path ".\.venv\Scripts\python.exe").Path +& $VenvPython -m pip install --upgrade pip +& $VenvPython -m pip install -e . + +& ".\.venv\Scripts\cm93db.exe" --help +Write-Host "修复完成。" diff --git a/scripts/setup_env.ps1 b/scripts/setup_env.ps1 new file mode 100644 index 0000000..c604def --- /dev/null +++ b/scripts/setup_env.ps1 @@ -0,0 +1,61 @@ +$ErrorActionPreference = "Stop" + +Write-Host "[1/6] 设置项目 Python 3.12.10" +pyenv local 3.12.10 + +Write-Host "[2/6] 通过 pyenv 获取项目 Python" +$Python312 = (pyenv which python | Select-Object -First 1).Trim() + +if (-not $Python312) { + throw "pyenv which python 未返回 Python 路径" +} +if (-not (Test-Path $Python312)) { + throw "pyenv 返回的 Python 不存在: $Python312" +} + +$Version = (& $Python312 --version 2>&1).Trim() +Write-Host "Python executable: $Python312" +Write-Host "Python version: $Version" + +if ($Version -ne "Python 3.12.10") { + throw "项目要求 Python 3.12.10，但 pyenv 当前返回: $Version" +} + +Write-Host "[3/6] 检查/创建虚拟环境" +if (Test-Path ".venv\Scripts\python.exe") { + $ExistingVersion = (& ".\.venv\Scripts\python.exe" --version 2>&1).Trim() + if ($ExistingVersion -ne "Python 3.12.10") { + Write-Host "发现错误虚拟环境 ($ExistingVersion)，正在删除..." + Remove-Item -Recurse -Force ".venv" + } +} + +if (-not (Test-Path ".venv\Scripts\python.exe")) { + & $Python312 -m venv .venv +} + +$VenvPython = (Resolve-Path ".\.venv\Scripts\python.exe").Path +$VenvVersion = (& $VenvPython --version 2>&1).Trim() +Write-Host "Venv Python: $VenvVersion" + +if ($VenvVersion -ne "Python 3.12.10") { + throw "虚拟环境 Python 版本错误: $VenvVersion" +} + +Write-Host "[4/6] 升级 pip" +& $VenvPython -m pip install --upgrade pip + +Write-Host "[5/6] 安装项目" +& $VenvPython -m pip install -e . + +$Cm93Db = (Resolve-Path ".\.venv\Scripts\cm93db.exe").Path + +Write-Host "[6/6] 基础命令检查" +& $Cm93Db --help + +Write-Host "" +Write-Host "Python 3.12.10 虚拟环境已建立。" +Write-Host "下一步：" +Write-Host " 1. Copy-Item .env.example .env" +Write-Host " 2. 修改 .env 中的 MySQL 5.7.34 参数" +Write-Host " 3. .\scripts\init_db.ps1" diff --git a/sql/001_schema_mysql57.sql b/sql/001_schema_mysql57.sql new file mode 100644 index 0000000..326140d --- /dev/null +++ b/sql/001_schema_mysql57.sql @@ -0,0 +1,281 @@ +SET NAMES utf8mb4; + +CREATE TABLE IF NOT EXISTS chart_dataset ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + source_type VARCHAR(16) NOT NULL DEFAULT 'CM93', + name VARCHAR(255) NOT NULL, + source_root TEXT NOT NULL, + source_fingerprint CHAR(64) NULL, + parser_version VARCHAR(32) NOT NULL, + normalized_srid INT UNSIGNED NOT NULL DEFAULT 4326, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (id), + KEY idx_dataset_source_type (source_type) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +CREATE TABLE IF NOT EXISTS chart_dictionary_object ( + dataset_id BIGINT UNSIGNED NOT NULL, + object_class_id SMALLINT UNSIGNED NOT NULL, + object_class VARCHAR(16) NOT NULL, + geometry_code VARCHAR(8) NULL, + display_code VARCHAR(8) NULL, + description VARCHAR(255) NULL, + raw_line TEXT NOT NULL, + PRIMARY KEY (dataset_id, object_class_id), + KEY idx_dict_object_class (object_class), + CONSTRAINT fk_dict_object_dataset FOREIGN KEY (dataset_id) + REFERENCES chart_dataset(id) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +CREATE TABLE IF NOT EXISTS chart_dictionary_attribute ( + dataset_id BIGINT UNSIGNED NOT NULL, + attribute_id SMALLINT UNSIGNED NOT NULL, + attribute_name VARCHAR(16) NOT NULL, + source_type_name VARCHAR(32) NOT NULL, + decoder_type CHAR(1) NOT NULL, + min_value DOUBLE NULL, + max_value DOUBLE NULL, + raw_line TEXT NOT NULL, + PRIMARY KEY (dataset_id, attribute_id), + KEY idx_dict_attribute_name (attribute_name), + CONSTRAINT fk_dict_attr_dataset FOREIGN KEY (dataset_id) + REFERENCES chart_dataset(id) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +CREATE TABLE IF NOT EXISTS chart_cell ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + dataset_id BIGINT UNSIGNED NOT NULL, + source_relative_path VARCHAR(1024) NOT NULL, + source_file_name VARCHAR(255) NOT NULL, + source_size BIGINT UNSIGNED NOT NULL, + source_sha256 CHAR(64) NULL, + scale_code CHAR(1) NOT NULL, + native_scale INT UNSIGNED NOT NULL, + cell_name VARCHAR(64) NOT NULL, + + prolog_word0 INT UNSIGNED NULL, + table1_length BIGINT UNSIGNED NULL, + table2_length BIGINT UNSIGNED NULL, + + easting_min DOUBLE NULL, + northing_min DOUBLE NULL, + easting_max DOUBLE NULL, + northing_max DOUBLE NULL, + + bbox_min_lon DOUBLE NULL, + bbox_min_lat DOUBLE NULL, + bbox_max_lon DOUBLE NULL, + bbox_max_lat DOUBLE NULL, + + vector_record_count INT UNSIGNED NULL, + vector_point_count BIGINT UNSIGNED NULL, + point2d_count INT UNSIGNED NULL, + point3d_count INT UNSIGNED NULL, + feature_count INT UNSIGNED NULL, + attribute_count BIGINT UNSIGNED NULL, + + import_status ENUM('DISCOVERED','IMPORTING','DONE','FAILED') NOT NULL DEFAULT 'DISCOVERED', + error_message TEXT NULL, + imported_at DATETIME NULL, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + PRIMARY KEY (id), + UNIQUE KEY uq_cell_dataset_path (dataset_id, source_relative_path(255)), + KEY idx_cell_scale (dataset_id, scale_code), + KEY idx_cell_status (dataset_id, import_status), + CONSTRAINT fk_cell_dataset FOREIGN KEY (dataset_id) + REFERENCES chart_dataset(id) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +CREATE TABLE IF NOT EXISTS chart_coverage ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + dataset_id BIGINT UNSIGNED NOT NULL, + cell_id BIGINT UNSIGNED NOT NULL, + source_record_index INT UNSIGNED NOT NULL, + object_class_id SMALLINT UNSIGNED NOT NULL, + compilation_scale INT UNSIGNED NULL, + horizontal_datum INT NULL, + vertical_datum INT NULL, + wgsox DOUBLE NULL, + wgsoy DOUBLE NULL, + attributes_json JSON NULL, + geom GEOMETRY NOT NULL, + geometry_repaired TINYINT(1) NOT NULL DEFAULT 0, + geometry_issue VARCHAR(255) NULL, + PRIMARY KEY (id), + UNIQUE KEY uq_cov_cell_record (cell_id, source_record_index), + KEY idx_cov_cell (cell_id), + SPATIAL KEY spx_cov_geom (geom), + CONSTRAINT fk_cov_dataset FOREIGN KEY (dataset_id) + REFERENCES chart_dataset(id) ON DELETE CASCADE, + CONSTRAINT fk_cov_cell FOREIGN KEY (cell_id) + REFERENCES chart_cell(id) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +CREATE TABLE IF NOT EXISTS chart_feature ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + dataset_id BIGINT UNSIGNED NOT NULL, + cell_id BIGINT UNSIGNED NOT NULL, + coverage_id BIGINT UNSIGNED NULL, + source_record_index INT UNSIGNED NOT NULL, + + object_class_id SMALLINT UNSIGNED NULL, + object_class VARCHAR(16) NOT NULL, + geometry_type ENUM('POINT','LINE','AREA','SOUNDING','META','UNKNOWN') NOT NULL, + + scale_code CHAR(1) NOT NULL, + native_scale INT UNSIGNED NOT NULL, + + raw_geotype SMALLINT UNSIGNED NULL, + normalized_srid INT UNSIGNED NOT NULL DEFAULT 4326, + + bbox_min_lon DOUBLE NULL, + bbox_min_lat DOUBLE NULL, + bbox_max_lon DOUBLE NULL, + bbox_max_lat DOUBLE NULL, + + attributes_json JSON NULL, + raw_feature_blob MEDIUMBLOB NULL, + raw_attribute_blob MEDIUMBLOB NULL, + + geometry_repaired TINYINT(1) NOT NULL DEFAULT 0, + geometry_issue VARCHAR(255) NULL, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + + PRIMARY KEY (id), + UNIQUE KEY uq_feature_cell_record (cell_id, source_record_index), + KEY idx_feature_class (dataset_id, object_class), + KEY idx_feature_scale (dataset_id, native_scale), + KEY idx_feature_cell (cell_id), + KEY idx_feature_coverage (coverage_id), + CONSTRAINT fk_feature_dataset FOREIGN KEY (dataset_id) + REFERENCES chart_dataset(id) ON DELETE CASCADE, + CONSTRAINT fk_feature_cell FOREIGN KEY (cell_id) + REFERENCES chart_cell(id) ON DELETE CASCADE, + CONSTRAINT fk_feature_coverage FOREIGN KEY (coverage_id) + REFERENCES chart_coverage(id) ON DELETE SET NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +CREATE TABLE IF NOT EXISTS chart_feature_attribute ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + feature_id BIGINT UNSIGNED NOT NULL, + ordinal_no SMALLINT UNSIGNED NOT NULL, + attribute_id SMALLINT UNSIGNED NOT NULL, + attribute_name VARCHAR(16) NOT NULL, + value_type ENUM('BYTE','INT','LONG','REAL','WORD10','STRING','LIST','COMPLEX','UNKNOWN') NOT NULL, + value_int BIGINT NULL, + value_real DOUBLE NULL, + value_text TEXT NULL, + value_json JSON NULL, + raw_value_blob BLOB NULL, + PRIMARY KEY (id), + UNIQUE KEY uq_feature_attr_ordinal (feature_id, ordinal_no), + KEY idx_feature_attr_name (attribute_name), + KEY idx_feature_attr_id (attribute_id), + CONSTRAINT fk_feature_attr_feature FOREIGN KEY (feature_id) + REFERENCES chart_feature(id) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +CREATE TABLE IF NOT EXISTS chart_point_geometry ( + feature_id BIGINT UNSIGNED NOT NULL, + geom POINT NOT NULL, + PRIMARY KEY (feature_id), + SPATIAL KEY spx_point_geom (geom), + CONSTRAINT fk_point_feature FOREIGN KEY (feature_id) + REFERENCES chart_feature(id) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +CREATE TABLE IF NOT EXISTS chart_line_geometry ( + feature_id BIGINT UNSIGNED NOT NULL, + geom GEOMETRY NOT NULL, + PRIMARY KEY (feature_id), + SPATIAL KEY spx_line_geom (geom), + CONSTRAINT fk_line_feature FOREIGN KEY (feature_id) + REFERENCES chart_feature(id) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +CREATE TABLE IF NOT EXISTS chart_area_geometry ( + feature_id BIGINT UNSIGNED NOT NULL, + geom GEOMETRY NOT NULL, + PRIMARY KEY (feature_id), + SPATIAL KEY spx_area_geom (geom), + CONSTRAINT fk_area_feature FOREIGN KEY (feature_id) + REFERENCES chart_feature(id) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +CREATE TABLE IF NOT EXISTS chart_sounding_point ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + feature_id BIGINT UNSIGNED NOT NULL, + point_index INT UNSIGNED NOT NULL, + raw_x INT UNSIGNED NOT NULL, + raw_y INT UNSIGNED NOT NULL, + raw_z INT UNSIGNED NOT NULL, + depth DOUBLE NOT NULL, + geom POINT NOT NULL, + PRIMARY KEY (id), + UNIQUE KEY uq_sounding_feature_point (feature_id, point_index), + KEY idx_sounding_depth (depth), + SPATIAL KEY spx_sounding_geom (geom), + CONSTRAINT fk_sounding_feature FOREIGN KEY (feature_id) + REFERENCES chart_feature(id) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +CREATE TABLE IF NOT EXISTS chart_raw_record ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + cell_id BIGINT UNSIGNED NOT NULL, + record_kind ENUM('HEADER','VECTOR','POINT2D','POINT3D','FEATURE','ATTRIBUTE','OTHER') NOT NULL, + source_record_index INT UNSIGNED NOT NULL DEFAULT 0, + raw_blob MEDIUMBLOB NOT NULL, + raw_metadata_json JSON NULL, + PRIMARY KEY (id), + KEY idx_raw_cell_kind (cell_id, record_kind, source_record_index), + CONSTRAINT fk_raw_cell FOREIGN KEY (cell_id) + REFERENCES chart_cell(id) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +CREATE TABLE IF NOT EXISTS chart_import_job ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + dataset_id BIGINT UNSIGNED NOT NULL, + status ENUM('RUNNING','DONE','FAILED','CANCELLED') NOT NULL DEFAULT 'RUNNING', + started_at DATETIME NOT NULL, + finished_at DATETIME NULL, + cells_discovered INT UNSIGNED NOT NULL DEFAULT 0, + cells_done INT UNSIGNED NOT NULL DEFAULT 0, + cells_failed INT UNSIGNED NOT NULL DEFAULT 0, + features_source BIGINT UNSIGNED NOT NULL DEFAULT 0, + features_imported BIGINT UNSIGNED NOT NULL DEFAULT 0, + attributes_source BIGINT UNSIGNED NOT NULL DEFAULT 0, + attributes_imported BIGINT UNSIGNED NOT NULL DEFAULT 0, + unknown_object_count BIGINT UNSIGNED NOT NULL DEFAULT 0, + unknown_attribute_count BIGINT UNSIGNED NOT NULL DEFAULT 0, + geometry_invalid_count BIGINT UNSIGNED NOT NULL DEFAULT 0, + geometry_repaired_count BIGINT UNSIGNED NOT NULL DEFAULT 0, + summary_json JSON NULL, + PRIMARY KEY (id), + KEY idx_job_dataset (dataset_id, started_at), + CONSTRAINT fk_job_dataset FOREIGN KEY (dataset_id) + REFERENCES chart_dataset(id) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +CREATE TABLE IF NOT EXISTS chart_import_event ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + job_id BIGINT UNSIGNED NULL, + cell_id BIGINT UNSIGNED NULL, + level ENUM('DEBUG','INFO','WARNING','ERROR') NOT NULL, + event_code VARCHAR(64) NOT NULL, + message TEXT NOT NULL, + details_json JSON NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (id), + KEY idx_event_job (job_id, id), + KEY idx_event_cell (cell_id, id), + KEY idx_event_code (event_code), + CONSTRAINT fk_event_job FOREIGN KEY (job_id) + REFERENCES chart_import_job(id) ON DELETE CASCADE, + CONSTRAINT fk_event_cell FOREIGN KEY (cell_id) + REFERENCES chart_cell(id) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; diff --git a/src/cm93db/__init__.py b/src/cm93db/__init__.py new file mode 100644 index 0000000..d3ec452 --- /dev/null +++ b/src/cm93db/__init__.py @@ -0,0 +1 @@ +__version__ = "0.2.0" diff --git a/src/cm93db/audit.py b/src/cm93db/audit.py new file mode 100644 index 0000000..a884ac8 --- /dev/null +++ b/src/cm93db/audit.py @@ -0,0 +1,294 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from collections import defaultdict +import math +from typing import Any + +from shapely import from_wkb + +from .cm93_binary import parse_cell +from .config import DbConfig +from .database import connect, assert_mysql57, server_version +from .dictionaries import load_object_dictionary, load_attribute_dictionary +from .geometry import build_coverages, build_feature +from .scanner import scan_cm93, CellFile + + +@dataclass(frozen=True) +class SourceCellStat: + rel: str + sha256: str + file_size: int + scale: str + features: int + attributes: int + points: int + lines: int + areas: int + coverages: int + sounding_features: int + sounding_points: int + unknown_objects: int + unknown_attributes: int + + +@dataclass +class PrecisionSample: + rel: str + record_index: int + geometry_type: str + expected_wkb: bytes + point_index: int | None = None + + +@dataclass +class AuditResult: + source_cells: int = 0 + source_parse_failed: int = 0 + db_missing_cells: int = 0 + db_not_done_cells: int = 0 + count_mismatches: int = 0 + hash_mismatches: int = 0 + unknown_objects: int = 0 + unknown_attributes: int = 0 + precision_checked: int = 0 + precision_failed: int = 0 + max_precision_delta_deg: float = 0.0 + raw_preservation_mismatches: int = 0 + geometry_validity_issues: int = 0 + messages: list[str] = field(default_factory=list) + + @property + def passed(self) -> bool: + return not any((self.source_parse_failed,self.db_missing_cells,self.db_not_done_cells,self.count_mismatches,self.hash_mismatches,self.raw_preservation_mismatches,self.unknown_objects,self.unknown_attributes,self.precision_failed)) + + def as_dict(self) -> dict[str, Any]: + return { + "passed": self.passed, + "source_cells": self.source_cells, + "source_parse_failed": self.source_parse_failed, + "db_missing_cells": self.db_missing_cells, + "db_not_done_cells": self.db_not_done_cells, + "count_mismatches": self.count_mismatches, + "hash_mismatches": self.hash_mismatches, + "raw_preservation_mismatches": self.raw_preservation_mismatches, + "geometry_validity_issues": self.geometry_validity_issues, + "unknown_objects": self.unknown_objects, + "unknown_attributes": self.unknown_attributes, + "precision_checked": self.precision_checked, + "precision_failed": self.precision_failed, + "max_precision_delta_deg": self.max_precision_delta_deg, + "messages": self.messages, + } + + +def _rel(cf: CellFile) -> str: + return str(cf.relative_path).replace("/", "\\") + + +def _select_cells(scan, limit: int | None, scales: set[str] | None, cell_name: str | None): + cells=list(scan.cells) + if scales: + cells=[c for c in cells if c.scale_code in scales] + if cell_name: + t=cell_name.lower() + cells=[c for c in cells if c.path.name.lower()==t or c.path.stem.lower()==t] + if limit is not None: + cells=cells[:limit] + return cells + + +def _source_stat(cell, cf: CellFile) -> SourceCellStat: + p=l=a=cov=sf=sp=0 + for f in cell.features: + if f.object_class == "_m_sor": + cov += 1 + continue + if f.geometry_primitive == 1: p += 1 + elif f.geometry_primitive == 2: l += 1 + elif f.geometry_primitive == 4: a += 1 + elif f.geometry_primitive == 8: + sf += 1 + if f.point3d_index is not None: + sp += len(cell.point3d[f.point3d_index].points) + return SourceCellStat( + rel=_rel(cf),sha256=cell.sha256,file_size=cell.file_size,scale=cf.scale_code,features=len(cell.features),attributes=cell.attribute_count, + points=p,lines=l,areas=a,coverages=cov,sounding_features=sf,sounding_points=sp, + unknown_objects=len(cell.unknown_object_ids),unknown_attributes=len(cell.unknown_attribute_ids), + ) + + +def collect_source(root: Path, limit: int | None = None, scales: set[str] | None = None, cell_name: str | None = None, + precision_samples: int = 0, validate_geometry: bool = False): + scan=scan_cm93(root) + if not scan.object_dictionary or not scan.attribute_dictionary: + raise RuntimeError("CM93 字典未找到") + objs={x.object_id:x for x in load_object_dictionary(scan.object_dictionary)} + attrs={x.attribute_id:x for x in load_attribute_dictionary(scan.attribute_dictionary)} + cells=_select_cells(scan,limit,scales,cell_name) + stats: dict[str,SourceCellStat]={} + failures: list[str]=[] + samples: list[PrecisionSample]=[] + geom_issues=0 + quota=max(1, math.ceil(precision_samples / max(1,len({c.scale_code for c in cells})))) if precision_samples else 0 + per_scale=defaultdict(int) + + for n,cf in enumerate(cells,1): + try: + cell=parse_cell(cf.path,objs,attrs) + st=_source_stat(cell,cf); stats[st.rel]=st + want = precision_samples and per_scale[cf.scale_code] < quota and len(samples) < precision_samples + if want or validate_geometry: + covs=build_coverages(cell) + for f in cell.features: + b=build_feature(cell,f,covs,validate=validate_geometry) + if validate_geometry and b.geometry_issue: + geom_issues += 1 + if want and per_scale[cf.scale_code] < quota and len(samples) < precision_samples: + if b.geom is not None and b.geometry_type in {"POINT","LINE","AREA"}: + samples.append(PrecisionSample(st.rel,f.index,b.geometry_type,b.geom.wkb)) + per_scale[cf.scale_code]+=1 + elif b.geometry_type=="SOUNDING" and b.soundings: + samples.append(PrecisionSample(st.rel,f.index,"SOUNDING",b.soundings[0][4].wkb,0)) + per_scale[cf.scale_code]+=1 + if n % 250 == 0 or n == len(cells): + print(f"source audit [{n}/{len(cells)}]") + except Exception as e: + failures.append(f"{_rel(cf)}: {e}") + return stats,failures,samples,geom_issues + + +def _group_counts(conn, sql: str) -> dict[int,int]: + with conn.cursor() as cur: + cur.execute(sql) + return {int(r["cell_id"]):int(r["n"]) for r in cur.fetchall()} + + +def _dataset_id(conn, root: Path) -> int | None: + with conn.cursor() as cur: + cur.execute("SELECT id FROM chart_dataset WHERE source_type='CM93' AND source_root=%s ORDER BY id LIMIT 1", (str(root.resolve()),)) + r=cur.fetchone(); return int(r["id"]) if r else None + + +def _precision_db_geom(conn, cell_id: int, sample: PrecisionSample): + if sample.geometry_type == "SOUNDING": + sql=("SELECT ST_AsWKB(s.geom) AS wkb FROM chart_feature f JOIN chart_sounding_point s ON s.feature_id=f.id " + "WHERE f.cell_id=%s AND f.source_record_index=%s AND s.point_index=%s") + params=(cell_id,sample.record_index,sample.point_index or 0) + else: + table={"POINT":"chart_point_geometry","LINE":"chart_line_geometry","AREA":"chart_area_geometry"}[sample.geometry_type] + sql=(f"SELECT ST_AsWKB(g.geom) AS wkb FROM chart_feature f JOIN {table} g ON g.feature_id=f.id " + "WHERE f.cell_id=%s AND f.source_record_index=%s") + params=(cell_id,sample.record_index) + with conn.cursor() as cur: + cur.execute(sql,params); r=cur.fetchone() + return bytes(r["wkb"]) if r and r["wkb"] is not None else None + + +def run_audit(cfg: DbConfig, root: Path, limit: int | None = None, scales: set[str] | None = None, cell_name: str | None = None, + precision_samples: int = 128, tolerance_deg: float = 1e-12, validate_geometry: bool = False, source_only: bool = False, require_raw: bool = True) -> AuditResult: + result=AuditResult() + stats,failures,samples,geom_issues=collect_source(root,limit,scales,cell_name,precision_samples if not source_only else 0,validate_geometry) + result.source_cells=len(stats)+len(failures) + result.source_parse_failed=len(failures) + result.messages.extend(f"SOURCE_PARSE_FAILED {m}" for m in failures[:100]) + result.unknown_objects=sum(s.unknown_objects for s in stats.values()) + result.unknown_attributes=sum(s.unknown_attributes for s in stats.values()) + print(f"Source cells parsed: {len(stats)}, failed: {len(failures)}") + print(f"Source features: {sum(s.features for s in stats.values())}") + print(f"Source attributes: {sum(s.attributes for s in stats.values())}") + result.geometry_validity_issues=geom_issues + if validate_geometry: + print(f"Source geometry issues (Shapely validity/structure): {geom_issues}") + if source_only: + return result + + conn=connect(cfg) + try: + assert_mysql57(server_version(conn)) + did=_dataset_id(conn,root) + if did is None: + result.db_missing_cells=len(stats) + result.messages.append("数据库中找不到该 source_root 对应 dataset") + return result + with conn.cursor() as cur: + cur.execute("SELECT id,source_relative_path,import_status,source_sha256,feature_count,attribute_count FROM chart_cell WHERE dataset_id=%s", (did,)) + rows=cur.fetchall() + db_by_rel={str(r["source_relative_path"]):r for r in rows} + cellids=[int(r["id"]) for r in rows] + fcnt=_group_counts(conn,f"SELECT cell_id,COUNT(*) n FROM chart_feature WHERE cell_id IN (SELECT id FROM chart_cell WHERE dataset_id={did}) GROUP BY cell_id") + acnt=_group_counts(conn,f"SELECT f.cell_id,COUNT(*) n FROM chart_feature_attribute a JOIN chart_feature f ON f.id=a.feature_id JOIN chart_cell c ON c.id=f.cell_id WHERE c.dataset_id={did} GROUP BY f.cell_id") + pcnt=_group_counts(conn,f"SELECT f.cell_id,COUNT(*) n FROM chart_point_geometry g JOIN chart_feature f ON f.id=g.feature_id JOIN chart_cell c ON c.id=f.cell_id WHERE c.dataset_id={did} GROUP BY f.cell_id") + lcnt=_group_counts(conn,f"SELECT f.cell_id,COUNT(*) n FROM chart_line_geometry g JOIN chart_feature f ON f.id=g.feature_id JOIN chart_cell c ON c.id=f.cell_id WHERE c.dataset_id={did} GROUP BY f.cell_id") + arcnt=_group_counts(conn,f"SELECT f.cell_id,COUNT(*) n FROM chart_area_geometry g JOIN chart_feature f ON f.id=g.feature_id JOIN chart_cell c ON c.id=f.cell_id WHERE c.dataset_id={did} GROUP BY f.cell_id") + ccnt=_group_counts(conn,f"SELECT cell_id,COUNT(*) n FROM chart_coverage WHERE cell_id IN (SELECT id FROM chart_cell WHERE dataset_id={did}) GROUP BY cell_id") + scnt=_group_counts(conn,f"SELECT f.cell_id,COUNT(*) n FROM chart_sounding_point s JOIN chart_feature f ON f.id=s.feature_id JOIN chart_cell c ON c.id=f.cell_id WHERE c.dataset_id={did} GROUP BY f.cell_id") + rawbytes=_group_counts(conn,f"SELECT cell_id,SUM(OCTET_LENGTH(raw_blob)) n FROM chart_raw_record WHERE cell_id IN (SELECT id FROM chart_cell WHERE dataset_id={did}) GROUP BY cell_id") + + for rel,s in stats.items(): + r=db_by_rel.get(rel) + if not r: + result.db_missing_cells+=1; result.messages.append(f"DB_MISSING {rel}"); continue + cid=int(r["id"]) + if r["import_status"]!="DONE": + result.db_not_done_cells+=1; result.messages.append(f"DB_NOT_DONE {rel} status={r['import_status']}") + if r.get("source_sha256") and str(r["source_sha256"]) != s.sha256: + result.hash_mismatches+=1; result.messages.append(f"HASH_MISMATCH {rel}") + actual=(fcnt.get(cid,0),acnt.get(cid,0),pcnt.get(cid,0),lcnt.get(cid,0),arcnt.get(cid,0),ccnt.get(cid,0),scnt.get(cid,0)) + expected=(s.features,s.attributes,s.points,s.lines,s.areas,s.coverages,s.sounding_points) + if actual != expected: + result.count_mismatches+=1 + result.messages.append(f"COUNT_MISMATCH {rel} expected={expected} actual={actual}") + if require_raw and rawbytes.get(cid,0) != s.file_size: + result.raw_preservation_mismatches += 1 + result.messages.append(f"RAW_BYTES_MISMATCH {rel} expected={s.file_size} actual={rawbytes.get(cid,0)}") + + # Precision audit: source-derived geometry versus geometry round-tripped through MySQL. + rel_to_id={k:int(v["id"]) for k,v in db_by_rel.items()} + for sample in samples: + cid=rel_to_id.get(sample.rel) + if not cid: + continue + dbwkb=_precision_db_geom(conn,cid,sample) + result.precision_checked+=1 + if dbwkb is None: + result.precision_failed+=1 + result.messages.append(f"PRECISION_MISSING {sample.rel} record={sample.record_index}") + continue + expected=from_wkb(sample.expected_wkb); actual=from_wkb(dbwkb) + try: + ok=expected.equals_exact(actual,tolerance_deg) + delta=expected.hausdorff_distance(actual) if not ok else 0.0 + except Exception: + ok=False; delta=float("inf") + if math.isfinite(delta): + result.max_precision_delta_deg=max(result.max_precision_delta_deg,float(delta)) + if not ok: + result.precision_failed+=1 + result.messages.append(f"PRECISION_FAIL {sample.rel} record={sample.record_index} type={sample.geometry_type} delta_deg={delta}") + finally: + conn.close() + return result + + +def print_result(r: AuditResult) -> None: + print("\n=== CM93 AUDIT ===") + print(f"source_cells : {r.source_cells}") + print(f"source_parse_failed : {r.source_parse_failed}") + print(f"db_missing_cells : {r.db_missing_cells}") + print(f"db_not_done_cells : {r.db_not_done_cells}") + print(f"count_mismatches : {r.count_mismatches}") + print(f"hash_mismatches : {r.hash_mismatches}") + print(f"raw_preservation_bad : {r.raw_preservation_mismatches}") + print(f"geometry_validity : {r.geometry_validity_issues} (informational)") + print(f"unknown_objects : {r.unknown_objects}") + print(f"unknown_attributes : {r.unknown_attributes}") + print(f"precision_checked : {r.precision_checked}") + print(f"precision_failed : {r.precision_failed}") + print(f"max_precision_delta° : {r.max_precision_delta_deg:.16g}") + print(f"RESULT : {'PASS' if r.passed else 'FAIL'}") + if r.messages: + print("\nDetails (max 100 shown):") + for m in r.messages[:100]: print(" -",m) diff --git a/src/cm93db/cell_parser.py b/src/cm93db/cell_parser.py new file mode 100644 index 0000000..a873bf3 --- /dev/null +++ b/src/cm93db/cell_parser.py @@ -0,0 +1,468 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +import hashlib +import struct +from typing import Any + +from .decoder import decode_cm93 +from .dictionaries import AttributeDef, ObjectDef +from .precision import CM93_COORD_MAX, CM93_SEMIMAJOR_M + + +class CM93ParseError(RuntimeError): + pass + + +@dataclass(frozen=True) +class CM93Header: + lon_min: float + lat_min: float + lon_max: float + lat_max: float + easting_min: float + northing_min: float + easting_max: float + northing_max: float + vector_records: int + vector_record_points: int + m_46: int + m_4a: int + point3d_records: int + point3d_points: int + m_54: int + point2d_records: int + m_5a: int + m_5c: int + feature_records: int + m_60: int + m_64: int + m_68: int + m_6a: int + m_6c: int + related_object_pointers: int + m_72: int + m_76: int + attribute_bytes: int + m_7c: int + + @property + def transform_x_rate(self) -> float: + delta_x = self.easting_max - self.easting_min + if delta_x < 0.0: + delta_x += CM93_SEMIMAJOR_M * 2.0 * 3.141592653589793 + return delta_x / CM93_COORD_MAX + + @property + def transform_y_rate(self) -> float: + return (self.northing_max - self.northing_min) / CM93_COORD_MAX + + +@dataclass(frozen=True) +class EdgeRecord: + index: int + points: tuple[tuple[int, int], ...] + raw: bytes + + +@dataclass(frozen=True) +class SoundingGroup: + index: int + points: tuple[tuple[int, int, int], ...] + raw: bytes + + +@dataclass(frozen=True) +class ParsedAttribute: + ordinal: int + attribute_id: int + name: str + source_type: str + decoder_type: str + value_type: str + value: Any + value_int: int | None + value_real: float | None + value_text: str | None + value_json: Any | None + raw_value: bytes + + +@dataclass(frozen=True) +class GeometryRef: + edge_index: int + segment_usage: int + + +@dataclass +class ParsedFeature: + record_index: int + object_type: int + object_class: str + raw_geotype: int + primitive: int + geometry_refs: tuple[GeometryRef, ...] = () + point2d_index: int | None = None + point3d_index: int | None = None + related_indices: tuple[int, ...] = () + related_count_hint: int | None = None + n_attributes: int = 0 + attributes: tuple[ParsedAttribute, ...] = () + raw_record: bytes = b"" + raw_attribute_block: bytes = b"" + + +@dataclass(frozen=True) +class ParsedCell: + path: Path + source_size: int + sha256: str + encoded_bytes: bytes + decoded_bytes: bytes + prolog_word0: int + table1_length: int + table2_length: int + header: CM93Header + edges: tuple[EdgeRecord, ...] + point2d: tuple[tuple[int, int], ...] + point3d: tuple[SoundingGroup, ...] + features: tuple[ParsedFeature, ...] + end_offset: int + table_offsets: dict[str, int] + attribute_count: int + + +class _Reader: + def __init__(self, data: bytes, offset: int = 0): + self.data = data + self.offset = offset + + def _need(self, n: int) -> None: + if self.offset + n > len(self.data): + raise CM93ParseError( + f"unexpected EOF at {self.offset}, need {n}, size={len(self.data)}" + ) + + def u8(self) -> int: + self._need(1) + v = self.data[self.offset] + self.offset += 1 + return v + + def u16(self) -> int: + self._need(2) + v = struct.unpack_from(" int: + self._need(4) + v = struct.unpack_from(" float: + self._need(4) + v = struct.unpack_from(" float: + self._need(8) + v = struct.unpack_from(" bytes: + self._need(n) + b = self.data[self.offset:self.offset+n] + self.offset += n + return b + + def cstring(self) -> bytes: + end = self.data.find(b"\x00", self.offset) + if end < 0: + raise CM93ParseError(f"unterminated string at {self.offset}") + b = self.data[self.offset:end] + self.offset = end + 1 + return b + + +def _parse_header(r: _Reader) -> CM93Header: + start = r.offset + vals: list[Any] = [r.f64() for _ in range(8)] + vector_records = r.u16() + vector_record_points = r.i32() + m_46 = r.i32() + m_4a = r.i32() + point3d_records = r.u16() + point3d_points = r.i32() + m_54 = r.i32() + point2d_records = r.u16() + m_5a = r.u16() + m_5c = r.u16() + feature_records = r.u16() + m_60 = r.i32() + m_64 = r.i32() + m_68 = r.u16() + m_6a = r.u16() + m_6c = r.u16() + related = r.i32() + m_72 = r.i32() + m_76 = r.u16() + attr_bytes = r.i32() + m_7c = r.i32() + if r.offset - start != 128: + raise CM93ParseError(f"header size != 128: {r.offset-start}") + if min(vector_records, vector_record_points, point3d_records, point3d_points, + point2d_records, feature_records, related, attr_bytes) < 0: + raise CM93ParseError("negative header count") + return CM93Header( + *vals, vector_records, vector_record_points, m_46, m_4a, + point3d_records, point3d_points, m_54, point2d_records, + m_5a, m_5c, feature_records, m_60, m_64, m_68, m_6a, m_6c, + related, m_72, m_76, attr_bytes, m_7c, + ) + + +def _text(b: bytes) -> str: + # CM93 data in the field is commonly ANSI/Windows text. Preserve bytes separately. + return b.decode("cp1252", errors="replace") + + +def _parse_attributes( + block: bytes, + nattr: int, + attr_defs: dict[int, AttributeDef], +) -> tuple[ParsedAttribute, ...]: + r = _Reader(block) + out: list[ParsedAttribute] = [] + for ordinal in range(nattr): + item_start = r.offset + aid = r.u8() + d = attr_defs.get(aid) + if d is None: + raise CM93ParseError(f"unknown attribute id={aid} at attr {ordinal}") + t = d.decoder_type + value: Any + vi: int | None = None + vr: float | None = None + vt: str | None = None + vj: Any | None = None + value_type = "UNKNOWN" + if t == "B": + vi = r.u8(); value = vi; value_type = "BYTE" + elif t == "S": + raw_s = r.cstring(); vt = _text(raw_s); value = vt; value_type = "STRING" + elif t == "R": + vr = r.f32(); value = vr; value_type = "REAL" + elif t == "W": + vi = r.u16(); vr = vi / 10.0; value = vr; value_type = "WORD10" + elif t == "G": + vi = r.i32(); value = vi; value_type = "LONG" + elif t == "C": + prefix = list(r.take(3)); raw_s = r.cstring(); vt = _text(raw_s) + vj = {"prefix": prefix, "text": vt}; value = vj; value_type = "COMPLEX" + elif t == "L": + n = r.u8(); arr = list(r.take(n)); vj = arr; value = arr; value_type = "LIST" + elif t == "I": + # Kept for compatibility with old CM93 decoders; not present in supplied dictionary. + vi = r.u16(); value = vi; value_type = "INT" + else: + raise CM93ParseError( + f"unsupported attribute type={d.source_type_name}/{t} id={aid}" + ) + raw_value = block[item_start+1:r.offset] + out.append(ParsedAttribute( + ordinal=ordinal, + attribute_id=aid, + name=d.name, + source_type=d.source_type_name, + decoder_type=t, + value_type=value_type, + value=value, + value_int=vi, + value_real=vr, + value_text=vt, + value_json=vj, + raw_value=raw_value, + )) + if r.offset != len(block): + raise CM93ParseError( + f"attribute block length mismatch: parsed={r.offset} actual={len(block)} nattr={nattr}" + ) + return tuple(out) + + +def parse_cell( + path: Path, + object_defs: dict[int, ObjectDef], + attr_defs: dict[int, AttributeDef], + *, + keep_source_bytes: bool = True, +) -> ParsedCell: + encoded = path.read_bytes() + decoded = decode_cm93(encoded) + r = _Reader(decoded) + word0 = r.u16() + table1 = r.i32() + table2 = r.i32() + if word0 + table1 + table2 != len(decoded): + raise CM93ParseError( + f"cell length check failed: {word0}+{table1}+{table2}!={len(decoded)}" + ) + if word0 != 138: + raise CM93ParseError(f"unexpected prolog+header length: {word0}") + header = _parse_header(r) + table_offsets = {"prolog": 0, "header": 10, "vector": r.offset} + + edges: list[EdgeRecord] = [] + total_vector_points = 0 + for i in range(header.vector_records): + st = r.offset + n = r.u16() + pts = tuple((r.u16(), r.u16()) for _ in range(n)) + total_vector_points += n + edges.append(EdgeRecord(i, pts, decoded[st:r.offset])) + if total_vector_points != header.vector_record_points: + raise CM93ParseError( + f"vector point count mismatch: parsed={total_vector_points} header={header.vector_record_points}" + ) + + table_offsets["point3d"] = r.offset + groups: list[SoundingGroup] = [] + total_3d = 0 + for i in range(header.point3d_records): + st = r.offset + n = r.u16() + pts = tuple((r.u16(), r.u16(), r.u16()) for _ in range(n)) + total_3d += n + groups.append(SoundingGroup(i, pts, decoded[st:r.offset])) + if total_3d != header.point3d_points: + raise CM93ParseError( + f"3D point count mismatch: parsed={total_3d} header={header.point3d_points}" + ) + + table_offsets["point2d"] = r.offset + point2d = tuple((r.u16(), r.u16()) for _ in range(header.point2d_records)) + + table_offsets["feature"] = r.offset + features: list[ParsedFeature] = [] + attr_count = 0 + total_attr_bytes = 0 + for i in range(header.feature_records): + st = r.offset + object_type = r.u8() + geotype = r.u8() + record_len = r.u16() + if record_len < 4: + raise CM93ParseError(f"feature {i}: record length {record_len} < 4") + record_end = st + record_len + if record_end > len(decoded): + raise CM93ParseError(f"feature {i}: record beyond EOF") + primitive = geotype & 0x0F + refs: list[GeometryRef] = [] + p2idx = None + p3idx = None + if primitive in (2, 4): + n = r.u16() + for _ in range(n): + raw_idx = r.u16() + edge_idx = raw_idx & 0x1FFF + usage = raw_idx >> 13 + if edge_idx >= len(edges): + raise CM93ParseError(f"feature {i}: edge index {edge_idx} out of range") + refs.append(GeometryRef(edge_idx, usage)) + elif primitive == 1: + p2idx = r.u16() + if p2idx >= len(point2d): + raise CM93ParseError(f"feature {i}: point2d index {p2idx} out of range") + elif primitive == 8: + p3idx = r.u16() + if p3idx >= len(groups): + raise CM93ParseError(f"feature {i}: point3d index {p3idx} out of range") + elif primitive == 0: + pass + else: + # Unknown low primitive cannot be skipped safely unless record boundary is honored. + # We preserve the record and mark geometry unknown; remaining bytes are consumed below. + pass + + related: list[int] = [] + related_hint: int | None = None + if geotype & 0x10: + nr = r.u8() + for _ in range(nr): + idx = r.u16() + if idx >= header.feature_records: + raise CM93ParseError(f"feature {i}: related index {idx} out of range") + related.append(idx) + if geotype & 0x20: + related_hint = r.u16() & 0xFF + + nattr = 0 + attrs: tuple[ParsedAttribute, ...] = () + attr_block = b"" + if geotype & 0x80: + nattr = r.u8() + attr_len = record_end - r.offset + if attr_len < 0: + raise CM93ParseError(f"feature {i}: negative attribute length") + attr_block = r.take(attr_len) + attrs = _parse_attributes(attr_block, nattr, attr_defs) + attr_count += nattr + total_attr_bytes += len(attr_block) + elif r.offset < record_end: + # Known flags consumed fewer bytes than descriptor length. Preserve but do not guess. + r.offset = record_end + + if r.offset != record_end: + raise CM93ParseError( + f"feature {i}: record boundary mismatch parsed={r.offset-st} expected={record_len}" + ) + obj = object_defs.get(object_type) + object_class = obj.name if obj else f"UNKNOWN_{object_type}" + features.append(ParsedFeature( + record_index=i, + object_type=object_type, + object_class=object_class, + raw_geotype=geotype, + primitive=primitive, + geometry_refs=tuple(refs), + point2d_index=p2idx, + point3d_index=p3idx, + related_indices=tuple(related), + related_count_hint=related_hint, + n_attributes=nattr, + attributes=attrs, + raw_record=decoded[st:record_end], + raw_attribute_block=attr_block, + )) + + if total_attr_bytes != header.attribute_bytes: + raise CM93ParseError( + f"attribute byte count mismatch: parsed={total_attr_bytes} header={header.attribute_bytes}" + ) + if r.offset != len(decoded): + raise CM93ParseError(f"cell end mismatch: parsed={r.offset} file={len(decoded)}") + + sha = hashlib.sha256(encoded).hexdigest() + return ParsedCell( + path=path, + source_size=len(encoded), + sha256=sha, + encoded_bytes=encoded if keep_source_bytes else b"", + decoded_bytes=decoded if keep_source_bytes else b"", + prolog_word0=word0, + table1_length=table1, + table2_length=table2, + header=header, + edges=tuple(edges), + point2d=point2d, + point3d=tuple(groups), + features=tuple(features), + end_offset=r.offset, + table_offsets=table_offsets, + attribute_count=attr_count, + ) diff --git a/src/cm93db/cli.py b/src/cm93db/cli.py new file mode 100644 index 0000000..a864f03 --- /dev/null +++ b/src/cm93db/cli.py @@ -0,0 +1,189 @@ +from __future__ import annotations +import argparse +import platform +import sys +import json +from datetime import datetime +from pathlib import Path + +from .config import load_config, load_source_dir +from .database import connect, server_version, assert_mysql57, create_database, apply_schema +from .scanner import scan_cm93 +from .dictionaries import load_object_dictionary, load_attribute_dictionary +from .schema_contract import REQUIRED_TABLES +from .importer import ImportOptions, run_import +from .audit import run_audit, print_result + + +def project_root() -> Path: + return Path(__file__).resolve().parents[2] + + +def _root(arg: str | None) -> Path: + return Path(arg) if arg else load_source_dir() + + +def _scales(values: list[str] | None) -> set[str] | None: + if not values: + return None + out=set() + for v in values: + for ch in v.upper().replace(",",""): + if ch.strip(): out.add(ch) + bad=out-set("ZABCDEFG") + if bad: + raise SystemExit(f"无效比例尺: {sorted(bad)}") + return out + + +def cmd_env_check(_args): + print(f"Python: {sys.version.split()[0]}") + if sys.version_info[:2] != (3, 12): + raise SystemExit("ERROR: 本项目固定 Python 3.12.x；推荐 3.12.10") + print(f"Platform: {platform.platform()}") + cfg = load_config() + conn = connect(cfg) + try: + v = server_version(conn) + print(f"MySQL: {v}") + assert_mysql57(v) + with conn.cursor() as cur: + cur.execute("SHOW VARIABLES LIKE 'max_allowed_packet'") + row=cur.fetchone() + if row: + val=int(row.get("Value",0)); print(f"max_allowed_packet: {val} bytes") + if val < 4*1024*1024: + print("WARNING: max_allowed_packet 较小；建议至少 4 MiB。raw table 已按 512 KiB 分块。") + print("MySQL 5.7 compatibility: OK") + finally: + conn.close() + + +def cmd_db_init(args): + cfg = load_config() + if args.create_database: + create_database(cfg) + print(f"Database ensured: {cfg.database}") + schema = project_root() / "sql" / "001_schema_mysql57.sql" + apply_schema(cfg, schema) + print(f"Schema initialized: {schema.name}") + + +def cmd_db_check(_args): + cfg = load_config() + conn = connect(cfg) + try: + v = server_version(conn) + assert_mysql57(v) + with conn.cursor() as cur: + cur.execute("SELECT table_name FROM information_schema.tables WHERE table_schema=%s", (cfg.database,)) + tables = {row["table_name"] for row in cur.fetchall()} + missing = sorted(REQUIRED_TABLES - tables) + print(f"MySQL: {v}") + print(f"Tables: {len(tables)}") + if missing: + print("Missing:") + for t in missing: print(f" - {t}") + raise SystemExit(2) + print("Schema contract: OK") + finally: + conn.close() + + +def cmd_scan(args): + result = scan_cm93(_root(args.root)) + by_scale = {} + for c in result.cells: by_scale[c.scale_code] = by_scale.get(c.scale_code, 0) + 1 + print(f"Root: {result.root}") + print(f"Cells: {len(result.cells)}") + for scale in sorted(by_scale): print(f" {scale}: {by_scale[scale]}") + print(f"CM93OBJ.DIC: {result.object_dictionary or 'NOT FOUND'}") + print(f"ATTR dictionary: {result.attribute_dictionary or 'NOT FOUND'}") + + +def cmd_inspect_dicts(args): + result = scan_cm93(_root(args.root)) + if not result.object_dictionary: raise SystemExit("CM93OBJ.DIC 未找到") + if not result.attribute_dictionary: raise SystemExit("CM93ATTR.DIC/ATTRLUT.DIC 未找到") + objects = load_object_dictionary(result.object_dictionary) + attrs = load_attribute_dictionary(result.attribute_dictionary) + print(f"Object dictionary: {len(objects)} records") + print(f"Attribute dictionary: {len(attrs)} records") + for key in ("DEPARE", "DEPCNT", "LIGHTS", "SOUNDG", "WRECKS", "_m_sor"): + item = next((x for x in objects if x.name == key), None) + print(f" {key}: {item.object_id if item else 'missing'}") + for key in ("DRVAL1", "VALSOU", "_wgsox", "_wgsoy"): + item = next((x for x in attrs if x.name == key), None) + print(f" {key}: {item.attribute_id if item else 'missing'} / {item.source_type_name if item else '-'}") + + +def cmd_import(args): + cfg=load_config(); root=_root(args.root) + opt=ImportOptions(root=root,limit=args.limit,scales=_scales(args.scale),cell_name=args.cell, + reimport=args.reimport,retry_failed=not args.skip_failed,fail_fast=args.fail_fast, + store_raw_tables=not args.no_raw_tables) + s=run_import(cfg,opt) + print("\n=== IMPORT SUMMARY ===") + for k in ("cells_discovered","cells_done","cells_skipped","cells_failed","features_source","features_imported","attributes_source","attributes_imported","unknown_object_count","unknown_attribute_count","geometry_issue_count","sounding_points"): + print(f"{k:24}: {getattr(s,k)}") + if s.cells_failed or s.unknown_object_count or s.unknown_attribute_count: + raise SystemExit(2) + + +def cmd_audit(args): + cfg=load_config() if not args.source_only else load_config() + r=run_audit(cfg,_root(args.root),limit=args.limit,scales=_scales(args.scale),cell_name=args.cell, + precision_samples=args.precision_samples,tolerance_deg=args.tolerance, + validate_geometry=args.validate_geometry,source_only=args.source_only,require_raw=not args.allow_missing_raw) + print_result(r) + report = Path(args.report) if args.report else (project_root() / "logs" / f"cm93_audit_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json") + report.parent.mkdir(parents=True,exist_ok=True) + report.write_text(json.dumps(r.as_dict(),ensure_ascii=False,indent=2),encoding="utf-8") + print(f"Audit report: {report}") + if not r.passed: raise SystemExit(2) + + +def _add_source_args(s, optional=True): + s.add_argument("root", nargs="?" if optional else None, help="CM93 根目录；省略时读取 .env 的 CM93_SOURCE_DIR") + + +def build_parser(): + p = argparse.ArgumentParser(prog="cm93db") + sub = p.add_subparsers(dest="cmd", required=True) + s = sub.add_parser("env-check"); s.set_defaults(func=cmd_env_check) + s = sub.add_parser("db-init"); s.add_argument("--create-database", action="store_true"); s.set_defaults(func=cmd_db_init) + s = sub.add_parser("db-check"); s.set_defaults(func=cmd_db_check) + s = sub.add_parser("scan"); _add_source_args(s); s.set_defaults(func=cmd_scan) + s = sub.add_parser("inspect-dicts"); _add_source_args(s); s.set_defaults(func=cmd_inspect_dicts) + + s = sub.add_parser("import", help="完整解析 CM93 并逐 cell 导入 MySQL") + _add_source_args(s) + s.add_argument("--limit", type=int) + s.add_argument("--scale", action="append", help="限定比例尺，例如 --scale G 或 --scale F,G") + s.add_argument("--cell", help="只导入指定 cell 文件名/不带扩展名") + s.add_argument("--reimport", action="store_true", help="重新导入已 DONE cell") + s.add_argument("--skip-failed", action="store_true", help="不重试之前 FAILED 的 cell") + s.add_argument("--fail-fast", action="store_true") + s.add_argument("--no-raw-tables", action="store_true", help="不保存 decoded 原始 table；不建议正式母库使用") + s.set_defaults(func=cmd_import) + + s = sub.add_parser("audit", help="重新解析源 CM93 并与数据库逐 cell 审计") + _add_source_args(s) + s.add_argument("--limit", type=int) + s.add_argument("--scale", action="append") + s.add_argument("--cell") + s.add_argument("--precision-samples", type=int, default=128, help="数据库 WKB 双精度回读样本数") + s.add_argument("--tolerance", type=float, default=1e-12, help="经纬度精度比较容差（degree）") + s.add_argument("--validate-geometry", action="store_true", help="额外执行 Shapely validity 全量检查，较慢") + s.add_argument("--source-only", action="store_true", help="只审计源文件解析，不查询数据库") + s.add_argument("--allow-missing-raw", action="store_true", help="不把 raw table 未保存视为审计失败") + s.add_argument("--report", help="JSON 审计报告路径；默认写入项目 logs/") + s.set_defaults(func=cmd_audit) + return p + + +def main(): + args = build_parser().parse_args() + args.func(args) + +if __name__ == "__main__": main() diff --git a/src/cm93db/cm93_binary.py b/src/cm93db/cm93_binary.py new file mode 100644 index 0000000..e3ba331 --- /dev/null +++ b/src/cm93db/cm93_binary.py @@ -0,0 +1,478 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +import hashlib +import math +import struct +from typing import Any + +from .dictionaries import AttributeDef, ObjectDef +from .precision import CM93_SEMIMAJOR_M + +TABLE_0 = bytes([ +0xCD,0xEA,0xDC,0x48,0x3E,0x6D,0xCA,0x7B,0x52,0xE1,0xA4,0x8E,0xAB,0x05,0xA7,0x97, +0xB9,0x60,0x39,0x85,0x7C,0x56,0x7A,0xBA,0x68,0x6E,0xF5,0x5D,0x02,0x4E,0x0F,0xA1, +0x27,0x24,0x41,0x34,0x00,0x5A,0xFE,0xCB,0xD0,0xFA,0xF8,0x6C,0x74,0x96,0x9E,0x0E, +0xC2,0x49,0xE3,0xE5,0xC0,0x3B,0x59,0x18,0xA9,0x86,0x8F,0x30,0xC3,0xA8,0x22,0x0A, +0x14,0x1A,0xB2,0xC9,0xC7,0xED,0xAA,0x29,0x94,0x75,0x0D,0xAC,0x0C,0xF4,0xBB,0xC5, +0x3F,0xFD,0xD9,0x9C,0x4F,0xD5,0x84,0x1E,0xB1,0x81,0x69,0xB4,0x09,0xB8,0x3C,0xAF, +0xA3,0x08,0xBF,0xE0,0x9A,0xD7,0xF7,0x8C,0x67,0x66,0xAE,0xD4,0x4C,0xA5,0xEC,0xF9, +0xB6,0x64,0x78,0x06,0x5B,0x9B,0xF2,0x99,0xCE,0xDB,0x53,0x55,0x65,0x8D,0x07,0x33, +0x04,0x37,0x92,0x26,0x23,0xB5,0x58,0xDA,0x2F,0xB3,0x40,0x5E,0x7F,0x4B,0x62,0x80, +0xE4,0x6F,0x73,0x1D,0xDF,0x17,0xCC,0x28,0x25,0x2D,0xEE,0x3A,0x98,0xE2,0x01,0xEB, +0xDD,0xBC,0x90,0xB0,0xFC,0x95,0x76,0x93,0x46,0x57,0x2C,0x2B,0x50,0x11,0x0B,0xC1, +0xF0,0xE7,0xD6,0x21,0x31,0xDE,0xFF,0xD8,0x12,0xA6,0x4D,0x8A,0x13,0x43,0x45,0x38, +0xD2,0x87,0xA0,0xEF,0x82,0xF1,0x47,0x89,0x6A,0xC8,0x54,0x1B,0x16,0x7E,0x79,0xBD, +0x6B,0x91,0xA2,0x71,0x36,0xB7,0x03,0x3D,0x72,0xC6,0x44,0x8B,0xCF,0x15,0x9F,0x32, +0xC4,0x77,0x83,0x63,0x20,0x88,0xF6,0xAD,0xF3,0xE8,0x4A,0xE9,0x35,0x1C,0x5F,0x19, +0x1F,0x7D,0x70,0xFB,0xD1,0x51,0x10,0xD3,0x2E,0x61,0x9D,0x5C,0x2A,0x42,0xBE,0xE6, +]) + +_encode = bytes((v ^ 8) for v in TABLE_0) +_decode_arr = bytearray(256) +for i, v in enumerate(_encode): + _decode_arr[v] = i +DECODE_TABLE = bytes(_decode_arr) + + +def decode_bytes(data: bytes) -> bytes: + return data.translate(DECODE_TABLE) + + +class CM93ParseError(RuntimeError): + pass + + +@dataclass(frozen=True) +class Header: + lon_min: float + lat_min: float + lon_max: float + lat_max: float + easting_min: float + northing_min: float + easting_max: float + northing_max: float + usn_vector_records: int + n_vector_record_points: int + m_46: int + m_4a: int + usn_point3d_records: int + m_50: int + m_54: int + usn_point2d_records: int + m_5a: int + m_5c: int + usn_feature_records: int + m_60: int + m_64: int + m_68: int + m_6a: int + m_6c: int + m_nrelated_object_pointers: int + m_72: int + m_76: int + m_78: int + m_7c: int + + @property + def x_rate(self) -> float: + delta = self.easting_max - self.easting_min + if delta < 0: + delta += CM93_SEMIMAJOR_M * 2.0 * math.pi + return delta / 65535.0 + + @property + def y_rate(self) -> float: + return (self.northing_max - self.northing_min) / 65535.0 + + def as_dict(self) -> dict[str, Any]: + return {k: getattr(self, k) for k in self.__dataclass_fields__} + + +@dataclass(frozen=True) +class Edge: + index: int + points: tuple[tuple[int, int], ...] + + +@dataclass(frozen=True) +class Point3DDescriptor: + index: int + points: tuple[tuple[int, int, int], ...] + + +@dataclass(frozen=True) +class ParsedAttribute: + ordinal: int + attribute_id: int + name: str + decoder_type: str + source_type_name: str + value: Any + raw_value: bytes + raw_entry: bytes + + +@dataclass +class FeatureRecord: + index: int + object_type: int + object_class: str + geotype: int + geometry_primitive: int + geometry_refs: list[tuple[int, int]] = field(default_factory=list) # (edge index, usage) + point2d_index: int | None = None + point3d_index: int | None = None + related_indices: list[int] = field(default_factory=list) + relation20_raw: int | None = None + attributes: list[ParsedAttribute] = field(default_factory=list) + raw_record: bytes = b"" + raw_attribute_blob: bytes = b"" + + +@dataclass(frozen=True) +class RawTables: + prolog: bytes + header: bytes + vectors: bytes + point3d: bytes + point2d: bytes + features: bytes + + +@dataclass +class ParsedCell: + path: Path + file_size: int + sha256: str + word0: int + table1_length: int + table2_length: int + header: Header + edges: list[Edge] + point3d: list[Point3DDescriptor] + point2d: list[tuple[int, int]] + features: list[FeatureRecord] + raw_tables: RawTables + unknown_object_ids: set[int] + unknown_attribute_ids: set[int] + attribute_count: int + + +class Cursor: + def __init__(self, data: bytes, pos: int = 0): + self.data = data + self.pos = pos + + def remaining(self) -> int: + return len(self.data) - self.pos + + def take(self, n: int) -> bytes: + if n < 0 or self.pos + n > len(self.data): + raise CM93ParseError(f"越界读取: pos={self.pos}, n={n}, size={len(self.data)}") + out = self.data[self.pos:self.pos+n] + self.pos += n + return out + + def u8(self) -> int: + return self.take(1)[0] + + def u16(self) -> int: + return struct.unpack(" int: + return struct.unpack(" float: + return struct.unpack(" float: + return struct.unpack(" Header: + if len(decoded_header) != 128: + raise CM93ParseError(f"CM93 header 长度应为 128，实际 {len(decoded_header)}") + c = Cursor(decoded_header) + vals = [c.f64() for _ in range(8)] + h = Header( + lon_min=vals[0], lat_min=vals[1], lon_max=vals[2], lat_max=vals[3], + easting_min=vals[4], northing_min=vals[5], easting_max=vals[6], northing_max=vals[7], + usn_vector_records=c.u16(), + n_vector_record_points=c.i32(), + m_46=c.i32(), + m_4a=c.i32(), + usn_point3d_records=c.u16(), + m_50=c.i32(), + m_54=c.i32(), + usn_point2d_records=c.u16(), + m_5a=c.u16(), + m_5c=c.u16(), + usn_feature_records=c.u16(), + m_60=c.i32(), + m_64=c.i32(), + m_68=c.u16(), + m_6a=c.u16(), + m_6c=c.u16(), + m_nrelated_object_pointers=c.i32(), + m_72=c.i32(), + m_76=c.u16(), + m_78=c.i32(), + m_7c=c.i32(), + ) + if c.remaining() != 0: + raise CM93ParseError(f"header 未完全消费: {c.remaining()} bytes") + return h + + +def _decode_text(raw: bytes) -> str: + # Byte-preserving fallback is intentional. raw_value is stored separately. + for enc in ("utf-8", "cp1252", "latin1"): + try: + return raw.decode(enc) + except UnicodeDecodeError: + pass + return raw.decode("latin1") + + +def _parse_attr_entry(c: Cursor, ordinal: int, attr_defs: dict[int, AttributeDef]) -> tuple[ParsedAttribute, bool]: + start = c.pos + aid = c.u8() + d = attr_defs.get(aid) + unknown = d is None + if d is None: + raise CM93ParseError(f"未知 attribute id={aid}，无法确定值长度") + vstart = c.pos + t = d.decoder_type + if t == "B": + value = c.u8() + elif t == "W": + value = c.u16() / 10.0 + elif t == "G": + value = c.i32() + elif t == "R": + value = c.f32() + elif t == "S": + z = c.data.find(b"\x00", c.pos) + if z < 0: + raise CM93ParseError(f"字符串属性 {d.name} 缺少 NUL 终止符") + raw = c.data[c.pos:z] + c.pos = z + 1 + value = _decode_text(raw) + elif t == "C": + prefix = list(c.take(3)) + z = c.data.find(b"\x00", c.pos) + if z < 0: + raise CM93ParseError(f"复杂属性 {d.name} 缺少 NUL 终止符") + raw = c.data[c.pos:z] + c.pos = z + 1 + value = {"prefix": prefix, "text": _decode_text(raw)} + elif t == "L": + n = c.u8() + value = list(c.take(n)) + elif t == "I": + value = c.u16() + else: + raise CM93ParseError(f"属性 {d.name}({aid}) 类型 {d.source_type_name}/{t} 无法解码") + vend = c.pos + return ParsedAttribute( + ordinal=ordinal, + attribute_id=aid, + name=d.name, + decoder_type=t, + source_type_name=d.source_type_name, + value=value, + raw_value=c.data[vstart:vend], + raw_entry=c.data[start:vend], + ), unknown + + +def parse_cell(path: Path, object_defs: dict[int, ObjectDef], attr_defs: dict[int, AttributeDef]) -> ParsedCell: + encoded = path.read_bytes() + decoded = decode_bytes(encoded) + if len(decoded) < 138: + raise CM93ParseError(f"文件过短: {path}") + c = Cursor(decoded) + prolog_start = c.pos + word0 = c.u16() + table1 = c.i32() + table2 = c.i32() + prolog = decoded[prolog_start:c.pos] + if word0 != 138: + raise CM93ParseError(f"prolog/header 长度异常: word0={word0}, expected=138") + if word0 + table1 + table2 != len(decoded): + raise CM93ParseError( + f"文件完整性失败: {word0}+{table1}+{table2}!={len(decoded)}" + ) + + header_start = c.pos + header = parse_header(c.take(128)) + raw_header = decoded[header_start:c.pos] + + vector_start = c.pos + edges: list[Edge] = [] + total_vector_points = 0 + for i in range(header.usn_vector_records): + n = c.u16() + pts = tuple((c.u16(), c.u16()) for _ in range(n)) + total_vector_points += n + edges.append(Edge(i, pts)) + vector_end = c.pos + if total_vector_points != header.n_vector_record_points: + raise CM93ParseError( + f"vector point count mismatch: header={header.n_vector_record_points}, parsed={total_vector_points}" + ) + + p3_start = c.pos + point3d: list[Point3DDescriptor] = [] + total_p3_points = 0 + for i in range(header.usn_point3d_records): + n = c.u16() + pts = tuple((c.u16(), c.u16(), c.u16()) for _ in range(n)) + total_p3_points += n + point3d.append(Point3DDescriptor(i, pts)) + p3_end = c.pos + if header.m_50 >= 0 and total_p3_points != header.m_50: + raise CM93ParseError(f"3D point count mismatch: header={header.m_50}, parsed={total_p3_points}") + + p2_start = c.pos + point2d = [(c.u16(), c.u16()) for _ in range(header.usn_point2d_records)] + p2_end = c.pos + + feature_start = c.pos + features: list[FeatureRecord] = [] + unknown_obj: set[int] = set() + unknown_attr: set[int] = set() + attr_total = 0 + + for fi in range(header.usn_feature_records): + rec_start = c.pos + object_type = c.u8() + geotype = c.u8() + desc_len = c.u16() + if desc_len < 4: + raise CM93ParseError(f"feature {fi}: obj_desc_bytes={desc_len}<4") + rec_end = rec_start + desc_len + if rec_end > len(decoded): + raise CM93ParseError(f"feature {fi}: record 越界") + od = object_defs.get(object_type) + if od is None: + unknown_obj.add(object_type) + object_class = f"UNKNOWN_{object_type}" + else: + object_class = od.name + prim = geotype & 0x0F + f = FeatureRecord(fi, object_type, object_class, geotype, prim) + + if prim in (2, 4): + n = c.u16() + for _ in range(n): + packed = c.u16() + edge_idx = packed & 0x1FFF + usage = packed >> 13 + if edge_idx >= len(edges): + raise CM93ParseError(f"feature {fi}: edge index {edge_idx} >= {len(edges)}") + f.geometry_refs.append((edge_idx, usage)) + elif prim == 1: + idx = c.u16() + if idx >= len(point2d): + raise CM93ParseError(f"feature {fi}: point2d index {idx} >= {len(point2d)}") + f.point2d_index = idx + elif prim == 8: + idx = c.u16() + if idx >= len(point3d): + raise CM93ParseError(f"feature {fi}: point3d index {idx} >= {len(point3d)}") + f.point3d_index = idx + + if geotype & 0x10: + nrel = c.u8() + f.related_indices = [c.u16() for _ in range(nrel)] + for rel in f.related_indices: + if rel >= header.usn_feature_records: + raise CM93ParseError(f"feature {fi}: related index {rel} 越界") + if geotype & 0x20: + f.relation20_raw = c.u16() + # 0x40 is observed as a flag without a documented payload in OpenCPN. + + if geotype & 0x80: + nattr = c.u8() + attr_blob_start = c.pos + # By format definition obj_desc_bytes is the total record length. + # Attribute payload is the remaining bytes in this record. + attr_blob = decoded[attr_blob_start:rec_end] + ac = Cursor(attr_blob) + attrs: list[ParsedAttribute] = [] + for ordinal in range(nattr): + try: + a, was_unknown = _parse_attr_entry(ac, ordinal, attr_defs) + except CM93ParseError as e: + # Capture the leading id when possible for audit diagnostics. + if ac.pos < len(attr_blob): + unknown_attr.add(attr_blob[ac.pos]) + raise CM93ParseError(f"feature {fi} attribute {ordinal}: {e}") from e + if was_unknown: + unknown_attr.add(a.attribute_id) + attrs.append(a) + if ac.remaining() != 0: + raise CM93ParseError( + f"feature {fi}: attribute blob 剩余 {ac.remaining()} bytes; nattr={nattr}" + ) + f.attributes = attrs + f.raw_attribute_blob = attr_blob + attr_total += len(attrs) + c.pos = rec_end + else: + # Preserve any unparsed tail, but do not silently cross the record boundary. + if c.pos > rec_end: + raise CM93ParseError(f"feature {fi}: 已解析字段超过 record 长度") + c.pos = rec_end + + f.raw_record = decoded[rec_start:rec_end] + features.append(f) + + feature_end = c.pos + if feature_end != len(decoded): + raise CM93ParseError(f"文件尾不匹配: parsed={feature_end}, size={len(decoded)}, remaining={len(decoded)-feature_end}") + parsed_attr_bytes = sum(len(f.raw_attribute_blob) for f in features) + if parsed_attr_bytes != header.m_78: + raise CM93ParseError(f"attribute byte count mismatch: header={header.m_78}, parsed={parsed_attr_bytes}") + parsed_geom_refs = sum(len(f.geometry_refs) for f in features) + if parsed_geom_refs != header.m_46 + header.m_4a: + raise CM93ParseError(f"geometry ref count mismatch: header={header.m_46 + header.m_4a}, parsed={parsed_geom_refs}") + parsed_related = sum(len(f.related_indices) for f in features) + if parsed_related != header.m_nrelated_object_pointers: + raise CM93ParseError(f"related pointer count mismatch: header={header.m_nrelated_object_pointers}, parsed={parsed_related}") + + # Table-1/Table-2 lengths are retained and audited at file level. The internal + # semantic subtable boundaries are determined from header record counts. + return ParsedCell( + path=path, + file_size=len(encoded), + sha256=hashlib.sha256(encoded).hexdigest(), + word0=word0, + table1_length=table1, + table2_length=table2, + header=header, + edges=edges, + point3d=point3d, + point2d=point2d, + features=features, + raw_tables=RawTables( + prolog=prolog, + header=raw_header, + vectors=decoded[vector_start:vector_end], + point3d=decoded[p3_start:p3_end], + point2d=decoded[p2_start:p2_end], + features=decoded[feature_start:feature_end], + ), + unknown_object_ids=unknown_obj, + unknown_attribute_ids=unknown_attr, + attribute_count=attr_total, + ) diff --git a/src/cm93db/config.py b/src/cm93db/config.py new file mode 100644 index 0000000..514104f --- /dev/null +++ b/src/cm93db/config.py @@ -0,0 +1,35 @@ +from __future__ import annotations +from dataclasses import dataclass +from pathlib import Path +import os +from dotenv import load_dotenv + +@dataclass(frozen=True) +class DbConfig: + host: str + port: int + database: str + user: str + password: str + charset: str = "utf8mb4" + +def load_config(require_database: bool = True) -> DbConfig: + load_dotenv() + database = os.getenv("CM93_DB_NAME", "cm93_chart") + if require_database and not database: + raise RuntimeError("CM93_DB_NAME 未配置") + return DbConfig( + host=os.getenv("CM93_DB_HOST", "127.0.0.1"), + port=int(os.getenv("CM93_DB_PORT", "3306")), + database=database, + user=os.getenv("CM93_DB_USER", "root"), + password=os.getenv("CM93_DB_PASSWORD", ""), + charset=os.getenv("CM93_DB_CHARSET", "utf8mb4"), + ) + +def load_source_dir() -> Path: + load_dotenv() + raw=os.getenv("CM93_SOURCE_DIR", "").strip() + if not raw: + raise RuntimeError("未传入 CM93 路径，且 .env 中 CM93_SOURCE_DIR 未配置") + return Path(raw) diff --git a/src/cm93db/database.py b/src/cm93db/database.py new file mode 100644 index 0000000..bf3aa71 --- /dev/null +++ b/src/cm93db/database.py @@ -0,0 +1,101 @@ +from __future__ import annotations +from pathlib import Path +from typing import Any +import re +from .config import DbConfig + + +def connect(cfg: DbConfig, with_database: bool = True): + import pymysql + kwargs = dict( + host=cfg.host, + port=cfg.port, + user=cfg.user, + password=cfg.password, + charset=cfg.charset, + autocommit=False, + cursorclass=pymysql.cursors.DictCursor, + ) + if with_database: + kwargs["database"] = cfg.database + return pymysql.connect(**kwargs) + + +def server_version(conn) -> str: + with conn.cursor() as cur: + cur.execute("SELECT VERSION() AS v") + return str(cur.fetchone()["v"]) + + +def assert_mysql57(version: str) -> None: + m = re.match(r"^(\d+)\.(\d+)\.(\d+)", version) + if not m: + raise RuntimeError(f"无法识别 MySQL 版本: {version}") + major, minor, _patch = map(int, m.groups()) + if (major, minor) != (5, 7): + raise RuntimeError( + f"项目数据库基线是 MySQL 5.7.34；当前服务器为 {version}。" + "不要直接继续初始化，先确认兼容策略。" + ) + + +def create_database(cfg: DbConfig) -> None: + safe = cfg.database.replace("`", "``") + conn = connect(cfg, with_database=False) + try: + assert_mysql57(server_version(conn)) + with conn.cursor() as cur: + cur.execute( + f"CREATE DATABASE IF NOT EXISTS `{safe}` " + "CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci" + ) + conn.commit() + finally: + conn.close() + + +def _split_sql(sql: str): + statements = [] + buf = [] + in_single = in_double = in_backtick = False + i = 0 + while i < len(sql): + ch = sql[i] + nxt = sql[i+1] if i + 1 < len(sql) else "" + if not (in_single or in_double or in_backtick) and ch == "-" and nxt == "-": + while i < len(sql) and sql[i] != "\n": i += 1 + continue + if not (in_single or in_double or in_backtick) and ch == "#": + while i < len(sql) and sql[i] != "\n": i += 1 + continue + if ch == "'" and not in_double and not in_backtick and (i == 0 or sql[i-1] != "\\"): + in_single = not in_single + elif ch == '"' and not in_single and not in_backtick and (i == 0 or sql[i-1] != "\\"): + in_double = not in_double + elif ch == "`" and not in_single and not in_double: + in_backtick = not in_backtick + if ch == ";" and not (in_single or in_double or in_backtick): + s = "".join(buf).strip() + if s: statements.append(s) + buf = [] + else: + buf.append(ch) + i += 1 + tail = "".join(buf).strip() + if tail: statements.append(tail) + return statements + + +def apply_schema(cfg: DbConfig, schema_path: Path) -> None: + sql = schema_path.read_text(encoding="utf-8") + conn = connect(cfg) + try: + assert_mysql57(server_version(conn)) + with conn.cursor() as cur: + for stmt in _split_sql(sql): cur.execute(stmt) + conn.commit() + except Exception: + conn.rollback() + raise + finally: + conn.close() diff --git a/src/cm93db/decoder.py b/src/cm93db/decoder.py new file mode 100644 index 0000000..4b530d1 --- /dev/null +++ b/src/cm93db/decoder.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +# CM93 byte substitution table. This is a format constant, not executable code. +# Decode table is derived as: encode[i] = TABLE_0[i] ^ 8; decode[encode[i]] = i. +TABLE_0 = bytes([ + 0xCD,0xEA,0xDC,0x48,0x3E,0x6D,0xCA,0x7B,0x52,0xE1,0xA4,0x8E,0xAB,0x05,0xA7,0x97, + 0xB9,0x60,0x39,0x85,0x7C,0x56,0x7A,0xBA,0x68,0x6E,0xF5,0x5D,0x02,0x4E,0x0F,0xA1, + 0x27,0x24,0x41,0x34,0x00,0x5A,0xFE,0xCB,0xD0,0xFA,0xF8,0x6C,0x74,0x96,0x9E,0x0E, + 0xC2,0x49,0xE3,0xE5,0xC0,0x3B,0x59,0x18,0xA9,0x86,0x8F,0x30,0xC3,0xA8,0x22,0x0A, + 0x14,0x1A,0xB2,0xC9,0xC7,0xED,0xAA,0x29,0x94,0x75,0x0D,0xAC,0x0C,0xF4,0xBB,0xC5, + 0x3F,0xFD,0xD9,0x9C,0x4F,0xD5,0x84,0x1E,0xB1,0x81,0x69,0xB4,0x09,0xB8,0x3C,0xAF, + 0xA3,0x08,0xBF,0xE0,0x9A,0xD7,0xF7,0x8C,0x67,0x66,0xAE,0xD4,0x4C,0xA5,0xEC,0xF9, + 0xB6,0x64,0x78,0x06,0x5B,0x9B,0xF2,0x99,0xCE,0xDB,0x53,0x55,0x65,0x8D,0x07,0x33, + 0x04,0x37,0x92,0x26,0x23,0xB5,0x58,0xDA,0x2F,0xB3,0x40,0x5E,0x7F,0x4B,0x62,0x80, + 0xE4,0x6F,0x73,0x1D,0xDF,0x17,0xCC,0x28,0x25,0x2D,0xEE,0x3A,0x98,0xE2,0x01,0xEB, + 0xDD,0xBC,0x90,0xB0,0xFC,0x95,0x76,0x93,0x46,0x57,0x2C,0x2B,0x50,0x11,0x0B,0xC1, + 0xF0,0xE7,0xD6,0x21,0x31,0xDE,0xFF,0xD8,0x12,0xA6,0x4D,0x8A,0x13,0x43,0x45,0x38, + 0xD2,0x87,0xA0,0xEF,0x82,0xF1,0x47,0x89,0x6A,0xC8,0x54,0x1B,0x16,0x7E,0x79,0xBD, + 0x6B,0x91,0xA2,0x71,0x36,0xB7,0x03,0x3D,0x72,0xC6,0x44,0x8B,0xCF,0x15,0x9F,0x32, + 0xC4,0x77,0x83,0x63,0x20,0x88,0xF6,0xAD,0xF3,0xE8,0x4A,0xE9,0x35,0x1C,0x5F,0x19, + 0x1F,0x7D,0x70,0xFB,0xD1,0x51,0x10,0xD3,0x2E,0x61,0x9D,0x5C,0x2A,0x42,0xBE,0xE6, +]) + + +def _make_decode_table() -> bytes: + decode = bytearray(256) + for plain, base in enumerate(TABLE_0): + encoded = base ^ 0x08 + decode[encoded] = plain + return bytes(decode) + + +DECODE_TABLE = _make_decode_table() + + +def decode_cm93(data: bytes) -> bytes: + """Decode a complete CM93 cell. Byte substitution is position-independent.""" + return data.translate(DECODE_TABLE) diff --git a/src/cm93db/dictionaries.py b/src/cm93db/dictionaries.py new file mode 100644 index 0000000..ade51d2 --- /dev/null +++ b/src/cm93db/dictionaries.py @@ -0,0 +1,94 @@ +from __future__ import annotations +from dataclasses import dataclass +from pathlib import Path + +TYPE_MAP = { + "aFLOAT": "R", + "aBYTE": "B", + "aSTRING": "S", + "aCMPLX": "C", + "aLIST": "L", + "aWORD10": "W", + "aLONG": "G", + "aNotUsed": "?", +} + +@dataclass(frozen=True) +class ObjectDef: + name: str + object_id: int + geometry_code: str + display_code: str | None + description: str | None + raw_line: str + +@dataclass(frozen=True) +class AttributeDef: + name: str + attribute_id: int + source_type_name: str + decoder_type: str + min_value: float | None + max_value: float | None + raw_line: str + +def _read_lines(path: Path): + for enc in ("utf-8-sig", "cp1252", "latin1"): + try: + return path.read_text(encoding=enc).splitlines() + except UnicodeDecodeError: + pass + raise UnicodeDecodeError("unknown", b"", 0, 1, f"无法读取 {path}") + +def load_object_dictionary(path: Path) -> list[ObjectDef]: + out = [] + for line in _read_lines(path): + if not line.strip(): + continue + parts = line.rstrip("\r\n").split("|") + if len(parts) < 3: + continue + try: + object_id = int(parts[1].strip()) + except ValueError: + continue + out.append(ObjectDef( + name=parts[0].strip(), + object_id=object_id, + geometry_code=parts[2].strip(), + display_code=parts[3].strip() if len(parts) > 3 else None, + description=parts[4].strip() if len(parts) > 4 else None, + raw_line=line, + )) + return out + +def load_attribute_dictionary(path: Path) -> list[AttributeDef]: + out = [] + for line in _read_lines(path): + if not line.strip(): + continue + parts = [p.strip() for p in line.rstrip("\r\n").split("|")] + if len(parts) < 3: + continue + try: + attribute_id = int(parts[1]) + except ValueError: + continue + source_type = parts[2] + def f(i): + if len(parts) <= i or parts[i] == "": + return None + try: + return float(parts[i]) + except ValueError: + return None + out.append(AttributeDef( + name=parts[0], + attribute_id=attribute_id, + source_type_name=source_type, + decoder_type=TYPE_MAP.get(source_type, "?"), + min_value=f(3), + max_value=f(4), + raw_line=line, + )) + return out diff --git a/src/cm93db/geometry.py b/src/cm93db/geometry.py new file mode 100644 index 0000000..90b93f2 --- /dev/null +++ b/src/cm93db/geometry.py @@ -0,0 +1,870 @@ +from __future__ import annotations + +from dataclasses import dataclass +import math +from typing import Iterable, Any + +try: + from shapely import normalize as _shapely_normalize +except ImportError: + _shapely_normalize = None + +from shapely.geometry import Point, LineString, MultiLineString, Polygon, MultiPolygon +from shapely.geometry.base import BaseGeometry +from shapely.ops import unary_union, polygonize_full +from shapely.validation import explain_validity + +from .cm93_binary import ParsedCell, FeatureRecord +from .precision import CM93_SEMIMAJOR_M + + +# 已通过 CM93 实际数据审计的 Area normalization 规则： +# 1. valid Polygon/MultiPolygon 完全原样 PASS-THROUGH； +# 2. invalid Polygon/MultiPolygon 仅使用原始 boundary 做 noding + polygonize_full； +# 3. planar face 采用 odd-even parity 选择； +# 4. 只对修复后的 candidate 做 GEOS canonical normalize； +# 5. 不使用 make_valid() / buffer(0) 覆盖原始 CM93 语义； +# 6. raw_feature_blob / chart_raw_record 仍由 importer 原样保留。 +_NORMALIZATION_BOUNDARY_TOLERANCE = 1e-10 +_POINT_ON_SEGMENT_EPS = 1e-12 + + +@dataclass(frozen=True) +class CoverageInfo: + source_record_index: int + geom_unshifted: BaseGeometry + wgsox: float + wgsoy: float + compilation_scale: int | None + horizontal_datum: int | None + vertical_datum: int | None + attributes: dict[str, Any] + geometry_repaired: bool + geometry_issue: str | None + + +@dataclass +class BuiltFeature: + feature: FeatureRecord + geometry_type: str + geom: BaseGeometry | None + coverage_record_index: int | None + geometry_issue: str | None + geometry_repaired: bool + soundings: list[tuple[int, int, int, float, Point, int | None]] + attributes_map: dict[str, Any] + + +def attr_map(feature: FeatureRecord) -> dict[str, Any]: + out: dict[str, Any] = {} + for a in feature.attributes: + if a.name in out: + prev = out[a.name] + if not isinstance(prev, list) or (prev and not isinstance(prev[0], (dict, list))): + out[a.name] = [prev, a.value] + else: + prev.append(a.value) + else: + out[a.name] = a.value + if feature.related_indices: + out["_cm93_related_indices"] = feature.related_indices + if feature.relation20_raw is not None: + out["_cm93_relation20_raw"] = feature.relation20_raw + return out + + +def raw_xy_to_mercator( + cell: ParsedCell, + x: int, + y: int, + wgsox: float = 0.0, + wgsoy: float = 0.0, +) -> tuple[float, float]: + h = cell.header + easting = x * h.x_rate + h.easting_min - wgsox + northing = y * h.y_rate + h.northing_min - wgsoy + return easting, northing + + +def mercator_to_lonlat(easting: float, northing: float) -> tuple[float, float]: + lon = math.degrees(easting / CM93_SEMIMAJOR_M) + lat = math.degrees( + 2.0 * math.atan(math.exp(northing / CM93_SEMIMAJOR_M)) + - math.pi / 2.0 + ) + # Normalize only after exact formula. This handles cells crossing the dateline. + while lon > 180.0: + lon -= 360.0 + while lon < -180.0: + lon += 360.0 + return lon, lat + + +def raw_xy_to_lonlat( + cell: ParsedCell, + x: int, + y: int, + wgsox: float = 0.0, + wgsoy: float = 0.0, +) -> tuple[float, float]: + return mercator_to_lonlat( + *raw_xy_to_mercator(cell, x, y, wgsox, wgsoy) + ) + + +def _oriented_edge_points( + cell: ParsedCell, + edge_idx: int, + usage: int, +) -> list[tuple[int, int]]: + pts = list(cell.edges[edge_idx].points) + if usage & 4: + pts.reverse() + return pts + + +def _append_joined( + dst: list[tuple[int, int]], + src: list[tuple[int, int]], +) -> None: + if not src: + return + if not dst: + dst.extend(src) + elif dst[-1] == src[0]: + dst.extend(src[1:]) + else: + # Preserve discontinuity; audit will report it through geometry validity/issue. + dst.extend(src) + + +def raw_line_parts( + cell: ParsedCell, + feature: FeatureRecord, +) -> list[list[tuple[int, int]]]: + parts: list[list[tuple[int, int]]] = [] + current: list[tuple[int, int]] = [] + for edge_idx, usage in feature.geometry_refs: + seg = _oriented_edge_points(cell, edge_idx, usage) + if not seg: + continue + if not current: + current = list(seg) + elif current[-1] == seg[0]: + current.extend(seg[1:]) + else: + if len(current) >= 2: + parts.append(current) + current = list(seg) + if len(current) >= 2: + parts.append(current) + return parts + + +def raw_line_points( + cell: ParsedCell, + feature: FeatureRecord, +) -> list[tuple[int, int]]: + # Compatibility helper for diagnostics. Disconnected parts are concatenated here; + # production geometry uses raw_line_parts() and preserves disconnection. + out: list[tuple[int, int]] = [] + for part in raw_line_parts(cell, feature): + out.extend(part) + return out + + +def raw_area_rings( + cell: ParsedCell, + feature: FeatureRecord, +) -> list[list[tuple[int, int]]]: + rings: list[list[tuple[int, int]]] = [] + current: list[tuple[int, int]] = [] + start: tuple[int, int] | None = None + for edge_idx, usage in feature.geometry_refs: + seg = _oriented_edge_points(cell, edge_idx, usage) + if not seg: + continue + if start is None: + start = seg[0] + _append_joined(current, seg) + if current and start is not None and current[-1] == start: + if len(current) >= 4: + rings.append(current) + current = [] + start = None + if current: + # Keep an open ring visible to audit. Shapely Polygon will close it, so the issue + # is returned explicitly by the Area builder. + rings.append(current) + return rings + + +def _transform_points( + cell: ParsedCell, + pts: Iterable[tuple[int, int]], + wgsox: float, + wgsoy: float, +) -> list[tuple[float, float]]: + return [ + raw_xy_to_lonlat(cell, x, y, wgsox, wgsoy) + for x, y in pts + ] + + +def _area_from_rings( + coord_rings: list[list[tuple[float, float]]], +) -> BaseGeometry | None: + valid_rings = [r for r in coord_rings if len(r) >= 3] + if not valid_rings: + return None + + # CM93 edge chains are emitted as closed contours. Most features are one exterior + # followed by holes. If a later contour is not inside the current exterior, treat it + # as another exterior rather than forcing an invalid hole. + polygons: list[Polygon] = [] + current_exterior = valid_rings[0] + holes: list[list[tuple[float, float]]] = [] + exterior_poly = Polygon(current_exterior) + + for r in valid_rings[1:]: + rp = Polygon(r) + try: + inside = exterior_poly.covers(rp.representative_point()) + except Exception: + inside = False + + if inside: + holes.append(r) + else: + polygons.append(Polygon(current_exterior, holes)) + current_exterior = r + holes = [] + exterior_poly = Polygon(current_exterior) + + polygons.append(Polygon(current_exterior, holes)) + + if len(polygons) == 1: + return polygons[0] + return MultiPolygon(polygons) + + +def _geom_issue( + geom: BaseGeometry | None, + extra: str | None = None, + validate: bool = False, +) -> str | None: + parts = [] + if extra: + parts.append(extra) + if geom is None: + parts.append("geometry missing") + elif geom.is_empty: + parts.append("geometry empty") + elif validate and not geom.is_valid: + parts.append(explain_validity(geom)) + return "; ".join(parts) if parts else None + + +def _polygon_rings( + geom: BaseGeometry, +) -> list[list[tuple[float, float]]]: + rings: list[list[tuple[float, float]]] = [] + + def add_polygon(poly: Polygon) -> None: + rings.append([ + (float(c[0]), float(c[1])) + for c in poly.exterior.coords + ]) + for ring in poly.interiors: + rings.append([ + (float(c[0]), float(c[1])) + for c in ring.coords + ]) + + if geom.geom_type == "Polygon": + add_polygon(geom) + elif geom.geom_type == "MultiPolygon": + for poly in geom.geoms: + add_polygon(poly) + + return rings + + +def _point_on_segment( + px: float, + py: float, + ax: float, + ay: float, + bx: float, + by: float, + eps: float = _POINT_ON_SEGMENT_EPS, +) -> bool: + vx = bx - ax + vy = by - ay + wx = px - ax + wy = py - ay + + cross_value = abs(vx * wy - vy * wx) + if cross_value > eps * max(1.0, abs(vx), abs(vy)): + return False + + dot_value = wx * vx + wy * vy + if dot_value < -eps: + return False + + vv = vx * vx + vy * vy + if dot_value - vv > eps: + return False + + return True + + +def _odd_even_inside( + point: tuple[float, float], + rings: list[list[tuple[float, float]]], +) -> bool: + x, y = point + crossings = 0 + + for ring in rings: + for i in range(len(ring) - 1): + x1, y1 = ring[i] + x2, y2 = ring[i + 1] + + if _point_on_segment(x, y, x1, y1, x2, y2): + # polygonize face representative_point should not normally lie + # on the input boundary. Treat a numerical boundary hit as inside. + return True + + if (y1 > y) != (y2 > y): + x_intersection = ( + x1 + + (y - y1) * (x2 - x1) / (y2 - y1) + ) + if x_intersection > x: + crossings += 1 + + return (crossings % 2) == 1 + + +def _canonicalize_polygonal( + geom: BaseGeometry, +) -> BaseGeometry: + if _shapely_normalize is not None: + return _shapely_normalize(geom) + + normalizer = getattr(geom, "normalize", None) + if callable(normalizer): + normalized = normalizer() + if normalized is not None: + return normalized + + raise RuntimeError( + "CM93 Area normalization requires Shapely/GEOS normalize support" + ) + + +def _normalize_cm93_area( + geom: BaseGeometry | None, + structural_issue: str | None = None, +) -> tuple[BaseGeometry | None, bool, str | None]: + """ + Normalize an OGC-invalid CM93 Area using the audited CM93 fill semantics. + + Valid geometry is returned byte/structure untouched. + Open/discontinuous source rings are not silently repaired. + """ + if geom is None: + return None, False, _geom_issue(None, structural_issue) + + if geom.is_empty: + return geom, False, _geom_issue(geom, structural_issue) + + if geom.geom_type not in ("Polygon", "MultiPolygon"): + return ( + geom, + False, + f"area geometry has unexpected type {geom.geom_type}", + ) + + # Structural source problems are evidence, not retraced-connector semantics. + # Preserve them for audit instead of silently normalizing them away. + if structural_issue: + issue = _geom_issue( + geom, + structural_issue, + validate=True, + ) + return geom, False, issue + + # Critical compatibility rule: already-valid geometry stays exactly as built. + if geom.is_valid: + return geom, False, None + + original_reason = explain_validity(geom) + + try: + original_rings = _polygon_rings(geom) + noded_boundary = unary_union(geom.boundary) + polygonized, _cuts, _dangles, _invalid_rings = polygonize_full( + noded_boundary + ) + + if polygonized.geom_type == "Polygon": + faces = [polygonized] + elif polygonized.geom_type in ("MultiPolygon", "GeometryCollection"): + faces = [ + g + for g in polygonized.geoms + if g.geom_type == "Polygon" + ] + else: + faces = [] + + selected: list[Polygon] = [] + + for face in faces: + rp = face.representative_point() + if _odd_even_inside( + (float(rp.x), float(rp.y)), + original_rings, + ): + selected.append(face) + + if not selected: + return ( + geom, + False, + f"{original_reason}; odd-even normalization produced no polygon face", + ) + + candidate = unary_union(selected) + + if candidate.geom_type not in ("Polygon", "MultiPolygon"): + return ( + geom, + False, + f"{original_reason}; odd-even normalization produced {candidate.geom_type}", + ) + + if candidate.is_empty: + return ( + geom, + False, + f"{original_reason}; odd-even normalization produced empty geometry", + ) + + if not candidate.is_valid: + return ( + geom, + False, + f"{original_reason}; normalized candidate invalid: " + f"{explain_validity(candidate)}", + ) + + # Safety guard learned from the dry-run audit: normalization may remove + # retraced zero-area connector linework, but must never invent a new + # boundary outside the original CM93 line network. + candidate_extra_boundary = candidate.boundary.difference( + noded_boundary + ) + if ( + not candidate_extra_boundary.is_empty + and float(candidate_extra_boundary.length) + > _NORMALIZATION_BOUNDARY_TOLERANCE + ): + return ( + geom, + False, + f"{original_reason}; normalized candidate introduces " + f"new boundary length={candidate_extra_boundary.length}", + ) + + candidate = _canonicalize_polygonal(candidate) + + if candidate.geom_type not in ("Polygon", "MultiPolygon"): + return ( + geom, + False, + f"{original_reason}; canonical candidate has type " + f"{candidate.geom_type}", + ) + + if candidate.is_empty or not candidate.is_valid: + return ( + geom, + False, + f"{original_reason}; canonical candidate is not valid", + ) + + return candidate, True, None + + except Exception as exc: + return ( + geom, + False, + f"{original_reason}; odd-even normalization failed: " + f"{type(exc).__name__}: {exc}", + ) + + +def build_unshifted_geometry( + cell: ParsedCell, + feature: FeatureRecord, + validate: bool = False, +) -> tuple[BaseGeometry | None, str | None]: + prim = feature.geometry_primitive + + if prim == 1 and feature.point2d_index is not None: + x, y = cell.point2d[feature.point2d_index] + return Point(raw_xy_to_lonlat(cell, x, y)), None + + if prim == 2: + parts = raw_line_parts(cell, feature) + if not parts: + return None, "line has fewer than 2 points" + coords = [ + _transform_points(cell, p, 0.0, 0.0) + for p in parts + ] + g = ( + LineString(coords[0]) + if len(coords) == 1 + else MultiLineString(coords) + ) + return g, _geom_issue(g, validate=validate) + + if prim == 4: + raw_rings = raw_area_rings(cell, feature) + open_count = sum( + 1 + for r in raw_rings + if r and r[0] != r[-1] + ) + rings = [ + _transform_points(cell, r, 0.0, 0.0) + for r in raw_rings + ] + g = _area_from_rings(rings) + extra = f"{open_count} open ring(s)" if open_count else None + return g, _geom_issue(g, extra, validate=validate) + + return None, None + + +def build_coverages(cell: ParsedCell) -> list[CoverageInfo]: + out: list[CoverageInfo] = [] + + for f in cell.features: + if f.object_class != "_m_sor": + continue + + geom, pre_issue = build_unshifted_geometry( + cell, + f, + validate=False, + ) + if geom is None: + continue + + raw_rings = raw_area_rings(cell, f) if f.geometry_primitive == 4 else [] + open_count = sum( + 1 + for r in raw_rings + if r and r[0] != r[-1] + ) + structural_issue = ( + f"{open_count} open ring(s)" + if open_count + else pre_issue + ) + + if f.geometry_primitive == 4: + geom, repaired, issue = _normalize_cm93_area( + geom, + structural_issue=structural_issue, + ) + else: + repaired = False + issue = _geom_issue( + geom, + pre_issue, + validate=False, + ) + + if geom is None: + continue + + a = attr_map(f) + + def number(name: str, default=0.0): + v = a.get(name, default) + if isinstance(v, list): + v = v[0] if v else default + try: + return float(v) + except (TypeError, ValueError): + return float(default) + + def integer(name: str): + if name not in a: + return None + try: + return int(a[name]) + except (TypeError, ValueError): + return None + + out.append(CoverageInfo( + source_record_index=f.index, + geom_unshifted=geom, + wgsox=number("_wgsox", 0.0), + wgsoy=number("_wgsoy", 0.0), + compilation_scale=integer("CSCALE"), + horizontal_datum=integer("HORDAT"), + vertical_datum=integer("VERDAT"), + attributes=a, + geometry_repaired=repaired, + geometry_issue=issue, + )) + + return out + + +def choose_coverage( + coverages: list[CoverageInfo], + geom_unshifted: BaseGeometry | None, +) -> CoverageInfo | None: + if not coverages: + return None + + if len(coverages) == 1: + return coverages[0] + + if geom_unshifted is None or geom_unshifted.is_empty: + return coverages[0] + + minx, miny, maxx, maxy = geom_unshifted.bounds + probe = Point( + (minx + maxx) / 2.0, + (miny + maxy) / 2.0, + ) + + for cov in coverages: + try: + if cov.geom_unshifted.covers(probe): + return cov + except Exception: + continue + + return coverages[0] + + +def _depth(raw_z: int) -> float: + return ( + float(raw_z - 12000) + if raw_z >= 12000 + else raw_z / 10.0 + ) + + +def build_feature( + cell: ParsedCell, + feature: FeatureRecord, + coverages: list[CoverageInfo], + validate: bool = False, +) -> BuiltFeature: + amap = attr_map(feature) + prim = feature.geometry_primitive + + # _m_sor is stored spatially in chart_coverage. Reuse the already-normalized + # coverage object here so chart_feature bbox/repair metadata matches it. + if feature.object_class == "_m_sor": + cov = next( + ( + c + for c in coverages + if c.source_record_index == feature.index + ), + None, + ) + if cov is not None: + return BuiltFeature( + feature=feature, + geometry_type="META", + geom=cov.geom_unshifted, + coverage_record_index=feature.index, + geometry_issue=cov.geometry_issue, + geometry_repaired=cov.geometry_repaired, + soundings=[], + attributes_map=amap, + ) + + g, issue = build_unshifted_geometry( + cell, + feature, + validate=validate, + ) + return BuiltFeature( + feature=feature, + geometry_type="META", + geom=g, + coverage_record_index=feature.index, + geometry_issue=issue, + geometry_repaired=False, + soundings=[], + attributes_map=amap, + ) + + # CM93 raw geotype 16 is a parent/no-geometry object. It is expected to + # have no own Point/Line/Area and must not be counted as geometry invalid. + if feature.geotype == 16: + return BuiltFeature( + feature=feature, + geometry_type="META", + geom=None, + coverage_record_index=None, + geometry_issue=None, + geometry_repaired=False, + soundings=[], + attributes_map=amap, + ) + + if prim == 8 and feature.point3d_index is not None: + desc = cell.point3d[feature.point3d_index] + soundings = [] + + for x, y, z in desc.points: + unshifted = Point(raw_xy_to_lonlat(cell, x, y)) + cov = choose_coverage(coverages, unshifted) + wx = cov.wgsox if cov else 0.0 + wy = cov.wgsoy if cov else 0.0 + p = Point( + raw_xy_to_lonlat( + cell, + x, + y, + wx, + wy, + ) + ) + soundings.append(( + x, + y, + z, + _depth(z), + p, + cov.source_record_index if cov else None, + )) + + return BuiltFeature( + feature=feature, + geometry_type="SOUNDING", + geom=None, + coverage_record_index=None, + geometry_issue=None, + geometry_repaired=False, + soundings=soundings, + attributes_map=amap, + ) + + # Unshifted geometry is used only to choose the WGS offset coverage. + # Do not let optional validation of this intermediate copy create a + # duplicate Area issue; final shifted Area is normalized below. + unshifted, pre_issue = build_unshifted_geometry( + cell, + feature, + validate=False, + ) + cov = choose_coverage(coverages, unshifted) + wx = cov.wgsox if cov else 0.0 + wy = cov.wgsoy if cov else 0.0 + + geometry_repaired = False + + if prim == 1 and feature.point2d_index is not None: + x, y = cell.point2d[feature.point2d_index] + geom = Point( + raw_xy_to_lonlat( + cell, + x, + y, + wx, + wy, + ) + ) + gtype = "POINT" + issue = _geom_issue( + geom, + pre_issue, + validate=validate, + ) + + elif prim == 2: + parts = raw_line_parts(cell, feature) + coords = [ + _transform_points(cell, p, wx, wy) + for p in parts + ] + geom = ( + ( + LineString(coords[0]) + if len(coords) == 1 + else MultiLineString(coords) + ) + if coords + else None + ) + gtype = "LINE" + issue = _geom_issue( + geom, + pre_issue, + validate=validate, + ) + + elif prim == 4: + raw_rings = raw_area_rings(cell, feature) + open_count = sum( + 1 + for r in raw_rings + if r and r[0] != r[-1] + ) + rings = [ + _transform_points(cell, r, wx, wy) + for r in raw_rings + ] + raw_geom = _area_from_rings(rings) + structural_issue = ( + f"{open_count} open ring(s)" + if open_count + else None + ) + + geom, geometry_repaired, issue = _normalize_cm93_area( + raw_geom, + structural_issue=structural_issue, + ) + gtype = "AREA" + + else: + geom = None + gtype = "META" if prim == 0 else "UNKNOWN" + issue = _geom_issue( + geom, + pre_issue, + validate=validate, + ) + + return BuiltFeature( + feature=feature, + geometry_type=gtype, + geom=geom, + coverage_record_index=( + cov.source_record_index + if cov + else None + ), + geometry_issue=issue, + geometry_repaired=geometry_repaired, + soundings=[], + attributes_map=amap, + ) diff --git a/src/cm93db/importer.py b/src/cm93db/importer.py new file mode 100644 index 0000000..c12fe2f --- /dev/null +++ b/src/cm93db/importer.py @@ -0,0 +1,1104 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime +import json +from pathlib import Path +from typing import Iterable, Any +import hashlib + +from shapely.geometry.base import BaseGeometry + +from . import __version__ +from .cm93_binary import ParsedCell, ParsedAttribute, parse_cell, CM93ParseError +from .config import DbConfig +from .database import connect, assert_mysql57, server_version +from .dictionaries import load_object_dictionary, load_attribute_dictionary, ObjectDef, AttributeDef +from .geometry import build_coverages, build_feature, BuiltFeature +from .scanner import scan_cm93, CellFile + +RAW_CHUNK = 512 * 1024 + + +@dataclass +class ImportOptions: + root: Path + limit: int | None = None + scales: set[str] | None = None + cell_name: str | None = None + reimport: bool = False + retry_failed: bool = True + fail_fast: bool = False + store_raw_tables: bool = True + + +@dataclass +class ImportSummary: + dataset_id: int + job_id: int + cells_discovered: int = 0 + cells_done: int = 0 + cells_skipped: int = 0 + cells_failed: int = 0 + features_source: int = 0 + features_imported: int = 0 + attributes_source: int = 0 + attributes_imported: int = 0 + unknown_object_count: int = 0 + unknown_attribute_count: int = 0 + geometry_issue_count: int = 0 + geometry_repaired_count: int = 0 + sounding_points: int = 0 + + +def _j(value: Any) -> str: + return json.dumps( + value, + ensure_ascii=False, + separators=(",", ":"), + allow_nan=False, + ) + + +def _dataset(conn, root: Path) -> int: + root_s = str(root.resolve()) + with conn.cursor() as cur: + cur.execute( + "SELECT id FROM chart_dataset " + "WHERE source_type='CM93' AND source_root=%s " + "ORDER BY id LIMIT 1", + (root_s,), + ) + row = cur.fetchone() + if row: + cur.execute( + "UPDATE chart_dataset SET parser_version=%s WHERE id=%s", + (__version__, row["id"]), + ) + return int(row["id"]) + + cur.execute( + "INSERT INTO chart_dataset" + "(source_type,name,source_root,parser_version,normalized_srid) " + "VALUES('CM93',%s,%s,%s,4326)", + ( + root.name or "CM93", + root_s, + __version__, + ), + ) + return int(cur.lastrowid) + + +def _store_dictionaries( + conn, + dataset_id: int, + objects: list[ObjectDef], + attrs: list[AttributeDef], +) -> None: + with conn.cursor() as cur: + cur.executemany( + "INSERT INTO chart_dictionary_object" + "(dataset_id,object_class_id,object_class,geometry_code," + "display_code,description,raw_line) " + "VALUES(%s,%s,%s,%s,%s,%s,%s) " + "ON DUPLICATE KEY UPDATE " + "object_class=VALUES(object_class)," + "geometry_code=VALUES(geometry_code)," + "display_code=VALUES(display_code)," + "description=VALUES(description)," + "raw_line=VALUES(raw_line)", + [ + ( + dataset_id, + o.object_id, + o.name, + o.geometry_code, + o.display_code, + o.description, + o.raw_line, + ) + for o in objects + ], + ) + cur.executemany( + "INSERT INTO chart_dictionary_attribute" + "(dataset_id,attribute_id,attribute_name,source_type_name," + "decoder_type,min_value,max_value,raw_line) " + "VALUES(%s,%s,%s,%s,%s,%s,%s,%s) " + "ON DUPLICATE KEY UPDATE " + "attribute_name=VALUES(attribute_name)," + "source_type_name=VALUES(source_type_name)," + "decoder_type=VALUES(decoder_type)," + "min_value=VALUES(min_value)," + "max_value=VALUES(max_value)," + "raw_line=VALUES(raw_line)", + [ + ( + dataset_id, + a.attribute_id, + a.name, + a.source_type_name, + a.decoder_type, + a.min_value, + a.max_value, + a.raw_line, + ) + for a in attrs + ], + ) + + +def _start_job( + conn, + dataset_id: int, + discovered: int, +) -> int: + with conn.cursor() as cur: + cur.execute( + "INSERT INTO chart_import_job" + "(dataset_id,status,started_at,cells_discovered) " + "VALUES(%s,'RUNNING',%s,%s)", + ( + dataset_id, + datetime.now(), + discovered, + ), + ) + return int(cur.lastrowid) + + +def _finish_job(conn, s: ImportSummary) -> None: + status = "FAILED" if s.cells_failed else "DONE" + payload = { + "cells_skipped": s.cells_skipped, + "geometry_issue_count": s.geometry_issue_count, + "geometry_repaired_count": s.geometry_repaired_count, + "sounding_points": s.sounding_points, + } + + with conn.cursor() as cur: + cur.execute( + "UPDATE chart_import_job SET " + "status=%s,finished_at=%s,cells_done=%s,cells_failed=%s," + "features_source=%s,features_imported=%s," + "attributes_source=%s,attributes_imported=%s," + "unknown_object_count=%s,unknown_attribute_count=%s," + "geometry_invalid_count=%s,geometry_repaired_count=%s," + "summary_json=%s WHERE id=%s", + ( + status, + datetime.now(), + s.cells_done, + s.cells_failed, + s.features_source, + s.features_imported, + s.attributes_source, + s.attributes_imported, + s.unknown_object_count, + s.unknown_attribute_count, + s.geometry_issue_count, + s.geometry_repaired_count, + _j(payload), + s.job_id, + ), + ) + + +def _event( + conn, + job_id: int, + cell_id: int | None, + level: str, + code: str, + message: str, + details: Any = None, +) -> None: + with conn.cursor() as cur: + cur.execute( + "INSERT INTO chart_import_event" + "(job_id,cell_id,level,event_code,message,details_json) " + "VALUES(%s,%s,%s,%s,%s,%s)", + ( + job_id, + cell_id, + level, + code, + message, + _j(details) if details is not None else None, + ), + ) + + +def _ensure_cell_row( + conn, + dataset_id: int, + cf: CellFile, +) -> tuple[int, str, int, str | None]: + rel = str(cf.relative_path).replace("/", "\\") + size = cf.path.stat().st_size + + with conn.cursor() as cur: + cur.execute( + "SELECT id,import_status,source_size,source_sha256 " + "FROM chart_cell " + "WHERE dataset_id=%s AND source_relative_path=%s", + ( + dataset_id, + rel, + ), + ) + row = cur.fetchone() + if row: + return ( + int(row["id"]), + str(row["import_status"]), + int(row["source_size"]), + ( + str(row["source_sha256"]) + if row.get("source_sha256") + else None + ), + ) + + cur.execute( + "INSERT INTO chart_cell" + "(dataset_id,source_relative_path,source_file_name," + "source_size,scale_code,native_scale,cell_name,import_status) " + "VALUES(%s,%s,%s,%s,%s,%s,%s,'DISCOVERED')", + ( + dataset_id, + rel, + cf.path.name, + size, + cf.scale_code, + cf.native_scale, + cf.path.stem, + ), + ) + return int(cur.lastrowid), "DISCOVERED", size, None + + +def _clear_cell_payload(conn, cell_id: int) -> None: + with conn.cursor() as cur: + cur.execute( + "DELETE FROM chart_feature WHERE cell_id=%s", + (cell_id,), + ) + cur.execute( + "DELETE FROM chart_coverage WHERE cell_id=%s", + (cell_id,), + ) + cur.execute( + "DELETE FROM chart_raw_record WHERE cell_id=%s", + (cell_id,), + ) + + +def _sha256_file(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter( + lambda: f.read(1024 * 1024), + b"", + ): + h.update(chunk) + return h.hexdigest() + + +def _mark_status( + cfg: DbConfig, + cell_id: int, + status: str, + message: str | None = None, +) -> None: + conn = connect(cfg) + try: + with conn.cursor() as cur: + cur.execute( + "UPDATE chart_cell SET " + "import_status=%s,error_message=%s,imported_at=%s " + "WHERE id=%s", + ( + status, + message, + datetime.now() if status == "DONE" else None, + cell_id, + ), + ) + conn.commit() + finally: + conn.close() + + +def _raw_chunks( + cell_id: int, + kind: str, + data: bytes, + meta: dict[str, Any] | None = None, +): + if not data: + return [] + + out = [] + count = (len(data) + RAW_CHUNK - 1) // RAW_CHUNK + + for i in range(count): + chunk = data[ + i * RAW_CHUNK: + (i + 1) * RAW_CHUNK + ] + md = { + "chunk_index": i, + "chunk_count": count, + "total_bytes": len(data), + } + if meta: + md.update(meta) + out.append(( + cell_id, + kind, + i, + chunk, + _j(md), + )) + + return out + + +def _insert_raw_tables( + conn, + cell_id: int, + cell: ParsedCell, +) -> None: + rows = [] + rows += _raw_chunks( + cell_id, + "OTHER", + cell.raw_tables.prolog, + {"name": "prolog"}, + ) + rows += _raw_chunks( + cell_id, + "HEADER", + cell.raw_tables.header, + {"header": cell.header.as_dict()}, + ) + rows += _raw_chunks( + cell_id, + "VECTOR", + cell.raw_tables.vectors, + {"name": "vector_table"}, + ) + rows += _raw_chunks( + cell_id, + "POINT3D", + cell.raw_tables.point3d, + {"name": "point3d_table"}, + ) + rows += _raw_chunks( + cell_id, + "POINT2D", + cell.raw_tables.point2d, + {"name": "point2d_table"}, + ) + rows += _raw_chunks( + cell_id, + "FEATURE", + cell.raw_tables.features, + {"name": "feature_table"}, + ) + + with conn.cursor() as cur: + cur.executemany( + "INSERT INTO chart_raw_record" + "(cell_id,record_kind,source_record_index," + "raw_blob,raw_metadata_json) " + "VALUES(%s,%s,%s,%s,%s)", + rows, + ) + + +def _bounds( + geom: BaseGeometry | None, +): + if geom is None or geom.is_empty: + return ( + None, + None, + None, + None, + ) + return tuple( + float(v) + for v in geom.bounds + ) + + +def _attr_row( + feature_id: int, + a: ParsedAttribute, +): + vtype = { + "B": "BYTE", + "I": "INT", + "G": "LONG", + "R": "REAL", + "W": "WORD10", + "S": "STRING", + "L": "LIST", + "C": "COMPLEX", + }.get(a.decoder_type, "UNKNOWN") + + vi = vr = vt = vj = None + + if a.decoder_type in ("B", "I", "G"): + vi = int(a.value) + elif a.decoder_type in ("R", "W"): + vr = float(a.value) + elif a.decoder_type == "S": + vt = str(a.value) + elif a.decoder_type == "L": + vj = _j(a.value) + elif a.decoder_type == "C": + if isinstance(a.value, dict): + vt = str(a.value.get("text", "")) + vj = _j(a.value) + + return ( + feature_id, + a.ordinal, + a.attribute_id, + a.name, + vtype, + vi, + vr, + vt, + vj, + a.raw_value, + ) + + +def _insert_parsed_cell( + conn, + dataset_id: int, + cell_id: int, + cf: CellFile, + cell: ParsedCell, + store_raw: bool, +) -> tuple[int, int, int, int, int]: + _clear_cell_payload(conn, cell_id) + h = cell.header + + with conn.cursor() as cur: + cur.execute( + "UPDATE chart_cell SET " + "source_size=%s,source_sha256=%s,scale_code=%s,native_scale=%s," + "cell_name=%s,prolog_word0=%s,table1_length=%s,table2_length=%s," + "easting_min=%s,northing_min=%s,easting_max=%s,northing_max=%s," + "bbox_min_lon=%s,bbox_min_lat=%s,bbox_max_lon=%s,bbox_max_lat=%s," + "vector_record_count=%s,vector_point_count=%s," + "point2d_count=%s,point3d_count=%s," + "feature_count=%s,attribute_count=%s,error_message=NULL " + "WHERE id=%s", + ( + cell.file_size, + cell.sha256, + cf.scale_code, + cf.native_scale, + cf.path.stem, + cell.word0, + cell.table1_length, + cell.table2_length, + h.easting_min, + h.northing_min, + h.easting_max, + h.northing_max, + h.lon_min, + h.lat_min, + h.lon_max, + h.lat_max, + h.usn_vector_records, + h.n_vector_record_points, + h.usn_point2d_records, + h.usn_point3d_records, + h.usn_feature_records, + cell.attribute_count, + cell_id, + ), + ) + + coverages = build_coverages(cell) + + if ( + sum( + 1 + for f in cell.features + if f.object_class == "_m_sor" + ) + != len(coverages) + ): + raise CM93ParseError( + "存在 _m_sor 但 coverage geometry 无法构建，" + "拒绝生成未经校正的母数据" + ) + + # Coverage controls WGS offset selection. If normalization cannot make it + # valid, reject the cell rather than silently storing an invalid coverage. + unresolved_coverages = [ + cov + for cov in coverages + if ( + cov.geometry_issue is not None + or cov.geom_unshifted.is_empty + or not cov.geom_unshifted.is_valid + ) + ] + if unresolved_coverages: + detail = "; ".join( + f"record={cov.source_record_index}: " + f"{cov.geometry_issue or 'invalid coverage geometry'}" + for cov in unresolved_coverages[:10] + ) + raise CM93ParseError( + "coverage geometry normalization 未解决，拒绝导入该 Cell: " + + detail + ) + + cov_rows = [] + + for cov in coverages: + cov_rows.append(( + dataset_id, + cell_id, + cov.source_record_index, + cell.features[cov.source_record_index].object_type, + cov.compilation_scale, + cov.horizontal_datum, + cov.vertical_datum, + cov.wgsox, + cov.wgsoy, + _j(cov.attributes), + cov.geom_unshifted.wkb, + int(cov.geometry_repaired), + cov.geometry_issue, + )) + + if cov_rows: + with conn.cursor() as cur: + cur.executemany( + "INSERT INTO chart_coverage" + "(dataset_id,cell_id,source_record_index,object_class_id," + "compilation_scale,horizontal_datum,vertical_datum," + "wgsox,wgsoy,attributes_json,geom," + "geometry_repaired,geometry_issue) " + "VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s," + "ST_GeomFromWKB(%s,4326),%s,%s)", + cov_rows, + ) + + with conn.cursor() as cur: + cur.execute( + "SELECT id,source_record_index " + "FROM chart_coverage WHERE cell_id=%s", + (cell_id,), + ) + cov_id = { + int(r["source_record_index"]): int(r["id"]) + for r in cur.fetchall() + } + + built: list[BuiltFeature] = [ + build_feature( + cell, + f, + coverages, + ) + for f in cell.features + ] + + feature_rows = [] + + for b in built: + f = b.feature + geom_for_bounds = b.geom + + if b.soundings: + xs = [s[4].x for s in b.soundings] + ys = [s[4].y for s in b.soundings] + bb = ( + min(xs), + min(ys), + max(xs), + max(ys), + ) + else: + # For normalized Area/Coverage-derived META this deliberately uses + # the normalized geometry bounds, so retraced zero-area connectors + # no longer enlarge feature bbox. + bb = _bounds(geom_for_bounds) + + feature_rows.append(( + dataset_id, + cell_id, + cov_id.get(b.coverage_record_index), + f.index, + f.object_type, + f.object_class, + b.geometry_type, + cf.scale_code, + cf.native_scale, + f.geotype, + 4326, + *bb, + _j(b.attributes_map), + f.raw_record, + f.raw_attribute_blob, + int(b.geometry_repaired), + b.geometry_issue, + )) + + with conn.cursor() as cur: + cur.executemany( + "INSERT INTO chart_feature" + "(dataset_id,cell_id,coverage_id,source_record_index," + "object_class_id,object_class,geometry_type,scale_code," + "native_scale,raw_geotype,normalized_srid," + "bbox_min_lon,bbox_min_lat,bbox_max_lon,bbox_max_lat," + "attributes_json,raw_feature_blob,raw_attribute_blob," + "geometry_repaired,geometry_issue) " + "VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s," + "%s,%s,%s,%s,%s,%s)", + feature_rows, + ) + + cur.execute( + "SELECT id,source_record_index " + "FROM chart_feature WHERE cell_id=%s", + (cell_id,), + ) + fid = { + int(r["source_record_index"]): int(r["id"]) + for r in cur.fetchall() + } + + attr_rows = [] + point_rows = [] + line_rows = [] + area_rows = [] + sounding_rows = [] + + issue_count = 0 + repaired_count = 0 + snd_count = 0 + + for b in built: + feature_id = fid[b.feature.index] + + attr_rows.extend( + _attr_row(feature_id, a) + for a in b.feature.attributes + ) + + if b.geometry_issue: + issue_count += 1 + + if b.geometry_repaired: + repaired_count += 1 + + if ( + b.geometry_type == "POINT" + and b.geom is not None + ): + point_rows.append(( + feature_id, + b.geom.wkb, + )) + + elif ( + b.geometry_type == "LINE" + and b.geom is not None + ): + line_rows.append(( + feature_id, + b.geom.wkb, + )) + + elif ( + b.geometry_type == "AREA" + and b.geom is not None + ): + area_rows.append(( + feature_id, + b.geom.wkb, + )) + + elif b.geometry_type == "SOUNDING": + for idx, ( + x, + y, + z, + depth, + p, + _cov_rec, + ) in enumerate(b.soundings): + sounding_rows.append(( + feature_id, + idx, + x, + y, + z, + depth, + p.wkb, + )) + snd_count += 1 + + with conn.cursor() as cur: + if attr_rows: + for i in range(0, len(attr_rows), 1000): + cur.executemany( + "INSERT INTO chart_feature_attribute" + "(feature_id,ordinal_no,attribute_id,attribute_name," + "value_type,value_int,value_real,value_text,value_json," + "raw_value_blob) " + "VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", + attr_rows[i:i + 1000], + ) + + for rows, sql in [ + ( + point_rows, + "INSERT INTO chart_point_geometry" + "(feature_id,geom) " + "VALUES(%s,ST_GeomFromWKB(%s,4326))", + ), + ( + line_rows, + "INSERT INTO chart_line_geometry" + "(feature_id,geom) " + "VALUES(%s,ST_GeomFromWKB(%s,4326))", + ), + ( + area_rows, + "INSERT INTO chart_area_geometry" + "(feature_id,geom) " + "VALUES(%s,ST_GeomFromWKB(%s,4326))", + ), + ]: + for i in range(0, len(rows), 500): + cur.executemany( + sql, + rows[i:i + 500], + ) + + for i in range(0, len(sounding_rows), 1000): + cur.executemany( + "INSERT INTO chart_sounding_point" + "(feature_id,point_index,raw_x,raw_y,raw_z,depth,geom) " + "VALUES(%s,%s,%s,%s,%s,%s," + "ST_GeomFromWKB(%s,4326))", + sounding_rows[i:i + 1000], + ) + + if store_raw: + _insert_raw_tables( + conn, + cell_id, + cell, + ) + + return ( + len(feature_rows), + len(attr_rows), + issue_count, + repaired_count, + snd_count, + ) + + +def _selected_cells( + scan, + opt: ImportOptions, +) -> list[CellFile]: + cells = list(scan.cells) + + if opt.scales: + cells = [ + c + for c in cells + if c.scale_code.upper() in opt.scales + ] + + if opt.cell_name: + target = opt.cell_name.lower() + cells = [ + c + for c in cells + if ( + c.path.name.lower() == target + or c.path.stem.lower() == target + ) + ] + + if opt.limit is not None: + cells = cells[:opt.limit] + + return cells + + +def run_import( + cfg: DbConfig, + opt: ImportOptions, +) -> ImportSummary: + scan = scan_cm93(opt.root) + + if ( + not scan.object_dictionary + or not scan.attribute_dictionary + ): + raise RuntimeError( + "CM93OBJ.DIC 或 CM93ATTR.DIC/ATTRLUT.DIC 未找到" + ) + + objects = load_object_dictionary( + scan.object_dictionary + ) + attrs = load_attribute_dictionary( + scan.attribute_dictionary + ) + object_map = { + x.object_id: x + for x in objects + } + attr_map = { + x.attribute_id: x + for x in attrs + } + cells = _selected_cells( + scan, + opt, + ) + + conn = connect(cfg) + try: + assert_mysql57( + server_version(conn) + ) + dataset_id = _dataset( + conn, + opt.root, + ) + _store_dictionaries( + conn, + dataset_id, + objects, + attrs, + ) + job_id = _start_job( + conn, + dataset_id, + len(cells), + ) + conn.commit() + finally: + conn.close() + + summary = ImportSummary( + dataset_id=dataset_id, + job_id=job_id, + cells_discovered=len(cells), + ) + + print(f"Dataset ID: {dataset_id}") + print(f"Cells selected: {len(cells)}") + + for n, cf in enumerate(cells, 1): + meta_conn = connect(cfg) + try: + ( + cell_id, + status, + old_size, + old_hash, + ) = _ensure_cell_row( + meta_conn, + dataset_id, + cf, + ) + meta_conn.commit() + finally: + meta_conn.close() + + cur_size = cf.path.stat().st_size + + if ( + not opt.reimport + and status == "DONE" + and old_size == cur_size + and old_hash + ): + current_hash = _sha256_file( + cf.path + ) + if current_hash == old_hash: + summary.cells_skipped += 1 + if ( + n % 100 == 0 + or n == len(cells) + ): + print( + f"[{n}/{len(cells)}] " + f"SKIP {cf.relative_path} " + f"(SHA-256 unchanged)" + ) + continue + + print( + f"[{n}/{len(cells)}] " + f"CHANGED {cf.relative_path}; " + f"SHA-256 differs, reimport" + ) + + if ( + not opt.reimport + and status == "FAILED" + and not opt.retry_failed + ): + summary.cells_skipped += 1 + continue + + _mark_status( + cfg, + cell_id, + "IMPORTING", + ) + + try: + cell = parse_cell( + cf.path, + object_map, + attr_map, + ) + + conn = connect(cfg) + try: + ( + fcnt, + acnt, + gissues, + grepaired, + snd, + ) = _insert_parsed_cell( + conn, + dataset_id, + cell_id, + cf, + cell, + opt.store_raw_tables, + ) + + with conn.cursor() as cur: + cur.execute( + "UPDATE chart_cell SET " + "import_status='DONE',error_message=NULL," + "imported_at=%s WHERE id=%s", + ( + datetime.now(), + cell_id, + ), + ) + + _event( + conn, + job_id, + cell_id, + "INFO", + "CELL_DONE", + f"导入完成 {cf.relative_path}", + { + "features": fcnt, + "attributes": acnt, + "soundings": snd, + "geometry_issues": gissues, + "geometry_repaired": grepaired, + }, + ) + conn.commit() + + except Exception: + conn.rollback() + raise + finally: + conn.close() + + summary.cells_done += 1 + summary.features_source += len(cell.features) + summary.features_imported += fcnt + summary.attributes_source += cell.attribute_count + summary.attributes_imported += acnt + summary.unknown_object_count += len( + cell.unknown_object_ids + ) + summary.unknown_attribute_count += len( + cell.unknown_attribute_ids + ) + summary.geometry_issue_count += gissues + summary.geometry_repaired_count += grepaired + summary.sounding_points += snd + + print( + f"[{n}/{len(cells)}] " + f"DONE {cf.relative_path} " + f"features={fcnt} " + f"attrs={acnt} " + f"snd={snd} " + f"geom_issues={gissues} " + f"geom_repaired={grepaired}" + ) + + except Exception as e: + summary.cells_failed += 1 + + _mark_status( + cfg, + cell_id, + "FAILED", + str(e)[:65000], + ) + + ec = connect(cfg) + try: + _event( + ec, + job_id, + cell_id, + "ERROR", + "CELL_FAILED", + f"导入失败 {cf.relative_path}", + { + "error": repr(e), + }, + ) + ec.commit() + finally: + ec.close() + + print( + f"[{n}/{len(cells)}] " + f"FAILED {cf.relative_path}: {e}" + ) + + if opt.fail_fast: + break + + conn = connect(cfg) + try: + _finish_job( + conn, + summary, + ) + conn.commit() + finally: + conn.close() + + return summary diff --git a/src/cm93db/precision.py b/src/cm93db/precision.py new file mode 100644 index 0000000..aa3fab3 --- /dev/null +++ b/src/cm93db/precision.py @@ -0,0 +1,39 @@ +from __future__ import annotations +import math +from dataclasses import dataclass + +CM93_COORD_MAX = 65535.0 +CM93_SEMIMAJOR_M = 6378388.0 + +@dataclass(frozen=True) +class CellTransform: + easting_min: float + northing_min: float + easting_max: float + northing_max: float + + @property + def x_rate(self) -> float: + delta=self.easting_max-self.easting_min + if delta < 0: + delta += CM93_SEMIMAJOR_M * 2.0 * math.pi + return delta / CM93_COORD_MAX + + @property + def y_rate(self) -> float: + return (self.northing_max - self.northing_min) / CM93_COORD_MAX + +def raw_xy_to_cm93_mercator(raw_x: int, raw_y: int, t: CellTransform) -> tuple[float, float]: + if not 0 <= raw_x <= 65535 or not 0 <= raw_y <= 65535: + raise ValueError("CM93 raw x/y 必须在 0..65535") + return float(raw_x*t.x_rate+t.easting_min), float(raw_y*t.y_rate+t.northing_min) + +def apply_mercator_offset(easting: float, northing: float, wgsox: float, wgsoy: float) -> tuple[float, float]: + # CM93 coverage attributes are metre offsets. CM93 Transform applies them by + # subtraction before the Mercator→lon/lat conversion. + return easting-wgsox, northing-wgsoy + +def cm93_mercator_to_lonlat(easting: float, northing: float) -> tuple[float, float]: + lon=math.degrees(easting/CM93_SEMIMAJOR_M) + lat=math.degrees(2.0*math.atan(math.exp(northing/CM93_SEMIMAJOR_M))-math.pi/2.0) + return lon,lat diff --git a/src/cm93db/scanner.py b/src/cm93db/scanner.py new file mode 100644 index 0000000..0a5140e --- /dev/null +++ b/src/cm93db/scanner.py @@ -0,0 +1,59 @@ +from __future__ import annotations +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable + +SCALE_NATIVE = { + "Z": 20_000_000, + "A": 3_000_000, + "B": 1_000_000, + "C": 200_000, + "D": 100_000, + "E": 50_000, + "F": 20_000, + "G": 7_500, +} + +@dataclass(frozen=True) +class CellFile: + path: Path + relative_path: Path + scale_code: str + native_scale: int + +@dataclass(frozen=True) +class ScanResult: + root: Path + cells: tuple[CellFile, ...] + object_dictionary: Path | None + attribute_dictionary: Path | None + +def _find_case_insensitive(root: Path, names: set[str]) -> Path | None: + lowers = {n.lower() for n in names} + for p in root.rglob("*"): + if p.is_file() and p.name.lower() in lowers: + return p + return None + +def scan_cm93(root: Path) -> ScanResult: + root = root.resolve() + if not root.exists(): + raise FileNotFoundError(root) + cells = [] + for p in root.rglob("*"): + if not p.is_file(): + continue + ext = p.suffix.upper().lstrip(".") + if ext in SCALE_NATIVE: + cells.append( + CellFile( + path=p, + relative_path=p.relative_to(root), + scale_code=ext, + native_scale=SCALE_NATIVE[ext], + ) + ) + cells.sort(key=lambda c: (c.scale_code, str(c.relative_path).lower())) + obj = _find_case_insensitive(root, {"CM93OBJ.DIC"}) + attr = _find_case_insensitive(root, {"ATTRLUT.DIC", "CM93ATTR.DIC"}) + return ScanResult(root, tuple(cells), obj, attr) diff --git a/src/cm93db/schema_contract.py b/src/cm93db/schema_contract.py new file mode 100644 index 0000000..5c5fe03 --- /dev/null +++ b/src/cm93db/schema_contract.py @@ -0,0 +1,22 @@ +SCHEMA_VERSION = 1 +TARGET_MYSQL_MAJOR = 5 +TARGET_MYSQL_MINOR = 7 +TARGET_MYSQL_PATCH = 34 +NORMALIZED_SRID = 4326 + +REQUIRED_TABLES = { + "chart_dataset", + "chart_dictionary_object", + "chart_dictionary_attribute", + "chart_cell", + "chart_coverage", + "chart_feature", + "chart_feature_attribute", + "chart_point_geometry", + "chart_line_geometry", + "chart_area_geometry", + "chart_sounding_point", + "chart_raw_record", + "chart_import_job", + "chart_import_event", +}