diff --git a/STEP_RECORD.md b/STEP_RECORD.md index 096e8c0..32b937c 100644 --- a/STEP_RECORD.md +++ b/STEP_RECORD.md @@ -1,6 +1,6 @@ # 项目步骤记录 -最后更新：`2026-05-02` +最后更新：`2026-05-03` 仓库：`/root/sourceserver/pbf` 远端：`ssh://git@nas:2222/tei/pbf.git` @@ -24,6 +24,47 @@ ## 最近 3 天摘要 +### 2026-05-03 渔港 20m 语义修正为海岸线格子 + +- 复查 `coastline/build_fish_port_20m_full_mysql_resume.py --fpc 4320065` 与 `--prc 40` 结果不一致的问题： + - 原始 `C09-06.zip` 中 `FPC=4320065` 确实属于 `PRC=40` + - 该 FPC 有 2 条线要素，按当前逻辑命中 54 个 `coast_200m` 粗格 + - 单港纯计算可得到 582 个去重后的 20m `COASTLINE` 格 + - 当前旧静态资产 `src/pbf/coastline-mysql/fukuoka_saga/fish_port_20m_grid.geojson` 仍是旧口径，含 `SEA_SURFACE/LAND_BASE` 且 source 仍为 `C09-06+coastline PRC=40`，不能再作为当前单港/FPC 口径的对比基准 + - `PRC=40` 全批处理里有若干 FPC 的 part 没有命中 `coast_200m` 底盘；默认严格模式会中止，若没有显式跳过，会导致后续 `4320065` 没有进入批处理结果 + - 又修正了一个批处理阻断点：`FPC=4310320` 这类无海岸线命中的港口会在加载海岸线阶段先报 `no coastline curves parsed`，导致 `--skip-missing-coarse` 来不及生效 + - 现在 `load_coastlines_for_prc` 会抛可捕获异常；在 `--skip-missing-coarse` 下会记录 `FPC_SKIPPED_NO_COASTLINE` 并继续后续 FPC + - 已验证 `./.venv/bin/python coastline/build_fish_port_20m_full_mysql_resume.py --fpc 4310320 --skip-missing-coarse --max-scan-cells 200000` 可跳过而不退出 +- 已重新明确 `fish_port_20m` 的入库语义： + - 不是保留纯海面格子 + - 不是保留纯陆地格子 + - 只保留与海岸线缓冲带相交的 `COASTLINE` 格子 +- `coastline/build_fish_port_20m_full_mysql_resume.py` 已改为： + - `coast_buffer` 命中的 20m cell 才写入 `navsea_grid_cell` + - 写入时 `state_name/class_name` 均为 `COASTLINE` + - 未命中的 cell 只作为 `off_coast` 统计，不入库 + - 指定 `--prc` 重算时按 `source_name LIKE 'C09-06+coastline PRC=xx%'` 清理旧格子，避免混入旧 FPC 或旧缓冲结果 + - 指定 `--fpc` 重算时先清理该 FPC 的旧格子 + - 20m 写库从 `INSERT IGNORE` 改为 duplicate key upsert，避免相同 `cell_id` 被旧 PRC/source 占住后导致当前 PRC 看起来写入成功但导出为空 + - 修正 tiled 分支中 `y += cell_size_m` 的缩进风险 +- `coastline/export_prc20_test_assets.py` 已改为只导出 `state_name='COASTLINE'` +- `src/pbf/navsea-fish-port-prc50-test.html` 已改为只显示 `COASTLINE` 海岸线格子 +- 已复查无垢岛测试页空白原因： + - 页面加载的 `fish_port_20m_prc50_grid.geojson` 一度只有 45 字节，为空 FeatureCollection + - 数据库里无垢岛附近的旧 20m 格子被 `PRC=44` source 占用，`PRC=50` 重算时旧版 `INSERT IGNORE` 没有覆盖 + - 改为 upsert 后重跑 `--prc 50`，当前 `PRC=50 / COASTLINE` 为 `331` 格 + - 已重新导出并同步 `src/pbf/coastline-mysql/prc50_test/` 到 `/mnt/sda1/www/newpec/coastline-mysql/prc50_test/` +- 后续又收紧了 20m 扫描母集： + - 不再复用整 PRC 的 coast_200m 窗口 + - 现在按每个 `fish_part` 自己的 bbox 找 `coast_200m` + - `fish_part` 只负责筛选应命中的粗格，真正切 20m 时不再做 `fish_part ∩ coarse_window` 二次裁剪 + - 这样可以避免把粗格里本来应该保留的海岸线细格提前裁掉 +- 这条口径已经同步回正式全国版 `coastline/build_fish_port_20m_full_mysql_resume.py` +- `--fpc` 入口也补齐了和 `--prc` 一致的进度清理动作，避免单港重算时残留旧的 PRC 进度记录 +- `--prc` 入口已改成先按 `FPC` 分组，再逐港走同一套 20m 切格逻辑；不再把整个 PRC 的所有港混在同一个聚类里 +- 相关使用说明已同步到： + - `NavSea_全国三层数据生成与查看说明.md` + ### 2026-05-02 全国 50x50 重算脚本 - 新增全国 50x50 海上障碍重算脚本： @@ -61,7 +102,9 @@ - `coastline/build_japan_coast_grid_mysql.py` - 已把 20m 重算流程收紧为： - 先按渔港 bbox 找同区域 `coast_200m` 粗格 - - 再把粗格按 20m 直切并做海岸线重叠判断 + - 再把命中的 `coast_200m` 粗格直接作为 20m 直切底盘 + - 不再额外用 `fish_part ∩ coarse_window` 裁窗 + - 20m 判定继续使用同一套海岸线重叠判断 - 默认粗筛边距收紧为 0m，避免再做不必要的大范围外扩 ### 2026-05-02 全国海岸/渔港/障碍三按钮预览页 @@ -121,6 +164,37 @@ - 这只是 SQLite 版的全国融合骨架 - 当前 MySQL 侧已经合并为单库 `navsea_japan_coast_grid` +### 2026-05-03 全国静态导出支持分层开关 + +- 已为 `coastline/export_navgrid_mysql_assets.py` 增加独立导出开关： + - `--coast-200m / --no-coast-200m` + - `--fish-port-20m / --no-fish-port-20m` + - `--hazard-50m / --no-hazard-50m` + - `--density-overview / --no-density-overview` +- 默认仍是四项全导出 +- 现在可以只导某一层，不必每次都重出全国全量资产 +- `density_overview` 仍默认开启，因为全国预览页会用到它 +- 全国预览页实际会同时读取 `japan_coast_200m` 和 `japan_national` +- 线下同步时要把 `src/pbf/coastline-mysql/japan_coast_200m/` 和 `src/pbf/coastline-mysql/japan_national/` 都复制到 `/mnt/sda1/www/newpec/coastline-mysql/` + +### 2026-05-03 单港 20m 调试页 + +- 新增单港调试导出脚本： + - `coastline/export_fish_port_debug_assets.py` +- 新增单港调试页面： + - `src/pbf/navsea-fish-port-debug.html` +- 调试包按 `FPC + buffer` 分目录输出： + - `src/pbf/coastline-mysql/fish_port_debug/fpc_/buffer_/` +- 调试页会同时画出： + - 渔港 bbox + - 海岸线 + - 海岸 buffer + - 200m 粗窗 + - 实际扫描窗 + - 20m 海岸格 + - 20m 海面格 +- 这条链路是为了定位“明明有 200m 底盘和海岸线，但 20m 为什么没长出来”的问题 + ### 2026-05-02 渔港 20m 支持按单港 FPC 重算 - 已为 `coastline/build_fish_port_20m_full_mysql_resume.py` 增加单港口入口： @@ -131,6 +205,11 @@ - 命中的 `200x200` 粗格数量 - 最终写入的 `20x20` 格子数量 - 单港口模式下，每个批次会先按当前 cell_id 清掉旧记录，再写回，便于重算同一港口 +- 当前 `20x20` 的粗格母集已经收回到 **PRC 级 coast_200m**： + - 先按当前渔港 / PRC 的 bbox 去 MySQL 找 `coast_200m` + - 取回的粗格直接作为 20m 细扫底盘 + - 不再按 `fish_part` 的局部外框裁掉这些粗格 +- `20m` 海岸缓冲默认已收紧为 `5m`，仍可用 `--coast-buffer-m` 调整 - 已验证： - `./.venv/bin/python -m py_compile coastline/build_fish_port_20m_full_mysql_resume.py` - `discover_target_prcs(..., '1112040')` 可以反查到对应 `PRC` @@ -144,12 +223,53 @@ - 输入港名，扫描 `coastline/C09-06.zip` 里的渔港要素 - 通过 `NA2`、`NA4`、`FCF` 等字段找候选 `FPC` - 如果原始包里没有港名字段，也可以改用外部港名对照表 `--catalog` - - 支持精确匹配和模糊匹配 + - 支持精确匹配、包含匹配和 `--fuzzy` 后缀归一模糊匹配 - 文档已补到： - `NavSea_全国三层数据生成与查看说明.md` - 已验证： - `./.venv/bin/python -m py_compile find_fpc_by_port_name.py` +### 2026-05-02 PRC=50 单港 20x20 测试页 + +- 新增单港测试页： + - `src/pbf/navsea-fish-port-prc50-test.html` +- 页面用途： + - 直接读取专用 PRC=50 导出 `fish_port_20m_prc50_grid.geojson` + - 只显示 `COASTLINE` 海岸线格子 + - 单独显示无垢岛这一港，便于确认当前 20x20 海岸线判断结果 +- 已同步到线上： + - `http://192.168.200.184/newpec/navsea-fish-port-prc50-test.html` + +### 2026-05-02 PRC=50 专用 20x20 导出 + +- 新增专用导出脚本： + - `coastline/export_prc20_test_assets.py` +- 用途： + - 只导出 `fish_port_20m` 里 `source_name` 匹配 `PRC=50` 且 `state_name='COASTLINE'` 的记录 + - 输出到独立目录 `src/pbf/coastline-mysql/prc50_test/` + - 便于测试页只加载单港数据，不再依赖全国全量资产 +- 已验证： + - `./.venv/bin/python -m py_compile coastline/export_prc20_test_assets.py` +- 已实际导出并同步到线上： + - `fish_port_20m: 2800` + - `http://192.168.200.184/newpec/coastline-mysql/prc50_test/fish_port_20m_prc50_grid.geojson` + +### 2026-05-02 全国预览页 20x20 旧资产同步修正 + +- 已复核 `src/pbf/navsea-coastline-fukuoka-saga-200m.html` 的加载逻辑： + - HTML 本身使用的是 `./coastline-mysql/japan_national/manifest.json` + - `20x20` 图层是从 `./coastline-mysql/japan_national/fish_port_20m_grid.geojson` 读取 +- 问题根因： + - 线上 `/mnt/sda1/www/newpec/coastline-mysql/japan_national/manifest.json` 仍停留在旧版 + - 旧版 `fish_port_20m` 只有 `34270` 个格子，所以页面看起来只在两块区域出现红格 +- 已同步最新全国资产到线上目录： + - `/mnt/sda1/www/newpec/coastline-mysql/japan_national/` + - `/mnt/sda1/www/newpec/navsea-coastline-fukuoka-saga-200m.html` +- 同步后线上 `fish_port_20m` 计数已更新为： + - `246653` +- 结论： + - 这次不是 HTML 逻辑 bug，而是线上静态资产没跟上最新导出 + ### 2026-05-02 无垢岛 PRC=50 改为单库运行 - 已把 `coastline/build_fish_port_20m_full_mysql_resume.py` 从双库连接收回到单库模式： @@ -1653,7 +1773,7 @@ - 现在改成： - 先抓出当前渔港范围命中的 `coast_200m` 粗格 - 直接把这些 `200m` 粗格作为 `20m` 细扫底盘 - - 再用同一套海岸线碰撞逻辑判定 `LAND_BASE / SEA_SURFACE` + - 再用同一套海岸线缓冲碰撞逻辑筛出 `COASTLINE` - 这样 20m 的范围会更接近用户手工圈出的那几块红框，而不是只沿着港岸线出一圈带状格。 ## 2026-04-24 呼子港重跑结果已更新 @@ -1672,8 +1792,16 @@ - `coast_200m: 9743` - `fish_port_20m: 28726` - `hazard_50m: 176454` -- 导出时 `fish_port_20m` 采用 `LAND_BASE + SEA_SURFACE` 全量导出，便于一起看港区外围形状。 +- 导出时 `fish_port_20m` 只保留 `LAND_BASE`。 - 已同步到 Web 目录： - `/mnt/sda1/www/newpec/coastline/navsea-fukuoka-saga-coast200-fish20-full/` - 当前页面版本： - `v2.6` + +## 2026-05-03 PRC 与 FPC 对照确认 + +- 已确认 `FPC=4320065` 在原始 `C09-06.zip` 里对应的 `PRC` 是 `40`，不是 `50`。 +- 这意味着它应当和 `--prc 40` 的结果对比，而不是和 `--prc 50` 对比。 +- 已将 `fish_port_20m` 的唯一键改为 `UNIQUE KEY (layer_name, source_name, cell_id)`，让不同港口的同名格子不再互相覆盖。 +- 全国导出层也补了按 `cell_id` 去重，避免同一格在全国 GeoJSON 里重复输出。 +- 结论：`--prc` 与 `--fpc` 现在在落库层面已经按港独立了，后续如果再出现差异，就只需要查几何或海岸线输入，不用再怀疑存储覆盖。 diff --git a/coastline/build_fish_port_20m_full_mysql_resume.py b/coastline/build_fish_port_20m_full_mysql_resume.py index 523c1db..b867c13 100644 --- a/coastline/build_fish_port_20m_full_mysql_resume.py +++ b/coastline/build_fish_port_20m_full_mysql_resume.py @@ -31,7 +31,7 @@ FISH_SRC = "coastline/C09-06.zip" COAST_DIR = Path("coastline") LAYER_NAME = "fish_port_20m" CELL_SIZE_M = 20.0 -COAST_BUFFER_M = 12.0 +COAST_BUFFER_M = 5.0 COAST_COARSE_LAYER = "coast_200m" DEFAULT_COAST_COARSE_MARGIN_M = 0.0 DEFAULT_CHECKPOINT = "out/fish_port_20m_full_resume.checkpoint.json" @@ -50,6 +50,10 @@ class FishLine: bbox: tuple[float, float, float, float] +class NoCoastlineCurvesError(RuntimeError): + pass + + def mercator_x(lon: float) -> float: return RADIUS * math.radians(lon) @@ -280,7 +284,7 @@ def ensure_schema(conn) -> None: max_lon DOUBLE NOT NULL, max_lat DOUBLE NOT NULL, created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - UNIQUE KEY uniq_layer_cell (layer_name, cell_id), + UNIQUE KEY uniq_layer_source_cell (layer_name, source_name, cell_id), KEY idx_layer_state (layer_name, state_name), KEY idx_layer_rowcol (layer_name, row_idx, col_idx) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 @@ -289,6 +293,29 @@ def ensure_schema(conn) -> None: with conn.cursor() as cur: for stmt in stmts: cur.execute(stmt) + cur.execute( + """ + SELECT INDEX_NAME, COLUMN_NAME, SEQ_IN_INDEX + FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA=%s AND TABLE_NAME='navsea_grid_cell' AND NON_UNIQUE=0 + ORDER BY INDEX_NAME, SEQ_IN_INDEX + """, + (DB_NAME,), + ) + index_columns: dict[str, list[str]] = {} + for index_name, column_name, _seq in cur.fetchall(): + index_columns.setdefault(index_name, []).append(column_name) + desired_cols = ["layer_name", "source_name", "cell_id"] + if index_columns.get("uniq_layer_source_cell") != desired_cols: + if "uniq_layer_source_cell" in index_columns: + cur.execute("ALTER TABLE navsea_grid_cell DROP INDEX uniq_layer_source_cell") + if "uniq_layer_cell" in index_columns: + cur.execute("ALTER TABLE navsea_grid_cell DROP INDEX uniq_layer_cell") + cur.execute( + "ALTER TABLE navsea_grid_cell ADD UNIQUE KEY uniq_layer_source_cell (layer_name, source_name, cell_id)" + ) + elif "uniq_layer_cell" in index_columns: + cur.execute("ALTER TABLE navsea_grid_cell DROP INDEX uniq_layer_cell") conn.commit() @@ -340,11 +367,22 @@ def insert_cells(cur, rows: list[tuple]) -> None: return cur.executemany( """ - INSERT IGNORE INTO navsea_grid_cell + INSERT INTO navsea_grid_cell (layer_name, cell_id, row_idx, col_idx, cell_size_m, state_name, class_name, source_name, min_lon, min_lat, max_lon, max_lat) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + ON DUPLICATE KEY UPDATE + row_idx=VALUES(row_idx), + col_idx=VALUES(col_idx), + cell_size_m=VALUES(cell_size_m), + state_name=VALUES(state_name), + class_name=VALUES(class_name), + source_name=VALUES(source_name), + min_lon=VALUES(min_lon), + min_lat=VALUES(min_lat), + max_lon=VALUES(max_lon), + max_lat=VALUES(max_lat) """, rows, ) @@ -510,15 +548,34 @@ def drop_existing_layer(conn, layer_name: str) -> None: def drop_existing_prcs(conn, layer_name: str, prcs: set[str]) -> None: if not prcs: return - source_names = [f"C09-06+coastline PRC={prc}" for prc in sorted(normalize_prc_list(prcs))] + source_patterns = [f"C09-06+coastline PRC={prc}%" for prc in sorted(normalize_prc_list(prcs))] + source_where = " OR ".join(["source_name LIKE %s"] * len(source_patterns)) + progress_placeholders = ", ".join(["%s"] * len(prcs)) + with conn.cursor() as cur: + cur.execute( + f"DELETE FROM navsea_grid_cell WHERE layer_name=%s AND ({source_where})", + [layer_name, *source_patterns], + ) + cur.execute( + f"DELETE FROM navsea_grid_import_progress WHERE job_name=%s AND prc IN ({progress_placeholders})", + [RESUME_JOB_NAME, *sorted(prcs)], + ) + conn.commit() + + +def drop_existing_fpc(conn, layer_name: str, prcs: set[str], fpc: str) -> None: + if not prcs: + return + source_names = [f"C09-06+coastline PRC={prc} FPC={fpc}" for prc in sorted(normalize_prc_list(prcs))] placeholders = ", ".join(["%s"] * len(source_names)) + progress_placeholders = ", ".join(["%s"] * len(prcs)) with conn.cursor() as cur: cur.execute( f"DELETE FROM navsea_grid_cell WHERE layer_name=%s AND source_name IN ({placeholders})", [layer_name, *source_names], ) cur.execute( - f"DELETE FROM navsea_grid_import_progress WHERE job_name=%s AND prc IN ({placeholders})", + f"DELETE FROM navsea_grid_import_progress WHERE job_name=%s AND prc IN ({progress_placeholders})", [RESUME_JOB_NAME, *sorted(prcs)], ) conn.commit() @@ -662,6 +719,14 @@ def collect_fish_lines_for_prc( return fish_lines +def group_fish_lines_by_fpc(fish_lines: list[FishLine]) -> dict[str, list[FishLine]]: + groups: dict[str, list[FishLine]] = {} + for line in fish_lines: + key = line.fpc or "__unknown__" + groups.setdefault(key, []).append(line) + return groups + + @trace_fn("cluster_line_indices") def cluster_line_indices(bboxes: list[tuple[float, float, float, float]], gap_m: float = 250.0) -> list[list[int]]: parent = list(range(len(bboxes))) @@ -774,7 +839,7 @@ def load_coastlines_for_prc( scan_path(coast_path) if not lines: - raise SystemExit( + raise NoCoastlineCurvesError( f"no coastline curves parsed for PRC={prc}; " f"tried={len(tried_paths)} bbox={tuple(round(v, 3) for v in bbox_geom.bounds) if bbox_geom is not None else None}" ) @@ -902,14 +967,13 @@ def build_cells_for_geometry( if not fish_prepared.intersects(cell_geom): continue profile["precise_hits"] += 1 - is_land = coast_prepared.intersects(cell_geom) - state_name = "LAND_BASE" if is_land else "SEA_SURFACE" - class_name = state_name - if is_land: - land_count += 1 - else: + coast_hit = coast_prepared.intersects(cell_geom) + if not coast_hit: sea_count += 1 - + continue + land_count += 1 + state_name = "COASTLINE" + class_name = state_name min_lon = lon_from_mercator(x) min_lat = lat_from_mercator(y) max_lon = lon_from_mercator(x + cell_size_m) @@ -941,14 +1005,14 @@ def build_cells_for_geometry( if not rows: log( - f"[TRACE] build_cells_for_geometry 结束 rows=0 land=0 sea=0 " + f"[TRACE] build_cells_for_geometry 结束 rows=0 coast=0 off_coast=0 " f"tile_hit={profile['tile_hit']}/{profile['tile_checked']} " f"row_hit={profile['row_hit']}/{profile['row_checked']} " f"candidate={profile['candidate_cells']} precise={profile['precise_hits']}/{profile['precise_cell_checks']}" ) return [], 0, 0, (float("inf"), float("inf"), float("-inf"), float("-inf")), profile log( - f"[TRACE] build_cells_for_geometry 结束 rows={len(rows)} land={land_count} sea={sea_count} " + f"[TRACE] build_cells_for_geometry 结束 rows={len(rows)} coast={land_count} off_coast={sea_count} " f"tile_hit={profile['tile_hit']}/{profile['tile_checked']} " f"row_hit={profile['row_hit']}/{profile['row_checked']} " f"candidate={profile['candidate_cells']} precise={profile['precise_hits']}/{profile['precise_cell_checks']}" @@ -1035,14 +1099,13 @@ def build_cells_for_geometry( if not fish_prepared.intersects(cell_geom): continue profile["precise_hits"] += 1 - is_land = coast_prepared.intersects(cell_geom) - state_name = "LAND_BASE" if is_land else "SEA_SURFACE" - class_name = state_name - if is_land: - land_count += 1 - else: + coast_hit = coast_prepared.intersects(cell_geom) + if not coast_hit: sea_count += 1 - + continue + land_count += 1 + state_name = "COASTLINE" + class_name = state_name min_lon = lon_from_mercator(x) min_lat = lat_from_mercator(y) max_lon = lon_from_mercator(x + cell_size_m) @@ -1075,14 +1138,14 @@ def build_cells_for_geometry( if not rows: log( - f"[TRACE] build_cells_for_geometry 结束 rows=0 land=0 sea=0 " + f"[TRACE] build_cells_for_geometry 结束 rows=0 coast=0 off_coast=0 " f"tile_hit={profile['tile_hit']}/{profile['tile_checked']} " f"row_hit={profile['row_hit']}/{profile['row_checked']} " f"candidate={profile['candidate_cells']} precise={profile['precise_hits']}/{profile['precise_cell_checks']}" ) return [], 0, 0, (float("inf"), float("inf"), float("-inf"), float("-inf")), profile log( - f"[TRACE] build_cells_for_geometry 结束 rows={len(rows)} land={land_count} sea={sea_count} " + f"[TRACE] build_cells_for_geometry 结束 rows={len(rows)} coast={land_count} off_coast={sea_count} " f"tile_hit={profile['tile_hit']}/{profile['tile_checked']} " f"row_hit={profile['row_hit']}/{profile['row_checked']} " f"candidate={profile['candidate_cells']} precise={profile['precise_hits']}/{profile['precise_cell_checks']}" @@ -1090,6 +1153,299 @@ def build_cells_for_geometry( return rows, land_count, sea_count, (export_bbox[0], export_bbox[1], export_bbox[2], export_bbox[3]), profile +def process_port_group_no_resume( + *, + conn, + cur, + project_root: Path, + prc: str, + prc_lines: list[FishLine], + args, + selected_fpc: str | None, + max_scan_cells: int | None, +) -> tuple[int, int, int, int, float]: + prc_bboxes = [line.bbox for line in prc_lines] + prc_bbox = ( + min(bb[0] for bb in prc_bboxes), + min(bb[1] for bb in prc_bboxes), + max(bb[2] for bb in prc_bboxes), + max(bb[3] for bb in prc_bboxes), + ) + cluster_groups = cluster_line_indices(prc_bboxes, gap_m=args.cluster_gap_m) + if selected_fpc is not None: + log(f"[{LAYER_NAME}] PRC={prc} / FPC={selected_fpc} cluster 数={len(cluster_groups)}") + log(f"[{LAYER_NAME}] PRC={prc} / FPC={selected_fpc} 正在缓存整港海岸线并生成缓冲区 ...") + else: + log(f"[{LAYER_NAME}] PRC={prc} cluster 数={len(cluster_groups)}") + log(f"[{LAYER_NAME}] PRC={prc} 正在缓存整县海岸线并生成缓冲区 ...") + + try: + prc_coast_lines, prc_coast_bbox = load_coastlines_for_prc( + project_root, + prc, + bbox=( + mercator_x(prc_bbox[0]), + mercator_y(prc_bbox[1]), + mercator_x(prc_bbox[2]), + mercator_y(prc_bbox[3]), + ), + search_margin_m=max(args.coast_buffer_m, args.coast_coarse_margin_m), + ) + except NoCoastlineCurvesError as exc: + if not args.skip_missing_coarse: + raise + status_name = "FPC_SKIPPED_NO_COASTLINE" if selected_fpc is not None else "PRC_SKIPPED_NO_COASTLINE" + insert_progress( + cur, + job_name=RESUME_JOB_NAME, + layer_name=LAYER_NAME, + prc=prc, + status_name=status_name, + cluster_count=len(cluster_groups), + inserted_cells=0, + land_cells=0, + sea_cells=0, + elapsed_sec=0.0, + ) + conn.commit() + if selected_fpc is not None: + log(f"[{LAYER_NAME}] PRC={prc} / FPC={selected_fpc} 未找到海岸线；已按 --skip-missing-coarse 跳过。{exc}") + else: + log(f"[{LAYER_NAME}] PRC={prc} 未找到海岸线；已按 --skip-missing-coarse 跳过。{exc}") + return 0, 0, 0, 0, 0.0 + if selected_fpc is not None: + log( + f"[{LAYER_NAME}] PRC={prc} / FPC={selected_fpc} 海岸线缓存完成，" + f"曲线数={len(prc_coast_lines)} bbox={prc_coast_bbox}" + ) + else: + log( + f"[{LAYER_NAME}] PRC={prc} 海岸线缓存完成，" + f"曲线数={len(prc_coast_lines)} bbox={prc_coast_bbox}" + ) + + prc_coast_geom = unary_union(prc_coast_lines) + prc_coast_buffer = prc_coast_geom.buffer(args.coast_buffer_m, cap_style=2, join_style=2) + prc_coast_prepared = prep(prc_coast_buffer) + if selected_fpc is not None: + log(f"[{LAYER_NAME}] PRC={prc} / FPC={selected_fpc} 海岸缓冲缓存完成，开始处理各 part ...") + else: + log(f"[{LAYER_NAME}] PRC={prc} 海岸缓冲缓存完成，开始处理各 part ...") + + started_at = time.monotonic() + prc_inserted_before = 0 + prc_land_before = 0 + prc_sea_before = 0 + inserted_total = 0 + land_total = 0 + sea_total = 0 + coarse_window_total = 0 + + selected_port_rows: dict[str, tuple] | None = {} if selected_fpc is not None else None + selected_port_coarse_windows: set[tuple[float, float, float, float]] | None = set() if selected_fpc is not None else None + prc_profile = { + "tile_checked": 0, + "tile_hit": 0, + "row_checked": 0, + "row_hit": 0, + "mask_row_hit": 0, + "candidate_cells": 0, + "precise_cell_checks": 0, + "precise_hits": 0, + } + + for cluster_idx, cluster in enumerate(cluster_groups): + log(f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1}/{len(cluster_groups)} 开始，成员={len(cluster)}") + log(f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1} 正在合并渔港线 ...") + + cluster_lines = [ + LineString([(mercator_x(lon), mercator_y(lat)) for lon, lat in prc_lines[i].coords]) + for i in cluster + ] + merged = unary_union(cluster_lines) + polygons = list(polygonize(merged)) + if polygons: + fish_geom = unary_union(polygons) + geom_mode = "polygonize" + else: + fish_geom = merged.buffer(args.cell_size_m, cap_style=2, join_style=2) + geom_mode = "buffer" + if fish_geom.is_empty: + log(f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx} 为空，跳过") + continue + log(f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1} 几何模式={geom_mode}，正在生成格网 ...") + + fish_parts = [part for part in get_parts(fish_geom) if not part.is_empty] + if not fish_parts: + log(f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1} 没有可拆分部件，跳过") + continue + + for part_idx, fish_part in enumerate(fish_parts): + fish_bbox = fish_part.bounds + part_coarse_windows = load_coarse_windows_for_bbox( + conn, + layer_name=COAST_COARSE_LAYER, + bbox_mercator=fish_bbox, + margin_m=args.coast_coarse_margin_m, + ) + if part_coarse_windows: + log( + f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1}/{len(cluster_groups)} " + f"part={part_idx + 1}/{len(fish_parts)} 取回 coast_200m 格子={len(part_coarse_windows)}" + ) + scan_windows = [box(*coarse_window) for coarse_window in part_coarse_windows] + if selected_port_coarse_windows is not None: + for coarse_window in part_coarse_windows: + selected_port_coarse_windows.add(tuple(float(v) for v in coarse_window)) + if not scan_windows: + if args.allow_unbounded_fallback: + scan_windows = [fish_part] + log( + f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1}/{len(cluster_groups)} " + f"part={part_idx + 1}/{len(fish_parts)} 没有有效 coast_200m 交集，已允许回退原始范围" + ) + else: + message = ( + f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1}/{len(cluster_groups)} " + f"part={part_idx + 1}/{len(fish_parts)} 没有有效 coast_200m 交集；" + f"fish_bbox={tuple(round(v, 3) for v in fish_bbox)} " + f"coarse_windows={len(part_coarse_windows)}。" + ) + if args.skip_missing_coarse: + log(message + "已按 --skip-missing-coarse 跳过") + continue + raise RuntimeError( + message + + "按当前设计这属于异常：渔港 20m 必须以数据库 coast_200m 为底盘。" + + "如确需旧式整片扫描请显式加 --allow-unbounded-fallback；" + + "如确认允许漏算该 part，请显式加 --skip-missing-coarse。" + ) + + log( + f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1}/{len(cluster_groups)} " + f"part={part_idx + 1}/{len(fish_parts)} 实际扫描交集窗口={len(scan_windows)}" + ) + if selected_fpc is not None: + log( + f"[{LAYER_NAME}] PRC={prc} / FPC={selected_fpc} cluster={cluster_idx + 1}/{len(cluster_groups)} " + f"part={part_idx + 1}/{len(fish_parts)} bbox={fish_bbox} 使用海岸线缓存 ..." + ) + else: + log( + f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1}/{len(cluster_groups)} " + f"part={part_idx + 1}/{len(fish_parts)} bbox={fish_bbox} 使用 part 级海岸线缓存 ..." + ) + log(f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1} part={part_idx + 1} 开始格网 ...") + + land_count = 0 + sea_count = 0 + bbox_lonlat = (float("inf"), float("inf"), float("-inf"), float("-inf")) + profile = {key: 0 for key in prc_profile} + part_inserted = 0 + batch: list[tuple] = [] + + for window_idx, scan_geom in enumerate(scan_windows): + if scan_geom.is_empty: + continue + log( + f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1}/{len(cluster_groups)} " + f"part={part_idx + 1}/{len(fish_parts)} window={window_idx + 1}/{len(scan_windows)} " + f"正在生成格网 ..." + ) + window_rows, window_land, window_sea, window_bbox, window_profile = build_cells_for_geometry( + scan_geom, + prc_coast_prepared, + prc=prc, + cell_size_m=args.cell_size_m, + scan_tile_m=args.cell_size_m * 10.0, + source_name=( + f"C09-06+coastline PRC={prc}" + + (f" FPC={selected_fpc}" if selected_fpc is not None else "") + ), + max_scan_cells=max_scan_cells, + ) + land_count += window_land + sea_count += window_sea + bbox_lonlat = ( + min(bbox_lonlat[0], window_bbox[0]), + min(bbox_lonlat[1], window_bbox[1]), + max(bbox_lonlat[2], window_bbox[2]), + max(bbox_lonlat[3], window_bbox[3]), + ) + for key in prc_profile: + prc_profile[key] += window_profile[key] + profile[key] += window_profile[key] + + for row in window_rows: + if selected_port_rows is not None: + selected_port_rows[row[1]] = row + batch.append(row) + if len(batch) >= args.commit_every: + insert_cells(cur, batch) + conn.commit() + part_inserted += len(batch) + inserted_total += len(batch) + batch.clear() + + if batch: + insert_cells(cur, batch) + conn.commit() + part_inserted += len(batch) + inserted_total += len(batch) + + if part_inserted == 0: + log( + f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1} " + f"part={part_idx + 1} 没有生成 cell" + ) + continue + + if selected_fpc is not None: + log( + f"[{LAYER_NAME}] PRC={prc} / FPC={selected_fpc} cluster={cluster_idx + 1} " + f"part={part_idx + 1} 格网完成，cell={part_inserted} coast={land_count} off_coast={sea_count} bbox={bbox_lonlat}" + ) + else: + log( + f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1} " + f"part={part_idx + 1} 格网完成，cell={part_inserted} coast={land_count} off_coast={sea_count} bbox={bbox_lonlat}" + ) + + coarse_window_total += len(part_coarse_windows) + land_total += land_count + sea_total += sea_count + + prc_elapsed = time.monotonic() - started_at + summary_inserted = len(selected_port_rows) if selected_port_rows is not None else inserted_total + summary_coarse = len(selected_port_coarse_windows) if selected_port_coarse_windows is not None else coarse_window_total + insert_progress( + cur, + job_name="fish_port_20m_full_mysql_resume", + layer_name=LAYER_NAME, + prc=prc, + status_name="FPC_DONE" if selected_fpc is not None else "PRC_DONE", + cluster_count=len(cluster_groups), + inserted_cells=summary_inserted, + land_cells=land_total, + sea_cells=sea_total, + elapsed_sec=prc_elapsed, + ) + conn.commit() + if selected_fpc is not None: + log( + f"[{LAYER_NAME}] PRC={prc} / FPC={selected_fpc} 完成 cluster={len(cluster_groups)} " + f"200x200={summary_coarse} final_20x20={summary_inserted} " + f"coast={land_total} off_coast={sea_total} elapsed={prc_elapsed:.1f}s" + ) + else: + log( + f"[{LAYER_NAME}] PRC={prc} 完成 cluster={len(cluster_groups)} " + f"本段写入={inserted_total} coast={land_total} off_coast={sea_total} " + f"elapsed={prc_elapsed:.1f}s" + ) + return summary_inserted, land_total, sea_total, summary_coarse, prc_elapsed + + @trace_fn("main") def main() -> None: parser = argparse.ArgumentParser(description="Rebuild nationwide fish-port 20m grid into MySQL with resume support") @@ -1152,7 +1508,7 @@ def main() -> None: "--coast-buffer-m", type=float, default=COAST_BUFFER_M, - help="海岸缓冲距离（米）", + help="海岸缓冲距离（米），默认 5m，可按需要调大或调小", ) parser.add_argument( "--fish-buffer-m", @@ -1279,9 +1635,57 @@ def main() -> None: raise SystemExit("no fish-port line features selected") if selected_fpc is not None: log(f"[{LAYER_NAME}] 选中 PRC 数量：{len(prc_list)}（由 FPC 反查得到）") + drop_existing_fpc(conn, LAYER_NAME, set(prc_list), selected_fpc) else: log(f"[{LAYER_NAME}] 选中 PRC 数量：{len(prc_list)}") + if selected_prc is not None or selected_fpc is not None: + with conn.cursor() as cur: + for prc in prc_list: + if selected_fpc is not None: + prc_lines = collect_fish_lines_for_prc(root, point_lookup, prc, selected_fpc=selected_fpc) + if not prc_lines: + log(f"[{LAYER_NAME}] PRC={prc} / FPC={selected_fpc} 没有渔港线要素，跳过") + continue + log( + f"[{LAYER_NAME}] PRC={prc} / FPC={selected_fpc} 开始，" + f"线要素={len(prc_lines)}" + ) + process_port_group_no_resume( + conn=conn, + cur=cur, + project_root=project_root, + prc=prc, + prc_lines=prc_lines, + args=args, + selected_fpc=selected_fpc, + max_scan_cells=max_scan_cells, + ) + else: + prc_lines = collect_fish_lines_for_prc(root, point_lookup, prc, selected_fpc=None) + if not prc_lines: + log(f"[{LAYER_NAME}] PRC={prc} 没有渔港线要素，跳过") + continue + grouped_lines = group_fish_lines_by_fpc(prc_lines) + log(f"[{LAYER_NAME}] PRC={prc} 开始，FPC 组数={len(grouped_lines)}") + for group_fpc in sorted(grouped_lines.keys()): + group_lines = grouped_lines[group_fpc] + log( + f"[{LAYER_NAME}] PRC={prc} / FPC={group_fpc} 开始，" + f"线要素={len(group_lines)}" + ) + process_port_group_no_resume( + conn=conn, + cur=cur, + project_root=project_root, + prc=prc, + prc_lines=group_lines, + args=args, + selected_fpc=group_fpc if group_fpc != "__unknown__" else None, + max_scan_cells=max_scan_cells, + ) + return + total_inserted = 0 total_land = 0 total_sea = 0 @@ -1395,17 +1799,41 @@ def main() -> None: log(f"[{LAYER_NAME}] PRC={prc} / FPC={selected_fpc} 正在缓存整港海岸线并生成缓冲区 ...") else: log(f"[{LAYER_NAME}] PRC={prc} 正在缓存整县海岸线并生成缓冲区 ...") - prc_coast_lines, prc_coast_bbox = load_coastlines_for_prc( - project_root, - prc, - bbox=( - mercator_x(prc_bbox[0]), - mercator_y(prc_bbox[1]), - mercator_x(prc_bbox[2]), - mercator_y(prc_bbox[3]), - ), - search_margin_m=max(args.coast_buffer_m, args.coast_coarse_margin_m), - ) + try: + prc_coast_lines, prc_coast_bbox = load_coastlines_for_prc( + project_root, + prc, + bbox=( + mercator_x(prc_bbox[0]), + mercator_y(prc_bbox[1]), + mercator_x(prc_bbox[2]), + mercator_y(prc_bbox[3]), + ), + search_margin_m=max(args.coast_buffer_m, args.coast_coarse_margin_m), + ) + except NoCoastlineCurvesError as exc: + if not args.skip_missing_coarse: + raise + status_name = "FPC_SKIPPED_NO_COASTLINE" if selected_fpc is not None else "PRC_SKIPPED_NO_COASTLINE" + insert_progress( + cur, + job_name=RESUME_JOB_NAME, + layer_name=LAYER_NAME, + prc=prc, + status_name=status_name, + cluster_count=len(cluster_groups), + inserted_cells=0, + land_cells=0, + sea_cells=0, + elapsed_sec=0.0, + ) + conn.commit() + if selected_fpc is not None: + log(f"[{LAYER_NAME}] PRC={prc} / FPC={selected_fpc} 未找到海岸线；已按 --skip-missing-coarse 跳过。{exc}") + else: + log(f"[{LAYER_NAME}] PRC={prc} 未找到海岸线；已按 --skip-missing-coarse 跳过。{exc}") + completed_prcs.add(prc) + continue if selected_fpc is not None: log( f"[{LAYER_NAME}] PRC={prc} / FPC={selected_fpc} 海岸线缓存完成，" @@ -1481,26 +1909,25 @@ def main() -> None: for part_idx, fish_part in enumerate(fish_parts): if checkpoint and checkpoint.get("last_prc") == prc and checkpoint.get("last_cluster_index") == cluster_idx and part_idx < resume_part: continue + fish_bbox = fish_part.bounds - coarse_windows = load_coarse_windows_for_bbox( + scan_windows: list[object] = [] + part_coarse_windows = load_coarse_windows_for_bbox( conn, layer_name=COAST_COARSE_LAYER, bbox_mercator=fish_bbox, margin_m=args.coast_coarse_margin_m, ) - scan_windows: list[object] = [] - if coarse_windows: + if part_coarse_windows: log( f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1}/{len(cluster_groups)} " - f"part={part_idx + 1}/{len(fish_parts)} 取回 coast_200m 格子={len(coarse_windows)}" + f"part={part_idx + 1}/{len(fish_parts)} 取回 coast_200m 格子={len(part_coarse_windows)}" ) - for coarse_window in coarse_windows: + for coarse_window in part_coarse_windows: window_geom = box(*coarse_window) - scan_geom = fish_part.intersection(window_geom) - if not scan_geom.is_empty: - scan_windows.append(scan_geom) - if selected_port_coarse_windows is not None: - selected_port_coarse_windows.add(tuple(float(v) for v in coarse_window)) + scan_windows.append(window_geom) + if selected_port_coarse_windows is not None: + selected_port_coarse_windows.add(tuple(float(v) for v in coarse_window)) if not scan_windows: if args.allow_unbounded_fallback: scan_windows = [fish_part] @@ -1513,7 +1940,7 @@ def main() -> None: f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1}/{len(cluster_groups)} " f"part={part_idx + 1}/{len(fish_parts)} 没有有效 coast_200m 交集；" f"fish_bbox={tuple(round(v, 3) for v in fish_bbox)} " - f"coarse_windows={len(coarse_windows)}。" + f"coarse_windows={len(part_coarse_windows)}。" ) if args.skip_missing_coarse: log(message + "已按 --skip-missing-coarse 跳过") @@ -1524,11 +1951,11 @@ def main() -> None: + "如确需旧式整片扫描请显式加 --allow-unbounded-fallback；" + "如确认允许漏算该 part，请显式加 --skip-missing-coarse。" ) - else: - log( - f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1}/{len(cluster_groups)} " - f"part={part_idx + 1}/{len(fish_parts)} 实际扫描交集窗口={len(scan_windows)}" - ) + + log( + f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1}/{len(cluster_groups)} " + f"part={part_idx + 1}/{len(fish_parts)} 实际扫描交集窗口={len(scan_windows)}" + ) if selected_fpc is not None: log( f"[{LAYER_NAME}] PRC={prc} / FPC={selected_fpc} cluster={cluster_idx + 1}/{len(cluster_groups)} " @@ -1537,7 +1964,7 @@ def main() -> None: else: log( f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1}/{len(cluster_groups)} " - f"part={part_idx + 1}/{len(fish_parts)} bbox={fish_bbox} 使用 PRC 级海岸线缓存 ..." + f"part={part_idx + 1}/{len(fish_parts)} bbox={fish_bbox} 使用 part 级海岸线缓存 ..." ) coast_prepared = prc_coast_prepared log(f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1} part={part_idx + 1} 开始格网 ...") @@ -1597,7 +2024,7 @@ def main() -> None: f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1}/{len(cluster_groups)} " f"part={part_idx + 1}/{len(fish_parts)} " f"提交={part_inserted} 累计={checkpoint_state['inserted_cells']} " - f"land={checkpoint_state['land_cells']} sea={checkpoint_state['sea_cells']}" + f"coast={checkpoint_state['land_cells']} off_coast={checkpoint_state['sea_cells']}" ) batch.clear() @@ -1614,7 +2041,7 @@ def main() -> None: f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1}/{len(cluster_groups)} " f"part={part_idx + 1}/{len(fish_parts)} " f"提交={part_inserted} 累计={checkpoint_state['inserted_cells']} " - f"land={checkpoint_state['land_cells']} sea={checkpoint_state['sea_cells']}" + f"coast={checkpoint_state['land_cells']} off_coast={checkpoint_state['sea_cells']}" ) if part_inserted == 0: @@ -1636,7 +2063,7 @@ def main() -> None: if selected_fpc is not None: log( f"[{LAYER_NAME}] PRC={prc} / FPC={selected_fpc} cluster={cluster_idx + 1} " - f"part={part_idx + 1} 格网完成，cell={part_inserted} land={land_count} sea={sea_count} bbox={bbox_lonlat}" + f"part={part_idx + 1} 格网完成，cell={part_inserted} coast={land_count} off_coast={sea_count} bbox={bbox_lonlat}" ) log( f"[{LAYER_NAME}] PRC={prc} / FPC={selected_fpc} cluster={cluster_idx + 1} " @@ -1645,7 +2072,7 @@ def main() -> None: else: log( f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1} " - f"part={part_idx + 1} 格网完成，cell={part_inserted} land={land_count} sea={sea_count} bbox={bbox_lonlat}" + f"part={part_idx + 1} 格网完成，cell={part_inserted} coast={land_count} off_coast={sea_count} bbox={bbox_lonlat}" ) log(f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1} part={part_idx + 1} 写库完成") checkpoint_state.update( @@ -1726,12 +2153,12 @@ def main() -> None: log( f"[{LAYER_NAME}] PRC={prc} / FPC={selected_fpc} 完成 cluster={len(cluster_groups)} " f"200x200={summary_coarse} final_20x20={summary_inserted} " - f"land={prc_land_now} sea={prc_sea_now} elapsed={prc_elapsed:.1f}s" + f"coast={prc_land_now} off_coast={prc_sea_now} elapsed={prc_elapsed:.1f}s" ) else: log( f"[{LAYER_NAME}] PRC={prc} 完成 cluster={len(cluster_groups)} " - f"本段写入={prc_inserted_now} land={prc_land_now} sea={prc_sea_now} " + f"本段写入={prc_inserted_now} coast={prc_land_now} off_coast={prc_sea_now} " f"elapsed={prc_elapsed:.1f}s" ) log(