tei/pbf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix fish port 20m skip missing coastline

Browse Source
This commit is contained in:
OpenAI Codex
2026-05-03 15:42:01 +08:00
parent 2513dd56f8
commit d678851f21
2 changed files with 619 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

545
coastline/build_fish_port_20m_full_mysql_resume.py
View File

@@ -31,7 +31,7 @@ FISH_SRC = "coastline/C09-06.zip"
COAST_DIR = Path("coastline")
LAYER_NAME = "fish_port_20m"
CELL_SIZE_M = 20.0
COAST_BUFFER_M = 12.0
COAST_BUFFER_M = 5.0
COAST_COARSE_LAYER = "coast_200m"
DEFAULT_COAST_COARSE_MARGIN_M = 0.0
DEFAULT_CHECKPOINT = "out/fish_port_20m_full_resume.checkpoint.json"
@@ -50,6 +50,10 @@ class FishLine:
bbox: tuple[float, float, float, float]
class NoCoastlineCurvesError(RuntimeError):
pass
def mercator_x(lon: float) -> float:
return RADIUS * math.radians(lon)
@@ -280,7 +284,7 @@ def ensure_schema(conn) -> None:
max_lon DOUBLE NOT NULL,
max_lat DOUBLE NOT NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
UNIQUE KEY uniq_layer_cell (layer_name, cell_id),
UNIQUE KEY uniq_layer_source_cell (layer_name, source_name, cell_id),
KEY idx_layer_state (layer_name, state_name),
KEY idx_layer_rowcol (layer_name, row_idx, col_idx)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
@@ -289,6 +293,29 @@ def ensure_schema(conn) -> None:
with conn.cursor() as cur:
for stmt in stmts:
cur.execute(stmt)
cur.execute(
"""
SELECT INDEX_NAME, COLUMN_NAME, SEQ_IN_INDEX
FROM information_schema.STATISTICS
WHERE TABLE_SCHEMA=%s AND TABLE_NAME='navsea_grid_cell' AND NON_UNIQUE=0
ORDER BY INDEX_NAME, SEQ_IN_INDEX
""",
(DB_NAME,),
)
index_columns: dict[str, list[str]] = {}
for index_name, column_name, _seq in cur.fetchall():
index_columns.setdefault(index_name, []).append(column_name)
desired_cols = ["layer_name", "source_name", "cell_id"]
if index_columns.get("uniq_layer_source_cell") != desired_cols:
if "uniq_layer_source_cell" in index_columns:
cur.execute("ALTER TABLE navsea_grid_cell DROP INDEX uniq_layer_source_cell")
if "uniq_layer_cell" in index_columns:
cur.execute("ALTER TABLE navsea_grid_cell DROP INDEX uniq_layer_cell")
cur.execute(
"ALTER TABLE navsea_grid_cell ADD UNIQUE KEY uniq_layer_source_cell (layer_name, source_name, cell_id)"
)
elif "uniq_layer_cell" in index_columns:
cur.execute("ALTER TABLE navsea_grid_cell DROP INDEX uniq_layer_cell")
conn.commit()
@@ -340,11 +367,22 @@ def insert_cells(cur, rows: list[tuple]) -> None:
return
cur.executemany(
"""
INSERT IGNORE INTO navsea_grid_cell
INSERT INTO navsea_grid_cell
(layer_name, cell_id, row_idx, col_idx, cell_size_m, state_name, class_name, source_name,
min_lon, min_lat, max_lon, max_lat)
VALUES
(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
ON DUPLICATE KEY UPDATE
row_idx=VALUES(row_idx),
col_idx=VALUES(col_idx),
cell_size_m=VALUES(cell_size_m),
state_name=VALUES(state_name),
class_name=VALUES(class_name),
source_name=VALUES(source_name),
min_lon=VALUES(min_lon),
min_lat=VALUES(min_lat),
max_lon=VALUES(max_lon),
max_lat=VALUES(max_lat)
""",
rows,
)
@@ -510,15 +548,34 @@ def drop_existing_layer(conn, layer_name: str) -> None:
def drop_existing_prcs(conn, layer_name: str, prcs: set[str]) -> None:
if not prcs:
return
source_names = [f"C09-06+coastline PRC={prc}" for prc in sorted(normalize_prc_list(prcs))]
source_patterns = [f"C09-06+coastline PRC={prc}%" for prc in sorted(normalize_prc_list(prcs))]
source_where = " OR ".join(["source_name LIKE %s"] * len(source_patterns))
progress_placeholders = ", ".join(["%s"] * len(prcs))
with conn.cursor() as cur:
cur.execute(
f"DELETE FROM navsea_grid_cell WHERE layer_name=%s AND ({source_where})",
[layer_name, *source_patterns],
)
cur.execute(
f"DELETE FROM navsea_grid_import_progress WHERE job_name=%s AND prc IN ({progress_placeholders})",
[RESUME_JOB_NAME, *sorted(prcs)],
)
conn.commit()
def drop_existing_fpc(conn, layer_name: str, prcs: set[str], fpc: str) -> None:
if not prcs:
return
source_names = [f"C09-06+coastline PRC={prc} FPC={fpc}" for prc in sorted(normalize_prc_list(prcs))]
placeholders = ", ".join(["%s"] * len(source_names))
progress_placeholders = ", ".join(["%s"] * len(prcs))
with conn.cursor() as cur:
cur.execute(
f"DELETE FROM navsea_grid_cell WHERE layer_name=%s AND source_name IN ({placeholders})",
[layer_name, *source_names],
)
cur.execute(
f"DELETE FROM navsea_grid_import_progress WHERE job_name=%s AND prc IN ({placeholders})",
f"DELETE FROM navsea_grid_import_progress WHERE job_name=%s AND prc IN ({progress_placeholders})",
[RESUME_JOB_NAME, *sorted(prcs)],
)
conn.commit()
@@ -662,6 +719,14 @@ def collect_fish_lines_for_prc(
return fish_lines
def group_fish_lines_by_fpc(fish_lines: list[FishLine]) -> dict[str, list[FishLine]]:
groups: dict[str, list[FishLine]] = {}
for line in fish_lines:
key = line.fpc or "__unknown__"
groups.setdefault(key, []).append(line)
return groups
@trace_fn("cluster_line_indices")
def cluster_line_indices(bboxes: list[tuple[float, float, float, float]], gap_m: float = 250.0) -> list[list[int]]:
parent = list(range(len(bboxes)))
@@ -774,7 +839,7 @@ def load_coastlines_for_prc(
scan_path(coast_path)
if not lines:
raise SystemExit(
raise NoCoastlineCurvesError(
f"no coastline curves parsed for PRC={prc}; "
f"tried={len(tried_paths)} bbox={tuple(round(v, 3) for v in bbox_geom.bounds) if bbox_geom is not None else None}"
)
@@ -902,14 +967,13 @@ def build_cells_for_geometry(
if not fish_prepared.intersects(cell_geom):
continue
profile["precise_hits"] += 1
is_land = coast_prepared.intersects(cell_geom)
state_name = "LAND_BASE" if is_land else "SEA_SURFACE"
class_name = state_name
if is_land:
land_count += 1
else:
coast_hit = coast_prepared.intersects(cell_geom)
if not coast_hit:
sea_count += 1
continue
land_count += 1
state_name = "COASTLINE"
class_name = state_name
min_lon = lon_from_mercator(x)
min_lat = lat_from_mercator(y)
max_lon = lon_from_mercator(x + cell_size_m)
@@ -941,14 +1005,14 @@ def build_cells_for_geometry(
if not rows:
log(
f"[TRACE] build_cells_for_geometry 结束 rows=0 land=0 sea=0 "
f"[TRACE] build_cells_for_geometry 结束 rows=0 coast=0 off_coast=0 "
f"tile_hit={profile['tile_hit']}/{profile['tile_checked']} "
f"row_hit={profile['row_hit']}/{profile['row_checked']} "
f"candidate={profile['candidate_cells']} precise={profile['precise_hits']}/{profile['precise_cell_checks']}"
)
return [], 0, 0, (float("inf"), float("inf"), float("-inf"), float("-inf")), profile
log(
f"[TRACE] build_cells_for_geometry 结束 rows={len(rows)} land={land_count} sea={sea_count} "
f"[TRACE] build_cells_for_geometry 结束 rows={len(rows)} coast={land_count} off_coast={sea_count} "
f"tile_hit={profile['tile_hit']}/{profile['tile_checked']} "
f"row_hit={profile['row_hit']}/{profile['row_checked']} "
f"candidate={profile['candidate_cells']} precise={profile['precise_hits']}/{profile['precise_cell_checks']}"
@@ -1035,14 +1099,13 @@ def build_cells_for_geometry(
if not fish_prepared.intersects(cell_geom):
continue
profile["precise_hits"] += 1
is_land = coast_prepared.intersects(cell_geom)
state_name = "LAND_BASE" if is_land else "SEA_SURFACE"
class_name = state_name
if is_land:
land_count += 1
else:
coast_hit = coast_prepared.intersects(cell_geom)
if not coast_hit:
sea_count += 1
continue
land_count += 1
state_name = "COASTLINE"
class_name = state_name
min_lon = lon_from_mercator(x)
min_lat = lat_from_mercator(y)
max_lon = lon_from_mercator(x + cell_size_m)
@@ -1075,14 +1138,14 @@ def build_cells_for_geometry(
if not rows:
log(
f"[TRACE] build_cells_for_geometry 结束 rows=0 land=0 sea=0 "
f"[TRACE] build_cells_for_geometry 结束 rows=0 coast=0 off_coast=0 "
f"tile_hit={profile['tile_hit']}/{profile['tile_checked']} "
f"row_hit={profile['row_hit']}/{profile['row_checked']} "
f"candidate={profile['candidate_cells']} precise={profile['precise_hits']}/{profile['precise_cell_checks']}"
)
return [], 0, 0, (float("inf"), float("inf"), float("-inf"), float("-inf")), profile
log(
f"[TRACE] build_cells_for_geometry 结束 rows={len(rows)} land={land_count} sea={sea_count} "
f"[TRACE] build_cells_for_geometry 结束 rows={len(rows)} coast={land_count} off_coast={sea_count} "
f"tile_hit={profile['tile_hit']}/{profile['tile_checked']} "
f"row_hit={profile['row_hit']}/{profile['row_checked']} "
f"candidate={profile['candidate_cells']} precise={profile['precise_hits']}/{profile['precise_cell_checks']}"
@@ -1090,6 +1153,299 @@ def build_cells_for_geometry(
return rows, land_count, sea_count, (export_bbox[0], export_bbox[1], export_bbox[2], export_bbox[3]), profile
def process_port_group_no_resume(
*,
conn,
cur,
project_root: Path,
prc: str,
prc_lines: list[FishLine],
args,
selected_fpc: str | None,
max_scan_cells: int | None,
) -> tuple[int, int, int, int, float]:
prc_bboxes = [line.bbox for line in prc_lines]
prc_bbox = (
min(bb[0] for bb in prc_bboxes),
min(bb[1] for bb in prc_bboxes),
max(bb[2] for bb in prc_bboxes),
max(bb[3] for bb in prc_bboxes),
)
cluster_groups = cluster_line_indices(prc_bboxes, gap_m=args.cluster_gap_m)
if selected_fpc is not None:
log(f"[{LAYER_NAME}] PRC={prc} / FPC={selected_fpc} cluster 数={len(cluster_groups)}")
log(f"[{LAYER_NAME}] PRC={prc} / FPC={selected_fpc} 正在缓存整港海岸线并生成缓冲区 ...")
else:
log(f"[{LAYER_NAME}] PRC={prc} cluster 数={len(cluster_groups)}")
log(f"[{LAYER_NAME}] PRC={prc} 正在缓存整县海岸线并生成缓冲区 ...")
try:
prc_coast_lines, prc_coast_bbox = load_coastlines_for_prc(
project_root,
prc,
bbox=(
mercator_x(prc_bbox[0]),
mercator_y(prc_bbox[1]),
mercator_x(prc_bbox[2]),
mercator_y(prc_bbox[3]),
),
search_margin_m=max(args.coast_buffer_m, args.coast_coarse_margin_m),
)
except NoCoastlineCurvesError as exc:
if not args.skip_missing_coarse:
raise
status_name = "FPC_SKIPPED_NO_COASTLINE" if selected_fpc is not None else "PRC_SKIPPED_NO_COASTLINE"
insert_progress(
cur,
job_name=RESUME_JOB_NAME,
layer_name=LAYER_NAME,
prc=prc,
status_name=status_name,
cluster_count=len(cluster_groups),
inserted_cells=0,
land_cells=0,
sea_cells=0,
elapsed_sec=0.0,
)
conn.commit()
if selected_fpc is not None:
log(f"[{LAYER_NAME}] PRC={prc} / FPC={selected_fpc} 未找到海岸线；已按 --skip-missing-coarse 跳过。{exc}")
else:
log(f"[{LAYER_NAME}] PRC={prc} 未找到海岸线；已按 --skip-missing-coarse 跳过。{exc}")
return 0, 0, 0, 0, 0.0
if selected_fpc is not None:
log(
f"[{LAYER_NAME}] PRC={prc} / FPC={selected_fpc} 海岸线缓存完成，"
f"曲线数={len(prc_coast_lines)} bbox={prc_coast_bbox}"
)
else:
log(
f"[{LAYER_NAME}] PRC={prc} 海岸线缓存完成，"
f"曲线数={len(prc_coast_lines)} bbox={prc_coast_bbox}"
)
prc_coast_geom = unary_union(prc_coast_lines)
prc_coast_buffer = prc_coast_geom.buffer(args.coast_buffer_m, cap_style=2, join_style=2)
prc_coast_prepared = prep(prc_coast_buffer)
if selected_fpc is not None:
log(f"[{LAYER_NAME}] PRC={prc} / FPC={selected_fpc} 海岸缓冲缓存完成，开始处理各 part ...")
else:
log(f"[{LAYER_NAME}] PRC={prc} 海岸缓冲缓存完成，开始处理各 part ...")
started_at = time.monotonic()
prc_inserted_before = 0
prc_land_before = 0
prc_sea_before = 0
inserted_total = 0
land_total = 0
sea_total = 0
coarse_window_total = 0
selected_port_rows: dict[str, tuple] | None = {} if selected_fpc is not None else None
selected_port_coarse_windows: set[tuple[float, float, float, float]] | None = set() if selected_fpc is not None else None
prc_profile = {
"tile_checked": 0,
"tile_hit": 0,
"row_checked": 0,
"row_hit": 0,
"mask_row_hit": 0,
"candidate_cells": 0,
"precise_cell_checks": 0,
"precise_hits": 0,
}
for cluster_idx, cluster in enumerate(cluster_groups):
log(f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1}/{len(cluster_groups)} 开始，成员={len(cluster)}")
log(f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1} 正在合并渔港线 ...")
cluster_lines = [
LineString([(mercator_x(lon), mercator_y(lat)) for lon, lat in prc_lines[i].coords])
for i in cluster
]
merged = unary_union(cluster_lines)
polygons = list(polygonize(merged))
if polygons:
fish_geom = unary_union(polygons)
geom_mode = "polygonize"
else:
fish_geom = merged.buffer(args.cell_size_m, cap_style=2, join_style=2)
geom_mode = "buffer"
if fish_geom.is_empty:
log(f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx} 为空，跳过")
continue
log(f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1} 几何模式={geom_mode}，正在生成格网 ...")
fish_parts = [part for part in get_parts(fish_geom) if not part.is_empty]
if not fish_parts:
log(f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1} 没有可拆分部件，跳过")
continue
for part_idx, fish_part in enumerate(fish_parts):
fish_bbox = fish_part.bounds
part_coarse_windows = load_coarse_windows_for_bbox(
conn,
layer_name=COAST_COARSE_LAYER,
bbox_mercator=fish_bbox,
margin_m=args.coast_coarse_margin_m,
)
if part_coarse_windows:
log(
f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1}/{len(cluster_groups)} "
f"part={part_idx + 1}/{len(fish_parts)} 取回 coast_200m 格子={len(part_coarse_windows)}"
)
scan_windows = [box(*coarse_window) for coarse_window in part_coarse_windows]
if selected_port_coarse_windows is not None:
for coarse_window in part_coarse_windows:
selected_port_coarse_windows.add(tuple(float(v) for v in coarse_window))
if not scan_windows:
if args.allow_unbounded_fallback:
scan_windows = [fish_part]
log(
f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1}/{len(cluster_groups)} "
f"part={part_idx + 1}/{len(fish_parts)} 没有有效 coast_200m 交集，已允许回退原始范围"
)
else:
message = (
f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1}/{len(cluster_groups)} "
f"part={part_idx + 1}/{len(fish_parts)} 没有有效 coast_200m 交集；"
f"fish_bbox={tuple(round(v, 3) for v in fish_bbox)} "
f"coarse_windows={len(part_coarse_windows)}。"
)
if args.skip_missing_coarse:
log(message + "已按 --skip-missing-coarse 跳过")
continue
raise RuntimeError(
message
+ "按当前设计这属于异常：渔港 20m 必须以数据库 coast_200m 为底盘。"
+ "如确需旧式整片扫描请显式加 --allow-unbounded-fallback；"
+ "如确认允许漏算该 part，请显式加 --skip-missing-coarse。"
)
log(
f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1}/{len(cluster_groups)} "
f"part={part_idx + 1}/{len(fish_parts)} 实际扫描交集窗口={len(scan_windows)}"
)
if selected_fpc is not None:
log(
f"[{LAYER_NAME}] PRC={prc} / FPC={selected_fpc} cluster={cluster_idx + 1}/{len(cluster_groups)} "
f"part={part_idx + 1}/{len(fish_parts)} bbox={fish_bbox} 使用海岸线缓存 ..."
)
else:
log(
f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1}/{len(cluster_groups)} "
f"part={part_idx + 1}/{len(fish_parts)} bbox={fish_bbox} 使用 part 级海岸线缓存 ..."
)
log(f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1} part={part_idx + 1} 开始格网 ...")
land_count = 0
sea_count = 0
bbox_lonlat = (float("inf"), float("inf"), float("-inf"), float("-inf"))
profile = {key: 0 for key in prc_profile}
part_inserted = 0
batch: list[tuple] = []
for window_idx, scan_geom in enumerate(scan_windows):
if scan_geom.is_empty:
continue
log(
f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1}/{len(cluster_groups)} "
f"part={part_idx + 1}/{len(fish_parts)} window={window_idx + 1}/{len(scan_windows)} "
f"正在生成格网 ..."
)
window_rows, window_land, window_sea, window_bbox, window_profile = build_cells_for_geometry(
scan_geom,
prc_coast_prepared,
prc=prc,
cell_size_m=args.cell_size_m,
scan_tile_m=args.cell_size_m * 10.0,
source_name=(
f"C09-06+coastline PRC={prc}"
+ (f" FPC={selected_fpc}" if selected_fpc is not None else "")
),
max_scan_cells=max_scan_cells,
)
land_count += window_land
sea_count += window_sea
bbox_lonlat = (
min(bbox_lonlat[0], window_bbox[0]),
min(bbox_lonlat[1], window_bbox[1]),
max(bbox_lonlat[2], window_bbox[2]),
max(bbox_lonlat[3], window_bbox[3]),
)
for key in prc_profile:
prc_profile[key] += window_profile[key]
profile[key] += window_profile[key]
for row in window_rows:
if selected_port_rows is not None:
selected_port_rows[row[1]] = row
batch.append(row)
if len(batch) >= args.commit_every:
insert_cells(cur, batch)
conn.commit()
part_inserted += len(batch)
inserted_total += len(batch)
batch.clear()
if batch:
insert_cells(cur, batch)
conn.commit()
part_inserted += len(batch)
inserted_total += len(batch)
if part_inserted == 0:
log(
f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1} "
f"part={part_idx + 1} 没有生成 cell"
)
continue
if selected_fpc is not None:
log(
f"[{LAYER_NAME}] PRC={prc} / FPC={selected_fpc} cluster={cluster_idx + 1} "
f"part={part_idx + 1} 格网完成，cell={part_inserted} coast={land_count} off_coast={sea_count} bbox={bbox_lonlat}"
)
else:
log(
f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1} "
f"part={part_idx + 1} 格网完成，cell={part_inserted} coast={land_count} off_coast={sea_count} bbox={bbox_lonlat}"
)
coarse_window_total += len(part_coarse_windows)
land_total += land_count
sea_total += sea_count
prc_elapsed = time.monotonic() - started_at
summary_inserted = len(selected_port_rows) if selected_port_rows is not None else inserted_total
summary_coarse = len(selected_port_coarse_windows) if selected_port_coarse_windows is not None else coarse_window_total
insert_progress(
cur,
job_name="fish_port_20m_full_mysql_resume",
layer_name=LAYER_NAME,
prc=prc,
status_name="FPC_DONE" if selected_fpc is not None else "PRC_DONE",
cluster_count=len(cluster_groups),
inserted_cells=summary_inserted,
land_cells=land_total,
sea_cells=sea_total,
elapsed_sec=prc_elapsed,
)
conn.commit()
if selected_fpc is not None:
log(
f"[{LAYER_NAME}] PRC={prc} / FPC={selected_fpc} 完成 cluster={len(cluster_groups)} "
f"200x200={summary_coarse} final_20x20={summary_inserted} "
f"coast={land_total} off_coast={sea_total} elapsed={prc_elapsed:.1f}s"
)
else:
log(
f"[{LAYER_NAME}] PRC={prc} 完成 cluster={len(cluster_groups)} "
f"本段写入={inserted_total} coast={land_total} off_coast={sea_total} "
f"elapsed={prc_elapsed:.1f}s"
)
return summary_inserted, land_total, sea_total, summary_coarse, prc_elapsed
@trace_fn("main")
def main() -> None:
parser = argparse.ArgumentParser(description="Rebuild nationwide fish-port 20m grid into MySQL with resume support")
@@ -1152,7 +1508,7 @@ def main() -> None:
"--coast-buffer-m",
type=float,
default=COAST_BUFFER_M,
help="海岸缓冲距离（米）",
help="海岸缓冲距离（米），默认 5m，可按需要调大或调小",
)
parser.add_argument(
"--fish-buffer-m",
@@ -1279,9 +1635,57 @@ def main() -> None:
raise SystemExit("no fish-port line features selected")
if selected_fpc is not None:
log(f"[{LAYER_NAME}] 选中 PRC 数量：{len(prc_list)}（由 FPC 反查得到）")
drop_existing_fpc(conn, LAYER_NAME, set(prc_list), selected_fpc)
else:
log(f"[{LAYER_NAME}] 选中 PRC 数量：{len(prc_list)}")
if selected_prc is not None or selected_fpc is not None:
with conn.cursor() as cur:
for prc in prc_list:
if selected_fpc is not None:
prc_lines = collect_fish_lines_for_prc(root, point_lookup, prc, selected_fpc=selected_fpc)
if not prc_lines:
log(f"[{LAYER_NAME}] PRC={prc} / FPC={selected_fpc} 没有渔港线要素，跳过")
continue
log(
f"[{LAYER_NAME}] PRC={prc} / FPC={selected_fpc} 开始，"
f"线要素={len(prc_lines)}"
)
process_port_group_no_resume(
conn=conn,
cur=cur,
project_root=project_root,
prc=prc,
prc_lines=prc_lines,
args=args,
selected_fpc=selected_fpc,
max_scan_cells=max_scan_cells,
)
else:
prc_lines = collect_fish_lines_for_prc(root, point_lookup, prc, selected_fpc=None)
if not prc_lines:
log(f"[{LAYER_NAME}] PRC={prc} 没有渔港线要素，跳过")
continue
grouped_lines = group_fish_lines_by_fpc(prc_lines)
log(f"[{LAYER_NAME}] PRC={prc} 开始，FPC 组数={len(grouped_lines)}")
for group_fpc in sorted(grouped_lines.keys()):
group_lines = grouped_lines[group_fpc]
log(
f"[{LAYER_NAME}] PRC={prc} / FPC={group_fpc} 开始，"
f"线要素={len(group_lines)}"
)
process_port_group_no_resume(
conn=conn,
cur=cur,
project_root=project_root,
prc=prc,
prc_lines=group_lines,
args=args,
selected_fpc=group_fpc if group_fpc != "__unknown__" else None,
max_scan_cells=max_scan_cells,
)
return
total_inserted = 0
total_land = 0
total_sea = 0
@@ -1395,17 +1799,41 @@ def main() -> None:
log(f"[{LAYER_NAME}] PRC={prc} / FPC={selected_fpc} 正在缓存整港海岸线并生成缓冲区 ...")
else:
log(f"[{LAYER_NAME}] PRC={prc} 正在缓存整县海岸线并生成缓冲区 ...")
prc_coast_lines, prc_coast_bbox = load_coastlines_for_prc(
project_root,
prc,
bbox=(
mercator_x(prc_bbox[0]),
mercator_y(prc_bbox[1]),
mercator_x(prc_bbox[2]),
mercator_y(prc_bbox[3]),
),
search_margin_m=max(args.coast_buffer_m, args.coast_coarse_margin_m),
)
try:
prc_coast_lines, prc_coast_bbox = load_coastlines_for_prc(
project_root,
prc,
bbox=(
mercator_x(prc_bbox[0]),
mercator_y(prc_bbox[1]),
mercator_x(prc_bbox[2]),
mercator_y(prc_bbox[3]),
),
search_margin_m=max(args.coast_buffer_m, args.coast_coarse_margin_m),
)
except NoCoastlineCurvesError as exc:
if not args.skip_missing_coarse:
raise
status_name = "FPC_SKIPPED_NO_COASTLINE" if selected_fpc is not None else "PRC_SKIPPED_NO_COASTLINE"
insert_progress(
cur,
job_name=RESUME_JOB_NAME,
layer_name=LAYER_NAME,
prc=prc,
status_name=status_name,
cluster_count=len(cluster_groups),
inserted_cells=0,
land_cells=0,
sea_cells=0,
elapsed_sec=0.0,
)
conn.commit()
if selected_fpc is not None:
log(f"[{LAYER_NAME}] PRC={prc} / FPC={selected_fpc} 未找到海岸线；已按 --skip-missing-coarse 跳过。{exc}")
else:
log(f"[{LAYER_NAME}] PRC={prc} 未找到海岸线；已按 --skip-missing-coarse 跳过。{exc}")
completed_prcs.add(prc)
continue
if selected_fpc is not None:
log(
f"[{LAYER_NAME}] PRC={prc} / FPC={selected_fpc} 海岸线缓存完成，"
@@ -1481,26 +1909,25 @@ def main() -> None:
for part_idx, fish_part in enumerate(fish_parts):
if checkpoint and checkpoint.get("last_prc") == prc and checkpoint.get("last_cluster_index") == cluster_idx and part_idx < resume_part:
continue
fish_bbox = fish_part.bounds
coarse_windows = load_coarse_windows_for_bbox(
scan_windows: list[object] = []
part_coarse_windows = load_coarse_windows_for_bbox(
conn,
layer_name=COAST_COARSE_LAYER,
bbox_mercator=fish_bbox,
margin_m=args.coast_coarse_margin_m,
)
scan_windows: list[object] = []
if coarse_windows:
if part_coarse_windows:
log(
f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1}/{len(cluster_groups)} "
f"part={part_idx + 1}/{len(fish_parts)} 取回 coast_200m 格子={len(coarse_windows)}"
f"part={part_idx + 1}/{len(fish_parts)} 取回 coast_200m 格子={len(part_coarse_windows)}"
)
for coarse_window in coarse_windows:
for coarse_window in part_coarse_windows:
window_geom = box(*coarse_window)
scan_geom = fish_part.intersection(window_geom)
if not scan_geom.is_empty:
scan_windows.append(scan_geom)
if selected_port_coarse_windows is not None:
selected_port_coarse_windows.add(tuple(float(v) for v in coarse_window))
scan_windows.append(window_geom)
if selected_port_coarse_windows is not None:
selected_port_coarse_windows.add(tuple(float(v) for v in coarse_window))
if not scan_windows:
if args.allow_unbounded_fallback:
scan_windows = [fish_part]
@@ -1513,7 +1940,7 @@ def main() -> None:
f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1}/{len(cluster_groups)} "
f"part={part_idx + 1}/{len(fish_parts)} 没有有效 coast_200m 交集；"
f"fish_bbox={tuple(round(v, 3) for v in fish_bbox)} "
f"coarse_windows={len(coarse_windows)}。"
f"coarse_windows={len(part_coarse_windows)}。"
)
if args.skip_missing_coarse:
log(message + "已按 --skip-missing-coarse 跳过")
@@ -1524,11 +1951,11 @@ def main() -> None:
+ "如确需旧式整片扫描请显式加 --allow-unbounded-fallback；"
+ "如确认允许漏算该 part，请显式加 --skip-missing-coarse。"
)
else:
log(
f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1}/{len(cluster_groups)} "
f"part={part_idx + 1}/{len(fish_parts)} 实际扫描交集窗口={len(scan_windows)}"
)
log(
f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1}/{len(cluster_groups)} "
f"part={part_idx + 1}/{len(fish_parts)} 实际扫描交集窗口={len(scan_windows)}"
)
if selected_fpc is not None:
log(
f"[{LAYER_NAME}] PRC={prc} / FPC={selected_fpc} cluster={cluster_idx + 1}/{len(cluster_groups)} "
@@ -1537,7 +1964,7 @@ def main() -> None:
else:
log(
f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1}/{len(cluster_groups)} "
f"part={part_idx + 1}/{len(fish_parts)} bbox={fish_bbox} 使用 PRC 级海岸线缓存 ..."
f"part={part_idx + 1}/{len(fish_parts)} bbox={fish_bbox} 使用 part 级海岸线缓存 ..."
)
coast_prepared = prc_coast_prepared
log(f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1} part={part_idx + 1} 开始格网 ...")
@@ -1597,7 +2024,7 @@ def main() -> None:
f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1}/{len(cluster_groups)} "
f"part={part_idx + 1}/{len(fish_parts)} "
f"提交={part_inserted} 累计={checkpoint_state['inserted_cells']} "
f"land={checkpoint_state['land_cells']} sea={checkpoint_state['sea_cells']}"
f"coast={checkpoint_state['land_cells']} off_coast={checkpoint_state['sea_cells']}"
)
batch.clear()
@@ -1614,7 +2041,7 @@ def main() -> None:
f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1}/{len(cluster_groups)} "
f"part={part_idx + 1}/{len(fish_parts)} "
f"提交={part_inserted} 累计={checkpoint_state['inserted_cells']} "
f"land={checkpoint_state['land_cells']} sea={checkpoint_state['sea_cells']}"
f"coast={checkpoint_state['land_cells']} off_coast={checkpoint_state['sea_cells']}"
)
if part_inserted == 0:
@@ -1636,7 +2063,7 @@ def main() -> None:
if selected_fpc is not None:
log(
f"[{LAYER_NAME}] PRC={prc} / FPC={selected_fpc} cluster={cluster_idx + 1} "
f"part={part_idx + 1} 格网完成，cell={part_inserted} land={land_count} sea={sea_count} bbox={bbox_lonlat}"
f"part={part_idx + 1} 格网完成，cell={part_inserted} coast={land_count} off_coast={sea_count} bbox={bbox_lonlat}"
)
log(
f"[{LAYER_NAME}] PRC={prc} / FPC={selected_fpc} cluster={cluster_idx + 1} "
@@ -1645,7 +2072,7 @@ def main() -> None:
else:
log(
f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1} "
f"part={part_idx + 1} 格网完成，cell={part_inserted} land={land_count} sea={sea_count} bbox={bbox_lonlat}"
f"part={part_idx + 1} 格网完成，cell={part_inserted} coast={land_count} off_coast={sea_count} bbox={bbox_lonlat}"
)
log(f"[{LAYER_NAME}] PRC={prc} cluster={cluster_idx + 1} part={part_idx + 1} 写库完成")
checkpoint_state.update(
@@ -1726,12 +2153,12 @@ def main() -> None:
log(
f"[{LAYER_NAME}] PRC={prc} / FPC={selected_fpc} 完成 cluster={len(cluster_groups)} "
f"200x200={summary_coarse} final_20x20={summary_inserted} "
f"land={prc_land_now} sea={prc_sea_now} elapsed={prc_elapsed:.1f}s"
f"coast={prc_land_now} off_coast={prc_sea_now} elapsed={prc_elapsed:.1f}s"
)
else:
log(
f"[{LAYER_NAME}] PRC={prc} 完成 cluster={len(cluster_groups)} "
f"本段写入={prc_inserted_now} land={prc_land_now} sea={prc_sea_now} "
f"本段写入={prc_inserted_now} coast={prc_land_now} off_coast={prc_sea_now} "
f"elapsed={prc_elapsed:.1f}s"
)
log(