feat: normalize CM93 objects for S-52 presentation

This commit is contained in:
OpenAI Codex
2026-09-17 14:56:19 +08:00
parent 089e7af6ee
commit c72310f35a
3 changed files with 451 additions and 4 deletions

View File

@@ -0,0 +1,120 @@
# CM95 S-52 全国对象规范化阶段报告 v1
## 1. 目标
本阶段沿 OpenCPN CM93 `CreateS57Obj()` 与 S-52 Presentation Lookup 流程,补齐 CM93 私有/扩展对象类到标准 S-57/S-52 Presentation 语义的规范化。
范围为 Dataset 1 日本数据。本阶段不重建正式 release,只进行全库只读审计、单 Tile 回归和 Presentation 映射实现。
## 2. 参考资源
- CM93 对象字典:`/root/sourceserver/cm95-data/CM93JAPAN/CM93OBJ.DIC`
- CM93 属性字典:`/root/sourceserver/cm95-data/CM93JAPAN/CM93ATTR.DIC`
- OpenCPN `chartsymbols.xml`
- OpenCPN `S52RAZDS.RLE`
- OpenCPN `cm93.cpp` 中的对象替代逻辑
- S-57 object class catalog 与 `attdecode.csv`
- 完整 3057 条 Presentation Lookup IR
## 3. 本阶段新增的主要规范化
- `ITDARE → DEPARE`
- `FSHHAV → FSHFAC`
- `BRTFAC → BERTHS`
- `TNKCON → SILTNK`
- `PRDINS → PRDARE`
- `ROADPT → ROADWY`
- `VEGARE → VEGATN`
- `SLTPAN → LNDRGN + CATLND=15`
- `TELPHC → CONVYR + CATCON=1`
- `DIFFUS → OBSTRN + CATOBS=3`
- `SLIPWY → SLCONS + CATSLC=13`
- `LNDSTS → SLCONS + CATSLC=11`
- `RMPARE → SLCONS + CATSLC=12`
- `DYKARE / DYKCRW → DYKCON`
- `SILBUI → SILTNK + CATSIL=1`
Landmark 类按 S-52 `LNDMRK + CATLMK` 规范化:
- `CHIMNY → CATLMK=3`
- `DSHAER → CATLMK=4`
- `FLGSTF → CATLMK=5`
- `FLASTK → CATLMK=6`
- `MSTCON → CATLMK=7`
- `WNDMIL → CATLMK=18`
- `WIMCON → CATLMK=19`
另外保留 OpenCPN Presentation Library 的扩展类大小写别名:
- `RIVBNK → rivbnk`
- `CANBNK → canbnk`
## 4. 全国审计结果
Dataset 1 共扫描 `1,282,431` 个 `chart_feature`。
最终 v7 审计:
- S-52 Lookup 命中:`1,246,817`
- 全对象表观命中率:`97.2229%`
- `unsupported_csp_counts = {}`
- 条件存在但属性不匹配:`0`
- no-candidate:`35,614`
其中几何可绘制对象总数为 `1,251,374`。
## 5. 不应计入普通 Presentation 缺口的对象
以下三类合计 `4,502` 个,不能作为“漏画”处理:
- `NATARE`:`1,516`,OpenCPN Presentation Library 明确为 `No data` 且 instruction 为空。
- `$COMPS`:`1,494`,属于制图罗盘辅助对象。
- `M_PROD`:`1,492`,属于生产信息元数据对象。
剔除上述对象后,需要普通 Presentation 的对象数为 `1,246,872`,其中已命中 `1,246,817`。
有效覆盖率:`99.9956%`。
剩余未安全规范化的普通地物仅 `55` 个。
## 6. 剩余 55 个语义复核项
- `TREPNT` Point:19
- `BUIREL` Area:13
- `TOWERS` Area:8
- `HILARE` Area:5
- `CEMTRY` Point/Area:7
- `RADDOM` Point:3
这些对象没有足够证据支持无损的一对一标准化。本阶段明确保留,不为追求数字上的 100% 而制造猜测映射。
后续如处理,应继续以 CM93 原始属性、S-57 类定义、S-52 Lookup 条件和 OpenCPN 实际行为共同确认。
## 7. 福冈湾回归
重新生成 `z12/3531/1641` v10 normalized PBF:
- Feature:`3293`
- Layer:`23`
- `hard_gate_unmapped = []`
- unresolved CSP:`0`
- `SOUNDG02 = 1881`
- `LIGHTS05 = 62`
- `OBSTRN04 = 19`
浏览器实际加载 `presentation_v10.html` 后进入 READY;灯火、灯弧、测深、危险物、岸线构造和原有主图层均正常。
浏览器唯一 404 为 `/favicon.ico`,与地图资源无关。
## 8. 结论
本阶段确认:CM95 当前主要 Presentation 缺口已经不再是大规模对象类未映射问题。
经过 CM93→S-57/S-52 对象规范化后,全国普通可绘制对象的有效 S-52 Lookup 覆盖率达到 `99.9956%`,且当前实际出现的 CSP 类型均已被 Resolver 支持。
下一阶段重点应从“补对象类”转向:
1. 对剩余 55 个小样本对象做语义复核;
2. 检查已命中 Lookup 的 `TE/TX/LC/AP` 是否全部被 MapLibre 后端完整执行;
3. 检查 Sprite/vector-only symbol 资源闭合;
4. 用 OpenCPN 对照图做视觉差异审计,而不是继续扩大类替换表。

View File

@@ -6,6 +6,84 @@ from .presentation import static_rule_hints
from .csp import S52MarinerSettings, resolve_csp_hints
OPENCPN_CM93_CLASS_SUBSTITUTIONS: dict[str, str] = {
"ITDARE": "DEPARE",
"_m_sor": "M_COVR",
"SPOGRD": "DMPGRD",
"FSHHAV": "FSHFAC",
"OFSPRD": "CTNARE",
"BRTFAC": "BERTHS",
"TNKCON": "SILTNK",
"ROADPT": "ROADWY",
"VEGARE": "VEGATN",
"PRDINS": "PRDARE",
"DYKARE": "DYKCON",
"DYKCRW": "DYKCON",
"SLTPAN": "LNDRGN",
"TELPHC": "CONVYR",
"DIFFUS": "OBSTRN",
"SLIPWY": "SLCONS",
"LNDSTS": "SLCONS",
"RMPARE": "SLCONS",
}
# OpenCPN's current Presentation Library contains these extension Lookup names
# in lowercase, while this CM93 dictionary stores the corresponding classes
# in uppercase. Keep the aliases explicit: lookup matching must remain
# case-sensitive because other extension classes (for example TOPMAR/topmar)
# have distinct semantics.
CM93_PRESENTATION_CLASS_ALIASES: dict[str, str] = {
"RIVBNK": "rivbnk",
"CANBNK": "canbnk",
}
CM93_LANDMARK_CLASS_TO_CATLMK: dict[str, int] = {
"CHIMNY": 3,
"DSHAER": 4,
"FLGSTF": 5,
"FLASTK": 6,
"MSTCON": 7,
"WNDMIL": 18,
"WIMCON": 19,
}
CM93_CLASS_SYNTHETIC_ATTRIBUTES: dict[str, dict[str, Any]] = {
"SILBUI": {"CATSIL": 1},
"SLTPAN": {"CATLND": 15},
"TELPHC": {"CATCON": 1},
"DIFFUS": {"CATOBS": 3},
"SLIPWY": {"CATSLC": 13},
"LNDSTS": {"CATSLC": 11},
"RMPARE": {"CATSLC": 12},
}
def normalize_cm93_object_class_for_s52(object_class: str) -> str:
"""Return the OpenCPN-compatible Presentation Library object class."""
name = str(object_class or "")
if name in CM93_LANDMARK_CLASS_TO_CATLMK:
return "LNDMRK"
if name == "SILBUI":
return "SILTNK"
name = OPENCPN_CM93_CLASS_SUBSTITUTIONS.get(name, name)
return CM93_PRESENTATION_CLASS_ALIASES.get(name, name)
def normalize_cm93_object_for_s52(
object_class: str, geometry_type: str, attrs: dict[str, Any]
) -> tuple[str, dict[str, Any]]:
"""Normalize CM93 class/attributes into the S-57/S-52 presentation view."""
source_class = str(object_class or "")
out = normalize_cm93_attributes_for_s52(attrs)
if str(geometry_type).upper() == "POINT":
catlmk = CM93_LANDMARK_CLASS_TO_CATLMK.get(source_class)
if catlmk is not None:
out.setdefault("CATLMK", catlmk)
for key, value in CM93_CLASS_SYNTHETIC_ATTRIBUTES.get(source_class, {}).items():
out.setdefault(key, value)
return normalize_cm93_object_class_for_s52(source_class), out
CM93_COLMAR_TO_S57_COLOUR: dict[int, str] = {
1: "4", # green
2: "2", # black
@@ -326,9 +404,12 @@ class S52LookupRegistry:
if table_geometry is None:
return None
table_name, lookup_geometry = table_geometry
lookup_class, lookup_attrs = normalize_cm93_object_for_s52(
object_class, geometry_type, attrs
)
return find_best_lookup(
self.by_class.get(str(object_class), []),
attrs,
self.by_class.get(lookup_class, []),
lookup_attrs,
table_name=table_name,
geometry_type=lookup_geometry,
)
@@ -343,6 +424,9 @@ class S52LookupRegistry:
sounding_depth_m: float | None = None,
associated_depth_ranges: list[dict[str, Any]] | None = None,
) -> dict[str, Any]:
lookup_class, normalized_attrs = normalize_cm93_object_for_s52(
object_class, geometry_type, attrs
)
row = self.resolve(object_class, geometry_type, attrs)
if row is None:
return {}
@@ -354,17 +438,18 @@ class S52LookupRegistry:
"s52_table_name": str(row.get("table_name") or ""),
}
out.update(dict(row.get("_static_hints") or {}))
if lookup_class != str(object_class):
out["s52_object_class"] = lookup_class
procedures = [
token.strip()
for token in str(out.get("s52_csp") or "").split(",")
if token.strip()
]
normalized_attrs = normalize_cm93_attributes_for_s52(attrs)
unresolved: list[str] = []
for procedure in procedures:
resolved = resolve_csp_hints(
procedure,
str(object_class),
lookup_class,
str(geometry_type),
normalized_attrs,
self.settings,

View File

@@ -0,0 +1,242 @@
from __future__ import annotations
import argparse
from collections import Counter, defaultdict
import json
import os
from pathlib import Path
from typing import Any
from dotenv import load_dotenv
import pymysql
from cm95pbf.s52 import (
lookup_condition_matches,
normalize_cm93_attributes_for_s52,
normalize_cm93_object_for_s52,
)
IMPLEMENTED_CSP = frozenset({
"DEPARE01", "DEPARE02", "DEPCNT02", "LIGHTS05", "SOUNDG02",
"OBSTRN04", "WRECKS02", "QUAPOS01", "SLCONS03", "RESTRN01",
"RESARE02", "TOPMAR01",
})
def _json_object(value: Any) -> dict[str, Any]:
if value is None:
return {}
if isinstance(value, dict):
return value
try:
parsed = json.loads(value)
except (TypeError, json.JSONDecodeError):
return {}
return parsed if isinstance(parsed, dict) else {}
def _geometry_key(geometry_type: str, point_table: str, boundary_table: str):
return {
"POINT": (point_table, "Point"),
"SOUNDING": (point_table, "Point"),
"LINE": ("Lines", "Line"),
"AREA": (boundary_table, "Area"),
}.get(str(geometry_type).upper())
def _prepare_lookups(rows: list[dict[str, Any]]):
index: dict[tuple[str, str, str], list[dict[str, Any]]] = defaultdict(list)
for row in rows:
key = (
str(row.get("object_class") or ""),
str(row.get("table_name") or ""),
str(row.get("geometry_type") or ""),
)
index[key].append(row)
for key, candidates in index.items():
candidates.sort(key=lambda r: (-len(r.get("attribute_conditions_raw") or []), int(r.get("id") or 0)))
return index
def _resolve(candidates: list[dict[str, Any]], attrs: dict[str, Any]):
normalized = normalize_cm93_attributes_for_s52(attrs)
fallback = None
for row in candidates:
conditions = row.get("attribute_conditions_raw") or []
if not conditions:
if fallback is None:
fallback = row
continue
if all(lookup_condition_matches(c, normalized) for c in conditions):
return row
return fallback
def main() -> int:
parser = argparse.ArgumentParser(description="流式审计整个 CM95 Dataset 的 S-52 Lookup/CSP 覆盖率。")
parser.add_argument("--lookups", required=True, type=Path)
parser.add_argument("--output", required=True, type=Path)
parser.add_argument("--env-file", default=Path(".env"), type=Path)
parser.add_argument("--dataset-id", default=1, type=int)
parser.add_argument("--point-table", default="Paper")
parser.add_argument("--boundary-table", default="Plain")
args = parser.parse_args()
lookup_rows = json.loads(args.lookups.read_text(encoding="utf-8"))
lookup_index = _prepare_lookups(lookup_rows)
load_dotenv(args.env_file)
conn = pymysql.connect(
host=os.environ["CM93_DB_HOST"],
port=int(os.environ.get("CM93_DB_PORT", "3306")),
user=os.environ["CM93_DB_USER"],
password=os.environ["CM93_DB_PASSWORD"],
database=os.environ["CM93_DB_NAME"],
charset=os.environ.get("CM93_DB_CHARSET", "utf8mb4"),
autocommit=True,
cursorclass=pymysql.cursors.SSCursor,
)
total = 0
matched = 0
no_lookup = 0
empty_candidates = 0
by_geometry = Counter()
matched_by_geometry = Counter()
no_lookup_by_key = Counter()
no_candidate_by_key = Counter()
condition_miss_by_key = Counter()
matched_by_key = Counter()
rcids = Counter()
csp_counts = Counter()
implemented_csp_counts = Counter()
unsupported_csp_counts = Counter()
opcode_counts = Counter()
direct_symbol_counts = Counter()
substitution_counts = Counter()
sql = (
"SELECT object_class, geometry_type, attributes_json "
"FROM chart_feature WHERE dataset_id=%s ORDER BY id"
)
with conn.cursor() as cur:
cur.execute(sql, (args.dataset_id,))
for object_class, geometry_type, attrs_raw in cur:
total += 1
object_class = str(object_class)
geometry_type = str(geometry_type)
attrs = _json_object(attrs_raw)
lookup_class, lookup_attrs = normalize_cm93_object_for_s52(
object_class, geometry_type, attrs
)
if lookup_class != object_class:
substitution_counts[(object_class, lookup_class, geometry_type)] += 1
by_geometry[geometry_type] += 1
tg = _geometry_key(geometry_type, args.point_table, args.boundary_table)
if tg is None:
no_lookup += 1
empty_candidates += 1
no_lookup_by_key[(object_class, geometry_type)] += 1
no_candidate_by_key[(object_class, geometry_type)] += 1
continue
table_name, lookup_geometry = tg
candidates = lookup_index.get((lookup_class, table_name, lookup_geometry), [])
if not candidates:
no_lookup += 1
empty_candidates += 1
no_lookup_by_key[(object_class, geometry_type)] += 1
no_candidate_by_key[(object_class, geometry_type)] += 1
continue
hit = _resolve(candidates, lookup_attrs)
if hit is None:
no_lookup += 1
no_lookup_by_key[(object_class, geometry_type)] += 1
condition_miss_by_key[(object_class, geometry_type)] += 1
continue
matched += 1
matched_by_geometry[geometry_type] += 1
matched_by_key[(object_class, geometry_type)] += 1
rcids[str(hit.get("rcid") or "")] += 1
rules = hit.get("rules") or []
for rule in rules:
opcode = str(rule.get("opcode") or "")
if opcode:
opcode_counts[opcode] += 1
args_list = rule.get("args") or []
if opcode == "SY" and args_list:
direct_symbol_counts[str(args_list[0])[:8]] += 1
if opcode == "CS" and args_list:
name = str(args_list[0])[:8]
csp_counts[name] += 1
if name in IMPLEMENTED_CSP:
implemented_csp_counts[name] += 1
else:
unsupported_csp_counts[name] += 1
conn.close()
top_no_lookup = [
{"object_class": k[0], "geometry_type": k[1], "count": v}
for k, v in no_lookup_by_key.most_common()
]
top_matched = [
{"object_class": k[0], "geometry_type": k[1], "count": v}
for k, v in matched_by_key.most_common()
]
report = {
"schema": "navsea.cm95.s52-full-database-audit/v1",
"dataset_id": args.dataset_id,
"point_table": args.point_table,
"boundary_table": args.boundary_table,
"feature_count": total,
"matched_lookup_count": matched,
"no_lookup_count": no_lookup,
"no_candidate_count": empty_candidates,
"condition_no_match_count": no_lookup - empty_candidates,
"match_rate": matched / total if total else None,
"geometry_counts": dict(by_geometry),
"matched_geometry_counts": dict(matched_by_geometry),
"lookup_rcid_top100": rcids.most_common(100),
"opcode_counts": dict(opcode_counts),
"csp_counts": dict(csp_counts),
"implemented_csp_counts": dict(implemented_csp_counts),
"unsupported_csp_counts": dict(unsupported_csp_counts),
"direct_symbol_top100": direct_symbol_counts.most_common(100),
"class_substitution_counts": [
{"source_object_class": k[0], "s52_object_class": k[1], "geometry_type": k[2], "count": v}
for k, v in substitution_counts.most_common()
],
"no_lookup_by_object_geometry": top_no_lookup,
"no_candidate_by_object_geometry": [
{"object_class": k[0], "geometry_type": k[1], "count": v}
for k, v in no_candidate_by_key.most_common()
],
"condition_no_match_by_object_geometry": [
{"object_class": k[0], "geometry_type": k[1], "count": v}
for k, v in condition_miss_by_key.most_common()
],
"matched_by_object_geometry": top_matched,
"notes": {
"context_dependent_csp": "本报告只统计 Lookup 实际触发量,不执行需要邻接几何、sounding depth、TOPMAR 平台关系等上下文的 CSP。",
"implemented_csp": sorted(IMPLEMENTED_CSP),
},
}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
print(json.dumps({
"feature_count": total,
"matched_lookup_count": matched,
"no_lookup_count": no_lookup,
"no_candidate_count": empty_candidates,
"condition_no_match_count": no_lookup - empty_candidates,
"match_rate": report["match_rate"],
"csp_counts": report["csp_counts"],
"unsupported_csp_counts": report["unsupported_csp_counts"],
"class_substitution_counts": report["class_substitution_counts"],
"top_no_lookup": top_no_lookup[:30],
}, ensure_ascii=False, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())