tei/pbf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Stage 1: standardize delivery field names

Browse Source
This commit is contained in:
OpenAI Codex
2026-03-24 08:36:16 +08:00
parent 5f3f7409b3
commit 047833ac6c
2 changed files with 123 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
navsea_mapping_registry.py
View File

@@ -27,6 +27,16 @@ class FieldValueRule:
return True
@dataclass(frozen=True)
class FieldNameRule:
field_name_jp: str
field_name_std: str
field_group_name: str
keep_in_delivery: bool
keep_in_engineering: bool
normalization_class: str
@dataclass(frozen=True)
class RenderRule:
rule_id: str
@@ -86,12 +96,14 @@ class NavSeaMappingRegistry:
bundle_id: str,
bundle_version: str,
source_layer_rules: dict[str, dict[str, str]],
field_name_rules: dict[str, FieldNameRule],
field_value_rules: dict[tuple[str, str], list[FieldValueRule]],
render_rules: list[RenderRule],
) -> None:
self.bundle_id = bundle_id
self.bundle_version = bundle_version
self.source_layer_rules = source_layer_rules
self.field_name_rules = field_name_rules
self.field_value_rules = field_value_rules
self.render_rules = render_rules
@@ -145,6 +157,32 @@ class NavSeaMappingRegistry:
for row in cur.fetchall()
}
cur.execute(
"""
SELECT
field_name_jp,
field_name_std,
field_group_name,
keep_in_delivery,
keep_in_engineering,
normalization_class
FROM navsea_field_name_rules
WHERE bundle_id = %s AND bundle_version = %s
""",
(resolved_bundle_id, resolved_bundle_version),
)
field_name_rules = {
str(row["field_name_jp"]): FieldNameRule(
field_name_jp=str(row["field_name_jp"]),
field_name_std=str(row["field_name_std"]),
field_group_name=str(row["field_group_name"]),
keep_in_delivery=bool(row["keep_in_delivery"]),
keep_in_engineering=bool(row["keep_in_engineering"]),
normalization_class=str(row["normalization_class"]),
)
for row in cur.fetchall()
}
cur.execute(
"""
SELECT
@@ -200,6 +238,7 @@ class NavSeaMappingRegistry:
bundle_id=resolved_bundle_id,
bundle_version=resolved_bundle_version,
source_layer_rules=source_layer_rules,
field_name_rules=field_name_rules,
field_value_rules=field_value_rules,
render_rules=render_rules,
)
@@ -210,6 +249,9 @@ class NavSeaMappingRegistry:
return source_layer_jp, f"SLR:{source_layer_jp}:identity"
return rule["source_layer_std"], rule["source_layer_rule_id"]
def get_field_name_rule(self, field_name: str) -> FieldNameRule | None:
return self.field_name_rules.get(field_name)
def standardize_field_value(
self,
field_name_jp: str,

81
navsea_tile_builder.py
View File

@@ -51,6 +51,8 @@ LIGHT_COLOR_REMARK_MAP = {
"O": "orange",
"A": "amber",
}
@dataclass(frozen=True)
class DbConfig:
host: str = "localhost"
@@ -144,6 +146,10 @@ def text_or_none(value: object) -> str | None:
return text or None
def contains_non_ascii(text: str) -> bool:
return any(ord(ch) > 127 for ch in text)
def parse_number(value: object) -> float | None:
text = text_or_none(value)
if not text:
@@ -208,6 +214,7 @@ class NavSeaTileBuilder:
workers: int,
all_tiles: bool,
engineering_mode: bool,
strip_legacy_japanese_delivery: bool,
fid_codec: NavSeaFidCodec | None,
fid_key_id: str | None,
bundle_id: str | None,
@@ -223,6 +230,7 @@ class NavSeaTileBuilder:
self.workers = workers
self.all_tiles = all_tiles
self.engineering_mode = engineering_mode
self.strip_legacy_japanese_delivery = strip_legacy_japanese_delivery
self.fid_codec = fid_codec
self.fid_key_id = fid_key_id
self.bundle_id = bundle_id
@@ -473,7 +481,17 @@ class NavSeaTileBuilder:
properties.update(chart_properties)
if self.engineering_mode:
properties["trace_status"] = trace_status
properties, field_name_unresolved = self.standardize_output_properties(
properties=properties,
source_layer=str(row["source_layer"]),
canonical_object_type=text_or_none(row["canonical_object_type"]) or "",
canonical_family=text_or_none(row["canonical_family"]) or "",
geom_type=str(source_feature.get("geometry", {}).get("type", "")),
feature_id=int(row["feature_id"]),
tile=job,
)
unresolved.extend(feature_unresolved)
unresolved.extend(field_name_unresolved)
output_feature = {
"geometry": source_feature["geometry"],
@@ -745,6 +763,63 @@ class NavSeaTileBuilder:
return chart, unresolved, trace_status
def standardize_output_properties(
self,
*,
properties: dict[str, object],
source_layer: str,
canonical_object_type: str,
canonical_family: str,
geom_type: str,
feature_id: int,
tile: TileJob,
) -> tuple[dict[str, object], list[dict]]:
normalized: dict[str, object] = {}
unresolved: list[dict] = []
for key, value in properties.items():
if key == "fid":
# By the time we normalize output properties, fid already means
# the NavSea public ID, not the original source fid field.
normalized[key] = value
continue
rule = self.mapping_registry.get_field_name_rule(str(key))
if rule is None:
normalized[key] = value
if (
not self.engineering_mode
and self.strip_legacy_japanese_delivery
and contains_non_ascii(str(key))
):
unresolved.append(
self.make_unresolved_event(
issue_type="field_name_unmapped",
reason="delivery output still contains an unmapped non-ASCII field name",
source_layer=source_layer,
canonical_object_type=canonical_object_type,
canonical_family=canonical_family,
geom_type=geom_type,
feature_id=feature_id,
tile=tile,
fid=properties.get("fid"),
field_name=str(key),
)
)
continue
if self.engineering_mode:
if rule.keep_in_engineering:
normalized[key] = value
normalized[rule.field_name_std] = value
continue
if not self.strip_legacy_japanese_delivery:
normalized[key] = value
if rule.keep_in_delivery:
normalized[rule.field_name_std] = value
return normalized, unresolved
@staticmethod
def make_unresolved_event(
*,
@@ -1302,6 +1377,11 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--workers", type=int, default=4)
parser.add_argument("--all-tiles", action="store_true")
parser.add_argument("--engineering", action="store_true")
parser.add_argument(
"--strip-legacy-japanese-delivery",
action="store_true",
help="drop mapped Japanese structural keys from delivery tiles after writing standardized aliases",
)
parser.add_argument("--fid-key")
parser.add_argument("--fid-key-id", default="navsea-fid-key-v1")
parser.add_argument("--bundle-id", default="navsea-reversible-v1")
@@ -1338,6 +1418,7 @@ def main() -> None:
workers=max(1, args.workers),
all_tiles=args.all_tiles,
engineering_mode=args.engineering,
strip_legacy_japanese_delivery=args.strip_legacy_japanese_delivery,
fid_codec=fid_codec,
fid_key_id=args.fid_key_id,
bundle_id=args.bundle_id,