Stage 1: standardize delivery field names

navsea_mapping_registry.py

@@ -27,6 +27,16 @@ class FieldValueRule:
         return True
 
 
+@dataclass(frozen=True)
+class FieldNameRule:
+    field_name_jp: str
+    field_name_std: str
+    field_group_name: str
+    keep_in_delivery: bool
+    keep_in_engineering: bool
+    normalization_class: str
+
+
 @dataclass(frozen=True)
 class RenderRule:
     rule_id: str
@@ -86,12 +96,14 @@ class NavSeaMappingRegistry:
         bundle_id: str,
         bundle_version: str,
         source_layer_rules: dict[str, dict[str, str]],
+        field_name_rules: dict[str, FieldNameRule],
         field_value_rules: dict[tuple[str, str], list[FieldValueRule]],
         render_rules: list[RenderRule],
     ) -> None:
         self.bundle_id = bundle_id
         self.bundle_version = bundle_version
         self.source_layer_rules = source_layer_rules
+        self.field_name_rules = field_name_rules
         self.field_value_rules = field_value_rules
         self.render_rules = render_rules
 
@@ -145,6 +157,32 @@ class NavSeaMappingRegistry:
                 for row in cur.fetchall()
             }
 
+            cur.execute(
+                """
+                SELECT
+                  field_name_jp,
+                  field_name_std,
+                  field_group_name,
+                  keep_in_delivery,
+                  keep_in_engineering,
+                  normalization_class
+                FROM navsea_field_name_rules
+                WHERE bundle_id = %s AND bundle_version = %s
+                """,
+                (resolved_bundle_id, resolved_bundle_version),
+            )
+            field_name_rules = {
+                str(row["field_name_jp"]): FieldNameRule(
+                    field_name_jp=str(row["field_name_jp"]),
+                    field_name_std=str(row["field_name_std"]),
+                    field_group_name=str(row["field_group_name"]),
+                    keep_in_delivery=bool(row["keep_in_delivery"]),
+                    keep_in_engineering=bool(row["keep_in_engineering"]),
+                    normalization_class=str(row["normalization_class"]),
+                )
+                for row in cur.fetchall()
+            }
+
             cur.execute(
                 """
                 SELECT
@@ -200,6 +238,7 @@ class NavSeaMappingRegistry:
             bundle_id=resolved_bundle_id,
             bundle_version=resolved_bundle_version,
             source_layer_rules=source_layer_rules,
+            field_name_rules=field_name_rules,
             field_value_rules=field_value_rules,
             render_rules=render_rules,
         )
@@ -210,6 +249,9 @@ class NavSeaMappingRegistry:
             return source_layer_jp, f"SLR:{source_layer_jp}:identity"
         return rule["source_layer_std"], rule["source_layer_rule_id"]
 
+    def get_field_name_rule(self, field_name: str) -> FieldNameRule | None:
+        return self.field_name_rules.get(field_name)
+
     def standardize_field_value(
         self,
         field_name_jp: str,

navsea_tile_builder.py

@@ -51,6 +51,8 @@ LIGHT_COLOR_REMARK_MAP = {
     "O": "orange",
     "A": "amber",
 }
+
+
 @dataclass(frozen=True)
 class DbConfig:
     host: str = "localhost"
@@ -144,6 +146,10 @@ def text_or_none(value: object) -> str | None:
     return text or None
 
 
+def contains_non_ascii(text: str) -> bool:
+    return any(ord(ch) > 127 for ch in text)
+
+
 def parse_number(value: object) -> float | None:
     text = text_or_none(value)
     if not text:
@@ -208,6 +214,7 @@ class NavSeaTileBuilder:
         workers: int,
         all_tiles: bool,
         engineering_mode: bool,
+        strip_legacy_japanese_delivery: bool,
         fid_codec: NavSeaFidCodec | None,
         fid_key_id: str | None,
         bundle_id: str | None,
@@ -223,6 +230,7 @@ class NavSeaTileBuilder:
         self.workers = workers
         self.all_tiles = all_tiles
         self.engineering_mode = engineering_mode
+        self.strip_legacy_japanese_delivery = strip_legacy_japanese_delivery
         self.fid_codec = fid_codec
         self.fid_key_id = fid_key_id
         self.bundle_id = bundle_id
@@ -473,7 +481,17 @@ class NavSeaTileBuilder:
                 properties.update(chart_properties)
                 if self.engineering_mode:
                     properties["trace_status"] = trace_status
+                properties, field_name_unresolved = self.standardize_output_properties(
+                    properties=properties,
+                    source_layer=str(row["source_layer"]),
+                    canonical_object_type=text_or_none(row["canonical_object_type"]) or "",
+                    canonical_family=text_or_none(row["canonical_family"]) or "",
+                    geom_type=str(source_feature.get("geometry", {}).get("type", "")),
+                    feature_id=int(row["feature_id"]),
+                    tile=job,
+                )
                 unresolved.extend(feature_unresolved)
+                unresolved.extend(field_name_unresolved)
 
                 output_feature = {
                     "geometry": source_feature["geometry"],
@@ -745,6 +763,63 @@ class NavSeaTileBuilder:
 
         return chart, unresolved, trace_status
 
+    def standardize_output_properties(
+        self,
+        *,
+        properties: dict[str, object],
+        source_layer: str,
+        canonical_object_type: str,
+        canonical_family: str,
+        geom_type: str,
+        feature_id: int,
+        tile: TileJob,
+    ) -> tuple[dict[str, object], list[dict]]:
+        normalized: dict[str, object] = {}
+        unresolved: list[dict] = []
+
+        for key, value in properties.items():
+            if key == "fid":
+                # By the time we normalize output properties, fid already means
+                # the NavSea public ID, not the original source fid field.
+                normalized[key] = value
+                continue
+            rule = self.mapping_registry.get_field_name_rule(str(key))
+            if rule is None:
+                normalized[key] = value
+                if (
+                    not self.engineering_mode
+                    and self.strip_legacy_japanese_delivery
+                    and contains_non_ascii(str(key))
+                ):
+                    unresolved.append(
+                        self.make_unresolved_event(
+                            issue_type="field_name_unmapped",
+                            reason="delivery output still contains an unmapped non-ASCII field name",
+                            source_layer=source_layer,
+                            canonical_object_type=canonical_object_type,
+                            canonical_family=canonical_family,
+                            geom_type=geom_type,
+                            feature_id=feature_id,
+                            tile=tile,
+                            fid=properties.get("fid"),
+                            field_name=str(key),
+                        )
+                    )
+                continue
+
+            if self.engineering_mode:
+                if rule.keep_in_engineering:
+                    normalized[key] = value
+                normalized[rule.field_name_std] = value
+                continue
+
+            if not self.strip_legacy_japanese_delivery:
+                normalized[key] = value
+            if rule.keep_in_delivery:
+                normalized[rule.field_name_std] = value
+
+        return normalized, unresolved
+
     @staticmethod
     def make_unresolved_event(
         *,
@@ -1302,6 +1377,11 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--workers", type=int, default=4)
     parser.add_argument("--all-tiles", action="store_true")
     parser.add_argument("--engineering", action="store_true")
+    parser.add_argument(
+        "--strip-legacy-japanese-delivery",
+        action="store_true",
+        help="drop mapped Japanese structural keys from delivery tiles after writing standardized aliases",
+    )
     parser.add_argument("--fid-key")
     parser.add_argument("--fid-key-id", default="navsea-fid-key-v1")
     parser.add_argument("--bundle-id", default="navsea-reversible-v1")
@@ -1338,6 +1418,7 @@ def main() -> None:
         workers=max(1, args.workers),
         all_tiles=args.all_tiles,
         engineering_mode=args.engineering,
+        strip_legacy_japanese_delivery=args.strip_legacy_japanese_delivery,
         fid_codec=fid_codec,
         fid_key_id=args.fid_key_id,
         bundle_id=args.bundle_id,