diff --git a/navsea_mapping_registry.py b/navsea_mapping_registry.py index 3847f2f..2b03c11 100644 --- a/navsea_mapping_registry.py +++ b/navsea_mapping_registry.py @@ -27,6 +27,16 @@ class FieldValueRule: return True +@dataclass(frozen=True) +class FieldNameRule: + field_name_jp: str + field_name_std: str + field_group_name: str + keep_in_delivery: bool + keep_in_engineering: bool + normalization_class: str + + @dataclass(frozen=True) class RenderRule: rule_id: str @@ -86,12 +96,14 @@ class NavSeaMappingRegistry: bundle_id: str, bundle_version: str, source_layer_rules: dict[str, dict[str, str]], + field_name_rules: dict[str, FieldNameRule], field_value_rules: dict[tuple[str, str], list[FieldValueRule]], render_rules: list[RenderRule], ) -> None: self.bundle_id = bundle_id self.bundle_version = bundle_version self.source_layer_rules = source_layer_rules + self.field_name_rules = field_name_rules self.field_value_rules = field_value_rules self.render_rules = render_rules @@ -145,6 +157,32 @@ class NavSeaMappingRegistry: for row in cur.fetchall() } + cur.execute( + """ + SELECT + field_name_jp, + field_name_std, + field_group_name, + keep_in_delivery, + keep_in_engineering, + normalization_class + FROM navsea_field_name_rules + WHERE bundle_id = %s AND bundle_version = %s + """, + (resolved_bundle_id, resolved_bundle_version), + ) + field_name_rules = { + str(row["field_name_jp"]): FieldNameRule( + field_name_jp=str(row["field_name_jp"]), + field_name_std=str(row["field_name_std"]), + field_group_name=str(row["field_group_name"]), + keep_in_delivery=bool(row["keep_in_delivery"]), + keep_in_engineering=bool(row["keep_in_engineering"]), + normalization_class=str(row["normalization_class"]), + ) + for row in cur.fetchall() + } + cur.execute( """ SELECT @@ -200,6 +238,7 @@ class NavSeaMappingRegistry: bundle_id=resolved_bundle_id, bundle_version=resolved_bundle_version, source_layer_rules=source_layer_rules, + field_name_rules=field_name_rules, field_value_rules=field_value_rules, render_rules=render_rules, ) @@ -210,6 +249,9 @@ class NavSeaMappingRegistry: return source_layer_jp, f"SLR:{source_layer_jp}:identity" return rule["source_layer_std"], rule["source_layer_rule_id"] + def get_field_name_rule(self, field_name: str) -> FieldNameRule | None: + return self.field_name_rules.get(field_name) + def standardize_field_value( self, field_name_jp: str, diff --git a/navsea_tile_builder.py b/navsea_tile_builder.py index 55bd01c..7a5bb2e 100644 --- a/navsea_tile_builder.py +++ b/navsea_tile_builder.py @@ -51,6 +51,8 @@ LIGHT_COLOR_REMARK_MAP = { "O": "orange", "A": "amber", } + + @dataclass(frozen=True) class DbConfig: host: str = "localhost" @@ -144,6 +146,10 @@ def text_or_none(value: object) -> str | None: return text or None +def contains_non_ascii(text: str) -> bool: + return any(ord(ch) > 127 for ch in text) + + def parse_number(value: object) -> float | None: text = text_or_none(value) if not text: @@ -208,6 +214,7 @@ class NavSeaTileBuilder: workers: int, all_tiles: bool, engineering_mode: bool, + strip_legacy_japanese_delivery: bool, fid_codec: NavSeaFidCodec | None, fid_key_id: str | None, bundle_id: str | None, @@ -223,6 +230,7 @@ class NavSeaTileBuilder: self.workers = workers self.all_tiles = all_tiles self.engineering_mode = engineering_mode + self.strip_legacy_japanese_delivery = strip_legacy_japanese_delivery self.fid_codec = fid_codec self.fid_key_id = fid_key_id self.bundle_id = bundle_id @@ -473,7 +481,17 @@ class NavSeaTileBuilder: properties.update(chart_properties) if self.engineering_mode: properties["trace_status"] = trace_status + properties, field_name_unresolved = self.standardize_output_properties( + properties=properties, + source_layer=str(row["source_layer"]), + canonical_object_type=text_or_none(row["canonical_object_type"]) or "", + canonical_family=text_or_none(row["canonical_family"]) or "", + geom_type=str(source_feature.get("geometry", {}).get("type", "")), + feature_id=int(row["feature_id"]), + tile=job, + ) unresolved.extend(feature_unresolved) + unresolved.extend(field_name_unresolved) output_feature = { "geometry": source_feature["geometry"], @@ -745,6 +763,63 @@ class NavSeaTileBuilder: return chart, unresolved, trace_status + def standardize_output_properties( + self, + *, + properties: dict[str, object], + source_layer: str, + canonical_object_type: str, + canonical_family: str, + geom_type: str, + feature_id: int, + tile: TileJob, + ) -> tuple[dict[str, object], list[dict]]: + normalized: dict[str, object] = {} + unresolved: list[dict] = [] + + for key, value in properties.items(): + if key == "fid": + # By the time we normalize output properties, fid already means + # the NavSea public ID, not the original source fid field. + normalized[key] = value + continue + rule = self.mapping_registry.get_field_name_rule(str(key)) + if rule is None: + normalized[key] = value + if ( + not self.engineering_mode + and self.strip_legacy_japanese_delivery + and contains_non_ascii(str(key)) + ): + unresolved.append( + self.make_unresolved_event( + issue_type="field_name_unmapped", + reason="delivery output still contains an unmapped non-ASCII field name", + source_layer=source_layer, + canonical_object_type=canonical_object_type, + canonical_family=canonical_family, + geom_type=geom_type, + feature_id=feature_id, + tile=tile, + fid=properties.get("fid"), + field_name=str(key), + ) + ) + continue + + if self.engineering_mode: + if rule.keep_in_engineering: + normalized[key] = value + normalized[rule.field_name_std] = value + continue + + if not self.strip_legacy_japanese_delivery: + normalized[key] = value + if rule.keep_in_delivery: + normalized[rule.field_name_std] = value + + return normalized, unresolved + @staticmethod def make_unresolved_event( *, @@ -1302,6 +1377,11 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--workers", type=int, default=4) parser.add_argument("--all-tiles", action="store_true") parser.add_argument("--engineering", action="store_true") + parser.add_argument( + "--strip-legacy-japanese-delivery", + action="store_true", + help="drop mapped Japanese structural keys from delivery tiles after writing standardized aliases", + ) parser.add_argument("--fid-key") parser.add_argument("--fid-key-id", default="navsea-fid-key-v1") parser.add_argument("--bundle-id", default="navsea-reversible-v1") @@ -1338,6 +1418,7 @@ def main() -> None: workers=max(1, args.workers), all_tiles=args.all_tiles, engineering_mode=args.engineering, + strip_legacy_japanese_delivery=args.strip_legacy_japanese_delivery, fid_codec=fid_codec, fid_key_id=args.fid_key_id, bundle_id=args.bundle_id,