diff --git a/STEP_RECORD.md b/STEP_RECORD.md index b8834e7..8ae128d 100644 --- a/STEP_RECORD.md +++ b/STEP_RECORD.md @@ -2400,3 +2400,10 @@ - 已明确当前 Chart Profile Tools 尚无 ZIP Import 入口，第一轮应增加统一 ProviderPackageImporter / import-package 薄层，再进入现有 Profile Validator / ProviderCompiler。 - 文档包含两个当前 ZIP 的路径与 SHA-256、9 文件固定结构、安全校验、Contract 校验、Package→Profile 映射、NavSea/CM95 双 Provider 试跑顺序、第一轮验收清单及禁止事项。 - 第一轮明确不覆盖正式 Profile，不同时实施 UI Provider 切换或 Runtime 多 Provider 原子切换；先证明 ZIP → Profile → ProviderCompiler → PLAN_READY。 + +## 2026-10-07 NAVSEA-PBF-MULTISOURCE-004A Provider Mapping v2 + +- Provider Pipeline 为声明 `executionEvidenceCatalog` 的 manifest 生成 Mapping v2；CM95 1.1.0 从 Japan 全量 Actual Catalog 附加 Style Layer、z-order、角色和资源证据，不重新分类人工 assignment。 +- 正式 CM95 102 条 assignment 均有 renderEvidence，status/standardLayer 与旧 1.0.0 逐条一致；Style 原文件、intake 与 Package SHA-256 对齐。 +- 新 ZIP `dist/cm95/s52-full/cm95-s52-full-1.1.0.navsea-provider.zip` 固定 9 文件，重复运行 SHA-256 稳定为 `fb5bc2606ab61662c16eccd22225532779830779e931f623bac4cf049e6b737f`；旧 1.0.0 保留。 +- Validation 为 PASS_WITH_WARNINGS，警告为原有 unresolved 和 limitations；审计记录在 `docs/报告/NAVSEA-PBF-MULTISOURCE-004A_CM95_MappingV2审计_v1.json`。本任务止于 004A，不进入 004B。 diff --git a/dist/cm95/s52-full/cm95-s52-full-1.1.0.navsea-provider.zip b/dist/cm95/s52-full/cm95-s52-full-1.1.0.navsea-provider.zip new file mode 100644 index 0000000..f25bee0 Binary files /dev/null and b/dist/cm95/s52-full/cm95-s52-full-1.1.0.navsea-provider.zip differ diff --git a/docs/报告/NAVSEA-PBF-MULTISOURCE-004A_CM95_MappingV2审计_v1.json b/docs/报告/NAVSEA-PBF-MULTISOURCE-004A_CM95_MappingV2审计_v1.json new file mode 100644 index 0000000..0b557a6 --- /dev/null +++ b/docs/报告/NAVSEA-PBF-MULTISOURCE-004A_CM95_MappingV2审计_v1.json @@ -0,0 +1,71 @@ +{ + "任务": "NAVSEA-PBF-MULTISOURCE-004A", + "packageFilename": "cm95-s52-full-1.1.0.navsea-provider.zip", + "packageSha256": "fb5bc2606ab61662c16eccd22225532779830779e931f623bac4cf049e6b737f", + "mappingSchemaVersion": "navsea-standard-layer-mapping/v2", + "assignmentCount": 102, + "statusCounts": { + "AMBIGUOUS": 2, + "LAYER_ONLY": 47, + "UNKNOWN": 44, + "VISUAL_OBJECT_MATCH": 9 + }, + "assignmentsWithRenderEvidence": 102, + "assignmentsMissingRenderEvidence": 0, + "fullActualCatalogSummary": { + "tileCount": 177383, + "decoded_feature_count_in_styled_layers": 2699258, + "matched_feature_occurrence_count": 976534, + "distinct_visual_object_count": 475, + "group_count": 26, + "unsupported_component_value_count": 0, + "missing_sprite_component_count": 0 + }, + "styleSha256": "b1d3d0060012a8e4b55a6053231231d742ec415b3a8398f9ae0867bbdfa5290b", + "originalStyleSha256": "b1d3d0060012a8e4b55a6053231231d742ec415b3a8398f9ae0867bbdfa5290b", + "distinctReferencedStyleLayerCount": 72, + "componentRoleCounts": { + "fill": 15, + "icon": 72, + "line": 39, + "text": 19 + }, + "navigationMarksAssignmentCount": 20, + "navigationMarksReferencedStyleLayerIds": [ + "s52-p03-sy-facility-boundary-point", + "s52-p03-text1-facility-boundary-point", + "s52-p08-lights05-sy-navigation-marks", + "s52-p08-sy-navigation-marks", + "s52-p08-sy-s52-points", + "s52-p08-text1-navigation-marks", + "s52-p08-text1-s52-points" + ], + "knownConflictCount": 5, + "knownConflictResources": [ + "BRTHNO01", + "FSHHAV01", + "MORFAC03", + "PILPNT02", + "ap-FSHHAV02" + ], + "manualStatusAndLayerUnchanged": true, + "mappingPolicyPackageAgree": true, + "zipFileCount": 9, + "zipFiles": [ + "cm95-s52-full-1.1.0/provider-package.json", + "cm95-s52-full-1.1.0/provider-policy.json", + "cm95-s52-full-1.1.0/mapping-layer-assignment.json", + "cm95-s52-full-1.1.0/resources-style.json", + "cm95-s52-full-1.1.0/resources-sprite.json", + "cm95-s52-full-1.1.0/resources-sprite.png", + "cm95-s52-full-1.1.0/validation.json", + "cm95-s52-full-1.1.0/validation.md", + "cm95-s52-full-1.1.0/MANIFEST.json" + ], + "warnings": [ + "limitations:present", + "mapping:unresolved_present" + ], + "errors": [], + "finalResult": "PASS" +} diff --git a/providers/cm95/manifest.json b/providers/cm95/manifest.json index e248e3a..7d780ca 100644 --- a/providers/cm95/manifest.json +++ b/providers/cm95/manifest.json @@ -2,7 +2,7 @@ "schemaVersion": "pbf-provider-source-manifest/v1", "provider": {"providerId": "cm95", "name": "CM95", "version": "full-style"}, "dataset": {"datasetId": "s52-full", "name": "S-52 Full Style"}, - "package": {"packageId": "cm95-s52-full", "version": "1.0.0"}, + "package": {"packageId": "cm95-s52-full", "version": "1.1.0"}, "resources": { "style": {"uri": "http://192.168.200.184:18889/s52/full-style.json"}, "sprite": {"uri": "http://192.168.200.184:18889/s52/sprite"} @@ -10,7 +10,7 @@ "tileSource": {"template": "http://192.168.200.184:18889/pbf/{z}/{x}/{y}.pbf"}, "sampleTiles": ["12/3531/1641", "12/3526/1643", "12/3638/1613"], "probeGlyph": false, - "mapping": {"mode": "manual-assignment", "assignmentFile": "../../tools/style-visual-object/output/cm95-manual-review/assignment/cm95-layer-assignment-v1.json"}, + "mapping": {"mode": "manual-assignment", "assignmentFile": "../../tools/style-visual-object/output/cm95-manual-review/assignment/cm95-layer-assignment-v1.json", "executionEvidenceCatalog": "../../tools/visual-object-instantiation/output/cm95-current/actual-anonymous-catalog.json"}, "policy": { "display": {"enabled": true, "source": "provider-style"}, "pickup": {"enabled": false, "rules": []}, @@ -18,5 +18,5 @@ "danger": {"enabled": false, "rules": []}, "capabilities": {"vectorTiles": true, "sprite": true, "glyphs": false, "display": true, "pickup": false, "planning": false, "danger": false} }, - "limitations": ["人工 mapping 依 CM95 resource key 与 occurrence 视觉组件对照；本次实例化仅覆盖 manifest 样本瓦片。"] + "limitations": ["人工 mapping 依资源键与视觉组件的已确认 review；occurrenceIds 仅覆盖 manifest 样本瓦片，全量覆盖计数以 renderEvidenceSource Catalog 为准。"] } diff --git a/tests/test_provider_pipeline.py b/tests/test_provider_pipeline.py index 1d75b30..496888d 100644 --- a/tests/test_provider_pipeline.py +++ b/tests/test_provider_pipeline.py @@ -1,6 +1,8 @@ from __future__ import annotations import json +import importlib.util +import hashlib import sys import tempfile import unittest @@ -11,8 +13,84 @@ ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tools/provider-policy-compiler")) from provider_policy import DELIVERY_FILES, build_policy, create_delivery_zip, export_package, fingerprint, validate_package, validate_policy +spec = importlib.util.spec_from_file_location("pipeline", ROOT / "tools/provider-pipeline/run_provider_pipeline.py") +pipeline = importlib.util.module_from_spec(spec) +assert spec and spec.loader +spec.loader.exec_module(pipeline) + class ProviderPipelineContractTests(unittest.TestCase): + def test_v2_full_catalog_evidence_and_fail_closed(self): + style = {"layers": [{"id": "layer-a", "source-layer": "points"}, {"id": "layer-b", "source-layer": "points"}]} + components = [ + {"style_layer_id": "layer-a", "style_index": 0, "role": "icon", "resource_key": "ICON", "resource_visual_hash": "a" * 64}, + {"style_layer_id": "layer-b", "style_index": 1, "role": "text", "resource_key": None, "resource_visual_hash": None}, + ] + catalog = {"visual_objects": [{"visual_object_id": "VO-a", "provenance": {"source_layer": "points"}, "geometry_role": "Point", "occurrence_count": 123, "components": components}]} + rows = [{"visualObjectId": "VO-a", "status": "LAYER_ONLY", "standardLayer": "navigation_marks"}, {"visualObjectId": "VO-missing", "status": "UNKNOWN", "standardLayer": None}] + self.assertEqual(pipeline.attach_render_evidence(rows, catalog), ["VO-missing"]) + self.assertIsNone(rows[1]["renderEvidence"]) + self.assertEqual([x["role"] for x in rows[0]["renderEvidence"]["components"]], ["icon", "text"]) + self.assertIsNone(rows[0]["renderEvidence"]["components"][1]["resourceKey"]) + self.assertEqual(rows[0]["renderEvidence"]["occurrenceCount"], 123) + from provider_policy import validate_mapping + mapping = {"schemaVersion": "navsea-standard-layer-mapping/v2", "renderEvidenceSource": {"schema": "actual-visual-object-catalog/v1"}, "assignments": rows} + self.assertEqual(validate_mapping(mapping, style), []) + for key, value, expected in (("styleLayerId", "missing", "styleLayerId:missing"), ("styleIndex", 9, "styleIndex:mismatch")): + rows[0]["renderEvidence"]["components"][0][key] = value + self.assertTrue(any(expected in x for x in validate_mapping(mapping, style))) + rows[0]["renderEvidence"]["components"][0][key] = components[0][{"styleLayerId": "style_layer_id", "styleIndex": "style_index"}[key]] + rows[0]["renderEvidence"]["sourceLayer"] = "wrong" + self.assertTrue(any("sourceLayer:mismatch" in x for x in validate_mapping(mapping, style))) + + def test_execution_catalog_style_hash_and_schema(self): + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) + style = base / "style.json" + style.write_bytes(b"{}") + catalog = base / "catalog.json" + catalog.write_text(json.dumps({"schema": "actual-visual-object-catalog/v1", "inputs": {"style": str(style)}, "summary": {}, "visual_objects": []})) + manifest = base / "manifest.json" + expected = hashlib.sha256(b"{}").hexdigest() + self.assertEqual(pipeline.load_execution_catalog(manifest, "catalog.json", expected)[1]["styleSha256"], expected) + with self.assertRaisesRegex(ValueError, "SHA-256"): + pipeline.load_execution_catalog(manifest, "catalog.json", "0" * 64) + + def test_cm95_existing_semantics_and_specific_visual_objects(self): + base = ROOT / "dist/cm95/s52-full/1.0.0/mapping-layer-assignment.json" + catalog_file = ROOT / "tools/visual-object-instantiation/output/cm95-current/actual-anonymous-catalog.json" + if not base.exists() or not catalog_file.exists(): + self.skipTest("本机未提供 CM95 样本与全量 Catalog") + previous = json.loads(base.read_text())["assignments"] + rows = json.loads(json.dumps(previous)) + catalog = json.loads(catalog_file.read_text()) + self.assertEqual(len(rows), 102) + self.assertEqual(pipeline.attach_render_evidence(rows, catalog), []) + self.assertEqual([(x["visualObjectId"], x["status"], x["standardLayer"]) for x in rows], [(x["visualObjectId"], x["status"], x["standardLayer"]) for x in previous]) + boy = next(x for x in rows if x["visualObjectId"] == "VO-2329594761200ffd") + self.assertEqual(boy["standardLayer"], "navigation_marks") + self.assertEqual(boy["renderEvidence"]["sourceLayer"], "s52_points") + self.assertTrue(any(c["styleLayerId"] == "s52-p08-sy-s52-points" and c["role"] == "icon" and c["resourceKey"] == "BOYCON61" for c in boy["renderEvidence"]["components"])) + self.assertTrue(any(x["standardLayer"] == "navigation_marks" and len(x["renderEvidence"]["components"]) > 1 and any((c["resourceKey"] or "").startswith("LIGHTS") or c["role"] == "text" for c in x["renderEvidence"]["components"]) for x in rows)) + + def test_generated_mapping_policy_package_and_zip_agree(self): + base = ROOT / "dist/cm95/s52-full/1.1.0" + if not base.exists(): + self.skipTest("正式 CM95 1.1.0 Package 尚未生成") + mapping = json.loads((base / "mapping-layer-assignment.json").read_text()) + policy = json.loads((base / "provider-policy.json").read_text()) + package = json.loads((base / "provider-package.json").read_text()) + self.assertEqual(mapping["schemaVersion"], "navsea-standard-layer-mapping/v2") + self.assertEqual({k: v for k, v in mapping.items() if k != "source"}, policy["mapping"]) + self.assertEqual(policy["mapping"], package["standardLayerMapping"]) + z = base.parent / "cm95-s52-full-1.1.0.navsea-provider.zip" + with zipfile.ZipFile(z) as archive: + self.assertEqual(len(archive.namelist()), 9) + self.assertIsNone(archive.testzip()) + first = fingerprint(mapping) + second = fingerprint(json.loads((base / "mapping-layer-assignment.json").read_text())) + self.assertEqual(first, second) + def data(self, provider_id: str) -> dict: return { "provider": {"providerId": provider_id, "name": provider_id, "version": "1"}, diff --git a/tools/provider-pipeline/run_provider_pipeline.py b/tools/provider-pipeline/run_provider_pipeline.py index c9e6a04..e2ed77c 100644 --- a/tools/provider-pipeline/run_provider_pipeline.py +++ b/tools/provider-pipeline/run_provider_pipeline.py @@ -140,6 +140,52 @@ def manual_assignment(actual: dict[str, Any], assignment_doc: dict[str, Any]) -> return rows +def load_execution_catalog(manifest_path: Path, relative: str, style_hash: str) -> tuple[dict[str, Any], dict[str, Any]]: + catalog_path = (manifest_path.parent / relative).resolve() + catalog = json.loads(catalog_path.read_text(encoding="utf-8")) + if catalog.get("schema") != "actual-visual-object-catalog/v1" or not isinstance(catalog.get("visual_objects"), list): + raise ValueError("executionEvidenceCatalog schema 无效") + style_path = Path(catalog.get("inputs", {}).get("style", "")) + if not style_path.is_file(): + raise ValueError("executionEvidenceCatalog Style 输入不存在") + catalog_style_hash = sha(style_path.read_bytes()) + if catalog_style_hash != style_hash: + raise ValueError("executionEvidenceCatalog Style SHA-256 与 intake Style 不一致") + summary = catalog.get("summary", {}) + source = { + "schema": catalog["schema"], "catalog": relative, + "styleSha256": catalog_style_hash, + "tileCount": catalog.get("inputs", {}).get("tile_count"), + "matchedOccurrenceCount": summary.get("matched_feature_occurrence_count"), + "distinctVisualObjectCount": summary.get("distinct_visual_object_count"), + } + return catalog, source + + +def attach_render_evidence(assignments: list[dict[str, Any]], catalog: dict[str, Any]) -> list[str]: + objects = {obj["visual_object_id"]: obj for obj in catalog["visual_objects"]} + missing = [] + for row in assignments: + obj = objects.get(row["visualObjectId"]) + if obj is None: + row["renderEvidence"] = None + missing.append(row["visualObjectId"]) + continue + row["renderEvidence"] = { + "sourceLayer": obj.get("provenance", {}).get("source_layer"), + "geometryRole": obj.get("geometry_role"), + "occurrenceCount": obj.get("occurrence_count"), + "components": [{ + "styleLayerId": component.get("style_layer_id"), + "styleIndex": component.get("style_index"), + "role": component.get("role"), + "resourceKey": component.get("resource_key"), + "resourceVisualHash": component.get("resource_visual_hash"), + } for component in obj.get("components", [])], + } + return missing + + def main() -> int: ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("--manifest", required=True, help="Provider manifest JSON") @@ -250,12 +296,22 @@ def main() -> int: occurrence_lookup.setdefault(occ["visual_object_id"], []).append(occ["occurrence_id"]) for row in assignments: row["occurrenceIds"] = occurrence_lookup.get(row["visualObjectId"], []) - mapping_doc = {"schemaVersion": "navsea-standard-layer-mapping/v1", "source": mapping_source, "assignments": assignments} + evidence_path = manifest.get("mapping", {}).get("executionEvidenceCatalog") + evidence_source = None + evidence_missing: list[str] = [] + if evidence_path: + execution_catalog, evidence_source = load_execution_catalog(manifest_path, evidence_path, style_info["sha256"]) + evidence_missing = attach_render_evidence(assignments, execution_catalog) + mapping_doc = {"schemaVersion": "navsea-standard-layer-mapping/v2" if evidence_path else "navsea-standard-layer-mapping/v1", "source": mapping_source, "assignments": assignments} + if evidence_source: + mapping_doc["renderEvidenceSource"] = evidence_source + mapping_doc["diagnostics"] = [{"code": "renderEvidence:catalog_missing_visual_object", "visualObjectId": x} for x in evidence_missing] mapping_path = work / "pbf-04-mapping" / "provider-layer-assignment.json" mapping_path.parent.mkdir(parents=True, exist_ok=True) mapping_path.write_text(json.dumps(mapping_doc, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") unresolved = [{"visualObjectId": x["visualObjectId"], "status": x["status"], "candidateStandardLayers": x["candidateStandardLayers"]} for x in assignments if x["status"] in {"AMBIGUOUS", "UNKNOWN", "COMPONENT_ONLY"}] - record["stages"]["PBF-04"] = {"status": "PASS", "mappingMode": mapping_mode, "assignments": len(assignments), "unresolved": len(unresolved), "source": mapping_source} + unresolved += [{"visualObjectId": x, "status": "MISSING_RENDER_EVIDENCE"} for x in evidence_missing] + record["stages"]["PBF-04"] = {"status": "PASS_WITH_WARNINGS" if evidence_missing else "PASS", "mappingMode": mapping_mode, "assignments": len(assignments), "unresolved": len(unresolved), "missingRenderEvidence": evidence_missing, "source": mapping_source} fingerprints = { "source": source_manifest["sourceFingerprint"], "style": style_info["sha256"], @@ -273,24 +329,24 @@ def main() -> int: "sprite": {"uri": sprite_base, "jsonSha256": sprite_info.get("sha256"), "pngSha256": sprite_info.get("png", {}).get("sha256")}, "glyphs": glyph_info, }, - "mapping": assignments, + "mapping": assignments, "mappingSchemaVersion": mapping_doc["schemaVersion"], "renderEvidenceSource": evidence_source, "mappingDiagnostics": mapping_doc.get("diagnostics", []), "display": manifest.get("policy", {}).get("display", {}), "pickup": manifest.get("policy", {}).get("pickup", {}), "planning": manifest.get("policy", {}).get("planning", {}), "danger": manifest.get("policy", {}).get("danger", {}), "capabilities": manifest.get("policy", {}).get("capabilities", {}), "validationRequirements": {"requireStyle": True, "requirePbfSample": True, "requireSprite": True, "minimumMappingCount": 1}, "unresolved": unresolved, "limitations": manifest.get("limitations", []), - "provenance": {"manifest": str(manifest_path), "mappingSource": mapping_source, "pipelineVersion": "1.1.0", "generatedAt": datetime.now(timezone.utc).isoformat()}, + "provenance": {"manifest": str(manifest_path), "mappingSource": mapping_source, "pipelineVersion": "1.1.0", "generatedAt": None if evidence_path else datetime.now(timezone.utc).isoformat()}, "fingerprints": fingerprints, } record["stages"]["PBF-05"] = {"status": "PASS", "schemaVersion": "navsea-provider-policy/v1"} policy = build_policy(data) - policy_validation = validate_policy(policy) + policy_validation = validate_policy(policy, style) if policy_validation["result"] == "FAIL": raise RuntimeError(f"Provider Policy 校验 FAIL: {policy_validation['errors']}") (work / "pbf-05-policy.json").write_text(json.dumps(policy, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") record["stages"]["PBF-05"]["validation"] = policy_validation package_doc = build_package(data, policy, policy_validation) - validation = validate_package(package_doc) + validation = validate_package(package_doc, style) package_doc["validation"] = validation record["stages"]["PBF-06"] = {"status": validation["result"], "schemaVersion": "navsea-provider-package/v1", "validation": validation} record["validationResult"] = validation["result"] diff --git a/tools/provider-policy-compiler/provider_policy.py b/tools/provider-policy-compiler/provider_policy.py index 9e8d84d..55cd9f0 100644 --- a/tools/provider-policy-compiler/provider_policy.py +++ b/tools/provider-policy-compiler/provider_policy.py @@ -51,7 +51,52 @@ def file_fingerprint(path: Path) -> str: return h.hexdigest() -def validate_policy(policy: dict[str, Any]) -> dict[str, Any]: +def validate_mapping(mapping: dict[str, Any], style: dict[str, Any] | None = None) -> list[str]: + errors: list[str] = [] + version = mapping.get("schemaVersion") + if version not in {"navsea-standard-layer-mapping/v1", "navsea-standard-layer-mapping/v2"}: + return ["mapping:schemaVersion:unsupported"] + if version.endswith("/v1"): + return errors + source = mapping.get("renderEvidenceSource") + if not isinstance(source, dict) or source.get("schema") != "actual-visual-object-catalog/v1": + errors.append("mapping:renderEvidenceSource:invalid") + layers = style.get("layers", []) if style is not None else None + by_id = {layer.get("id"): (index, layer) for index, layer in enumerate(layers or [])} + for i, row in enumerate(mapping.get("assignments", [])): + evidence = row.get("renderEvidence") + if evidence is None: + continue + if not isinstance(evidence, dict) or not isinstance(evidence.get("sourceLayer"), str) or not isinstance(evidence.get("geometryRole"), str) or not isinstance(evidence.get("occurrenceCount"), int) or evidence["occurrenceCount"] < 0 or not isinstance(evidence.get("components"), list): + errors.append(f"mapping[{i}]:renderEvidence:invalid") + continue + previous = -1 + for j, component in enumerate(evidence["components"]): + if not isinstance(component, dict): + errors.append(f"mapping[{i}]:component[{j}]:invalid") + continue + index = component.get("styleIndex") + layer_id = component.get("styleLayerId") + if not isinstance(layer_id, str) or not isinstance(index, int) or isinstance(index, bool) or not isinstance(component.get("role"), str) or not component["role"] or (component.get("resourceKey") is not None and not isinstance(component["resourceKey"], str)) or (component.get("resourceVisualHash") is not None and not isinstance(component["resourceVisualHash"], str)): + errors.append(f"mapping[{i}]:component[{j}]:invalid") + continue + if any(token in value.lower() for value in (layer_id, component["role"], component.get("resourceKey") or "") for token in ("")): + errors.append(f"mapping[{i}]:component[{j}]:script_forbidden") + if index < previous: + errors.append(f"mapping[{i}]:component_order:invalid") + previous = index + if layers is not None: + entry = by_id.get(layer_id) + if entry is None: + errors.append(f"mapping[{i}]:component[{j}]:styleLayerId:missing") + elif entry[0] != index: + errors.append(f"mapping[{i}]:component[{j}]:styleIndex:mismatch") + elif entry[1].get("source-layer") != evidence["sourceLayer"]: + errors.append(f"mapping[{i}]:component[{j}]:sourceLayer:mismatch") + return errors + + +def validate_policy(policy: dict[str, Any], style: dict[str, Any] | None = None) -> dict[str, Any]: errors: list[str] = [] required = ("schemaVersion", "provider", "dataset", "package", "standardCatalog", "resources", "mapping", "display", "pickup", "planning", "danger", "capabilities", "validationRequirements", "unresolved", "limitations", "provenance", "fingerprints") errors += [f"missing:{key}" for key in required if key not in policy] @@ -65,6 +110,8 @@ def validate_policy(policy: dict[str, Any]) -> dict[str, Any]: if not isinstance(value, dict) or not isinstance(value.get(field), str) or not value[field].strip(): errors.append(f"identity:{section}.{field}:required") mapping = policy.get("mapping", {}).get("assignments", []) if isinstance(policy.get("mapping"), dict) else [] + if isinstance(policy.get("mapping"), dict): + errors += validate_mapping(policy["mapping"], style) if not isinstance(mapping, list): errors.append("type:mapping.assignments:array_required") else: @@ -83,7 +130,7 @@ def validate_policy(policy: dict[str, Any]) -> dict[str, Any]: return {"schemaVersion": "navsea-provider-validation-report/v1", "result": "FAIL" if errors else "PASS", "errors": errors, "warnings": []} -def validate_package(package: dict[str, Any]) -> dict[str, Any]: +def validate_package(package: dict[str, Any], style: dict[str, Any] | None = None) -> dict[str, Any]: errors: list[str] = [] warnings: list[str] = [] required = ("schemaVersion", "provider", "dataset", "package", "resources", "standardLayerMapping", "providerPolicy", "display", "pickup", "planning", "danger", "capabilities", "validationRequirements", "fingerprints", "unresolved", "limitations", "provenance") @@ -97,7 +144,7 @@ def validate_package(package: dict[str, Any]) -> dict[str, Any]: package_identity = package.get("package", {}) if not package_identity.get("version"): errors.append("identity:package.version:required") - policy_result = validate_policy(package.get("providerPolicy", {})) + policy_result = validate_policy(package.get("providerPolicy", {}), style) if policy_result["result"] == "FAIL": errors.append("providerPolicy:invalid") requirements = package.get("validationRequirements", {}) @@ -113,6 +160,14 @@ def validate_package(package: dict[str, Any]) -> dict[str, Any]: if not resource.get("sha256"): warnings.append("resource:pbfSample:hash_missing") mapping = package.get("standardLayerMapping", {}).get("assignments", []) + if package.get("standardLayerMapping") != package.get("providerPolicy", {}).get("mapping"): + errors.append("mapping:package_policy:mismatch") + if package.get("standardLayerMapping", {}).get("schemaVersion") == "navsea-standard-layer-mapping/v2": + source = package["standardLayerMapping"].get("renderEvidenceSource", {}) + if source.get("styleSha256") != package.get("resources", {}).get("style", {}).get("sha256"): + errors.append("mapping:styleSha256:mismatch") + if any(row.get("renderEvidence") is None for row in mapping): + warnings.append("mapping:renderEvidence_missing") if not mapping: warnings.append("mapping:empty") if package.get("unresolved"): @@ -127,7 +182,7 @@ def build_policy(data: dict[str, Any]) -> dict[str, Any]: "schemaVersion": POLICY_SCHEMA, "provider": data["provider"], "dataset": data["dataset"], "package": data["package"], "standardCatalog": data["standardCatalog"], "resources": data["resources"], - "mapping": {"schemaVersion": "navsea-standard-layer-mapping/v1", "assignments": data["mapping"]}, + "mapping": {"schemaVersion": data.get("mappingSchemaVersion", "navsea-standard-layer-mapping/v1"), "assignments": data["mapping"]}, "display": data.get("display", {}), "pickup": data.get("pickup", {}), "planning": data.get("planning", {}), "danger": data.get("danger", {}), "capabilities": data.get("capabilities", {}), @@ -135,6 +190,9 @@ def build_policy(data: dict[str, Any]) -> dict[str, Any]: "unresolved": data.get("unresolved", []), "limitations": data.get("limitations", []), "provenance": data.get("provenance", {}), "fingerprints": data["fingerprints"], } + if data.get("renderEvidenceSource"): + policy["mapping"]["renderEvidenceSource"] = data["renderEvidenceSource"] + policy["mapping"]["diagnostics"] = data.get("mappingDiagnostics", []) policy["fingerprints"]["policy"] = fingerprint({k: v for k, v in policy.items() if k != "fingerprints"}) return policy @@ -168,7 +226,7 @@ def export_package(output: Path, package: dict[str, Any], policy: dict[str, Any] manifest = { "manifestSchemaVersion": "navsea-provider-package-manifest/v1", "provider": package["provider"], "dataset": package["dataset"], "package": package["package"], - "createdAt": datetime.now(timezone.utc).isoformat(), "fingerprints": package["fingerprints"], + "createdAt": None if policy["mapping"]["schemaVersion"].endswith("/v2") else datetime.now(timezone.utc).isoformat(), "fingerprints": package["fingerprints"], "validationResult": validation["result"], "knownLimitations": package["limitations"], "files": {name: file_fingerprint(output / name) for name in ("provider-package.json", "provider-policy.json", "validation.json", "validation.md")}, } @@ -200,7 +258,10 @@ def create_delivery_zip(export_dir: Path, provider_id: str, dataset_id: str, ver with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=9) as archive: for name in DELIVERY_FILES: - archive.write(export_dir / name, arcname=f"{root_name}/{name}") + info = zipfile.ZipInfo(f"{root_name}/{name}", date_time=(2026, 1, 1, 0, 0, 0)) + info.compress_type = zipfile.ZIP_DEFLATED + info.external_attr = 0o644 << 16 + archive.writestr(info, (export_dir / name).read_bytes(), compress_type=zipfile.ZIP_DEFLATED, compresslevel=9) with zipfile.ZipFile(zip_path, "r") as archive: actual = archive.namelist() diff --git a/tools/provider-policy-compiler/schema/provider-policy-v1.schema.json b/tools/provider-policy-compiler/schema/provider-policy-v1.schema.json index 6fdcb65..8f8bde1 100644 --- a/tools/provider-policy-compiler/schema/provider-policy-v1.schema.json +++ b/tools/provider-policy-compiler/schema/provider-policy-v1.schema.json @@ -10,7 +10,7 @@ "dataset": {"type": "object", "required": ["datasetId"], "properties": {"datasetId": {"type": "string", "minLength": 1}}}, "package": {"type": "object", "required": ["packageId", "version"], "properties": {"packageId": {"type": "string"}, "version": {"type": "string"}}}, "standardCatalog": {"type": "object"}, "resources": {"type": "object"}, - "mapping": {"type": "object", "required": ["schemaVersion", "assignments"], "properties": {"schemaVersion": {"const": "navsea-standard-layer-mapping/v1"}, "assignments": {"type": "array"}}}, + "mapping": {"type": "object", "required": ["schemaVersion", "assignments"], "properties": {"schemaVersion": {"enum": ["navsea-standard-layer-mapping/v1", "navsea-standard-layer-mapping/v2"]}, "assignments": {"type": "array"}, "renderEvidenceSource": {"type": "object"}, "diagnostics": {"type": "array"}}}, "display": {"type": "object"}, "pickup": {"type": "object"}, "planning": {"type": "object"}, "danger": {"type": "object"}, "capabilities": {"type": "object"}, "validationRequirements": {"type": "object"}, "unresolved": {"type": "array"}, "limitations": {"type": "array"}, "provenance": {"type": "object"}, "fingerprints": {"type": "object"} },