292 lines
17 KiB
Python
292 lines
17 KiB
Python
#!/usr/bin/env python3
|
||
"""统一 Provider Policy / Package v1 编译、校验与导出。"""
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import json
|
||
import re
|
||
import zipfile
|
||
from datetime import datetime, timezone
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
POLICY_SCHEMA = "navsea-provider-policy/v1"
|
||
PACKAGE_SCHEMA = "navsea-provider-package/v1"
|
||
ASSIGNMENT_STATUSES = {"VISUAL_OBJECT_MATCH", "LAYER_ONLY", "COMPONENT_ONLY", "AMBIGUOUS", "UNKNOWN"}
|
||
|
||
DELIVERY_FILES = (
|
||
"provider-package.json",
|
||
"provider-policy.json",
|
||
"mapping-layer-assignment.json",
|
||
"resources-style.json",
|
||
"resources-sprite.json",
|
||
"resources-sprite.png",
|
||
"validation.json",
|
||
"validation.md",
|
||
"MANIFEST.json",
|
||
)
|
||
|
||
|
||
def canonical(value: Any) -> bytes:
|
||
return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||
|
||
|
||
def stable_projection(value: Any) -> Any:
|
||
if isinstance(value, dict):
|
||
return {k: stable_projection(v) for k, v in value.items() if k not in {"snapshot", "createdAt", "generatedAt"}}
|
||
if isinstance(value, list):
|
||
return [stable_projection(v) for v in value]
|
||
return value
|
||
|
||
|
||
def fingerprint(value: Any) -> str:
|
||
return hashlib.sha256(canonical(stable_projection(value))).hexdigest()
|
||
|
||
|
||
def file_fingerprint(path: Path) -> str:
|
||
h = hashlib.sha256()
|
||
with path.open("rb") as stream:
|
||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||
h.update(chunk)
|
||
return h.hexdigest()
|
||
|
||
|
||
def validate_mapping(mapping: dict[str, Any], style: dict[str, Any] | None = None) -> list[str]:
|
||
errors: list[str] = []
|
||
version = mapping.get("schemaVersion")
|
||
if version not in {"navsea-standard-layer-mapping/v1", "navsea-standard-layer-mapping/v2"}:
|
||
return ["mapping:schemaVersion:unsupported"]
|
||
if version.endswith("/v1"):
|
||
return errors
|
||
source = mapping.get("renderEvidenceSource")
|
||
if not isinstance(source, dict) or source.get("schema") != "actual-visual-object-catalog/v1":
|
||
errors.append("mapping:renderEvidenceSource:invalid")
|
||
layers = style.get("layers", []) if style is not None else None
|
||
by_id = {layer.get("id"): (index, layer) for index, layer in enumerate(layers or [])}
|
||
seen_evidence: dict[str, bytes] = {}
|
||
for i, row in enumerate(mapping.get("assignments", [])):
|
||
evidence = row.get("renderEvidence")
|
||
if evidence is None:
|
||
continue
|
||
visual_id = row.get("visualObjectId")
|
||
encoded = canonical(evidence)
|
||
if visual_id in seen_evidence and seen_evidence[visual_id] != encoded:
|
||
errors.append(f"mapping[{i}]:renderEvidence:nondeterministic_duplicate")
|
||
seen_evidence[visual_id] = encoded
|
||
if not isinstance(evidence, dict) or not isinstance(evidence.get("sourceLayer"), str) or not isinstance(evidence.get("geometryRole"), str) or not isinstance(evidence.get("occurrenceCount"), int) or evidence["occurrenceCount"] < 0 or not isinstance(evidence.get("components"), list):
|
||
errors.append(f"mapping[{i}]:renderEvidence:invalid")
|
||
continue
|
||
if re.search(r"<script|javascript:|eval\s*\(|function\s*\(|=>", json.dumps(evidence, ensure_ascii=False), re.IGNORECASE):
|
||
errors.append(f"mapping[{i}]:renderEvidence:script_forbidden")
|
||
previous = -1
|
||
for j, component in enumerate(evidence["components"]):
|
||
if not isinstance(component, dict):
|
||
errors.append(f"mapping[{i}]:component[{j}]:invalid")
|
||
continue
|
||
index = component.get("styleIndex")
|
||
layer_id = component.get("styleLayerId")
|
||
if not isinstance(layer_id, str) or not isinstance(index, int) or isinstance(index, bool) or not isinstance(component.get("role"), str) or not component["role"] or (component.get("resourceKey") is not None and not isinstance(component["resourceKey"], str)) or (component.get("resourceVisualHash") is not None and not isinstance(component["resourceVisualHash"], str)):
|
||
errors.append(f"mapping[{i}]:component[{j}]:invalid")
|
||
continue
|
||
if index < previous:
|
||
errors.append(f"mapping[{i}]:component_order:invalid")
|
||
previous = index
|
||
if layers is not None:
|
||
entry = by_id.get(layer_id)
|
||
if entry is None:
|
||
errors.append(f"mapping[{i}]:component[{j}]:styleLayerId:missing")
|
||
elif entry[0] != index:
|
||
errors.append(f"mapping[{i}]:component[{j}]:styleIndex:mismatch")
|
||
elif entry[1].get("source-layer") != evidence["sourceLayer"]:
|
||
errors.append(f"mapping[{i}]:component[{j}]:sourceLayer:mismatch")
|
||
return errors
|
||
|
||
|
||
def validate_policy(policy: dict[str, Any], style: dict[str, Any] | None = None) -> dict[str, Any]:
|
||
errors: list[str] = []
|
||
required = ("schemaVersion", "provider", "dataset", "package", "standardCatalog", "resources", "mapping", "display", "pickup", "planning", "danger", "capabilities", "validationRequirements", "unresolved", "limitations", "provenance", "fingerprints")
|
||
errors += [f"missing:{key}" for key in required if key not in policy]
|
||
if policy.get("schemaVersion") != POLICY_SCHEMA:
|
||
errors.append("schemaVersion:unsupported")
|
||
for key in ("provider", "dataset", "package", "standardCatalog", "resources", "mapping", "display", "pickup", "planning", "danger", "capabilities", "validationRequirements", "provenance", "fingerprints"):
|
||
if key in policy and not isinstance(policy[key], dict):
|
||
errors.append(f"type:{key}:object_required")
|
||
for section, field in (("provider", "providerId"), ("dataset", "datasetId"), ("package", "packageId"), ("package", "version")):
|
||
value = policy.get(section)
|
||
if not isinstance(value, dict) or not isinstance(value.get(field), str) or not value[field].strip():
|
||
errors.append(f"identity:{section}.{field}:required")
|
||
mapping = policy.get("mapping", {}).get("assignments", []) if isinstance(policy.get("mapping"), dict) else []
|
||
if isinstance(policy.get("mapping"), dict):
|
||
errors += validate_mapping(policy["mapping"], style)
|
||
if not isinstance(mapping, list):
|
||
errors.append("type:mapping.assignments:array_required")
|
||
else:
|
||
for i, row in enumerate(mapping):
|
||
if not isinstance(row, dict) or row.get("status") not in ASSIGNMENT_STATUSES:
|
||
errors.append(f"mapping[{i}]:invalid_status")
|
||
continue
|
||
if row["status"] in {"VISUAL_OBJECT_MATCH", "LAYER_ONLY"} and not row.get("standardLayer"):
|
||
errors.append(f"mapping[{i}]:standardLayer_required")
|
||
if row["status"] == "VISUAL_OBJECT_MATCH" and not row.get("standardVisualObjectRef"):
|
||
errors.append(f"mapping[{i}]:standardVisualObjectRef_required")
|
||
if row["status"] in {"AMBIGUOUS", "UNKNOWN", "COMPONENT_ONLY"} and row.get("standardLayer") is not None:
|
||
errors.append(f"mapping[{i}]:unresolved_status_cannot_set_standardLayer")
|
||
if row["status"] == "AMBIGUOUS" and len(row.get("candidateStandardLayers", [])) < 2:
|
||
errors.append(f"mapping[{i}]:ambiguous_requires_candidates")
|
||
return {"schemaVersion": "navsea-provider-validation-report/v1", "result": "FAIL" if errors else "PASS", "errors": errors, "warnings": []}
|
||
|
||
|
||
def validate_package(package: dict[str, Any], style: dict[str, Any] | None = None) -> dict[str, Any]:
|
||
errors: list[str] = []
|
||
warnings: list[str] = []
|
||
required = ("schemaVersion", "provider", "dataset", "package", "resources", "standardLayerMapping", "providerPolicy", "display", "pickup", "planning", "danger", "capabilities", "validationRequirements", "fingerprints", "unresolved", "limitations", "provenance")
|
||
errors += [f"missing:{key}" for key in required if key not in package]
|
||
if package.get("schemaVersion") != PACKAGE_SCHEMA:
|
||
errors.append("schemaVersion:unsupported")
|
||
for section, field in (("provider", "providerId"), ("dataset", "datasetId"), ("package", "packageId")):
|
||
value = package.get(section)
|
||
if not isinstance(value, dict) or not isinstance(value.get(field), str) or not value[field].strip():
|
||
errors.append(f"identity:{section}.{field}:required")
|
||
package_identity = package.get("package", {})
|
||
if not package_identity.get("version"):
|
||
errors.append("identity:package.version:required")
|
||
policy_result = validate_policy(package.get("providerPolicy", {}), style)
|
||
if policy_result["result"] == "FAIL":
|
||
errors.append("providerPolicy:invalid")
|
||
requirements = package.get("validationRequirements", {})
|
||
if requirements.get("requireStyle") and not package.get("resources", {}).get("style"):
|
||
errors.append("resource:style:required")
|
||
if requirements.get("requirePbfSample") and not package.get("resources", {}).get("pbfSamples"):
|
||
errors.append("resource:pbfSamples:required")
|
||
if requirements.get("requireSprite") and not package.get("resources", {}).get("sprite", {}).get("uri"):
|
||
errors.append("resource:sprite:required")
|
||
if requirements.get("requireGlyphs") and not package.get("resources", {}).get("glyphs", {}).get("uriTemplate"):
|
||
errors.append("resource:glyphs:required")
|
||
for resource in package.get("resources", {}).get("pbfSamples", []):
|
||
if not resource.get("sha256"):
|
||
warnings.append("resource:pbfSample:hash_missing")
|
||
mapping = package.get("standardLayerMapping", {}).get("assignments", [])
|
||
if package.get("standardLayerMapping") != package.get("providerPolicy", {}).get("mapping"):
|
||
errors.append("mapping:package_policy:mismatch")
|
||
if package.get("standardLayerMapping", {}).get("schemaVersion") == "navsea-standard-layer-mapping/v2":
|
||
source = package["standardLayerMapping"].get("renderEvidenceSource", {})
|
||
if source.get("styleSha256") != package.get("resources", {}).get("style", {}).get("sha256"):
|
||
errors.append("mapping:styleSha256:mismatch")
|
||
if any(row.get("renderEvidence") is None for row in mapping):
|
||
warnings.append("mapping:renderEvidence_missing")
|
||
if not mapping:
|
||
warnings.append("mapping:empty")
|
||
if package.get("unresolved"):
|
||
warnings.append("mapping:unresolved_present")
|
||
if package.get("limitations"):
|
||
warnings.append("limitations:present")
|
||
return {"schemaVersion": "navsea-provider-validation-report/v1", "result": "FAIL" if errors else ("PASS_WITH_WARNINGS" if warnings else "PASS"), "errors": errors, "warnings": sorted(set(warnings))}
|
||
|
||
|
||
def build_policy(data: dict[str, Any]) -> dict[str, Any]:
|
||
policy = {
|
||
"schemaVersion": POLICY_SCHEMA,
|
||
"provider": data["provider"], "dataset": data["dataset"], "package": data["package"],
|
||
"standardCatalog": data["standardCatalog"], "resources": data["resources"],
|
||
"mapping": {"schemaVersion": data.get("mappingSchemaVersion", "navsea-standard-layer-mapping/v1"), "assignments": data["mapping"]},
|
||
"display": data.get("display", {}), "pickup": data.get("pickup", {}),
|
||
"planning": data.get("planning", {}), "danger": data.get("danger", {}),
|
||
"capabilities": data.get("capabilities", {}),
|
||
"validationRequirements": data.get("validationRequirements", {}),
|
||
"unresolved": data.get("unresolved", []), "limitations": data.get("limitations", []),
|
||
"provenance": data.get("provenance", {}), "fingerprints": data["fingerprints"],
|
||
}
|
||
if data.get("renderEvidenceSource"):
|
||
policy["mapping"]["renderEvidenceSource"] = data["renderEvidenceSource"]
|
||
policy["mapping"]["diagnostics"] = data.get("mappingDiagnostics", [])
|
||
policy["fingerprints"]["policy"] = fingerprint({k: v for k, v in policy.items() if k != "fingerprints"})
|
||
return policy
|
||
|
||
|
||
def build_package(data: dict[str, Any], policy: dict[str, Any], validation: dict[str, Any]) -> dict[str, Any]:
|
||
return {
|
||
"schemaVersion": PACKAGE_SCHEMA,
|
||
"provider": data["provider"], "dataset": data["dataset"], "package": data["package"],
|
||
"resources": data["resources"],
|
||
"standardLayerMapping": policy["mapping"], "providerPolicy": policy,
|
||
"display": policy["display"], "pickup": policy["pickup"], "planning": policy["planning"], "danger": policy["danger"],
|
||
"capabilities": policy["capabilities"], "validationRequirements": policy["validationRequirements"],
|
||
"fingerprints": {**data["fingerprints"], "policy": policy["fingerprints"]["policy"]},
|
||
"unresolved": policy["unresolved"], "limitations": policy["limitations"], "provenance": policy["provenance"],
|
||
"validation": validation,
|
||
}
|
||
|
||
|
||
def export_package(output: Path, package: dict[str, Any], policy: dict[str, Any], validation: dict[str, Any]) -> Path:
|
||
if validation.get("result") not in {"PASS", "PASS_WITH_WARNINGS"}:
|
||
raise ValueError("只有 PASS 或 PASS_WITH_WARNINGS 可以导出 Provider Package")
|
||
output.mkdir(parents=True, exist_ok=True)
|
||
(output / "provider-package.json").write_text(json.dumps(package, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
(output / "provider-policy.json").write_text(json.dumps(policy, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
(output / "validation.json").write_text(json.dumps(validation, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
md = ["# Provider Package 验证报告", "", f"- 结果：**{validation['result']}**", f"- Provider：`{package['provider']['providerId']}`", f"- 数据集：`{package['dataset']['datasetId']}`", "", "## 错误", ""]
|
||
md += [f"- {x}" for x in validation.get("errors", [])] or ["- 无"]
|
||
md += ["", "## 警告", ""]
|
||
md += [f"- {x}" for x in validation.get("warnings", [])] or ["- 无"]
|
||
(output / "validation.md").write_text("\n".join(md) + "\n", encoding="utf-8")
|
||
manifest = {
|
||
"manifestSchemaVersion": "navsea-provider-package-manifest/v1",
|
||
"provider": package["provider"], "dataset": package["dataset"], "package": package["package"],
|
||
"createdAt": None if policy["mapping"]["schemaVersion"].endswith("/v2") else datetime.now(timezone.utc).isoformat(), "fingerprints": package["fingerprints"],
|
||
"validationResult": validation["result"], "knownLimitations": package["limitations"],
|
||
"files": {name: file_fingerprint(output / name) for name in ("provider-package.json", "provider-policy.json", "validation.json", "validation.md")},
|
||
}
|
||
(output / "MANIFEST.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
return output
|
||
|
||
|
||
def _safe_delivery_component(value: str) -> str:
|
||
cleaned = re.sub(r"[^A-Za-z0-9._-]+", "-", str(value)).strip("-._")
|
||
return cleaned or "provider"
|
||
|
||
|
||
def create_delivery_zip(export_dir: Path, provider_id: str, dataset_id: str, version: str) -> dict[str, Any]:
|
||
"""把 PBF-07 的正式 NavSea 输入打成一个严格白名单 ZIP。"""
|
||
export_dir = Path(export_dir)
|
||
missing = [name for name in DELIVERY_FILES if not (export_dir / name).is_file()]
|
||
if missing:
|
||
raise ValueError(f"PBF-08 缺少正式交付文件，禁止生成 ZIP: {missing}")
|
||
|
||
validation = json.loads((export_dir / "validation.json").read_text(encoding="utf-8"))
|
||
if validation.get("result") not in {"PASS", "PASS_WITH_WARNINGS"}:
|
||
raise ValueError(f"PBF-08 只接受 PASS/PASS_WITH_WARNINGS，当前为 {validation.get('result')}")
|
||
|
||
stem = "-".join(_safe_delivery_component(x) for x in (provider_id, dataset_id, version))
|
||
zip_path = export_dir.parent / f"{stem}.navsea-provider.zip"
|
||
root_name = stem
|
||
if zip_path.exists():
|
||
zip_path.unlink()
|
||
|
||
with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=9) as archive:
|
||
for name in DELIVERY_FILES:
|
||
info = zipfile.ZipInfo(f"{root_name}/{name}", date_time=(2026, 1, 1, 0, 0, 0))
|
||
info.compress_type = zipfile.ZIP_DEFLATED
|
||
info.external_attr = 0o644 << 16
|
||
archive.writestr(info, (export_dir / name).read_bytes(), compress_type=zipfile.ZIP_DEFLATED, compresslevel=9)
|
||
|
||
with zipfile.ZipFile(zip_path, "r") as archive:
|
||
actual = archive.namelist()
|
||
expected = [f"{root_name}/{name}" for name in DELIVERY_FILES]
|
||
if actual != expected:
|
||
zip_path.unlink(missing_ok=True)
|
||
raise ValueError(f"PBF-08 ZIP 内容校验失败: expected={expected}, actual={actual}")
|
||
bad = archive.testzip()
|
||
if bad is not None:
|
||
zip_path.unlink(missing_ok=True)
|
||
raise ValueError(f"PBF-08 ZIP CRC 校验失败: {bad}")
|
||
|
||
return {
|
||
"path": str(zip_path),
|
||
"filename": zip_path.name,
|
||
"sha256": file_fingerprint(zip_path),
|
||
"bytes": zip_path.stat().st_size,
|
||
"root": root_name,
|
||
"files": list(DELIVERY_FILES),
|
||
"validationResult": validation["result"],
|
||
}
|