427 lines
18 KiB
Python
427 lines
18 KiB
Python
#!/usr/bin/env python3
|
||
"""CM95 → NavSea Layer 分类基线一致性校验。
|
||
|
||
只读取已冻结/已完成人工确认的产物，不修改分类结果。
|
||
失败时退出码为 1；全部硬校验通过时退出码为 0。
|
||
语义建议与人工结果冲突仅记 warning，因为人工结果是最终权威。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import hashlib
|
||
import json
|
||
import re
|
||
import sys
|
||
from collections import Counter, defaultdict
|
||
from datetime import datetime, timezone
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
ROOT = Path(__file__).resolve().parent
|
||
OUTPUT = ROOT / "output"
|
||
|
||
DEFAULT_PATHS = {
|
||
"cm95_catalog": OUTPUT / "cm95-anonymous" / "anonymous-catalog.json",
|
||
"standard_catalog": OUTPUT / "navsea-standard" / "standard-catalog.json",
|
||
"manual_review": OUTPUT / "cm95-manual-review" / "manual-review-v1.json",
|
||
"manual_anchors": OUTPUT / "cm95-manual-review" / "reviewed-anchors-manual-v1.json",
|
||
"assignment": OUTPUT / "cm95-manual-review" / "assignment" / "cm95-layer-assignment-v1.json",
|
||
"reference_names": OUTPUT / "reference-name-enrichment-v1.json",
|
||
"semantic_rules": ROOT / "cm95_semantic_layer_rules_v1.json",
|
||
}
|
||
|
||
VALID_STATUS = {"MATCH", "PARTIAL", "AMBIGUOUS", "COMPONENT", "UNKNOWN"}
|
||
|
||
|
||
class Validation:
|
||
def __init__(self) -> None:
|
||
self.checks: list[dict[str, Any]] = []
|
||
self.warnings: list[dict[str, Any]] = []
|
||
|
||
def check(self, name: str, condition: bool, detail: Any = None) -> None:
|
||
row = {"name": name, "status": "PASS" if condition else "FAIL"}
|
||
if detail is not None:
|
||
row["detail"] = detail
|
||
self.checks.append(row)
|
||
|
||
def warn(self, name: str, detail: Any) -> None:
|
||
self.warnings.append({"name": name, "status": "WARN", "detail": detail})
|
||
|
||
@property
|
||
def ok(self) -> bool:
|
||
return all(x["status"] == "PASS" for x in self.checks)
|
||
|
||
|
||
def load(path: Path) -> dict[str, Any]:
|
||
return json.loads(path.read_text(encoding="utf-8"))
|
||
|
||
|
||
def anonymous_resource_id(resource_key: str, visual_hash: str | None) -> str:
|
||
seed = f"{resource_key}|{visual_hash or ''}".encode("utf-8")
|
||
return "CM95-VO-" + hashlib.sha256(seed).hexdigest()[:10].upper()
|
||
|
||
|
||
def build_resource_index(cm95: dict[str, Any]) -> tuple[dict[str, dict[str, Any]], dict[str, str]]:
|
||
by_key: dict[str, dict[str, Any]] = {}
|
||
for obj in cm95.get("visual_objects", []):
|
||
key = obj.get("resource_key")
|
||
if not key:
|
||
continue
|
||
row = by_key.setdefault(
|
||
key,
|
||
{
|
||
"resource_key": key,
|
||
"visual_hash": obj.get("visual_hash"),
|
||
"occurrences": 0,
|
||
},
|
||
)
|
||
row["occurrences"] += 1
|
||
if not row.get("visual_hash") and obj.get("visual_hash"):
|
||
row["visual_hash"] = obj.get("visual_hash")
|
||
by_id = {
|
||
anonymous_resource_id(key, row.get("visual_hash")): key
|
||
for key, row in by_key.items()
|
||
}
|
||
return by_key, by_id
|
||
|
||
|
||
def main() -> int:
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument(
|
||
"--report",
|
||
default=str(OUTPUT / "cm95-manual-review" / "validation-report-v1.json"),
|
||
help="校验报告输出路径",
|
||
)
|
||
args = ap.parse_args()
|
||
|
||
v = Validation()
|
||
|
||
missing = [str(p) for p in DEFAULT_PATHS.values() if not p.exists()]
|
||
v.check("required_files_exist", not missing, {"missing": missing})
|
||
if missing:
|
||
report = {
|
||
"schema": "cm95-layer-classification-validation/v1",
|
||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||
"ok": False,
|
||
"checks": v.checks,
|
||
"warnings": v.warnings,
|
||
}
|
||
Path(args.report).parent.mkdir(parents=True, exist_ok=True)
|
||
Path(args.report).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||
return 1
|
||
|
||
cm95 = load(DEFAULT_PATHS["cm95_catalog"])
|
||
std = load(DEFAULT_PATHS["standard_catalog"])
|
||
manual = load(DEFAULT_PATHS["manual_review"])
|
||
anchors = load(DEFAULT_PATHS["manual_anchors"])
|
||
assignment = load(DEFAULT_PATHS["assignment"])
|
||
names = load(DEFAULT_PATHS["reference_names"])
|
||
semantic_rules = load(DEFAULT_PATHS["semantic_rules"])
|
||
|
||
cm_resources, anon_to_key = build_resource_index(cm95)
|
||
resource_keys = set(cm_resources)
|
||
decisions = manual.get("decisions", {})
|
||
decision_ids = set(decisions)
|
||
|
||
std_layers = {
|
||
str(o["standard_layer_name"])
|
||
for o in std.get("visual_objects", [])
|
||
if o.get("standard_layer_name")
|
||
}
|
||
std_resource_layers: dict[str, set[str]] = defaultdict(set)
|
||
for o in std.get("visual_objects", []):
|
||
key = o.get("resource_key")
|
||
layer = o.get("standard_layer_name")
|
||
if key and layer:
|
||
std_resource_layers[str(key)].add(str(layer))
|
||
|
||
v.check("standard_catalog_frozen", std.get("catalog_status") == "FROZEN", std.get("catalog_status"))
|
||
v.check(
|
||
"cm95_anonymous_catalog_has_no_standard_layer",
|
||
not any("standard_layer_name" in o for o in cm95.get("visual_objects", [])),
|
||
)
|
||
v.check("cm95_unique_visual_resources_150", len(resource_keys) == 150, len(resource_keys))
|
||
v.check("manual_decisions_150", len(decisions) == 150, len(decisions))
|
||
v.check("manual_covers_all_resources", decision_ids == set(anon_to_key), {
|
||
"missing": sorted(set(anon_to_key) - decision_ids),
|
||
"extra": sorted(decision_ids - set(anon_to_key)),
|
||
})
|
||
v.check(
|
||
"style_hash_cm95_matches",
|
||
manual.get("cm95_style_sha256") == cm95.get("style", {}).get("sha256"),
|
||
{
|
||
"manual": manual.get("cm95_style_sha256"),
|
||
"catalog": cm95.get("style", {}).get("sha256"),
|
||
},
|
||
)
|
||
v.check(
|
||
"style_hash_standard_matches",
|
||
manual.get("standard_style_sha256") == std.get("style", {}).get("sha256"),
|
||
{
|
||
"manual": manual.get("standard_style_sha256"),
|
||
"catalog": std.get("style", {}).get("sha256"),
|
||
},
|
||
)
|
||
|
||
status_counts = Counter()
|
||
bad_decisions: list[dict[str, Any]] = []
|
||
for anon_id, d in decisions.items():
|
||
status = d.get("status")
|
||
status_counts[status] += 1
|
||
expected_key = anon_to_key.get(anon_id)
|
||
layers = [str(x) for x in d.get("standard_layers", [])]
|
||
std_key = d.get("standard_resource_key")
|
||
mode = d.get("assignment_mode")
|
||
component_kind = d.get("component_kind")
|
||
|
||
reasons: list[str] = []
|
||
if status not in VALID_STATUS:
|
||
reasons.append("invalid_status")
|
||
if d.get("resource_key_internal") != expected_key:
|
||
reasons.append("resource_key_mismatch")
|
||
if any(layer not in std_layers for layer in layers):
|
||
reasons.append("unknown_standard_layer")
|
||
if std_key and std_key not in std_resource_layers:
|
||
reasons.append("unknown_standard_resource")
|
||
|
||
if status in {"MATCH", "PARTIAL"}:
|
||
if len(layers) != 1:
|
||
reasons.append("match_partial_requires_one_layer")
|
||
if mode not in {"VISUAL_OBJECT_MATCH", "LAYER_ONLY"}:
|
||
reasons.append("invalid_match_partial_mode")
|
||
if std_key and layers and layers[0] not in std_resource_layers.get(std_key, set()):
|
||
reasons.append("standard_resource_layer_mismatch")
|
||
elif status == "AMBIGUOUS":
|
||
if len(layers) < 2:
|
||
reasons.append("ambiguous_requires_two_layers")
|
||
if component_kind:
|
||
reasons.append("ambiguous_has_component_kind")
|
||
elif status == "COMPONENT":
|
||
if layers or std_key is not None or mode != "COMPONENT_ONLY":
|
||
reasons.append("component_must_have_no_layer_or_standard_resource")
|
||
elif status == "UNKNOWN":
|
||
if layers or std_key is not None or mode != "UNKNOWN":
|
||
reasons.append("unknown_must_have_no_layer_or_standard_resource")
|
||
|
||
if reasons:
|
||
bad_decisions.append({"anonymous_id": anon_id, "reasons": reasons, "decision": d})
|
||
|
||
v.check("manual_decision_state_constraints", not bad_decisions, bad_decisions[:20])
|
||
v.check(
|
||
"manual_status_counts_total_150",
|
||
sum(status_counts.values()) == 150,
|
||
dict(status_counts),
|
||
)
|
||
|
||
# 人工导出 anchors：UNKNOWN 不导出；其余必须一一存在。
|
||
anchor_by_key = {
|
||
str(a["cm95_resource_key"]): a
|
||
for a in anchors.get("anchors", [])
|
||
if a.get("cm95_resource_key")
|
||
}
|
||
expected_anchor_keys = {
|
||
d["resource_key_internal"]
|
||
for d in decisions.values()
|
||
if d.get("status") != "UNKNOWN"
|
||
}
|
||
v.check(
|
||
"manual_anchor_export_matches_non_unknown_decisions",
|
||
set(anchor_by_key) == expected_anchor_keys,
|
||
{
|
||
"expected": len(expected_anchor_keys),
|
||
"actual": len(anchor_by_key),
|
||
"missing": sorted(expected_anchor_keys - set(anchor_by_key)),
|
||
"extra": sorted(set(anchor_by_key) - expected_anchor_keys),
|
||
},
|
||
)
|
||
|
||
anchor_mismatches: list[dict[str, Any]] = []
|
||
decision_by_key = {d["resource_key_internal"]: d for d in decisions.values()}
|
||
for key, a in anchor_by_key.items():
|
||
d = decision_by_key[key]
|
||
if a.get("status") != d.get("status"):
|
||
anchor_mismatches.append({"resource": key, "field": "status", "decision": d.get("status"), "anchor": a.get("status")})
|
||
if a.get("standard_resource_key") != d.get("standard_resource_key"):
|
||
anchor_mismatches.append({"resource": key, "field": "standard_resource_key"})
|
||
if a.get("assignment_mode") != d.get("assignment_mode"):
|
||
anchor_mismatches.append({"resource": key, "field": "assignment_mode"})
|
||
if d.get("status") in {"MATCH", "PARTIAL"}:
|
||
expected_layer = (d.get("standard_layers") or [None])[0]
|
||
if a.get("standard_layer") != expected_layer:
|
||
anchor_mismatches.append({"resource": key, "field": "standard_layer"})
|
||
elif d.get("status") == "AMBIGUOUS":
|
||
if sorted(a.get("standard_layers", [])) != sorted(d.get("standard_layers", [])):
|
||
anchor_mismatches.append({"resource": key, "field": "standard_layers"})
|
||
elif d.get("status") == "COMPONENT":
|
||
if a.get("component_kind") != d.get("component_kind"):
|
||
anchor_mismatches.append({"resource": key, "field": "component_kind"})
|
||
v.check("manual_anchor_fields_match_decisions", not anchor_mismatches, anchor_mismatches[:20])
|
||
|
||
# Reference names 必须覆盖全部当前 CM95 资源。
|
||
name_keys = set(names.get("cm95", {}))
|
||
v.check(
|
||
"reference_names_cover_all_cm95_resources",
|
||
name_keys == resource_keys,
|
||
{
|
||
"missing": sorted(resource_keys - name_keys),
|
||
"extra": sorted(name_keys - resource_keys),
|
||
},
|
||
)
|
||
|
||
# 声明式语义规则：正则可编译，Layer 必须存在于 Standard Catalog。
|
||
rule_errors: list[dict[str, Any]] = []
|
||
for rule in semantic_rules.get("rules", []):
|
||
try:
|
||
re.compile(str(rule.get("pattern") or ""), re.IGNORECASE)
|
||
except re.error as exc:
|
||
rule_errors.append({"id": rule.get("id"), "error": f"regex: {exc}"})
|
||
for item in rule.get("layer_candidates", []):
|
||
if not isinstance(item, list) or len(item) < 2:
|
||
rule_errors.append({"id": rule.get("id"), "error": "invalid layer_candidates shape"})
|
||
continue
|
||
if str(item[0]) not in std_layers:
|
||
rule_errors.append({"id": rule.get("id"), "error": f"unknown standard layer: {item[0]}"})
|
||
v.check("semantic_rules_are_valid", not rule_errors, rule_errors)
|
||
|
||
# 最终 Assignment 的状态/Layer 必须逐 occurrence 服从人工资源级决定。
|
||
assignments = assignment.get("assignments", [])
|
||
expected_total = len(cm95.get("visual_objects", [])) + len(cm95.get("unresolved_visual_objects", []))
|
||
v.check("assignment_total_count", len(assignments) == expected_total, {
|
||
"expected": expected_total,
|
||
"actual": len(assignments),
|
||
})
|
||
|
||
assignment_errors: list[dict[str, Any]] = []
|
||
assignment_status_counts = Counter()
|
||
for row in assignments:
|
||
status = row.get("assignment_status")
|
||
assignment_status_counts[status] += 1
|
||
key = row.get("cm95_resource_key")
|
||
if key:
|
||
d = decision_by_key.get(key)
|
||
if not d:
|
||
assignment_errors.append({"visual_object_id": row.get("visual_object_id"), "error": "resource_without_manual_decision"})
|
||
continue
|
||
expected_status = d.get("status")
|
||
if status != expected_status:
|
||
assignment_errors.append({"visual_object_id": row.get("visual_object_id"), "error": "status_mismatch", "expected": expected_status, "actual": status})
|
||
continue
|
||
if status in {"MATCH", "PARTIAL"}:
|
||
expected_layer = (d.get("standard_layers") or [None])[0]
|
||
if row.get("assigned_standard_layer") != expected_layer:
|
||
assignment_errors.append({"visual_object_id": row.get("visual_object_id"), "error": "layer_mismatch"})
|
||
elif status == "AMBIGUOUS":
|
||
if sorted(row.get("candidate_standard_layers", [])) != sorted(d.get("standard_layers", [])):
|
||
assignment_errors.append({"visual_object_id": row.get("visual_object_id"), "error": "ambiguous_layers_mismatch"})
|
||
elif status == "COMPONENT":
|
||
if row.get("assigned_standard_layer") is not None or row.get("component_kind") != d.get("component_kind"):
|
||
assignment_errors.append({"visual_object_id": row.get("visual_object_id"), "error": "component_assignment_mismatch"})
|
||
elif status == "UNKNOWN":
|
||
if row.get("assigned_standard_layer") is not None:
|
||
assignment_errors.append({"visual_object_id": row.get("visual_object_id"), "error": "unknown_has_layer"})
|
||
else:
|
||
# Style-only 未展开动态对象没有对应具体资源，必须保持 UNKNOWN。
|
||
if status != "UNKNOWN":
|
||
assignment_errors.append({"visual_object_id": row.get("visual_object_id"), "error": "unresolved_dynamic_not_unknown"})
|
||
|
||
v.check("final_assignment_matches_manual_decisions", not assignment_errors, assignment_errors[:20])
|
||
|
||
summary = assignment.get("summary", {})
|
||
recomputed = {
|
||
"MATCH": assignment_status_counts["MATCH"],
|
||
"PARTIAL": assignment_status_counts["PARTIAL"],
|
||
"AMBIGUOUS": assignment_status_counts["AMBIGUOUS"],
|
||
"COMPONENT": assignment_status_counts["COMPONENT"],
|
||
"UNKNOWN": assignment_status_counts["UNKNOWN"],
|
||
}
|
||
summary_expected = {
|
||
"MATCH": summary.get("match_visual_objects"),
|
||
"PARTIAL": summary.get("partial_visual_objects"),
|
||
"AMBIGUOUS": summary.get("ambiguous_visual_objects"),
|
||
"COMPONENT": summary.get("component_visual_objects"),
|
||
"UNKNOWN": summary.get("unknown_visual_objects"),
|
||
}
|
||
v.check("assignment_summary_counts_match_rows", recomputed == summary_expected, {
|
||
"recomputed": recomputed,
|
||
"summary": summary_expected,
|
||
})
|
||
v.check(
|
||
"assignment_policy_preserves_architecture",
|
||
assignment.get("policy", {}).get("cm95_source_layer_used_for_assignment") is False
|
||
and assignment.get("policy", {}).get("provider_field_semantics_used_for_assignment") is False
|
||
and assignment.get("policy", {}).get("anonymous_catalog_modified") is False,
|
||
assignment.get("policy"),
|
||
)
|
||
|
||
# 新语义分类器与历史人工结果的冲突只作 warning。
|
||
# 这里复现最小规则匹配，只比较高置信（>=0.72）首选 Layer。
|
||
high_conflicts: list[dict[str, Any]] = []
|
||
cm_name_map = names.get("cm95", {})
|
||
rules = semantic_rules.get("rules", [])
|
||
for d in decisions.values():
|
||
if d.get("status") not in {"MATCH", "PARTIAL"}:
|
||
continue
|
||
selected = d.get("standard_layers") or []
|
||
if len(selected) != 1:
|
||
continue
|
||
key = d["resource_key_internal"]
|
||
info = cm_name_map.get(key, {})
|
||
desc = str(info.get("official_description_en") or "")
|
||
rtype = str(info.get("resource_type") or "")
|
||
top: tuple[str, float] | None = None
|
||
for rule in rules:
|
||
if rule.get("resource_type") and rule.get("resource_type") != rtype:
|
||
continue
|
||
if not re.search(str(rule.get("pattern") or ""), desc, flags=re.IGNORECASE):
|
||
continue
|
||
candidates = rule.get("layer_candidates", [])
|
||
if candidates:
|
||
top = (str(candidates[0][0]), float(candidates[0][1]))
|
||
break
|
||
if top and top[1] >= 0.72 and top[0] != selected[0]:
|
||
high_conflicts.append({
|
||
"anonymous_id": d.get("anonymous_id"),
|
||
"resource_key": key,
|
||
"display_name_zh": info.get("display_name_zh"),
|
||
"manual_layer": selected[0],
|
||
"semantic_suggestion": top[0],
|
||
"semantic_score": top[1],
|
||
})
|
||
if high_conflicts:
|
||
v.warn("manual_vs_semantic_classifier_conflicts", high_conflicts)
|
||
|
||
report = {
|
||
"schema": "cm95-layer-classification-validation/v1",
|
||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||
"ok": v.ok,
|
||
"baseline": {
|
||
"cm95_style_sha256": cm95.get("style", {}).get("sha256"),
|
||
"standard_style_sha256": std.get("style", {}).get("sha256"),
|
||
"standard_catalog_status": std.get("catalog_status"),
|
||
"unique_cm95_resources": len(resource_keys),
|
||
"manual_status_counts": dict(sorted(status_counts.items())),
|
||
"final_assignment_status_counts": recomputed,
|
||
"final_assignment_total": len(assignments),
|
||
},
|
||
"checks": v.checks,
|
||
"warnings": v.warnings,
|
||
}
|
||
|
||
report_path = Path(args.report)
|
||
report_path.parent.mkdir(parents=True, exist_ok=True)
|
||
report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
|
||
print("CM95 → NavSea Layer 分类基线校验")
|
||
for row in v.checks:
|
||
print(f"[{row['status']}] {row['name']}")
|
||
for row in v.warnings:
|
||
print(f"[WARN] {row['name']}: {json.dumps(row['detail'], ensure_ascii=False)}")
|
||
print(f"report: {report_path}")
|
||
print("RESULT:", "PASS" if v.ok else "FAIL")
|
||
return 0 if v.ok else 1
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|