749 lines
32 KiB
Python
749 lines
32 KiB
Python
#!/usr/bin/env python3
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import hashlib
|
||
import difflib
|
||
import re
|
||
import json
|
||
import mimetypes
|
||
import subprocess
|
||
import sys
|
||
import urllib.parse
|
||
from datetime import datetime, timezone
|
||
from collections import Counter
|
||
from http import HTTPStatus
|
||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
from PIL import Image
|
||
|
||
|
||
TOOL_DIR = Path(__file__).resolve().parents[1]
|
||
OUTPUT_DIR = TOOL_DIR / "output"
|
||
CM95_CATALOG_PATH = OUTPUT_DIR / "cm95-anonymous" / "anonymous-catalog.json"
|
||
STD_CATALOG_PATH = OUTPUT_DIR / "navsea-standard" / "standard-catalog.json"
|
||
SUGGESTION_PATH = OUTPUT_DIR / "cm95-visual-anchor-match" / "reviewed-anchors-v1.json"
|
||
VISUAL_CANDIDATE_PATH = OUTPUT_DIR / "cm95-visual-anchor-match" / "visual-anchor-matches.json"
|
||
REFERENCE_NAME_PATH = OUTPUT_DIR / "reference-name-enrichment-v1.json"
|
||
SEMANTIC_RULES_PATH = TOOL_DIR / "cm95_semantic_layer_rules_v1.json"
|
||
MANUAL_DIR = OUTPUT_DIR / "cm95-manual-review"
|
||
MANUAL_REVIEW_PATH = MANUAL_DIR / "manual-review-v1.json"
|
||
MANUAL_ANCHOR_PATH = MANUAL_DIR / "reviewed-anchors-manual-v1.json"
|
||
MANUAL_ASSIGNMENT_DIR = MANUAL_DIR / "assignment"
|
||
WEB_DIR = Path(__file__).resolve().parent / "web"
|
||
|
||
VALID_STATUS = {"MATCH", "PARTIAL", "AMBIGUOUS", "COMPONENT", "UNKNOWN"}
|
||
|
||
|
||
def _normalize_name(text: str | None) -> str:
|
||
if not text:
|
||
return ""
|
||
return re.sub(r"[^0-9a-zA-Z\u4e00-\u9fff]+", "", str(text).lower())
|
||
|
||
|
||
def _text_similarity(a: str | None, b: str | None) -> float:
|
||
na = _normalize_name(a)
|
||
nb = _normalize_name(b)
|
||
if not na or not nb:
|
||
return 0.0
|
||
seq = difflib.SequenceMatcher(None, na, nb).ratio()
|
||
sa, sb = set(na), set(nb)
|
||
jac = len(sa & sb) / max(1, len(sa | sb))
|
||
contain = 0.0
|
||
if na in nb or nb in na:
|
||
contain = min(len(na), len(nb)) / max(len(na), len(nb))
|
||
contain = 0.70 + 0.30 * contain
|
||
return max(seq, jac, contain)
|
||
|
||
COMPONENT_KIND_ZH = {
|
||
"topmark": "顶标",
|
||
"buoy_body": "浮标主体",
|
||
"beacon_body": "立标主体",
|
||
"light": "灯光",
|
||
"auxiliary_mark": "附加标记",
|
||
"fill_pattern": "面填充图案",
|
||
"line_pattern": "线型图案",
|
||
"text": "文字",
|
||
"other": "其他组件",
|
||
}
|
||
|
||
|
||
def read_json(path: Path) -> dict[str, Any]:
|
||
return json.loads(path.read_text(encoding="utf-8"))
|
||
|
||
|
||
def write_json_atomic(path: Path, value: Any) -> None:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
tmp = path.with_suffix(path.suffix + ".tmp")
|
||
tmp.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||
tmp.replace(path)
|
||
|
||
|
||
class ReviewState:
|
||
def __init__(self) -> None:
|
||
self.cm = read_json(CM95_CATALOG_PATH)
|
||
self.std = read_json(STD_CATALOG_PATH)
|
||
self.suggestions = read_json(SUGGESTION_PATH) if SUGGESTION_PATH.exists() else {"anchors": []}
|
||
self.visual_candidates = read_json(VISUAL_CANDIDATE_PATH) if VISUAL_CANDIDATE_PATH.exists() else {"top_candidates": []}
|
||
self.reference_names = read_json(REFERENCE_NAME_PATH) if REFERENCE_NAME_PATH.exists() else {"cm95": {}, "navsea_standard": {}, "standard_layer_zh": {}}
|
||
self.semantic_rules = read_json(SEMANTIC_RULES_PATH).get("rules", []) if SEMANTIC_RULES_PATH.exists() else []
|
||
self.cm_names = self.reference_names.get("cm95", {})
|
||
self.std_names = self.reference_names.get("navsea_standard", {})
|
||
self.layer_zh = self.reference_names.get("standard_layer_zh", {})
|
||
|
||
self.cm_atlas = Image.open(OUTPUT_DIR / "cm95-anonymous" / "_sprite" / "sprite.png").convert("RGBA")
|
||
self.std_atlas = Image.open(OUTPUT_DIR / "navsea-anonymous" / "_sprite" / "sprite.png").convert("RGBA")
|
||
|
||
self.cm_resources = self._build_cm_resources()
|
||
self.cm_by_id = {x["anonymous_id"]: x for x in self.cm_resources}
|
||
self.cm_by_key = {x["resource_key"]: x for x in self.cm_resources}
|
||
|
||
self.std_resources = self._build_std_resources()
|
||
self.std_by_key = {x["resource_key"]: x for x in self.std_resources}
|
||
self.standard_layers = sorted({
|
||
str(obj.get("standard_layer_name"))
|
||
for obj in self.std.get("visual_objects", [])
|
||
if obj.get("standard_layer_name")
|
||
})
|
||
|
||
self.suggestion_by_key = {
|
||
x["cm95_resource_key"]: x
|
||
for x in self.suggestions.get("anchors", [])
|
||
}
|
||
self.candidate_by_key = {
|
||
x["cm95_resource_key"]: x
|
||
for x in self.visual_candidates.get("top_candidates", [])
|
||
}
|
||
|
||
self.review = self._load_or_initialize_review()
|
||
|
||
def _anonymous_id(self, resource_key: str, visual_hash: str | None) -> str:
|
||
seed = f"{resource_key}|{visual_hash or ''}".encode("utf-8")
|
||
return "CM95-VO-" + hashlib.sha256(seed).hexdigest()[:10].upper()
|
||
|
||
def _build_cm_resources(self) -> list[dict[str, Any]]:
|
||
grouped: dict[str, dict[str, Any]] = {}
|
||
for obj in self.cm.get("visual_objects", []):
|
||
key = obj.get("resource_key")
|
||
meta = obj.get("resource_meta")
|
||
if not key or not meta:
|
||
continue
|
||
item = grouped.setdefault(key, {
|
||
"resource_key": key,
|
||
"resource_meta": meta,
|
||
"visual_hash": obj.get("visual_hash"),
|
||
"component_roles": set(),
|
||
"geometry_roles": set(),
|
||
"occurrence_count": 0,
|
||
"group_ids": set(),
|
||
"provenance": [],
|
||
})
|
||
if obj.get("component_role"):
|
||
item["component_roles"].add(obj["component_role"])
|
||
if obj.get("geometry_role"):
|
||
item["geometry_roles"].add(obj["geometry_role"])
|
||
item["occurrence_count"] += 1
|
||
item["group_ids"].add(obj["group_id"])
|
||
if obj.get("provenance"):
|
||
item["provenance"].append(obj["provenance"])
|
||
|
||
out = []
|
||
for key in sorted(grouped):
|
||
item = grouped[key]
|
||
item["anonymous_id"] = self._anonymous_id(key, item.get("visual_hash"))
|
||
item["component_roles"] = sorted(item["component_roles"])
|
||
item["geometry_roles"] = sorted(item["geometry_roles"])
|
||
item["group_ids"] = sorted(item["group_ids"])
|
||
out.append(item)
|
||
return out
|
||
|
||
def _build_std_resources(self) -> list[dict[str, Any]]:
|
||
grouped: dict[tuple[str, str], dict[str, Any]] = {}
|
||
for obj in self.std.get("visual_objects", []):
|
||
key = obj.get("resource_key")
|
||
layer = obj.get("standard_layer_name")
|
||
meta = obj.get("resource_meta")
|
||
if not key or not layer or not meta:
|
||
continue
|
||
gkey = (key, layer)
|
||
item = grouped.setdefault(gkey, {
|
||
"resource_key": key,
|
||
"standard_layer": layer,
|
||
"resource_meta": meta,
|
||
"visual_hash": obj.get("visual_hash"),
|
||
"component_role": obj.get("component_role"),
|
||
"geometry_role": obj.get("geometry_role"),
|
||
"visual_object_id": obj.get("visual_object_id"),
|
||
})
|
||
return sorted(grouped.values(), key=lambda x: (x["standard_layer"], x["resource_key"]))
|
||
|
||
def _semantic_info(self, resource_key: str) -> dict[str, Any]:
|
||
info = self.cm_names.get(resource_key, {}) or {}
|
||
desc = str(info.get("official_description_en") or "")
|
||
resource_type = str(info.get("resource_type") or "")
|
||
for rule in self.semantic_rules:
|
||
rule_type = rule.get("resource_type")
|
||
if rule_type and rule_type != resource_type:
|
||
continue
|
||
pattern = str(rule.get("pattern") or "")
|
||
if pattern and not re.search(pattern, desc, flags=re.IGNORECASE):
|
||
continue
|
||
return {
|
||
"rule_id": rule.get("id"),
|
||
"category_zh": rule.get("category_zh") or "未分类",
|
||
"reason_zh": rule.get("reason_zh") or "",
|
||
"component_kind": rule.get("component_kind"),
|
||
"layer_candidates": [
|
||
{"layer": str(x[0]), "semantic_score": float(x[1])}
|
||
for x in rule.get("layer_candidates", [])
|
||
if isinstance(x, list) and len(x) >= 2
|
||
],
|
||
}
|
||
return {
|
||
"rule_id": None,
|
||
"category_zh": "尚未建立语义分类",
|
||
"reason_zh": "当前 S-52 说明没有命中已声明的 Layer 语义规则，需人工确认。",
|
||
"component_kind": None,
|
||
"layer_candidates": [],
|
||
}
|
||
|
||
def _layer_suggestions(
|
||
self,
|
||
semantic: dict[str, Any],
|
||
candidates: list[dict[str, Any]],
|
||
suggestion: dict[str, Any] | None,
|
||
) -> list[dict[str, Any]]:
|
||
valid_layers = set(self.standard_layers)
|
||
semantic_map = {
|
||
x["layer"]: float(x.get("semantic_score") or 0.0)
|
||
for x in semantic.get("layer_candidates", [])
|
||
if x.get("layer") in valid_layers
|
||
}
|
||
visual_map: dict[str, tuple[float, str | None]] = {}
|
||
for c in candidates:
|
||
layer = c.get("standard_layer")
|
||
if layer not in valid_layers:
|
||
continue
|
||
score = float(c.get("score") or 0.0)
|
||
prev = visual_map.get(layer)
|
||
if prev is None or score > prev[0]:
|
||
visual_map[layer] = (score, c.get("resource_key"))
|
||
|
||
prior_layers: set[str] = set()
|
||
if suggestion:
|
||
if suggestion.get("standard_layer") in valid_layers:
|
||
prior_layers.add(str(suggestion.get("standard_layer")))
|
||
for layer in suggestion.get("standard_layers") or []:
|
||
if layer in valid_layers:
|
||
prior_layers.add(str(layer))
|
||
|
||
layers = set(semantic_map) | set(visual_map) | prior_layers
|
||
rows: list[dict[str, Any]] = []
|
||
has_semantic = bool(semantic_map)
|
||
for layer in layers:
|
||
sem = semantic_map.get(layer, 0.0)
|
||
vis, best_resource = visual_map.get(layer, (0.0, None))
|
||
prior = 1.0 if layer in prior_layers else 0.0
|
||
if has_semantic:
|
||
combined = 0.75 * sem + 0.15 * vis + 0.10 * prior
|
||
else:
|
||
combined = 0.72 * vis + 0.28 * prior
|
||
reasons = []
|
||
if sem > 0:
|
||
reasons.append(f"S-52 语义 {sem*100:.0f}%")
|
||
if vis > 0:
|
||
reasons.append(f"视觉证据 {vis*100:.0f}%")
|
||
if prior > 0:
|
||
reasons.append("已有 reviewed suggestion 支持")
|
||
rows.append({
|
||
"standard_layer": layer,
|
||
"standard_layer_zh": self.layer_zh.get(layer, layer),
|
||
"semantic_score": round(sem, 4),
|
||
"visual_score": round(vis, 4),
|
||
"prior_support": bool(prior),
|
||
"combined_score": round(min(1.0, combined), 4),
|
||
"best_standard_resource_key": best_resource,
|
||
"reason_zh": "；".join(reasons) or "辅助证据不足",
|
||
})
|
||
rows.sort(key=lambda x: (x["combined_score"], x["semantic_score"], x["visual_score"]), reverse=True)
|
||
return rows[:8]
|
||
|
||
def _load_or_initialize_review(self) -> dict[str, Any]:
|
||
if MANUAL_REVIEW_PATH.exists():
|
||
data = read_json(MANUAL_REVIEW_PATH)
|
||
return data
|
||
data = {
|
||
"schema": "cm95-manual-review/v1",
|
||
"created_at": datetime.now(timezone.utc).isoformat(),
|
||
"updated_at": datetime.now(timezone.utc).isoformat(),
|
||
"cm95_style_sha256": self.cm.get("style", {}).get("sha256"),
|
||
"standard_style_sha256": self.std.get("style", {}).get("sha256"),
|
||
"standard_catalog_status": self.std.get("catalog_status"),
|
||
"policy": {
|
||
"source_layer_hidden_from_main_review": True,
|
||
"provider_field_semantics_used": False,
|
||
"standard_catalog_mutated": False,
|
||
"cm95_anonymous_catalog_mutated": False,
|
||
},
|
||
"decisions": {},
|
||
}
|
||
write_json_atomic(MANUAL_REVIEW_PATH, data)
|
||
return data
|
||
|
||
def save_decision(self, payload: dict[str, Any]) -> dict[str, Any]:
|
||
anonymous_id = str(payload.get("anonymous_id") or "")
|
||
item = self.cm_by_id.get(anonymous_id)
|
||
if not item:
|
||
raise ValueError("未知 anonymous_id")
|
||
|
||
status = str(payload.get("status") or "")
|
||
if status not in VALID_STATUS:
|
||
raise ValueError("status 必须是 MATCH / PARTIAL / AMBIGUOUS / COMPONENT / UNKNOWN")
|
||
|
||
layers = payload.get("standard_layers") or []
|
||
if not isinstance(layers, list):
|
||
raise ValueError("standard_layers 必须是数组")
|
||
layers = sorted({str(x) for x in layers if x})
|
||
|
||
std_resource = payload.get("standard_resource_key")
|
||
if std_resource is not None:
|
||
std_resource = str(std_resource)
|
||
if std_resource not in self.std_by_key:
|
||
raise ValueError("未知 standard_resource_key")
|
||
|
||
valid_layers = set(self.standard_layers)
|
||
if any(x not in valid_layers for x in layers):
|
||
raise ValueError("standard_layers 包含未知标准 Layer")
|
||
|
||
if status in {"MATCH", "PARTIAL"} and len(layers) != 1:
|
||
raise ValueError(f"{status} 必须且只能选择 1 个标准 Layer")
|
||
if status == "AMBIGUOUS" and len(layers) < 2:
|
||
raise ValueError("AMBIGUOUS 至少选择 2 个标准 Layer")
|
||
component_kind = payload.get("component_kind")
|
||
if component_kind is not None:
|
||
component_kind = str(component_kind)
|
||
if component_kind not in COMPONENT_KIND_ZH:
|
||
raise ValueError("未知 component_kind")
|
||
|
||
if status in {"UNKNOWN", "COMPONENT"}:
|
||
layers = []
|
||
std_resource = None
|
||
if status != "COMPONENT":
|
||
component_kind = None
|
||
|
||
if std_resource:
|
||
std_layer = self.std_by_key[std_resource]["standard_layer"]
|
||
if status in {"MATCH", "PARTIAL"} and layers and std_layer != layers[0]:
|
||
raise ValueError("选择的 Standard Visual Object 与 Layer 不一致")
|
||
|
||
now = datetime.now(timezone.utc).isoformat()
|
||
decision = {
|
||
"anonymous_id": anonymous_id,
|
||
"resource_key_internal": item["resource_key"],
|
||
"status": status,
|
||
"standard_layers": layers,
|
||
"standard_resource_key": std_resource,
|
||
"component_kind": component_kind,
|
||
"assignment_mode": (
|
||
"COMPONENT_ONLY" if status == "COMPONENT"
|
||
else ("VISUAL_OBJECT_MATCH" if std_resource else ("LAYER_ONLY" if layers else "UNKNOWN"))
|
||
),
|
||
"note": str(payload.get("note") or ""),
|
||
"reviewed_at": now,
|
||
}
|
||
self.review["decisions"][anonymous_id] = decision
|
||
self.review["updated_at"] = now
|
||
write_json_atomic(MANUAL_REVIEW_PATH, self.review)
|
||
return decision
|
||
|
||
def delete_decision(self, anonymous_id: str) -> None:
|
||
self.review["decisions"].pop(anonymous_id, None)
|
||
self.review["updated_at"] = datetime.now(timezone.utc).isoformat()
|
||
write_json_atomic(MANUAL_REVIEW_PATH, self.review)
|
||
|
||
def abandon_unreviewed(self, dry_run: bool = False) -> dict[str, Any]:
|
||
decisions = self.review.get("decisions", {})
|
||
pending = [x for x in self.cm_resources if x["anonymous_id"] not in decisions]
|
||
if dry_run:
|
||
return {"affected": len(pending), "applied": False}
|
||
|
||
now = datetime.now(timezone.utc).isoformat()
|
||
for item in pending:
|
||
anonymous_id = item["anonymous_id"]
|
||
decisions[anonymous_id] = {
|
||
"anonymous_id": anonymous_id,
|
||
"resource_key_internal": item["resource_key"],
|
||
"status": "UNKNOWN",
|
||
"standard_layers": [],
|
||
"standard_resource_key": None,
|
||
"component_kind": None,
|
||
"assignment_mode": "UNKNOWN",
|
||
"note": "批量放弃未确认项",
|
||
"reviewed_at": now,
|
||
"bulk_action": "ABANDON_UNREVIEWED",
|
||
}
|
||
self.review["updated_at"] = now
|
||
write_json_atomic(MANUAL_REVIEW_PATH, self.review)
|
||
return {"affected": len(pending), "applied": True}
|
||
|
||
def bootstrap(self) -> dict[str, Any]:
|
||
decisions = self.review.get("decisions", {})
|
||
items = []
|
||
for item in self.cm_resources:
|
||
anon = item["anonymous_id"]
|
||
suggestion = self.suggestion_by_key.get(item["resource_key"])
|
||
candidate = self.candidate_by_key.get(item["resource_key"])
|
||
candidates = []
|
||
if candidate:
|
||
best = candidate.get("best_standard_resource_key")
|
||
best_layer = candidate.get("best_standard_layer")
|
||
if best and best_layer:
|
||
name_info_std = self.std_names.get(best, {}) or {}
|
||
candidates.append({
|
||
"resource_key": best,
|
||
"standard_layer": best_layer,
|
||
"standard_layer_zh": self.layer_zh.get(best_layer, best_layer),
|
||
"display_name_zh": name_info_std.get("display_name_zh") or self.layer_zh.get(best_layer, best_layer),
|
||
"score": candidate.get("score"),
|
||
"margin": candidate.get("margin"),
|
||
"source": "visual_candidate",
|
||
})
|
||
for alt in candidate.get("alternatives", []):
|
||
key = alt.get("standard_resource_key")
|
||
layer = alt.get("standard_layer")
|
||
if key and layer:
|
||
name_info_std = self.std_names.get(key, {}) or {}
|
||
candidates.append({
|
||
"resource_key": key,
|
||
"standard_layer": layer,
|
||
"standard_layer_zh": self.layer_zh.get(layer, layer),
|
||
"display_name_zh": name_info_std.get("display_name_zh") or self.layer_zh.get(layer, layer),
|
||
"score": alt.get("score"),
|
||
"source": "visual_candidate",
|
||
})
|
||
|
||
name_info = self.cm_names.get(item["resource_key"], {})
|
||
cm_name_zh = name_info.get("display_name_zh") or "未命名视觉对象"
|
||
|
||
suggestion_layers = set()
|
||
suggestion_resource = None
|
||
if suggestion:
|
||
suggestion_resource = suggestion.get("standard_resource_key")
|
||
if suggestion.get("standard_layer"):
|
||
suggestion_layers.add(str(suggestion.get("standard_layer")))
|
||
for layer_name in suggestion.get("standard_layers") or []:
|
||
if layer_name:
|
||
suggestion_layers.add(str(layer_name))
|
||
|
||
# UI-only recommendation: actual visual score + Chinese name similarity +
|
||
# existing reviewed visual suggestion. This never writes Layer Assignment.
|
||
for c in candidates:
|
||
visual_score = float(c.get("score") or 0.0)
|
||
text_score = _text_similarity(cm_name_zh, c.get("display_name_zh"))
|
||
bonus = 0.0
|
||
if suggestion_resource and c.get("resource_key") == suggestion_resource:
|
||
bonus += 0.16
|
||
elif c.get("standard_layer") in suggestion_layers:
|
||
bonus += 0.08
|
||
c["text_score"] = round(text_score, 4)
|
||
c["recommendation_score"] = round(min(1.0, 0.62 * visual_score + 0.38 * text_score + bonus), 4)
|
||
|
||
candidates.sort(key=lambda x: (float(x.get("recommendation_score") or 0.0), float(x.get("score") or 0.0)), reverse=True)
|
||
recommended_resource_key = candidates[0]["resource_key"] if candidates else suggestion_resource
|
||
semantic = self._semantic_info(item["resource_key"])
|
||
layer_suggestions = self._layer_suggestions(semantic, candidates, suggestion)
|
||
top_layer = layer_suggestions[0]["standard_layer"] if layer_suggestions else None
|
||
current_decision = decisions.get(anon)
|
||
decision_conflict = False
|
||
if current_decision and top_layer and layer_suggestions and layer_suggestions[0]["combined_score"] >= 0.72:
|
||
selected = current_decision.get("standard_layers") or []
|
||
if current_decision.get("status") in {"MATCH", "PARTIAL"} and len(selected) == 1 and selected[0] != top_layer:
|
||
decision_conflict = True
|
||
|
||
items.append({
|
||
"anonymous_id": anon,
|
||
"display_name_zh": cm_name_zh,
|
||
"official_description_en": name_info.get("official_description_en"),
|
||
"name_source": name_info.get("display_name_source"),
|
||
"semantic": semantic,
|
||
"layer_suggestions": layer_suggestions,
|
||
"recommended_layer": top_layer,
|
||
"decision_conflict": decision_conflict,
|
||
"component_roles": item["component_roles"],
|
||
"geometry_roles": item["geometry_roles"],
|
||
"occurrence_count": item["occurrence_count"],
|
||
"suggestion": {
|
||
"status": suggestion.get("status"),
|
||
"standard_layer": suggestion.get("standard_layer"),
|
||
"standard_layers": suggestion.get("standard_layers"),
|
||
"standard_resource_key": suggestion.get("standard_resource_key"),
|
||
"standard_name_zh": (self.std_names.get(suggestion.get("standard_resource_key") or "", {}) or {}).get("display_name_zh"),
|
||
"standard_layer_zh": self.layer_zh.get(suggestion.get("standard_layer") or ""),
|
||
"review_stage": suggestion.get("review_stage"),
|
||
} if suggestion else None,
|
||
"candidates": candidates,
|
||
"recommended_resource_key": recommended_resource_key,
|
||
"decision": current_decision,
|
||
})
|
||
|
||
std_layers = list(self.standard_layers)
|
||
std_resources = [
|
||
{
|
||
"resource_key": x["resource_key"],
|
||
"standard_layer": x["standard_layer"],
|
||
"standard_layer_zh": self.layer_zh.get(x["standard_layer"], x["standard_layer"]),
|
||
"display_name_zh": (self.std_names.get(x["resource_key"], {}) or {}).get("display_name_zh") or self.layer_zh.get(x["standard_layer"], x["standard_layer"]),
|
||
"component_role": x.get("component_role"),
|
||
"geometry_role": x.get("geometry_role"),
|
||
"visual_object_id": x.get("visual_object_id"),
|
||
}
|
||
for x in self.std_resources
|
||
]
|
||
|
||
summary = Counter()
|
||
for d in decisions.values():
|
||
summary[d.get("status", "UNKNOWN")] += 1
|
||
|
||
return {
|
||
"items": items,
|
||
"standard_layers": std_layers,
|
||
"standard_layer_zh": {x: self.layer_zh.get(x, x) for x in std_layers},
|
||
"standard_resources": std_resources,
|
||
"progress": {
|
||
"total": len(items),
|
||
"reviewed": len(decisions),
|
||
"unreviewed": len(items) - len(decisions),
|
||
"MATCH": summary["MATCH"],
|
||
"PARTIAL": summary["PARTIAL"],
|
||
"AMBIGUOUS": summary["AMBIGUOUS"],
|
||
"COMPONENT": summary["COMPONENT"],
|
||
"UNKNOWN": summary["UNKNOWN"],
|
||
},
|
||
"component_kinds": [
|
||
{"id": key, "name_zh": value}
|
||
for key, value in COMPONENT_KIND_ZH.items()
|
||
],
|
||
"policy": self.review.get("policy"),
|
||
}
|
||
|
||
def audit_info(self, anonymous_id: str) -> dict[str, Any]:
|
||
item = self.cm_by_id.get(anonymous_id)
|
||
if not item:
|
||
raise ValueError("未知 anonymous_id")
|
||
unique_prov = []
|
||
seen = set()
|
||
for p in item["provenance"]:
|
||
key = json.dumps(p, ensure_ascii=False, sort_keys=True)
|
||
if key not in seen:
|
||
seen.add(key)
|
||
unique_prov.append(p)
|
||
return {
|
||
"anonymous_id": anonymous_id,
|
||
"resource_key": item["resource_key"],
|
||
"visual_hash": item.get("visual_hash"),
|
||
"group_ids": item["group_ids"],
|
||
"provenance": unique_prov,
|
||
}
|
||
|
||
def crop_png(self, which: str, identifier: str) -> bytes:
|
||
if which == "cm95":
|
||
item = self.cm_by_id.get(identifier)
|
||
if not item:
|
||
raise KeyError(identifier)
|
||
atlas = self.cm_atlas
|
||
meta = item["resource_meta"]
|
||
elif which == "std":
|
||
item = self.std_by_key.get(identifier)
|
||
if not item:
|
||
raise KeyError(identifier)
|
||
atlas = self.std_atlas
|
||
meta = item["resource_meta"]
|
||
else:
|
||
raise KeyError(which)
|
||
|
||
crop = atlas.crop((
|
||
int(meta["x"]), int(meta["y"]),
|
||
int(meta["x"]) + int(meta["width"]),
|
||
int(meta["y"]) + int(meta["height"]),
|
||
)).convert("RGBA")
|
||
from io import BytesIO
|
||
buf = BytesIO()
|
||
crop.save(buf, format="PNG")
|
||
return buf.getvalue()
|
||
|
||
def finalize(self) -> dict[str, Any]:
|
||
decisions = self.review.get("decisions", {})
|
||
anchors = []
|
||
for anon, d in decisions.items():
|
||
item = self.cm_by_id.get(anon)
|
||
if not item:
|
||
continue
|
||
status = d["status"]
|
||
if status == "UNKNOWN":
|
||
continue
|
||
layers = d.get("standard_layers") or []
|
||
row: dict[str, Any] = {
|
||
"cm95_resource_key": item["resource_key"],
|
||
"status": status,
|
||
"standard_resource_key": d.get("standard_resource_key"),
|
||
"component_kind": d.get("component_kind"),
|
||
"assignment_mode": d.get("assignment_mode") or (
|
||
"COMPONENT_ONLY" if status == "COMPONENT"
|
||
else ("VISUAL_OBJECT_MATCH" if d.get("standard_resource_key") else "LAYER_ONLY")
|
||
),
|
||
"review_stage": "user_manual_ui_review",
|
||
"note": d.get("note", ""),
|
||
}
|
||
if status in {"MATCH", "PARTIAL"} and len(layers) == 1:
|
||
row["standard_layer"] = layers[0]
|
||
elif status == "AMBIGUOUS":
|
||
row["standard_layers"] = layers
|
||
anchors.append(row)
|
||
|
||
exported = {
|
||
"schema": "cm95-reviewed-visual-anchors/manual-v1",
|
||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||
"source_manual_review": str(MANUAL_REVIEW_PATH),
|
||
"rules": {
|
||
"source_layer_used": False,
|
||
"provider_field_semantics_used": False,
|
||
"manual_review_is_authoritative": True,
|
||
"unreviewed_resources_are_unknown": True,
|
||
},
|
||
"anchors": anchors,
|
||
}
|
||
write_json_atomic(MANUAL_ANCHOR_PATH, exported)
|
||
|
||
script = TOOL_DIR / "apply_reviewed_layer_assignment.py"
|
||
cmd = [
|
||
sys.executable, str(script),
|
||
"--anonymous", str(CM95_CATALOG_PATH),
|
||
"--reviewed-anchors", str(MANUAL_ANCHOR_PATH),
|
||
"--output", str(MANUAL_ASSIGNMENT_DIR),
|
||
]
|
||
proc = subprocess.run(cmd, text=True, capture_output=True, timeout=120)
|
||
if proc.returncode != 0:
|
||
raise RuntimeError(proc.stderr or proc.stdout or "assignment 生成失败")
|
||
|
||
return {
|
||
"ok": True,
|
||
"manual_review_path": str(MANUAL_REVIEW_PATH),
|
||
"anchor_path": str(MANUAL_ANCHOR_PATH),
|
||
"assignment_dir": str(MANUAL_ASSIGNMENT_DIR),
|
||
"stdout": proc.stdout,
|
||
}
|
||
|
||
|
||
STATE = ReviewState()
|
||
|
||
|
||
class Handler(BaseHTTPRequestHandler):
|
||
server_version = "StyleVisualObjectReviewer/1.0"
|
||
|
||
def _send_json(self, obj: Any, status: int = 200) -> None:
|
||
data = json.dumps(obj, ensure_ascii=False).encode("utf-8")
|
||
self.send_response(status)
|
||
self.send_header("Content-Type", "application/json; charset=utf-8")
|
||
self.send_header("Content-Length", str(len(data)))
|
||
self.send_header("Cache-Control", "no-store")
|
||
self.end_headers()
|
||
self.wfile.write(data)
|
||
|
||
def _send_bytes(self, data: bytes, content_type: str, status: int = 200) -> None:
|
||
self.send_response(status)
|
||
self.send_header("Content-Type", content_type)
|
||
self.send_header("Content-Length", str(len(data)))
|
||
self.send_header("Cache-Control", "no-store")
|
||
self.end_headers()
|
||
self.wfile.write(data)
|
||
|
||
def _read_json(self) -> dict[str, Any]:
|
||
length = int(self.headers.get("Content-Length", "0") or 0)
|
||
if length <= 0 or length > 2_000_000:
|
||
raise ValueError("请求体大小不合法")
|
||
return json.loads(self.rfile.read(length).decode("utf-8"))
|
||
|
||
def do_GET(self) -> None:
|
||
try:
|
||
parsed = urllib.parse.urlparse(self.path)
|
||
path = parsed.path
|
||
|
||
if path == "/api/bootstrap":
|
||
return self._send_json(STATE.bootstrap())
|
||
|
||
if path == "/api/audit":
|
||
q = urllib.parse.parse_qs(parsed.query)
|
||
anon = (q.get("id") or [""])[0]
|
||
return self._send_json(STATE.audit_info(anon))
|
||
|
||
if path.startswith("/sprite/cm95/") and path.endswith(".png"):
|
||
anon = urllib.parse.unquote(path[len("/sprite/cm95/"):-4])
|
||
return self._send_bytes(STATE.crop_png("cm95", anon), "image/png")
|
||
|
||
if path.startswith("/sprite/std/") and path.endswith(".png"):
|
||
key = urllib.parse.unquote(path[len("/sprite/std/"):-4])
|
||
return self._send_bytes(STATE.crop_png("std", key), "image/png")
|
||
|
||
if path == "/" or path == "/index.html":
|
||
file_path = WEB_DIR / "index.html"
|
||
else:
|
||
rel = path.lstrip("/")
|
||
if ".." in Path(rel).parts:
|
||
return self._send_json({"error": "invalid path"}, 400)
|
||
file_path = WEB_DIR / rel
|
||
|
||
if not file_path.exists() or not file_path.is_file():
|
||
return self._send_json({"error": "not found"}, 404)
|
||
content_type = mimetypes.guess_type(str(file_path))[0] or "application/octet-stream"
|
||
return self._send_bytes(file_path.read_bytes(), content_type)
|
||
except Exception as exc:
|
||
return self._send_json({"error": str(exc)}, 500)
|
||
|
||
def do_POST(self) -> None:
|
||
try:
|
||
parsed = urllib.parse.urlparse(self.path)
|
||
payload = self._read_json()
|
||
|
||
if parsed.path == "/api/review":
|
||
decision = STATE.save_decision(payload)
|
||
return self._send_json({"ok": True, "decision": decision, "progress": STATE.bootstrap()["progress"]})
|
||
|
||
if parsed.path == "/api/unreview":
|
||
anon = str(payload.get("anonymous_id") or "")
|
||
STATE.delete_decision(anon)
|
||
return self._send_json({"ok": True, "progress": STATE.bootstrap()["progress"]})
|
||
|
||
if parsed.path == "/api/abandon-unreviewed":
|
||
dry_run = bool(payload.get("dry_run"))
|
||
result = STATE.abandon_unreviewed(dry_run=dry_run)
|
||
return self._send_json({"ok": True, **result, "progress": STATE.bootstrap()["progress"]})
|
||
|
||
if parsed.path == "/api/finalize":
|
||
return self._send_json(STATE.finalize())
|
||
|
||
return self._send_json({"error": "not found"}, 404)
|
||
except ValueError as exc:
|
||
return self._send_json({"error": str(exc)}, 400)
|
||
except Exception as exc:
|
||
return self._send_json({"error": str(exc)}, 500)
|
||
|
||
def log_message(self, fmt: str, *args: Any) -> None:
|
||
sys.stdout.write("%s - %s\n" % (self.address_string(), fmt % args))
|
||
sys.stdout.flush()
|
||
|
||
|
||
def main() -> None:
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--host", default="0.0.0.0")
|
||
ap.add_argument("--port", type=int, default=18083)
|
||
args = ap.parse_args()
|
||
server = ThreadingHTTPServer((args.host, args.port), Handler)
|
||
print(f"CM95 Visual Object Reviewer: http://{args.host}:{args.port}/")
|
||
server.serve_forever()
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|