Add unified strict audit workflow
This commit is contained in:
OpenAI Codex
281
navsea_strict_audit.py
Normal file
281
navsea_strict_audit.py
Normal file
@@ -0,0 +1,281 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
from PIL import Image, ImageChops, ImageStat
|
||||
|
||||
|
||||
DEFAULT_COMPARE_URL = "http://192.168.200.184/newpec/navsea-compare-karatsu-20nm.html"
|
||||
DEFAULT_BACKEND_AUDIT_JSON = Path(
|
||||
"/root/sourceserver/pbf/NavSea_Original_vs_Delivery_Render_Audit_Karatsu_20nm_2026-03-31.r7.json"
|
||||
)
|
||||
DEFAULT_OUTPUT_DIR = Path("/root/sourceserver/pbf/report/strict_audit")
|
||||
|
||||
|
||||
@dataclass
|
||||
class CropBox:
|
||||
left: int
|
||||
top: int
|
||||
right: int
|
||||
bottom: int
|
||||
|
||||
def as_tuple(self) -> tuple[int, int, int, int]:
|
||||
return (self.left, self.top, self.right, self.bottom)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Run strict audit by combining browser-rendered visual diff with backend render audit summary."
|
||||
)
|
||||
parser.add_argument("--compare-url", default=DEFAULT_COMPARE_URL)
|
||||
parser.add_argument("--backend-audit-json", type=Path, default=DEFAULT_BACKEND_AUDIT_JSON)
|
||||
parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR)
|
||||
parser.add_argument("--width", type=int, default=1600)
|
||||
parser.add_argument("--height", type=int, default=900)
|
||||
parser.add_argument("--toolbar-height", type=int, default=150)
|
||||
parser.add_argument("--bottom-trim", type=int, default=170)
|
||||
parser.add_argument("--timeout-sec", type=int, default=90)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def fetch_compare_version(compare_url: str) -> str | None:
|
||||
try:
|
||||
text = requests.get(compare_url, timeout=20).text
|
||||
except Exception:
|
||||
return None
|
||||
match = re.search(r'const\s+COMPARE_VERSION\s*=\s*"([^"]+)"', text)
|
||||
return match.group(1) if match else None
|
||||
|
||||
|
||||
def run_browser_screenshot(compare_url: str, output_png: Path, width: int, height: int, timeout_sec: int) -> None:
|
||||
cmd = [
|
||||
"timeout",
|
||||
f"{timeout_sec}s",
|
||||
"google-chrome",
|
||||
"--headless=new",
|
||||
"--disable-gpu",
|
||||
"--enable-unsafe-swiftshader",
|
||||
"--no-sandbox",
|
||||
"--virtual-time-budget=20000",
|
||||
f"--window-size={width},{height}",
|
||||
f"--screenshot={output_png}",
|
||||
compare_url,
|
||||
]
|
||||
subprocess.run(cmd, check=True, capture_output=True, text=True)
|
||||
|
||||
|
||||
def build_crop_boxes(image_width: int, image_height: int, toolbar_height: int, bottom_trim: int) -> tuple[CropBox, CropBox]:
|
||||
usable_top = toolbar_height
|
||||
usable_bottom = max(usable_top + 1, image_height - bottom_trim)
|
||||
mid_x = image_width // 2
|
||||
left_box = CropBox(0, usable_top, mid_x, usable_bottom)
|
||||
right_box = CropBox(mid_x, usable_top, image_width, usable_bottom)
|
||||
return left_box, right_box
|
||||
|
||||
|
||||
def compute_visual_metrics(left_img: Image.Image, right_img: Image.Image) -> dict[str, Any]:
|
||||
if left_img.size != right_img.size:
|
||||
raise ValueError(f"image sizes differ: {left_img.size} vs {right_img.size}")
|
||||
|
||||
diff = ImageChops.difference(left_img, right_img)
|
||||
diff_rgb = diff.convert("RGB")
|
||||
diff_gray = diff.convert("L")
|
||||
stat = ImageStat.Stat(diff_rgb)
|
||||
mean_abs = [round(v, 4) for v in stat.mean]
|
||||
rms = [round(v, 4) for v in stat.rms]
|
||||
|
||||
bbox = diff_gray.getbbox()
|
||||
nonzero_pixels = 0
|
||||
if bbox:
|
||||
mask = diff_gray.point(lambda value: 255 if value else 0)
|
||||
histogram = mask.histogram()
|
||||
nonzero_pixels = histogram[255] if len(histogram) > 255 else 0
|
||||
|
||||
total_pixels = left_img.size[0] * left_img.size[1]
|
||||
changed_ratio = round(nonzero_pixels / total_pixels, 6) if total_pixels else 0.0
|
||||
|
||||
return {
|
||||
"image_width": left_img.size[0],
|
||||
"image_height": left_img.size[1],
|
||||
"total_pixels": total_pixels,
|
||||
"changed_pixels": nonzero_pixels,
|
||||
"changed_ratio": changed_ratio,
|
||||
"mean_abs_rgb": mean_abs,
|
||||
"rms_rgb": rms,
|
||||
"diff_bbox": list(bbox) if bbox else None,
|
||||
"diff_image": diff,
|
||||
}
|
||||
|
||||
|
||||
def save_visual_outputs(full_png: Path, left_png: Path, right_png: Path, diff_png: Path, toolbar_height: int, bottom_trim: int) -> dict[str, Any]:
|
||||
image = Image.open(full_png).convert("RGBA")
|
||||
left_box, right_box = build_crop_boxes(image.width, image.height, toolbar_height, bottom_trim)
|
||||
left = image.crop(left_box.as_tuple())
|
||||
right = image.crop(right_box.as_tuple())
|
||||
left.save(left_png)
|
||||
right.save(right_png)
|
||||
|
||||
metrics = compute_visual_metrics(left, right)
|
||||
diff_img = metrics.pop("diff_image")
|
||||
diff_boost = diff_img.convert("RGB").point(lambda value: min(255, value * 4))
|
||||
diff_boost.save(diff_png)
|
||||
|
||||
metrics["full_crop_left"] = list(left_box.as_tuple())
|
||||
metrics["full_crop_right"] = list(right_box.as_tuple())
|
||||
return metrics
|
||||
|
||||
|
||||
def load_backend_summary(report_json_path: Path) -> dict[str, Any]:
|
||||
data = json.loads(report_json_path.read_text())
|
||||
|
||||
def top_list(items: list[dict[str, Any]], limit: int = 10) -> list[dict[str, Any]]:
|
||||
return sorted(items, key=lambda item: item["count"], reverse=True)[:limit]
|
||||
|
||||
return {
|
||||
"original_feature_instances": data["original_feature_instances"],
|
||||
"engineering_feature_instances": data["engineering_feature_instances"],
|
||||
"result_count": data["result_count"],
|
||||
"status_counts": data["status_counts"],
|
||||
"top_annotation_issues": top_list(data.get("annotation_issue_counts", [])),
|
||||
"top_style_semantic_issues": top_list(data.get("style_semantic_issue_counts", [])),
|
||||
"top_source_layer_issues": top_list(data.get("source_layer_issue_counts", [])),
|
||||
}
|
||||
|
||||
|
||||
def write_reports(
|
||||
output_dir: Path,
|
||||
compare_url: str,
|
||||
compare_version: str | None,
|
||||
backend_audit_json: Path,
|
||||
visual_metrics: dict[str, Any],
|
||||
backend_summary: dict[str, Any],
|
||||
) -> tuple[Path, Path]:
|
||||
report_json = output_dir / "strict_audit_summary.json"
|
||||
report_md = output_dir / "strict_audit_summary.md"
|
||||
|
||||
summary = {
|
||||
"compare_url": compare_url,
|
||||
"compare_version": compare_version,
|
||||
"backend_audit_json": str(backend_audit_json),
|
||||
"visual_metrics": visual_metrics,
|
||||
"backend_summary": backend_summary,
|
||||
"notes": [
|
||||
"Visual metrics are browser-rendered screenshot diff metrics from the fixed compare page.",
|
||||
"Backend metrics are current render-audit summary metrics and may remain blind to sprite-load failures.",
|
||||
],
|
||||
}
|
||||
report_json.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
lines = [
|
||||
"# NavSea Strict Audit Summary",
|
||||
"",
|
||||
"## Scope",
|
||||
"",
|
||||
f"- Compare page: `{compare_url}`",
|
||||
f"- Compare version: `{compare_version or 'unknown'}`",
|
||||
f"- Backend render audit: `{backend_audit_json}`",
|
||||
"",
|
||||
"## Visual Audit",
|
||||
"",
|
||||
f"- changed pixels: `{visual_metrics['changed_pixels']}` / `{visual_metrics['total_pixels']}`",
|
||||
f"- changed ratio: `{visual_metrics['changed_ratio']}`",
|
||||
f"- mean abs rgb: `{visual_metrics['mean_abs_rgb']}`",
|
||||
f"- rms rgb: `{visual_metrics['rms_rgb']}`",
|
||||
f"- left crop: `{visual_metrics['full_crop_left']}`",
|
||||
f"- right crop: `{visual_metrics['full_crop_right']}`",
|
||||
"",
|
||||
"Artifacts:",
|
||||
"",
|
||||
f"- `{output_dir / 'compare_full.png'}`",
|
||||
f"- `{output_dir / 'compare_left.png'}`",
|
||||
f"- `{output_dir / 'compare_right.png'}`",
|
||||
f"- `{output_dir / 'compare_diff.png'}`",
|
||||
"",
|
||||
"## Backend Render Audit",
|
||||
"",
|
||||
f"- original feature instances: `{backend_summary['original_feature_instances']}`",
|
||||
f"- delivery feature instances: `{backend_summary['engineering_feature_instances']}`",
|
||||
f"- result count: `{backend_summary['result_count']}`",
|
||||
"",
|
||||
"Status counts:",
|
||||
"",
|
||||
]
|
||||
for key, value in backend_summary["status_counts"].items():
|
||||
lines.append(f"- `{key}`: `{value}`")
|
||||
|
||||
lines.extend(["", "Top annotation issues:", ""])
|
||||
for item in backend_summary["top_annotation_issues"][:10]:
|
||||
lines.append(f"- `{item['issue']}` | `{item['source_layer']}` | `{item['count']}`")
|
||||
|
||||
lines.extend(["", "Top style semantic issues:", ""])
|
||||
for item in backend_summary["top_style_semantic_issues"][:10]:
|
||||
lines.append(f"- `{item['issue']}` | `{item['source_layer']}` | `{item['count']}`")
|
||||
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Interpretation",
|
||||
"",
|
||||
"- This report intentionally puts browser-rendered visual output and backend render-audit summary in one place.",
|
||||
"- If the browser screenshot improves but backend counts do not, the current backend audit is likely blind to a browser/runtime issue such as sprite resolution.",
|
||||
]
|
||||
)
|
||||
report_md.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
return report_md, report_json
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
args.output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
compare_version = fetch_compare_version(args.compare_url)
|
||||
full_png = args.output_dir / "compare_full.png"
|
||||
left_png = args.output_dir / "compare_left.png"
|
||||
right_png = args.output_dir / "compare_right.png"
|
||||
diff_png = args.output_dir / "compare_diff.png"
|
||||
|
||||
run_browser_screenshot(args.compare_url, full_png, args.width, args.height, args.timeout_sec)
|
||||
visual_metrics = save_visual_outputs(
|
||||
full_png=full_png,
|
||||
left_png=left_png,
|
||||
right_png=right_png,
|
||||
diff_png=diff_png,
|
||||
toolbar_height=args.toolbar_height,
|
||||
bottom_trim=args.bottom_trim,
|
||||
)
|
||||
backend_summary = load_backend_summary(args.backend_audit_json)
|
||||
report_md, report_json = write_reports(
|
||||
output_dir=args.output_dir,
|
||||
compare_url=args.compare_url,
|
||||
compare_version=compare_version,
|
||||
backend_audit_json=args.backend_audit_json,
|
||||
visual_metrics=visual_metrics,
|
||||
backend_summary=backend_summary,
|
||||
)
|
||||
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"compare_version": compare_version,
|
||||
"report_md": str(report_md),
|
||||
"report_json": str(report_json),
|
||||
"changed_ratio": visual_metrics["changed_ratio"],
|
||||
"changed_pixels": visual_metrics["changed_pixels"],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user