361 lines
14 KiB
Python
361 lines
14 KiB
Python
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
from pathlib import Path
|
||
|
||
from cm93db.config import load_config
|
||
|
||
from .builder import Cm95TileBuilder, TileJob
|
||
from .identity import build_manifest
|
||
from .policy import child_tiles
|
||
from .production import (
|
||
create_production_plan,
|
||
migrate_production_release,
|
||
parse_zooms,
|
||
production_status,
|
||
run_production_plan,
|
||
)
|
||
from .release import finalize_release
|
||
|
||
|
||
DEFAULT_MAPPING = Path("config/mappings/cm95_to_navsea_v1.yaml")
|
||
DEFAULT_POLICY = Path("config/pbf/cm95_tile_policy_v1.yaml")
|
||
DEFAULT_MANIFEST = Path("build/cm95_fid_manifest_v1.sqlite")
|
||
|
||
|
||
def _add_common_db(p):
|
||
p.add_argument("--dataset-id", type=int, default=1)
|
||
|
||
|
||
def build_parser() -> argparse.ArgumentParser:
|
||
p = argparse.ArgumentParser(
|
||
prog="cm95pbf",
|
||
description="CM95 MySQL → NavSea-compatible MVT/PBF pilot generator",
|
||
)
|
||
sub = p.add_subparsers(dest="command", required=True)
|
||
|
||
manifest = sub.add_parser(
|
||
"build-fid-manifest",
|
||
help="建立全 Dataset 确定性 int32 MVT Feature ID 清单",
|
||
)
|
||
_add_common_db(manifest)
|
||
manifest.add_argument("--manifest", default=str(DEFAULT_MANIFEST))
|
||
manifest.add_argument("--rebuild-fid-manifest", action="store_true")
|
||
|
||
tile = sub.add_parser(
|
||
"build-tile",
|
||
help="生成单张 CM95 NavSea-compatible PBF",
|
||
)
|
||
_add_common_db(tile)
|
||
tile.add_argument("--z", type=int, required=True)
|
||
tile.add_argument("--x", type=int, required=True)
|
||
tile.add_argument("--y", type=int, required=True)
|
||
tile.add_argument("--profile", choices=["engineering", "delivery"], default="delivery")
|
||
tile.add_argument("--mapping", default=str(DEFAULT_MAPPING))
|
||
tile.add_argument("--policy", default=str(DEFAULT_POLICY))
|
||
tile.add_argument("--manifest", default=str(DEFAULT_MANIFEST))
|
||
tile.add_argument("--output-root", default="build/pbf")
|
||
tile.add_argument("--report-root", default="build/pbf-reports")
|
||
|
||
sample = sub.add_parser(
|
||
"build-sample-aoi",
|
||
help="生成 z10 parent + z11 children + z12 grandchildren，共 21 张测试 tile",
|
||
)
|
||
_add_common_db(sample)
|
||
sample.add_argument("--z10-x", type=int, default=914)
|
||
sample.add_argument("--z10-y", type=int, default=385)
|
||
sample.add_argument(
|
||
"--profiles",
|
||
default="engineering,delivery",
|
||
help="逗号分隔：engineering,delivery",
|
||
)
|
||
sample.add_argument("--mapping", default=str(DEFAULT_MAPPING))
|
||
sample.add_argument("--policy", default=str(DEFAULT_POLICY))
|
||
sample.add_argument("--manifest", default=str(DEFAULT_MANIFEST))
|
||
sample.add_argument("--output-root", default="build/pbf")
|
||
sample.add_argument("--report-root", default="build/pbf-reports")
|
||
|
||
|
||
prod_plan = sub.add_parser(
|
||
"plan-production",
|
||
help="按 Dataset coverage 枚举生产 tile，并冻结不可变生成计划",
|
||
)
|
||
_add_common_db(prod_plan)
|
||
prod_plan.add_argument("--zooms", default="0-12", help="缩放级别，支持 0-12 或 0,1,...,12")
|
||
prod_plan.add_argument("--profile", choices=["engineering", "delivery"], default="delivery")
|
||
prod_plan.add_argument("--mapping", default=str(DEFAULT_MAPPING))
|
||
prod_plan.add_argument("--policy", default=str(DEFAULT_POLICY))
|
||
prod_plan.add_argument("--manifest", default=str(DEFAULT_MANIFEST))
|
||
prod_plan.add_argument("--output", default="build/production/cm95_production_plan_v1.json")
|
||
|
||
prod_build = sub.add_parser(
|
||
"build-production",
|
||
help="按冻结计划批量生成，支持断点续跑、并发和失败重试",
|
||
)
|
||
prod_build.add_argument("--plan", default="build/production/cm95_production_plan_v1.json")
|
||
prod_build.add_argument("--release-root", default="build/releases")
|
||
prod_build.add_argument("--workers", type=int, default=4)
|
||
prod_build.add_argument("--max-attempts", type=int, default=3)
|
||
|
||
prod_migrate = sub.add_parser(
|
||
"migrate-production-release",
|
||
help="把旧 release 的 delivery-equivalent PASS tile 复用到新计划，只重建受影响/失败任务（支持 builder contract 升级）",
|
||
)
|
||
prod_migrate.add_argument("--old-release-dir", required=True)
|
||
prod_migrate.add_argument("--new-plan", default="build/production/cm95_production_plan_v1_4_5.json")
|
||
prod_migrate.add_argument("--release-root", default="build/releases")
|
||
prod_migrate.add_argument("--copy-mode", choices=["hardlink", "copy"], default="hardlink")
|
||
prod_migrate.add_argument(
|
||
"--invalidate-tiles-file",
|
||
default=None,
|
||
help="每行一个 z/x/y；这些旧 PASS tile 强制留为 PENDING，适用于 builder contract 的定向重建",
|
||
)
|
||
|
||
prod_status = sub.add_parser(
|
||
"status-production",
|
||
help="查看生产计划当前账本状态",
|
||
)
|
||
prod_status.add_argument("--plan", default="build/production/cm95_production_plan_v1.json")
|
||
prod_status.add_argument("--release-root", default="build/releases")
|
||
|
||
prod_final = sub.add_parser(
|
||
"finalize-production",
|
||
help="对完成的生产产物做三方计数/FID/哈希盘点并生成 release manifest",
|
||
)
|
||
prod_final.add_argument("--plan", default="build/production/cm95_production_plan_v1.json")
|
||
prod_final.add_argument("--release-root", default="build/releases")
|
||
prod_final.add_argument("--style", default="config/style/style.navsea-delivery-full-semantic-unified-20260808-cm95-v1.json")
|
||
prod_final.add_argument("--sprite-json", default="config/sprite/semantic-sprite-20260808.json")
|
||
prod_final.add_argument("--gap-disposition", default="config/mappings/cm95_gap_disposition_v1.yaml")
|
||
|
||
return p
|
||
|
||
|
||
def ensure_manifest(dataset_id: int, manifest_path: Path) -> dict:
|
||
cfg = load_config()
|
||
with __import__("pymysql").connect(
|
||
host=cfg.host,
|
||
port=cfg.port,
|
||
user=cfg.user,
|
||
password=cfg.password,
|
||
database=cfg.database,
|
||
charset=cfg.charset,
|
||
cursorclass=__import__("pymysql").cursors.DictCursor,
|
||
autocommit=False,
|
||
connect_timeout=20,
|
||
read_timeout=900,
|
||
) as conn:
|
||
return build_manifest(
|
||
conn,
|
||
dataset_id,
|
||
manifest_path,
|
||
rebuild=False,
|
||
)
|
||
|
||
|
||
def main() -> None:
|
||
args = build_parser().parse_args()
|
||
cfg = load_config()
|
||
|
||
if args.command == "plan-production":
|
||
manifest_path = Path(args.manifest)
|
||
if not manifest_path.exists():
|
||
print("FID manifest 不存在，先自动建立。")
|
||
print(json.dumps(ensure_manifest(args.dataset_id, manifest_path), ensure_ascii=False, indent=2))
|
||
result = create_production_plan(
|
||
db=cfg,
|
||
dataset_id=args.dataset_id,
|
||
mapping_path=Path(args.mapping),
|
||
policy_path=Path(args.policy),
|
||
manifest_path=manifest_path,
|
||
output_path=Path(args.output),
|
||
zooms=parse_zooms(args.zooms),
|
||
profile=args.profile,
|
||
)
|
||
print(json.dumps({
|
||
"plan_id": result["plan_id"],
|
||
"release_id": result["release_id"],
|
||
"job_count": result["job_count"],
|
||
"source_stats": result["source_stats"],
|
||
"output": str(Path(args.output)),
|
||
}, ensure_ascii=False, indent=2))
|
||
return
|
||
|
||
if args.command == "build-production":
|
||
result = run_production_plan(
|
||
db=cfg,
|
||
plan_path=Path(args.plan),
|
||
release_root=Path(args.release_root),
|
||
workers=args.workers,
|
||
max_attempts=args.max_attempts,
|
||
)
|
||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||
states = result.get("states") or {}
|
||
if states.get("FAILED", 0) or states.get("PENDING", 0) or states.get("RUNNING", 0):
|
||
raise SystemExit(2)
|
||
return
|
||
|
||
if args.command == "migrate-production-release":
|
||
result = migrate_production_release(
|
||
old_release_dir=Path(args.old_release_dir),
|
||
new_plan_path=Path(args.new_plan),
|
||
release_root=Path(args.release_root),
|
||
copy_mode=args.copy_mode,
|
||
invalidate_tiles_file=(Path(args.invalidate_tiles_file) if args.invalidate_tiles_file else None),
|
||
)
|
||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||
return
|
||
|
||
if args.command == "status-production":
|
||
print(json.dumps(production_status(Path(args.plan), Path(args.release_root)), ensure_ascii=False, indent=2))
|
||
return
|
||
|
||
if args.command == "finalize-production":
|
||
result = finalize_release(
|
||
Path(args.plan),
|
||
Path(args.release_root),
|
||
Path(args.style) if args.style else None,
|
||
Path(args.sprite_json) if args.sprite_json else None,
|
||
Path(args.gap_disposition) if args.gap_disposition else None,
|
||
)
|
||
print(json.dumps({
|
||
"status": result["status"],
|
||
"release_id": result["release_id"],
|
||
"planned_tile_jobs": result["planned_tile_jobs"],
|
||
"pbf_file_count": result["pbf_file_count"],
|
||
"planned_output_features": result["planned_output_features"],
|
||
"decoded_pbf_features": result["decoded_pbf_features"],
|
||
"semantic_completeness": result.get("semantic_completeness"),
|
||
"semantic_backlog": result.get("semantic_backlog"),
|
||
"non_renderable_omissions": result.get("non_renderable_omissions"),
|
||
"non_renderable_omission_layers": result.get("non_renderable_omission_layers"),
|
||
"style_sprite_contract": {
|
||
"overall": (result.get("style_sprite_contract") or {}).get("overall"),
|
||
"contract_unmatched_count": (result.get("style_sprite_contract") or {}).get("contract_unmatched_count"),
|
||
"sprite_missing_count": (result.get("style_sprite_contract") or {}).get("sprite_missing_count"),
|
||
},
|
||
"quantization_fallbacks": result.get("quantization_fallbacks"),
|
||
"quantization_omissions": result.get("quantization_omissions"),
|
||
"nav_name_conflict_audit": {
|
||
"overall": (result.get("nav_name_conflict_audit") or {}).get("overall"),
|
||
"policy": (result.get("nav_name_conflict_audit") or {}).get("policy"),
|
||
"report_occurrences": (result.get("nav_name_conflict_audit") or {}).get("report_occurrences"),
|
||
"trace_occurrences": (result.get("nav_name_conflict_audit") or {}).get("trace_occurrences"),
|
||
"safe_omission_occurrences": (result.get("nav_name_conflict_audit") or {}).get("safe_omission_occurrences"),
|
||
"unique_conflict_outputs": (result.get("nav_name_conflict_audit") or {}).get("unique_conflict_outputs"),
|
||
"affected_tiles": (result.get("nav_name_conflict_audit") or {}).get("affected_tiles"),
|
||
},
|
||
"failures": result["failures"],
|
||
}, ensure_ascii=False, indent=2))
|
||
if result["status"] != "PASS":
|
||
raise SystemExit(2)
|
||
return
|
||
|
||
if args.command == "build-fid-manifest":
|
||
import pymysql
|
||
with pymysql.connect(
|
||
host=cfg.host,
|
||
port=cfg.port,
|
||
user=cfg.user,
|
||
password=cfg.password,
|
||
database=cfg.database,
|
||
charset=cfg.charset,
|
||
cursorclass=pymysql.cursors.DictCursor,
|
||
autocommit=False,
|
||
connect_timeout=20,
|
||
read_timeout=900,
|
||
) as conn:
|
||
result = build_manifest(
|
||
conn,
|
||
args.dataset_id,
|
||
Path(args.manifest),
|
||
rebuild=bool(args.rebuild_fid_manifest),
|
||
)
|
||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||
return
|
||
|
||
manifest_path = Path(args.manifest)
|
||
if not manifest_path.exists():
|
||
print("FID manifest 不存在，先自动建立。")
|
||
print(
|
||
json.dumps(
|
||
ensure_manifest(args.dataset_id, manifest_path),
|
||
ensure_ascii=False,
|
||
indent=2,
|
||
)
|
||
)
|
||
|
||
if args.command == "build-tile":
|
||
builder = Cm95TileBuilder(
|
||
db=cfg,
|
||
dataset_id=args.dataset_id,
|
||
mapping_path=Path(args.mapping),
|
||
policy_path=Path(args.policy),
|
||
fid_manifest_path=manifest_path,
|
||
output_root=Path(args.output_root) / args.profile,
|
||
report_root=Path(args.report_root),
|
||
profile=args.profile,
|
||
)
|
||
result = builder.build_tile(
|
||
TileJob(args.z, args.x, args.y)
|
||
)
|
||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||
return
|
||
|
||
if args.command == "build-sample-aoi":
|
||
profiles = [
|
||
token.strip()
|
||
for token in str(args.profiles).split(",")
|
||
if token.strip()
|
||
]
|
||
for profile in profiles:
|
||
if profile not in {"engineering", "delivery"}:
|
||
raise SystemExit(f"invalid profile: {profile}")
|
||
|
||
jobs = [(10, args.z10_x, args.z10_y)]
|
||
jobs.extend(child_tiles(10, args.z10_x, args.z10_y, 11))
|
||
jobs.extend(child_tiles(10, args.z10_x, args.z10_y, 12))
|
||
|
||
results = []
|
||
for profile in profiles:
|
||
builder = Cm95TileBuilder(
|
||
db=cfg,
|
||
dataset_id=args.dataset_id,
|
||
mapping_path=Path(args.mapping),
|
||
policy_path=Path(args.policy),
|
||
fid_manifest_path=manifest_path,
|
||
output_root=Path(args.output_root) / profile,
|
||
report_root=Path(args.report_root),
|
||
profile=profile,
|
||
)
|
||
for z, x, y in jobs:
|
||
result = builder.build_tile(TileJob(z, x, y))
|
||
results.append(result)
|
||
print(
|
||
f"[{profile}] {z}/{x}/{y} "
|
||
f"features={result['output_feature_count']} "
|
||
f"layers={result['output_layer_count']} "
|
||
f"hard_gate_unmapped={len(result['hard_gate_unmapped'])}"
|
||
)
|
||
|
||
summary = {
|
||
"dataset_id": args.dataset_id,
|
||
"root_tile": f"10/{args.z10_x}/{args.z10_y}",
|
||
"profiles": profiles,
|
||
"tile_count_per_profile": len(jobs),
|
||
"results": results,
|
||
}
|
||
summary_path = Path(args.report_root) / "sample_aoi_summary_v1.json"
|
||
summary_path.parent.mkdir(parents=True, exist_ok=True)
|
||
summary_path.write_text(
|
||
json.dumps(summary, ensure_ascii=False, indent=2),
|
||
encoding="utf-8",
|
||
)
|
||
print(f"SUMMARY: {summary_path}")
|
||
return
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|