Checkpoint CM95 PBF production code and docs for Linux migration

2026-09-11 18:10:56 +08:00
parent 0ca1f63320
commit b52b3447cf
116 changed files with 55495 additions and 2 deletions
--- a/scripts/cm95_japan_v1_4_6_continue.ps1
+++ b/scripts/cm95_japan_v1_4_6_continue.ps1
@@ -0,0 +1,280 @@
+param(
+    [string]$ProjectRoot = "D:\React\cm93-mysql-importer-starter",
+    [string]$SourceRoot  = "D:\Ship\CM93V2\CM93JAPAN",
+    [int]$DatasetId = 1,
+    [int]$ExpectedCells = 1492,
+    [int]$Workers = 4,
+    [switch]$SkipQueryEquivalence
+)
+
+Set-StrictMode -Version Latest
+$ErrorActionPreference = "Stop"
+
+function Step([string]$Text) {
+    Write-Host ""
+    Write-Host "======================================================================"
+    Write-Host $Text
+    Write-Host "======================================================================"
+}
+
+function Run([string]$Label, [scriptblock]$Action) {
+    Step $Label
+    & $Action
+    $Code = $LASTEXITCODE
+    if ($null -ne $Code -and $Code -ne 0) {
+        throw "$Label FAILED (exit=$Code)"
+    }
+}
+
+function StateCount($States, [string]$Name) {
+    if ($null -eq $States) { return 0 }
+    $P = $States.PSObject.Properties[$Name]
+    if ($null -eq $P) { return 0 }
+    return [int]$P.Value
+}
+
+Set-Location $ProjectRoot
+
+$Python  = ".\.venv\Scripts\python.exe"
+$Cm95Pbf = ".\.venv\Scripts\cm95pbf.exe"
+$Mapping = ".\config\mappings\cm95_to_navsea_v1.yaml"
+$Disposition = ".\config\mappings\cm95_gap_disposition_japan_dataset_1_v1.yaml"
+$Policy = ".\config\pbf\cm95_tile_policy_japan_full_detail_v1.yaml"
+$Manifest = ".\build\cm95_fid_manifest_japan_dataset_1_v1.sqlite"
+$GapAudit = ".\build\production\cm95_global_mapping_gap_audit_dataset_1.json"
+$Closure = ".\build\production\cm95_global_mapping_gap_closure_dataset_1_v1_4_6.json"
+$MappingContract = ".\build\production\cm95_mapping_contract_check_japan_dataset_1_v1_4_6.json"
+$Plan = ".\build\production\cm95_production_plan_japan_full_detail_v1_4_6.json"
+$PlanAudit = ".\build\production\cm95_production_plan_audit_japan_full_detail_v1_4_6.json"
+
+foreach ($Path in @($Python,$Cm95Pbf,$Mapping,$Disposition,$Policy)) {
+    if (-not (Test-Path $Path)) {
+        throw "Missing required file: $Path"
+    }
+}
+
+Run "STEP 0A - pip install v1.4.6 editable" {
+    & $Python -m pip install -e .
+}
+
+Run "STEP 0B - CM95 v1.4.6 preflight" {
+    & $Python ".\tools\cm95_v1_4_6_preflight.py"
+}
+
+# Verify the existing fresh Japan Dataset. No import / no DROP database.
+Step "STEP 1 - Verify existing Japan Dataset"
+
+$Tmp = Join-Path $env:TEMP ("cm95_japan_v146_" + $PID)
+Remove-Item $Tmp -Recurse -Force -ErrorAction SilentlyContinue
+New-Item -ItemType Directory -Path $Tmp | Out-Null
+
+try {
+    $Helper = Join-Path $Tmp "verify_dataset.py"
+@'
+from pathlib import Path
+from dotenv import load_dotenv
+import os, sys, json, pymysql
+
+project_root = Path(sys.argv[1]).resolve()
+dataset_id = int(sys.argv[2])
+expected_root = str(Path(sys.argv[3]).resolve())
+load_dotenv(dotenv_path=project_root / ".env")
+
+conn = pymysql.connect(
+    host=os.getenv("CM93_DB_HOST", "127.0.0.1"),
+    port=int(os.getenv("CM93_DB_PORT", "3306")),
+    user=os.getenv("CM93_DB_USER", "root"),
+    password=os.getenv("CM93_DB_PASSWORD", ""),
+    database=os.getenv("CM93_DB_NAME", "cm93_chart"),
+    charset=os.getenv("CM93_DB_CHARSET", "utf8mb4"),
+    cursorclass=pymysql.cursors.DictCursor,
+)
+try:
+    with conn.cursor() as cur:
+        cur.execute(
+            """
+            SELECT d.id,d.name,d.source_root,
+                   COUNT(c.id) cells,
+                   SUM(c.import_status='DONE') done_cells,
+                   SUM(c.import_status='FAILED') failed_cells
+            FROM chart_dataset d
+            LEFT JOIN chart_cell c ON c.dataset_id=d.id
+            WHERE d.id=%s
+            GROUP BY d.id,d.name,d.source_root
+            """,
+            (dataset_id,),
+        )
+        row = cur.fetchone()
+        if not row:
+            raise RuntimeError(f"dataset {dataset_id} not found")
+        actual_root = str(Path(row["source_root"]).resolve())
+        if actual_root.lower() != expected_root.lower():
+            raise RuntimeError(f"source_root mismatch: {actual_root} != {expected_root}")
+        print(json.dumps({
+            "id": int(row["id"]),
+            "name": row["name"],
+            "source_root": actual_root,
+            "cells": int(row["cells"] or 0),
+            "done_cells": int(row["done_cells"] or 0),
+            "failed_cells": int(row["failed_cells"] or 0),
+        }, ensure_ascii=False))
+finally:
+    conn.close()
+'@ | Set-Content $Helper -Encoding UTF8
+
+    $DatasetText = (& $Python $Helper $ProjectRoot $DatasetId $SourceRoot | Out-String).Trim()
+    if ($LASTEXITCODE -ne 0) { throw "Dataset verification failed" }
+    $Dataset = $DatasetText | ConvertFrom-Json
+    if ([int]$Dataset.cells -ne $ExpectedCells -or
+        [int]$Dataset.done_cells -ne $ExpectedCells -or
+        [int]$Dataset.failed_cells -ne 0) {
+        throw "Dataset state mismatch: $DatasetText"
+    }
+    Write-Host $DatasetText
+}
+finally {
+    Remove-Item $Tmp -Recurse -Force -ErrorAction SilentlyContinue
+}
+
+if (-not $SkipQueryEquivalence) {
+    Run "STEP 2 - v1.4.5 vs v1.4.6 candidate query equivalence + SPATIAL plan" {
+        & $Python ".\tools\cm95_v1_4_6_query_equivalence.py" `
+            --dataset-id $DatasetId
+    }
+}
+else {
+    Step "STEP 2 - Query equivalence SKIPPED by user"
+}
+
+Run "STEP 3A - Re-run mapping gap audit" {
+    & $Python ".\tools\cm95_global_mapping_gap_audit_v1.py" `
+        --dataset-id $DatasetId `
+        --mapping $Mapping `
+        --output $GapAudit
+}
+
+Run "STEP 3B - Japan gap closure" {
+    & $Python ".\tools\cm95_global_mapping_gap_closure_audit_v2.py" `
+        --gap-audit $GapAudit `
+        --mapping $Mapping `
+        --disposition $Disposition `
+        --output $Closure
+}
+
+$C = Get-Content $Closure -Raw | ConvertFrom-Json
+if ([string]$C.overall -ne "PASS") { throw "Mapping gap closure != PASS" }
+if (@($C.unclassified_variants).Count -ne 0) { throw "Unclassified gap variants remain" }
+
+Run "STEP 3C - Mapping / Style / Sprite contract" {
+    & $Python ".\tools\cm95_mapping_contract_check_v1.py" `
+        --mapping $Mapping `
+        --style ".\config\style\style.navsea-delivery-full-semantic-unified-20260808-cm95-v1.json" `
+        --sprite-json ".\config\sprite\semantic-sprite-20260808.json" `
+        --output $MappingContract
+}
+
+$MC = Get-Content $MappingContract -Raw | ConvertFrom-Json
+if ([string]$MC.overall -ne "PASS") { throw "Mapping/style/sprite contract != PASS" }
+
+# FID manifest is independent from builder query implementation; reuse when valid.
+if (-not (Test-Path $Manifest)) {
+    Run "STEP 4 - FID manifest missing; rebuild" {
+        & $Cm95Pbf build-fid-manifest `
+            --dataset-id $DatasetId `
+            --manifest $Manifest `
+            --rebuild-fid-manifest
+    }
+}
+else {
+    Step "STEP 4 - Reuse existing FID manifest"
+    Write-Host $Manifest
+}
+
+# v1.4.6 builder contract changes the immutable plan hash, so create a NEW plan/release.
+Run "STEP 5A - Create NEW v1.4.6 production plan" {
+    & $Cm95Pbf plan-production `
+        --dataset-id $DatasetId `
+        --zooms 0-12 `
+        --profile delivery `
+        --mapping $Mapping `
+        --policy $Policy `
+        --manifest $Manifest `
+        --output $Plan
+}
+
+Run "STEP 5B - Audit v1.4.6 production plan" {
+    & $Python ".\tools\cm95_production_plan_audit_v1.py" `
+        --plan $Plan `
+        --expected-zooms 0-12 `
+        --output $PlanAudit
+}
+
+$PA = Get-Content $PlanAudit -Raw | ConvertFrom-Json
+if ([string]$PA.overall -ne "PASS") { throw "Production plan audit != PASS" }
+
+$PlanObj = Get-Content $Plan -Raw | ConvertFrom-Json
+if ([string]$PlanObj.builder_contract_version -ne "cm95pbf-builder/0.4.6-spatial-candidate-query") {
+    throw "Wrong builder contract in plan: $($PlanObj.builder_contract_version)"
+}
+
+$ReleaseId = [string]$PlanObj.release_id
+Write-Host ""
+Write-Host "Plan ID    : $($PlanObj.plan_id)"
+Write-Host "Release ID : $ReleaseId"
+Write-Host "Jobs       : $($PlanObj.job_count)"
+Write-Host "Workers    : $Workers"
+
+# Deliberately do NOT migrate PASS jobs from v1.4.5 partial release.
+# The new query must receive a clean production run for output equivalence/final gate.
+Run "STEP 6 - Full v1.4.6 PBF production build" {
+    & $Cm95Pbf build-production `
+        --plan $Plan `
+        --release-root ".\build\releases" `
+        --workers $Workers `
+        --max-attempts 3
+}
+
+Step "STEP 7 - Production status"
+$StatusText = (
+    & $Cm95Pbf status-production `
+        --plan $Plan `
+        --release-root ".\build\releases" |
+    Out-String
+).Trim()
+if ($LASTEXITCODE -ne 0) { throw "status-production failed" }
+Write-Host $StatusText
+$S = $StatusText | ConvertFrom-Json
+$Pass = StateCount $S.states "PASS"
+$Fail = StateCount $S.states "FAILED"
+$Pending = StateCount $S.states "PENDING"
+$Running = StateCount $S.states "RUNNING"
+if ($Pass -ne [int]$S.jobs -or $Fail -ne 0 -or $Pending -ne 0 -or $Running -ne 0) {
+    throw "Production state gate failed"
+}
+
+Run "STEP 8 - Finalize v1.4.6 release" {
+    & $Cm95Pbf finalize-production `
+        --plan $Plan `
+        --release-root ".\build\releases" `
+        --style ".\config\style\style.navsea-delivery-full-semantic-unified-20260808-cm95-v1.json" `
+        --sprite-json ".\config\sprite\semantic-sprite-20260808.json" `
+        --gap-disposition $Disposition
+}
+
+$ReleaseManifest = ".\build\releases\$ReleaseId\release_manifest_v1.json"
+if (-not (Test-Path $ReleaseManifest)) { throw "Release manifest missing" }
+$Final = Get-Content $ReleaseManifest -Raw | ConvertFrom-Json
+if ([string]$Final.status -ne "PASS") { throw "Final release status != PASS" }
+
+Step "CM95 JAPAN v1.4.6 COMPLETE / PASS"
+Write-Host "Dataset ID       : $DatasetId"
+Write-Host "Release ID       : $ReleaseId"
+Write-Host "Jobs             : $($S.jobs)"
+Write-Host "PBF files        : $($S.pbf_files)"
+Write-Host "Output features  : $($S.output_features)"
+Write-Host "Final status     : $($Final.status)"
+Write-Host "Semantic status  : $($Final.semantic_completeness)"
+Write-Host ""
+Write-Host "Web test:"
+Write-Host "  cd `"$ProjectRoot`""
+Write-Host "  & `".\.venv\Scripts\python.exe`" .\web\cm95_web.py --release-id `"$ReleaseId`""