Checkpoint CM95 PBF production code and docs for Linux migration
This commit is contained in:
kaotei
392
scripts/cm95_japan_continue_from_step8.ps1
Normal file
392
scripts/cm95_japan_continue_from_step8.ps1
Normal file
@@ -0,0 +1,392 @@
|
||||
param(
|
||||
[string]$ProjectRoot = "D:\React\cm93-mysql-importer-starter",
|
||||
[string]$SourceRoot = "D:\Ship\CM93V2\CM93JAPAN",
|
||||
[int]$DatasetId = 1,
|
||||
[int]$ExpectedCells = 1492,
|
||||
[int]$Workers = 4
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function Step([string]$Text) {
|
||||
Write-Host ""
|
||||
Write-Host "======================================================================"
|
||||
Write-Host $Text
|
||||
Write-Host "======================================================================"
|
||||
}
|
||||
|
||||
function Run([string]$Label, [scriptblock]$Action) {
|
||||
Step $Label
|
||||
& $Action
|
||||
$Code = $LASTEXITCODE
|
||||
if ($null -ne $Code -and $Code -ne 0) {
|
||||
throw "$Label FAILED (exit=$Code)"
|
||||
}
|
||||
}
|
||||
|
||||
function StateCount($States, [string]$Name) {
|
||||
if ($null -eq $States) { return 0 }
|
||||
$P = $States.PSObject.Properties[$Name]
|
||||
if ($null -eq $P) { return 0 }
|
||||
return [int]$P.Value
|
||||
}
|
||||
|
||||
Set-Location $ProjectRoot
|
||||
|
||||
$Python = ".\.venv\Scripts\python.exe"
|
||||
$Cm95Pbf = ".\.venv\Scripts\cm95pbf.exe"
|
||||
|
||||
$Mapping = ".\config\mappings\cm95_to_navsea_v1.yaml"
|
||||
$Disposition = ".\config\mappings\cm95_gap_disposition_japan_dataset_1_v1.yaml"
|
||||
$Policy = ".\config\pbf\cm95_tile_policy_japan_full_detail_v1.yaml"
|
||||
|
||||
$GapAudit = ".\build\production\cm95_global_mapping_gap_audit_dataset_1.json"
|
||||
$Closure = ".\build\production\cm95_global_mapping_gap_closure_dataset_1_after_review.json"
|
||||
$MappingContract = ".\build\production\cm95_mapping_contract_check_japan_dataset_1.json"
|
||||
|
||||
$Manifest = ".\build\cm95_fid_manifest_japan_dataset_1_v1.sqlite"
|
||||
$Plan = ".\build\production\cm95_production_plan_japan_full_detail_v1.json"
|
||||
$PlanAudit = ".\build\production\cm95_production_plan_audit_japan_full_detail_v1.json"
|
||||
|
||||
foreach ($Path in @($Python,$Cm95Pbf,$Mapping,$Disposition,$Policy)) {
|
||||
if (-not (Test-Path $Path)) {
|
||||
throw "Missing required file: $Path"
|
||||
}
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# 0. Verify v1.4.5 runtime
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
Run "STEP 0 - CM95 v1.4.5 preflight" {
|
||||
& $Python ".\tools\cm95_v1_4_5_preflight.py"
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# 1. Verify fresh Japan Dataset in MySQL, no reimport
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
Step "STEP 1 - Verify existing fresh Japan Dataset"
|
||||
|
||||
$Tmp = Join-Path $env:TEMP ("cm95_japan_continue_" + $PID)
|
||||
Remove-Item $Tmp -Recurse -Force -ErrorAction SilentlyContinue
|
||||
New-Item -ItemType Directory -Path $Tmp | Out-Null
|
||||
|
||||
try {
|
||||
$Helper = Join-Path $Tmp "verify_dataset.py"
|
||||
|
||||
@'
|
||||
from pathlib import Path
|
||||
from dotenv import load_dotenv
|
||||
import os, sys, json, pymysql
|
||||
|
||||
project_root = Path(sys.argv[1]).resolve()
|
||||
dataset_id = int(sys.argv[2])
|
||||
expected_root = str(Path(sys.argv[3]).resolve())
|
||||
|
||||
load_dotenv(dotenv_path=project_root / ".env")
|
||||
|
||||
conn = pymysql.connect(
|
||||
host=os.getenv("CM93_DB_HOST", "127.0.0.1"),
|
||||
port=int(os.getenv("CM93_DB_PORT", "3306")),
|
||||
user=os.getenv("CM93_DB_USER", "root"),
|
||||
password=os.getenv("CM93_DB_PASSWORD", ""),
|
||||
database=os.getenv("CM93_DB_NAME", "cm93_chart"),
|
||||
charset=os.getenv("CM93_DB_CHARSET", "utf8mb4"),
|
||||
cursorclass=pymysql.cursors.DictCursor,
|
||||
)
|
||||
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT d.id,d.name,d.source_root,
|
||||
COUNT(c.id) cells,
|
||||
SUM(c.import_status='DONE') done_cells,
|
||||
SUM(c.import_status='FAILED') failed_cells
|
||||
FROM chart_dataset d
|
||||
LEFT JOIN chart_cell c ON c.dataset_id=d.id
|
||||
WHERE d.id=%s
|
||||
GROUP BY d.id,d.name,d.source_root
|
||||
""",
|
||||
(dataset_id,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
raise RuntimeError(f"dataset {dataset_id} not found")
|
||||
|
||||
actual_root = str(Path(row["source_root"]).resolve())
|
||||
|
||||
result = {
|
||||
"id": int(row["id"]),
|
||||
"name": row["name"],
|
||||
"source_root": actual_root,
|
||||
"cells": int(row["cells"] or 0),
|
||||
"done_cells": int(row["done_cells"] or 0),
|
||||
"failed_cells": int(row["failed_cells"] or 0),
|
||||
}
|
||||
|
||||
if actual_root.lower() != expected_root.lower():
|
||||
raise RuntimeError(
|
||||
f"source_root mismatch: {actual_root} != {expected_root}"
|
||||
)
|
||||
|
||||
print(json.dumps(result, ensure_ascii=False))
|
||||
finally:
|
||||
conn.close()
|
||||
'@ | Set-Content $Helper -Encoding UTF8
|
||||
|
||||
$DatasetText = (
|
||||
& $Python $Helper $ProjectRoot $DatasetId $SourceRoot |
|
||||
Out-String
|
||||
).Trim()
|
||||
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Dataset verification failed"
|
||||
}
|
||||
|
||||
$Dataset = $DatasetText | ConvertFrom-Json
|
||||
|
||||
if ([int]$Dataset.cells -ne $ExpectedCells -or
|
||||
[int]$Dataset.done_cells -ne $ExpectedCells -or
|
||||
[int]$Dataset.failed_cells -ne 0) {
|
||||
throw "Dataset state mismatch: $DatasetText"
|
||||
}
|
||||
|
||||
Write-Host $DatasetText
|
||||
|
||||
}
|
||||
finally {
|
||||
Remove-Item $Tmp -Recurse -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# 2. Re-run gap audit and closure with the new dataset-specific disposition
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
Run "STEP 2A - Re-run global mapping gap audit" {
|
||||
& $Python ".\tools\cm95_global_mapping_gap_audit_v1.py" `
|
||||
--dataset-id $DatasetId `
|
||||
--mapping $Mapping `
|
||||
--output $GapAudit
|
||||
}
|
||||
|
||||
Run "STEP 2B - Close 25 new Japan gap variants" {
|
||||
& $Python ".\tools\cm95_global_mapping_gap_closure_audit_v2.py" `
|
||||
--gap-audit $GapAudit `
|
||||
--mapping $Mapping `
|
||||
--disposition $Disposition `
|
||||
--output $Closure
|
||||
}
|
||||
|
||||
$C = Get-Content $Closure -Raw | ConvertFrom-Json
|
||||
|
||||
if ([string]$C.overall -ne "PASS") {
|
||||
throw "Mapping gap closure is not PASS"
|
||||
}
|
||||
if (@($C.unclassified_variants).Count -ne 0) {
|
||||
throw "Unclassified variants remain"
|
||||
}
|
||||
if ([int]$C.observed_gap_variants -ne 165) {
|
||||
throw "Unexpected gap variant count: $($C.observed_gap_variants)"
|
||||
}
|
||||
if (@($C.pending_semantic_review).Count -ne 137) {
|
||||
throw "Unexpected pending variant count: $(@($C.pending_semantic_review).Count)"
|
||||
}
|
||||
if ([int]$C.pending_semantic_review_feature_rows -ne 90009) {
|
||||
throw "Unexpected pending rows: $($C.pending_semantic_review_feature_rows)"
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "Expected closure confirmed:"
|
||||
Write-Host " observed = 165"
|
||||
Write-Host " unclassified = 0"
|
||||
Write-Host " pending = 137"
|
||||
Write-Host " pending rows = 90009"
|
||||
Write-Host " overall = PASS"
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# 3. Static mapping/style/sprite contract
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
Run "STEP 3 - Mapping / Style / Sprite static contract" {
|
||||
& $Python ".\tools\cm95_mapping_contract_check_v1.py" `
|
||||
--mapping $Mapping `
|
||||
--style ".\config\style\style.navsea-delivery-full-semantic-unified-20260808-cm95-v1.json" `
|
||||
--sprite-json ".\config\sprite\semantic-sprite-20260808.json" `
|
||||
--output $MappingContract
|
||||
}
|
||||
|
||||
$MC = Get-Content $MappingContract -Raw | ConvertFrom-Json
|
||||
if ([string]$MC.overall -ne "PASS") {
|
||||
throw "Mapping/style/sprite contract failed"
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# 4. Full-detail policy hard gate: G=7500 must be in z12
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
Step "STEP 4 - Japan full-detail policy check"
|
||||
|
||||
$PolicyObj = Get-Content $Policy -Raw |
|
||||
& $Python -c "import sys,yaml,json; print(json.dumps(yaml.safe_load(sys.stdin.read())))" |
|
||||
ConvertFrom-Json
|
||||
|
||||
# No multi-line Python is passed here; only a fixed one-line expression.
|
||||
$Enum12 = @($PolicyObj.enumeration_scales.'12')
|
||||
$Ladder12 = @($PolicyObj.scale_ladders.'12')
|
||||
|
||||
if (7500 -notin $Enum12 -or
|
||||
$Ladder12.Count -eq 0 -or
|
||||
[int]$Ladder12[0] -ne 7500) {
|
||||
throw "Japan full-detail policy does not prioritize G=7500 at z12"
|
||||
}
|
||||
|
||||
Write-Host "z12 enumeration : $($Enum12 -join ', ')"
|
||||
Write-Host "z12 ladder : $($Ladder12 -join ', ')"
|
||||
Write-Host "G=7500 gate : PASS"
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# 5. Reset PBF-side outputs only. MySQL remains untouched.
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
Step "STEP 5 - Reset PBF-side outputs only"
|
||||
|
||||
Remove-Item ".\build\releases" -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Remove-Item $Manifest -Force -ErrorAction SilentlyContinue
|
||||
Remove-Item $Plan -Force -ErrorAction SilentlyContinue
|
||||
Remove-Item $PlanAudit -Force -ErrorAction SilentlyContinue
|
||||
|
||||
New-Item -ItemType Directory -Force -Path ".\build\releases" | Out-Null
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# 6. Fresh deterministic FID manifest
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
Run "STEP 6 - Build fresh FID manifest" {
|
||||
& $Cm95Pbf build-fid-manifest `
|
||||
--dataset-id $DatasetId `
|
||||
--manifest $Manifest `
|
||||
--rebuild-fid-manifest
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# 7. Fresh production plan using G=7500-aware z12 policy
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
Run "STEP 7A - plan-production z0-z12 Japan full detail" {
|
||||
& $Cm95Pbf plan-production `
|
||||
--dataset-id $DatasetId `
|
||||
--zooms 0-12 `
|
||||
--profile delivery `
|
||||
--mapping $Mapping `
|
||||
--policy $Policy `
|
||||
--manifest $Manifest `
|
||||
--output $Plan
|
||||
}
|
||||
|
||||
Run "STEP 7B - production plan audit" {
|
||||
& $Python ".\tools\cm95_production_plan_audit_v1.py" `
|
||||
--plan $Plan `
|
||||
--expected-zooms 0-12 `
|
||||
--output $PlanAudit
|
||||
}
|
||||
|
||||
$PA = Get-Content $PlanAudit -Raw | ConvertFrom-Json
|
||||
if ([string]$PA.overall -ne "PASS") {
|
||||
throw "Production plan audit failed"
|
||||
}
|
||||
|
||||
$PlanObj = Get-Content $Plan -Raw | ConvertFrom-Json
|
||||
$ReleaseId = [string]$PlanObj.release_id
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "Plan ID : $($PlanObj.plan_id)"
|
||||
Write-Host "Release ID : $ReleaseId"
|
||||
Write-Host "Jobs : $($PlanObj.job_count)"
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# 8. Full production build
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
Run "STEP 8 - Full PBF production build" {
|
||||
& $Cm95Pbf build-production `
|
||||
--plan $Plan `
|
||||
--release-root ".\build\releases" `
|
||||
--workers $Workers `
|
||||
--max-attempts 3
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# 9. Status
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
Step "STEP 9 - Production status"
|
||||
|
||||
$StatusText = (
|
||||
& $Cm95Pbf status-production `
|
||||
--plan $Plan `
|
||||
--release-root ".\build\releases" |
|
||||
Out-String
|
||||
).Trim()
|
||||
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "status-production failed"
|
||||
}
|
||||
|
||||
Write-Host $StatusText
|
||||
|
||||
$S = $StatusText | ConvertFrom-Json
|
||||
|
||||
$Pass = StateCount $S.states "PASS"
|
||||
$Fail = StateCount $S.states "FAILED"
|
||||
$Pending = StateCount $S.states "PENDING"
|
||||
$Running = StateCount $S.states "RUNNING"
|
||||
|
||||
if ($Pass -ne [int]$S.jobs -or
|
||||
$Fail -ne 0 -or
|
||||
$Pending -ne 0 -or
|
||||
$Running -ne 0) {
|
||||
throw "Production state gate failed"
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# 10. Final release gate
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
Run "STEP 10 - Finalize production release" {
|
||||
& $Cm95Pbf finalize-production `
|
||||
--plan $Plan `
|
||||
--release-root ".\build\releases" `
|
||||
--style ".\config\style\style.navsea-delivery-full-semantic-unified-20260808-cm95-v1.json" `
|
||||
--sprite-json ".\config\sprite\semantic-sprite-20260808.json" `
|
||||
--gap-disposition $Disposition
|
||||
}
|
||||
|
||||
$ReleaseManifest = ".\build\releases\$ReleaseId\release_manifest_v1.json"
|
||||
|
||||
if (-not (Test-Path $ReleaseManifest)) {
|
||||
throw "Release manifest missing: $ReleaseManifest"
|
||||
}
|
||||
|
||||
$Final = Get-Content $ReleaseManifest -Raw | ConvertFrom-Json
|
||||
|
||||
if ([string]$Final.status -ne "PASS") {
|
||||
throw "Final release status is not PASS"
|
||||
}
|
||||
|
||||
Step "JAPAN CONTINUATION COMPLETE / PASS"
|
||||
|
||||
Write-Host "Dataset ID : $DatasetId"
|
||||
Write-Host "Release ID : $ReleaseId"
|
||||
Write-Host "Jobs : $($S.jobs)"
|
||||
Write-Host "PBF files : $($S.pbf_files)"
|
||||
Write-Host "Output features : $($S.output_features)"
|
||||
Write-Host "Semantic status : $($Final.semantic_completeness)"
|
||||
Write-Host "Pending variants : 137"
|
||||
Write-Host "Pending rows : 90009"
|
||||
Write-Host ""
|
||||
Write-Host "Web test:"
|
||||
Write-Host " cd `"$ProjectRoot`""
|
||||
Write-Host " & `".\.venv\Scripts\python.exe`" .\web\cm95_web.py --release-id `"$ReleaseId`""
|
||||
946
scripts/cm95_japan_full_rebuild_pipeline.ps1
Normal file
946
scripts/cm95_japan_full_rebuild_pipeline.ps1
Normal file
@@ -0,0 +1,946 @@
|
||||
param(
|
||||
[string]$ProjectRoot = "D:\React\cm93-mysql-importer-starter",
|
||||
[string]$SourceRoot = "D:\Ship\CM93V2\CM93JAPAN",
|
||||
[int]$ExpectedCells = 1492,
|
||||
[int]$Workers = 4,
|
||||
[switch]$Force
|
||||
)
|
||||
|
||||
# CM95 Japan Full Rebuild Pipeline v2
|
||||
#
|
||||
# 从零重建：
|
||||
# CM93 source -> fresh MySQL -> audit -> mapping gates
|
||||
# -> fresh FID manifest -> fresh production plan
|
||||
# -> full PBF build -> final release gate
|
||||
#
|
||||
# Windows 11 / PowerShell
|
||||
#
|
||||
# 破坏性范围：
|
||||
# - DROP .env 中 CM93_DB_NAME 整个数据库
|
||||
# - 删除旧 build\production
|
||||
# - 删除旧 build\releases
|
||||
# - 删除旧 build\pbf
|
||||
# - 删除旧 build\pbf-reports
|
||||
# - 删除旧 build\cm95_fid_manifest*.sqlite*
|
||||
#
|
||||
# 不会删除：
|
||||
# - D:\Ship\CM93V2\CM93JAPAN
|
||||
# - src\
|
||||
# - config\
|
||||
# - web\
|
||||
# - logs\ 历史文件
|
||||
#
|
||||
# v2 修正：
|
||||
# - 禁止 python.exe -c 多行代码，避免 Windows/PowerShell 引号损坏
|
||||
# - 临时 Python helper 一律落盘执行
|
||||
# - audit JSON gate 对齐当前 cm93db AuditResult
|
||||
# - geometry_validity_issues 保持 informational
|
||||
# - status JSON 使用 Out-String 后再 ConvertFrom-Json
|
||||
# - 生成当前 Dataset 专属 semantic disposition summary
|
||||
#
|
||||
# 如果新 Dataset 出现未分类 mapping gap，会在 PBF build 前 STOP。
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function Write-Step {
|
||||
param([string]$Text)
|
||||
Write-Host ""
|
||||
Write-Host "======================================================================"
|
||||
Write-Host $Text
|
||||
Write-Host "======================================================================"
|
||||
}
|
||||
|
||||
function Invoke-Checked {
|
||||
param(
|
||||
[string]$Label,
|
||||
[scriptblock]$Action
|
||||
)
|
||||
|
||||
Write-Step $Label
|
||||
|
||||
& $Action
|
||||
$ExitCode = $LASTEXITCODE
|
||||
|
||||
if ($null -ne $ExitCode -and $ExitCode -ne 0) {
|
||||
throw "$Label FAILED (exit code=$ExitCode)"
|
||||
}
|
||||
}
|
||||
|
||||
function Remove-PathIfExists {
|
||||
param([string]$PathValue)
|
||||
|
||||
if (Test-Path $PathValue) {
|
||||
Write-Host "DELETE: $PathValue"
|
||||
Remove-Item $PathValue -Recurse -Force
|
||||
}
|
||||
}
|
||||
|
||||
function Get-StateCount {
|
||||
param(
|
||||
[object]$States,
|
||||
[string]$Name
|
||||
)
|
||||
|
||||
if ($null -eq $States) {
|
||||
return 0
|
||||
}
|
||||
|
||||
$Prop = $States.PSObject.Properties[$Name]
|
||||
if ($null -eq $Prop) {
|
||||
return 0
|
||||
}
|
||||
|
||||
return [int]$Prop.Value
|
||||
}
|
||||
|
||||
if (-not (Test-Path $ProjectRoot -PathType Container)) {
|
||||
throw "ProjectRoot 不存在: $ProjectRoot"
|
||||
}
|
||||
|
||||
if (-not (Test-Path $SourceRoot -PathType Container)) {
|
||||
throw "SourceRoot 不存在: $SourceRoot"
|
||||
}
|
||||
|
||||
Set-Location $ProjectRoot
|
||||
|
||||
$ProjectRoot = (Resolve-Path $ProjectRoot).Path
|
||||
$SourceRoot = (Resolve-Path $SourceRoot).Path
|
||||
|
||||
$Python = Join-Path $ProjectRoot ".venv\Scripts\python.exe"
|
||||
$Cm93Db = Join-Path $ProjectRoot ".venv\Scripts\cm93db.exe"
|
||||
$Cm95Pbf = Join-Path $ProjectRoot ".venv\Scripts\cm95pbf.exe"
|
||||
$EnvPath = Join-Path $ProjectRoot ".env"
|
||||
|
||||
foreach ($RequiredExe in @($Python, $Cm93Db, $Cm95Pbf)) {
|
||||
if (-not (Test-Path $RequiredExe -PathType Leaf)) {
|
||||
throw "缺少运行文件: $RequiredExe"
|
||||
}
|
||||
}
|
||||
|
||||
if (-not (Test-Path $EnvPath -PathType Leaf)) {
|
||||
throw "缺少项目 .env: $EnvPath"
|
||||
}
|
||||
|
||||
# 临时 helper 放系统 TEMP，不受 build 清理影响。
|
||||
$TempRoot = Join-Path $env:TEMP ("cm95_japan_pipeline_v2_" + $PID)
|
||||
|
||||
Remove-Item $TempRoot -Recurse -Force -ErrorAction SilentlyContinue
|
||||
New-Item -ItemType Directory -Path $TempRoot -Force | Out-Null
|
||||
|
||||
try {
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# STEP 0 - Source hard gate
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
Write-Step "STEP 0 - Japan CM93 源目录检查"
|
||||
|
||||
$RequiredRegions = @(
|
||||
"03300360",
|
||||
"03300420",
|
||||
"03900360",
|
||||
"03900420"
|
||||
)
|
||||
|
||||
$ActualRegions = @(
|
||||
Get-ChildItem $SourceRoot -Directory -ErrorAction Stop |
|
||||
Where-Object { $_.Name -match '^\d{8}$' } |
|
||||
ForEach-Object { $_.Name } |
|
||||
Sort-Object
|
||||
)
|
||||
|
||||
$MissingRegions = @(
|
||||
$RequiredRegions |
|
||||
Where-Object { $_ -notin $ActualRegions }
|
||||
)
|
||||
|
||||
$ExtraRegions = @(
|
||||
$ActualRegions |
|
||||
Where-Object { $_ -notin $RequiredRegions }
|
||||
)
|
||||
|
||||
Write-Host "SourceRoot:"
|
||||
Write-Host " $SourceRoot"
|
||||
Write-Host ""
|
||||
Write-Host "Numeric regions:"
|
||||
$ActualRegions | ForEach-Object { Write-Host " $_" }
|
||||
|
||||
if ($MissingRegions.Count -gt 0) {
|
||||
throw "缺少 Japan 必需区域: $($MissingRegions -join ', ')"
|
||||
}
|
||||
|
||||
if ($ExtraRegions.Count -gt 0) {
|
||||
throw "发现 Japan 工作集之外的数字区域，停止: $($ExtraRegions -join ', ')"
|
||||
}
|
||||
|
||||
if (-not (Test-Path (Join-Path $SourceRoot "CM93OBJ.DIC"))) {
|
||||
throw "缺少 CM93OBJ.DIC"
|
||||
}
|
||||
|
||||
$HasAttrDic =
|
||||
(Test-Path (Join-Path $SourceRoot "CM93ATTR.DIC")) -or
|
||||
(Test-Path (Join-Path $SourceRoot "ATTRLUT.DIC"))
|
||||
|
||||
if (-not $HasAttrDic) {
|
||||
throw "缺少 CM93ATTR.DIC / ATTRLUT.DIC"
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "Japan source region gate: PASS"
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# STEP 1 - Current code baseline + read-only checks
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
Invoke-Checked "STEP 1A - pip install -e ." {
|
||||
& $Python -m pip install -e .
|
||||
}
|
||||
|
||||
if (-not (Test-Path ".\tools\cm95_v1_4_5_preflight.py")) {
|
||||
throw "缺少 tools\cm95_v1_4_5_preflight.py；当前项目不是 v1.4.5 baseline"
|
||||
}
|
||||
|
||||
Invoke-Checked "STEP 1B - CM95 v1.4.5 preflight" {
|
||||
& $Python ".\tools\cm95_v1_4_5_preflight.py"
|
||||
}
|
||||
|
||||
Invoke-Checked "STEP 1C - CM93 scan" {
|
||||
& $Cm93Db scan $SourceRoot
|
||||
}
|
||||
|
||||
Invoke-Checked "STEP 1D - CM93 dictionaries" {
|
||||
& $Cm93Db inspect-dicts $SourceRoot
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# STEP 2 - Read DB config with a real helper .py file
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
$ReadDbInfoPy = Join-Path $TempRoot "read_db_info.py"
|
||||
|
||||
@'
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
env_path = Path(sys.argv[1]).resolve()
|
||||
load_dotenv(dotenv_path=env_path)
|
||||
|
||||
result = {
|
||||
"host": os.getenv("CM93_DB_HOST", "127.0.0.1"),
|
||||
"port": int(os.getenv("CM93_DB_PORT", "3306")),
|
||||
"database": (os.getenv("CM93_DB_NAME", "cm93_chart") or "").strip(),
|
||||
"user": os.getenv("CM93_DB_USER", "root"),
|
||||
"charset": os.getenv("CM93_DB_CHARSET", "utf8mb4"),
|
||||
}
|
||||
|
||||
if not result["database"]:
|
||||
raise SystemExit("CM93_DB_NAME is empty")
|
||||
|
||||
print(json.dumps(result, ensure_ascii=False))
|
||||
'@ | Set-Content $ReadDbInfoPy -Encoding UTF8
|
||||
|
||||
$DbInfoText = (& $Python $ReadDbInfoPy $EnvPath | Out-String).Trim()
|
||||
|
||||
if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($DbInfoText)) {
|
||||
throw "无法读取数据库配置"
|
||||
}
|
||||
|
||||
$DbInfo = $DbInfoText | ConvertFrom-Json
|
||||
$DbName = [string]$DbInfo.database
|
||||
|
||||
$ForbiddenDbNames = @(
|
||||
"mysql",
|
||||
"information_schema",
|
||||
"performance_schema",
|
||||
"sys"
|
||||
)
|
||||
|
||||
if ($DbName.ToLowerInvariant() -in $ForbiddenDbNames) {
|
||||
throw "拒绝删除系统数据库: $DbName"
|
||||
}
|
||||
|
||||
Write-Step "STEP 2 - DESTRUCTIVE RESET CONFIRMATION"
|
||||
|
||||
Write-Host "将永久删除整个 MySQL database:"
|
||||
Write-Host " $DbName"
|
||||
Write-Host ""
|
||||
Write-Host "数据库服务器:"
|
||||
Write-Host " $($DbInfo.host):$($DbInfo.port)"
|
||||
Write-Host ""
|
||||
Write-Host "也会删除旧 CM95 build/release 产物。"
|
||||
Write-Host ""
|
||||
Write-Host "不会删除 CM93 source:"
|
||||
Write-Host " $SourceRoot"
|
||||
|
||||
if (-not $Force) {
|
||||
$Expected = "DELETE $DbName"
|
||||
$Answer = Read-Host "请输入 [$Expected] 继续"
|
||||
|
||||
if ($Answer -ne $Expected) {
|
||||
throw "用户取消。"
|
||||
}
|
||||
}
|
||||
else {
|
||||
Write-Host "-Force 已指定，跳过交互确认。"
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# STEP 3 - Delete old generated artifacts
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
Write-Step "STEP 3 - 删除旧 CM95 build/release 产物"
|
||||
|
||||
Remove-PathIfExists (Join-Path $ProjectRoot "build\production")
|
||||
Remove-PathIfExists (Join-Path $ProjectRoot "build\releases")
|
||||
Remove-PathIfExists (Join-Path $ProjectRoot "build\pbf")
|
||||
Remove-PathIfExists (Join-Path $ProjectRoot "build\pbf-reports")
|
||||
|
||||
$BuildRoot = Join-Path $ProjectRoot "build"
|
||||
|
||||
if (Test-Path $BuildRoot) {
|
||||
Get-ChildItem $BuildRoot -File -Filter "cm95_fid_manifest*.sqlite*" -ErrorAction SilentlyContinue |
|
||||
ForEach-Object {
|
||||
Write-Host "DELETE: $($_.FullName)"
|
||||
Remove-Item $_.FullName -Force
|
||||
}
|
||||
}
|
||||
|
||||
New-Item -ItemType Directory -Force -Path ".\build\production" | Out-Null
|
||||
New-Item -ItemType Directory -Force -Path ".\logs" | Out-Null
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# STEP 4 - DROP database using helper file, then let cm93db recreate schema
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
$DropDbPy = Join-Path $TempRoot "drop_cm93_database.py"
|
||||
|
||||
@'
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pymysql
|
||||
from dotenv import load_dotenv
|
||||
|
||||
env_path = Path(sys.argv[1]).resolve()
|
||||
load_dotenv(dotenv_path=env_path)
|
||||
|
||||
host = os.getenv("CM93_DB_HOST", "127.0.0.1")
|
||||
port = int(os.getenv("CM93_DB_PORT", "3306"))
|
||||
user = os.getenv("CM93_DB_USER", "root")
|
||||
password = os.getenv("CM93_DB_PASSWORD", "")
|
||||
database = (os.getenv("CM93_DB_NAME", "cm93_chart") or "").strip()
|
||||
charset = os.getenv("CM93_DB_CHARSET", "utf8mb4")
|
||||
|
||||
if not database:
|
||||
raise RuntimeError("CM93_DB_NAME is empty")
|
||||
|
||||
if database.lower() in {
|
||||
"mysql",
|
||||
"information_schema",
|
||||
"performance_schema",
|
||||
"sys",
|
||||
}:
|
||||
raise RuntimeError(f"refuse to drop system database: {database}")
|
||||
|
||||
conn = pymysql.connect(
|
||||
host=host,
|
||||
port=port,
|
||||
user=user,
|
||||
password=password,
|
||||
charset=charset,
|
||||
autocommit=False,
|
||||
cursorclass=pymysql.cursors.DictCursor,
|
||||
)
|
||||
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT VERSION() AS v")
|
||||
version = str(cur.fetchone()["v"])
|
||||
|
||||
match = re.match(r"^(\d+)\.(\d+)\.(\d+)", version)
|
||||
if not match or tuple(map(int, match.groups()[:2])) != (5, 7):
|
||||
raise RuntimeError(
|
||||
f"expected MySQL 5.7.x, got {version}; refusing destructive reset"
|
||||
)
|
||||
|
||||
safe = database.replace("`", "``")
|
||||
cur.execute(f"DROP DATABASE IF EXISTS `{safe}`")
|
||||
|
||||
conn.commit()
|
||||
print(f"DROP_DATABASE_OK database={database} mysql={version}")
|
||||
|
||||
finally:
|
||||
conn.close()
|
||||
'@ | Set-Content $DropDbPy -Encoding UTF8
|
||||
|
||||
Invoke-Checked "STEP 4A - DROP old database" {
|
||||
& $Python $DropDbPy $EnvPath
|
||||
}
|
||||
|
||||
Invoke-Checked "STEP 4B - CREATE database + initialize schema" {
|
||||
& $Cm93Db db-init --create-database
|
||||
}
|
||||
|
||||
Invoke-Checked "STEP 4C - DB contract check" {
|
||||
& $Cm93Db db-check
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# STEP 5 - Full import on fresh DB
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
Invoke-Checked "STEP 5 - 全量 CM93 Japan import（fresh DB）" {
|
||||
& $Cm93Db import $SourceRoot --fail-fast
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# STEP 6 - Full audit
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
$Timestamp = Get-Date -Format "yyyyMMdd_HHmmss"
|
||||
$AuditReport = ".\logs\cm93_audit_japan_full_rebuild_$Timestamp.json"
|
||||
|
||||
Invoke-Checked "STEP 6 - CM93 full audit + geometry validation" {
|
||||
& $Cm93Db audit `
|
||||
$SourceRoot `
|
||||
--validate-geometry `
|
||||
--report $AuditReport
|
||||
}
|
||||
|
||||
$Audit = Get-Content $AuditReport -Raw | ConvertFrom-Json
|
||||
|
||||
if (-not [bool]$Audit.passed) {
|
||||
throw "CM93 audit report passed=false；检查 $AuditReport"
|
||||
}
|
||||
|
||||
$AuditHardFailures = @(
|
||||
[int64]$Audit.source_parse_failed,
|
||||
[int64]$Audit.db_missing_cells,
|
||||
[int64]$Audit.db_not_done_cells,
|
||||
[int64]$Audit.count_mismatches,
|
||||
[int64]$Audit.hash_mismatches,
|
||||
[int64]$Audit.raw_preservation_mismatches,
|
||||
[int64]$Audit.unknown_objects,
|
||||
[int64]$Audit.unknown_attributes,
|
||||
[int64]$Audit.precision_failed
|
||||
)
|
||||
|
||||
if (($AuditHardFailures | Measure-Object -Sum).Sum -ne 0) {
|
||||
throw "CM93 audit hard gate failed；检查 $AuditReport"
|
||||
}
|
||||
|
||||
if ($ExpectedCells -gt 0 -and [int]$Audit.source_cells -ne $ExpectedCells) {
|
||||
throw "Source cell count 不符合当前 Japan baseline: $($Audit.source_cells) != $ExpectedCells"
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "CM93 AUDIT HARD GATE: PASS"
|
||||
Write-Host "Source cells : $($Audit.source_cells)"
|
||||
Write-Host "Geometry validity issues : $($Audit.geometry_validity_issues) (informational)"
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# STEP 7 - Resolve fresh Dataset
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
$DatasetInfoPy = Join-Path $TempRoot "dataset_info.py"
|
||||
|
||||
@'
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pymysql
|
||||
from dotenv import load_dotenv
|
||||
|
||||
env_path = Path(sys.argv[1]).resolve()
|
||||
source_root = str(Path(sys.argv[2]).resolve())
|
||||
|
||||
load_dotenv(dotenv_path=env_path)
|
||||
|
||||
conn = pymysql.connect(
|
||||
host=os.getenv("CM93_DB_HOST", "127.0.0.1"),
|
||||
port=int(os.getenv("CM93_DB_PORT", "3306")),
|
||||
user=os.getenv("CM93_DB_USER", "root"),
|
||||
password=os.getenv("CM93_DB_PASSWORD", ""),
|
||||
database=os.getenv("CM93_DB_NAME", "cm93_chart"),
|
||||
charset=os.getenv("CM93_DB_CHARSET", "utf8mb4"),
|
||||
cursorclass=pymysql.cursors.DictCursor,
|
||||
)
|
||||
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT COUNT(*) AS n FROM chart_dataset")
|
||||
dataset_count = int(cur.fetchone()["n"])
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
d.id,
|
||||
d.name,
|
||||
d.source_root,
|
||||
COUNT(c.id) AS cells,
|
||||
SUM(c.import_status='DONE') AS done_cells,
|
||||
SUM(c.import_status='FAILED') AS failed_cells
|
||||
FROM chart_dataset d
|
||||
LEFT JOIN chart_cell c
|
||||
ON c.dataset_id=d.id
|
||||
WHERE d.source_type='CM93'
|
||||
AND d.source_root=%s
|
||||
GROUP BY d.id,d.name,d.source_root
|
||||
""",
|
||||
(source_root,),
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
|
||||
if dataset_count != 1:
|
||||
raise RuntimeError(
|
||||
f"fresh DB expected exactly 1 dataset, got {dataset_count}"
|
||||
)
|
||||
|
||||
if len(rows) != 1:
|
||||
raise RuntimeError(
|
||||
f"expected one dataset for source_root={source_root}, got {len(rows)}"
|
||||
)
|
||||
|
||||
row = rows[0]
|
||||
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"id": int(row["id"]),
|
||||
"name": row["name"],
|
||||
"source_root": row["source_root"],
|
||||
"cells": int(row["cells"] or 0),
|
||||
"done_cells": int(row["done_cells"] or 0),
|
||||
"failed_cells": int(row["failed_cells"] or 0),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
)
|
||||
|
||||
finally:
|
||||
conn.close()
|
||||
'@ | Set-Content $DatasetInfoPy -Encoding UTF8
|
||||
|
||||
$DatasetText = (& $Python $DatasetInfoPy $EnvPath $SourceRoot | Out-String).Trim()
|
||||
|
||||
if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($DatasetText)) {
|
||||
throw "无法读取 fresh Dataset 信息"
|
||||
}
|
||||
|
||||
$Dataset = $DatasetText | ConvertFrom-Json
|
||||
$DatasetId = [int]$Dataset.id
|
||||
|
||||
if ([int]$Dataset.cells -ne [int]$Audit.source_cells) {
|
||||
throw "DB cell count != audit source_cells ($($Dataset.cells) != $($Audit.source_cells))"
|
||||
}
|
||||
|
||||
if ([int]$Dataset.done_cells -ne [int]$Audit.source_cells) {
|
||||
throw "不是所有 Cell 都 DONE"
|
||||
}
|
||||
|
||||
if ([int]$Dataset.failed_cells -ne 0) {
|
||||
throw "存在 FAILED Cell"
|
||||
}
|
||||
|
||||
Write-Step "STEP 7 - Fresh Dataset confirmed"
|
||||
|
||||
Write-Host "Dataset ID : $DatasetId"
|
||||
Write-Host "Name : $($Dataset.name)"
|
||||
Write-Host "SourceRoot : $($Dataset.source_root)"
|
||||
Write-Host "Cells : $($Dataset.cells)"
|
||||
Write-Host "DONE : $($Dataset.done_cells)"
|
||||
Write-Host "FAILED : $($Dataset.failed_cells)"
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# STEP 8 - Dataset-aware mapping gates
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
$GapAudit = ".\build\production\cm95_global_mapping_gap_audit_dataset_${DatasetId}.json"
|
||||
$GapClosure = ".\build\production\cm95_global_mapping_gap_closure_dataset_${DatasetId}.json"
|
||||
$MappingContract = ".\build\production\cm95_mapping_contract_check_dataset_${DatasetId}.json"
|
||||
|
||||
Invoke-Checked "STEP 8A - Global mapping gap audit (fresh Dataset)" {
|
||||
& $Python ".\tools\cm95_global_mapping_gap_audit_v1.py" `
|
||||
--dataset-id $DatasetId `
|
||||
--mapping ".\config\mappings\cm95_to_navsea_v1.yaml" `
|
||||
--output $GapAudit
|
||||
}
|
||||
|
||||
Invoke-Checked "STEP 8B - Mapping gap classification closure" {
|
||||
& $Python ".\tools\cm95_global_mapping_gap_closure_audit_v2.py" `
|
||||
--gap-audit $GapAudit `
|
||||
--mapping ".\config\mappings\cm95_to_navsea_v1.yaml" `
|
||||
--disposition ".\config\mappings\cm95_gap_disposition_v1.yaml" `
|
||||
--output $GapClosure
|
||||
}
|
||||
|
||||
$Closure = Get-Content $GapClosure -Raw | ConvertFrom-Json
|
||||
|
||||
if ([string]$Closure.overall -ne "PASS") {
|
||||
Write-Host ""
|
||||
Write-Host "NEW DATASET MAPPING GATE STOPPED PIPELINE."
|
||||
Write-Host "新 Japan Dataset 有当前 disposition 尚未收口的 gap variant。"
|
||||
Write-Host ""
|
||||
Write-Host "检查:"
|
||||
Write-Host " $GapAudit"
|
||||
Write-Host " $GapClosure"
|
||||
throw "Mapping gap closure FAIL；PBF generation intentionally stopped."
|
||||
}
|
||||
|
||||
Invoke-Checked "STEP 8C - Mapping / Style / Sprite static contract" {
|
||||
& $Python ".\tools\cm95_mapping_contract_check_v1.py" `
|
||||
--mapping ".\config\mappings\cm95_to_navsea_v1.yaml" `
|
||||
--style ".\config\style\style.navsea-delivery-full-semantic-unified-20260808-cm95-v1.json" `
|
||||
--sprite-json ".\config\sprite\semantic-sprite-20260808.json" `
|
||||
--output $MappingContract
|
||||
}
|
||||
|
||||
$MapContract = Get-Content $MappingContract -Raw | ConvertFrom-Json
|
||||
|
||||
if ([string]$MapContract.overall -ne "PASS") {
|
||||
throw "Mapping / Style / Sprite static contract FAIL"
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# STEP 8D - Build Dataset-specific effective gap disposition
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
$EffectiveDisposition = ".\build\production\cm95_gap_disposition_dataset_${DatasetId}_effective.yaml"
|
||||
$EffectiveDispositionPy = Join-Path $TempRoot "build_effective_disposition.py"
|
||||
|
||||
@'
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
template_path = Path(sys.argv[1])
|
||||
gap_path = Path(sys.argv[2])
|
||||
closure_path = Path(sys.argv[3])
|
||||
dataset_id = int(sys.argv[4])
|
||||
out_path = Path(sys.argv[5])
|
||||
|
||||
template = yaml.safe_load(template_path.read_text(encoding="utf-8")) or {}
|
||||
gap = yaml.safe_load(gap_path.read_text(encoding="utf-8")) or {}
|
||||
closure = yaml.safe_load(closure_path.read_text(encoding="utf-8")) or {}
|
||||
|
||||
template_by_key = {
|
||||
(
|
||||
str(v.get("object_class") or ""),
|
||||
str(v.get("geometry_type") or "").upper(),
|
||||
): v
|
||||
for v in template.get("variants") or []
|
||||
if isinstance(v, dict)
|
||||
}
|
||||
|
||||
effective_variants = []
|
||||
|
||||
for item in gap.get("gap_variants") or []:
|
||||
key = (
|
||||
str(item.get("object_class") or ""),
|
||||
str(item.get("geometry_type") or "").upper(),
|
||||
)
|
||||
|
||||
src = template_by_key.get(key)
|
||||
if src is None:
|
||||
raise RuntimeError(
|
||||
f"unclassified gap variant while building effective disposition: {key}"
|
||||
)
|
||||
|
||||
row = copy.deepcopy(src)
|
||||
row["object_class"] = key[0]
|
||||
row["geometry_type"] = key[1]
|
||||
row["feature_count"] = int(item.get("feature_count") or 0)
|
||||
|
||||
if item.get("object_class_id") is not None:
|
||||
row["object_class_id"] = item.get("object_class_id")
|
||||
|
||||
if item.get("dictionary_description") is not None:
|
||||
row["dictionary_description"] = item.get("dictionary_description")
|
||||
|
||||
if item.get("gap_reason") is not None:
|
||||
row["gap_reason_v1"] = item.get("gap_reason")
|
||||
|
||||
effective_variants.append(row)
|
||||
|
||||
pending = [
|
||||
v for v in effective_variants
|
||||
if str(v.get("disposition") or "")
|
||||
== "ENGINEERING_ONLY_PENDING_SEMANTIC_REVIEW"
|
||||
]
|
||||
|
||||
delivery = [
|
||||
v for v in effective_variants
|
||||
if str(v.get("disposition") or "")
|
||||
== "DELIVERY_MAPPED_V1_4_3"
|
||||
]
|
||||
|
||||
existing = [
|
||||
v for v in effective_variants
|
||||
if str(v.get("disposition") or "").startswith("EXISTING_")
|
||||
]
|
||||
|
||||
result = {
|
||||
"schema": "navsea.cm95.gap-disposition/v1",
|
||||
"version": "v1.4.5-dataset-effective",
|
||||
"dataset_id": dataset_id,
|
||||
"source_audit": str(gap_path),
|
||||
"policy": (
|
||||
"Dataset-specific effective disposition generated from "
|
||||
"cm95_gap_disposition_v1.yaml after current-dataset closure PASS."
|
||||
),
|
||||
"summary": {
|
||||
"observed_gap_variants": len(effective_variants),
|
||||
"delivery_mapped_v1_4_3": len(delivery),
|
||||
"existing_explicit_non_delivery": len(existing),
|
||||
"pending_semantic_review": len(pending),
|
||||
"delivery_mapped_v1_4_3_feature_rows": sum(
|
||||
int(v.get("feature_count") or 0) for v in delivery
|
||||
),
|
||||
"existing_explicit_non_delivery_feature_rows": sum(
|
||||
int(v.get("feature_count") or 0) for v in existing
|
||||
),
|
||||
"pending_semantic_review_feature_rows": sum(
|
||||
int(v.get("feature_count") or 0) for v in pending
|
||||
),
|
||||
},
|
||||
"variants": effective_variants,
|
||||
}
|
||||
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
out_path.write_text(
|
||||
yaml.safe_dump(
|
||||
result,
|
||||
allow_unicode=True,
|
||||
sort_keys=False,
|
||||
width=120,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
print(f"Effective disposition: {out_path}")
|
||||
print(f"Observed gap variants : {len(effective_variants)}")
|
||||
print(f"Pending review : {len(pending)}")
|
||||
print(
|
||||
"Pending rows : "
|
||||
f"{result['summary']['pending_semantic_review_feature_rows']}"
|
||||
)
|
||||
'@ | Set-Content $EffectiveDispositionPy -Encoding UTF8
|
||||
|
||||
Invoke-Checked "STEP 8D - Dataset-specific semantic disposition summary" {
|
||||
& $Python $EffectiveDispositionPy `
|
||||
".\config\mappings\cm95_gap_disposition_v1.yaml" `
|
||||
$GapAudit `
|
||||
$GapClosure `
|
||||
$DatasetId `
|
||||
$EffectiveDisposition
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# STEP 9 - Fresh deterministic FID manifest
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
$Manifest = ".\build\cm95_fid_manifest_dataset_${DatasetId}_v1.sqlite"
|
||||
|
||||
Invoke-Checked "STEP 9 - Rebuild FID manifest from zero" {
|
||||
& $Cm95Pbf build-fid-manifest `
|
||||
--dataset-id $DatasetId `
|
||||
--manifest $Manifest `
|
||||
--rebuild-fid-manifest
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# STEP 10 - Fresh immutable production plan
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
$Plan = ".\build\production\cm95_production_plan_dataset_${DatasetId}_v1_4_5.json"
|
||||
$PlanAudit = ".\build\production\cm95_production_plan_audit_dataset_${DatasetId}_v1_4_5.json"
|
||||
|
||||
Invoke-Checked "STEP 10A - plan-production z0-z12" {
|
||||
& $Cm95Pbf plan-production `
|
||||
--dataset-id $DatasetId `
|
||||
--zooms 0-12 `
|
||||
--profile delivery `
|
||||
--manifest $Manifest `
|
||||
--output $Plan
|
||||
}
|
||||
|
||||
Invoke-Checked "STEP 10B - production plan audit" {
|
||||
& $Python ".\tools\cm95_production_plan_audit_v1.py" `
|
||||
--plan $Plan `
|
||||
--expected-zooms 0-12 `
|
||||
--output $PlanAudit
|
||||
}
|
||||
|
||||
$PlanObj = Get-Content $Plan -Raw | ConvertFrom-Json
|
||||
$PlanAuditObj = Get-Content $PlanAudit -Raw | ConvertFrom-Json
|
||||
|
||||
if ([string]$PlanAuditObj.overall -ne "PASS") {
|
||||
throw "Production plan audit FAIL"
|
||||
}
|
||||
|
||||
$ReleaseId = [string]$PlanObj.release_id
|
||||
|
||||
if ([string]::IsNullOrWhiteSpace($ReleaseId)) {
|
||||
throw "Plan 没有 release_id"
|
||||
}
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "Plan ID : $($PlanObj.plan_id)"
|
||||
Write-Host "Release ID : $ReleaseId"
|
||||
Write-Host "Jobs : $($PlanObj.job_count)"
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# STEP 11 - Full PBF build
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
Invoke-Checked "STEP 11 - build-production FULL" {
|
||||
& $Cm95Pbf build-production `
|
||||
--plan $Plan `
|
||||
--release-root ".\build\releases" `
|
||||
--workers $Workers `
|
||||
--max-attempts 3
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# STEP 12 - Production status
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
Write-Step "STEP 12 - Production status"
|
||||
|
||||
$StatusText = (
|
||||
& $Cm95Pbf status-production `
|
||||
--plan $Plan `
|
||||
--release-root ".\build\releases" |
|
||||
Out-String
|
||||
).Trim()
|
||||
|
||||
if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($StatusText)) {
|
||||
throw "status-production FAILED"
|
||||
}
|
||||
|
||||
Write-Host $StatusText
|
||||
|
||||
$Status = $StatusText | ConvertFrom-Json
|
||||
$States = $Status.states
|
||||
|
||||
$PassCount = Get-StateCount $States "PASS"
|
||||
$FailedCount = Get-StateCount $States "FAILED"
|
||||
$PendingCount = Get-StateCount $States "PENDING"
|
||||
$RunningCount = Get-StateCount $States "RUNNING"
|
||||
|
||||
if ($PassCount -ne [int]$Status.jobs -or
|
||||
$FailedCount -ne 0 -or
|
||||
$PendingCount -ne 0 -or
|
||||
$RunningCount -ne 0) {
|
||||
throw "Production state gate FAIL"
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# STEP 13 - Final release gate
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
Invoke-Checked "STEP 13 - finalize-production" {
|
||||
& $Cm95Pbf finalize-production `
|
||||
--plan $Plan `
|
||||
--release-root ".\build\releases" `
|
||||
--style ".\config\style\style.navsea-delivery-full-semantic-unified-20260808-cm95-v1.json" `
|
||||
--sprite-json ".\config\sprite\semantic-sprite-20260808.json" `
|
||||
--gap-disposition $EffectiveDisposition
|
||||
}
|
||||
|
||||
$ReleaseDir = Join-Path $ProjectRoot ("build\releases\" + $ReleaseId)
|
||||
$ReleaseManifest = Join-Path $ReleaseDir "release_manifest_v1.json"
|
||||
|
||||
if (-not (Test-Path $ReleaseManifest -PathType Leaf)) {
|
||||
throw "finalize 已运行，但 release manifest 不存在: $ReleaseManifest"
|
||||
}
|
||||
|
||||
$Final = Get-Content $ReleaseManifest -Raw | ConvertFrom-Json
|
||||
|
||||
if ([string]$Final.status -ne "PASS") {
|
||||
throw "Final release manifest status != PASS"
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# STEP 14 - Save compact baseline summary
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
$SummaryPath = ".\build\production\CM95_JAPAN_FULL_REBUILD_CURRENT.json"
|
||||
|
||||
$Summary = [ordered]@{
|
||||
schema = "navsea.cm95.japan-full-rebuild-pipeline/v2"
|
||||
completed_at = (Get-Date).ToString("o")
|
||||
project_root = $ProjectRoot
|
||||
source_root = $SourceRoot
|
||||
database = $DbName
|
||||
dataset_id = $DatasetId
|
||||
source_cells = [int]$Audit.source_cells
|
||||
source_geometry_validity_issues = [int]$Audit.geometry_validity_issues
|
||||
audit_report = $AuditReport
|
||||
mapping_gap_audit = $GapAudit
|
||||
mapping_gap_closure = [string]$Closure.overall
|
||||
effective_gap_disposition = $EffectiveDisposition
|
||||
semantic_completeness = [string]$Final.semantic_completeness
|
||||
semantic_backlog = $Final.semantic_backlog
|
||||
fid_manifest = $Manifest
|
||||
plan = $Plan
|
||||
plan_id = [string]$PlanObj.plan_id
|
||||
release_id = $ReleaseId
|
||||
release_dir = $ReleaseDir
|
||||
release_manifest = $ReleaseManifest
|
||||
jobs = [int]$Status.jobs
|
||||
pbf_files = [int]$Status.pbf_files
|
||||
output_features = [int64]$Status.output_features
|
||||
final_status = [string]$Final.status
|
||||
}
|
||||
|
||||
$Summary |
|
||||
ConvertTo-Json -Depth 12 |
|
||||
Set-Content $SummaryPath -Encoding UTF8
|
||||
|
||||
Write-Step "PIPELINE COMPLETE / PASS"
|
||||
|
||||
Write-Host "Database : $DbName"
|
||||
Write-Host "Dataset ID : $DatasetId"
|
||||
Write-Host "Source cells : $($Audit.source_cells)"
|
||||
Write-Host "Plan ID : $($PlanObj.plan_id)"
|
||||
Write-Host "Release ID : $ReleaseId"
|
||||
Write-Host "Jobs : $($Status.jobs) PASS"
|
||||
Write-Host "PBF files : $($Status.pbf_files)"
|
||||
Write-Host "Output features: $($Status.output_features)"
|
||||
Write-Host "Final status : $($Final.status)"
|
||||
Write-Host "Semantic : $($Final.semantic_completeness)"
|
||||
Write-Host ""
|
||||
Write-Host "Summary:"
|
||||
Write-Host " $SummaryPath"
|
||||
Write-Host ""
|
||||
Write-Host "Release:"
|
||||
Write-Host " $ReleaseDir"
|
||||
Write-Host ""
|
||||
Write-Host "Web test:"
|
||||
Write-Host " cd `"$ProjectRoot`""
|
||||
Write-Host " & `".\.venv\Scripts\python.exe`" .\web\cm95_web.py --release-id `"$ReleaseId`""
|
||||
|
||||
}
|
||||
finally {
|
||||
Remove-Item $TempRoot -Recurse -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
280
scripts/cm95_japan_v1_4_6_continue.ps1
Normal file
280
scripts/cm95_japan_v1_4_6_continue.ps1
Normal file
@@ -0,0 +1,280 @@
|
||||
param(
|
||||
[string]$ProjectRoot = "D:\React\cm93-mysql-importer-starter",
|
||||
[string]$SourceRoot = "D:\Ship\CM93V2\CM93JAPAN",
|
||||
[int]$DatasetId = 1,
|
||||
[int]$ExpectedCells = 1492,
|
||||
[int]$Workers = 4,
|
||||
[switch]$SkipQueryEquivalence
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function Step([string]$Text) {
|
||||
Write-Host ""
|
||||
Write-Host "======================================================================"
|
||||
Write-Host $Text
|
||||
Write-Host "======================================================================"
|
||||
}
|
||||
|
||||
function Run([string]$Label, [scriptblock]$Action) {
|
||||
Step $Label
|
||||
& $Action
|
||||
$Code = $LASTEXITCODE
|
||||
if ($null -ne $Code -and $Code -ne 0) {
|
||||
throw "$Label FAILED (exit=$Code)"
|
||||
}
|
||||
}
|
||||
|
||||
function StateCount($States, [string]$Name) {
|
||||
if ($null -eq $States) { return 0 }
|
||||
$P = $States.PSObject.Properties[$Name]
|
||||
if ($null -eq $P) { return 0 }
|
||||
return [int]$P.Value
|
||||
}
|
||||
|
||||
Set-Location $ProjectRoot
|
||||
|
||||
$Python = ".\.venv\Scripts\python.exe"
|
||||
$Cm95Pbf = ".\.venv\Scripts\cm95pbf.exe"
|
||||
$Mapping = ".\config\mappings\cm95_to_navsea_v1.yaml"
|
||||
$Disposition = ".\config\mappings\cm95_gap_disposition_japan_dataset_1_v1.yaml"
|
||||
$Policy = ".\config\pbf\cm95_tile_policy_japan_full_detail_v1.yaml"
|
||||
$Manifest = ".\build\cm95_fid_manifest_japan_dataset_1_v1.sqlite"
|
||||
$GapAudit = ".\build\production\cm95_global_mapping_gap_audit_dataset_1.json"
|
||||
$Closure = ".\build\production\cm95_global_mapping_gap_closure_dataset_1_v1_4_6.json"
|
||||
$MappingContract = ".\build\production\cm95_mapping_contract_check_japan_dataset_1_v1_4_6.json"
|
||||
$Plan = ".\build\production\cm95_production_plan_japan_full_detail_v1_4_6.json"
|
||||
$PlanAudit = ".\build\production\cm95_production_plan_audit_japan_full_detail_v1_4_6.json"
|
||||
|
||||
foreach ($Path in @($Python,$Cm95Pbf,$Mapping,$Disposition,$Policy)) {
|
||||
if (-not (Test-Path $Path)) {
|
||||
throw "Missing required file: $Path"
|
||||
}
|
||||
}
|
||||
|
||||
Run "STEP 0A - pip install v1.4.6 editable" {
|
||||
& $Python -m pip install -e .
|
||||
}
|
||||
|
||||
Run "STEP 0B - CM95 v1.4.6 preflight" {
|
||||
& $Python ".\tools\cm95_v1_4_6_preflight.py"
|
||||
}
|
||||
|
||||
# Verify the existing fresh Japan Dataset. No import / no DROP database.
|
||||
Step "STEP 1 - Verify existing Japan Dataset"
|
||||
|
||||
$Tmp = Join-Path $env:TEMP ("cm95_japan_v146_" + $PID)
|
||||
Remove-Item $Tmp -Recurse -Force -ErrorAction SilentlyContinue
|
||||
New-Item -ItemType Directory -Path $Tmp | Out-Null
|
||||
|
||||
try {
|
||||
$Helper = Join-Path $Tmp "verify_dataset.py"
|
||||
@'
|
||||
from pathlib import Path
|
||||
from dotenv import load_dotenv
|
||||
import os, sys, json, pymysql
|
||||
|
||||
project_root = Path(sys.argv[1]).resolve()
|
||||
dataset_id = int(sys.argv[2])
|
||||
expected_root = str(Path(sys.argv[3]).resolve())
|
||||
load_dotenv(dotenv_path=project_root / ".env")
|
||||
|
||||
conn = pymysql.connect(
|
||||
host=os.getenv("CM93_DB_HOST", "127.0.0.1"),
|
||||
port=int(os.getenv("CM93_DB_PORT", "3306")),
|
||||
user=os.getenv("CM93_DB_USER", "root"),
|
||||
password=os.getenv("CM93_DB_PASSWORD", ""),
|
||||
database=os.getenv("CM93_DB_NAME", "cm93_chart"),
|
||||
charset=os.getenv("CM93_DB_CHARSET", "utf8mb4"),
|
||||
cursorclass=pymysql.cursors.DictCursor,
|
||||
)
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT d.id,d.name,d.source_root,
|
||||
COUNT(c.id) cells,
|
||||
SUM(c.import_status='DONE') done_cells,
|
||||
SUM(c.import_status='FAILED') failed_cells
|
||||
FROM chart_dataset d
|
||||
LEFT JOIN chart_cell c ON c.dataset_id=d.id
|
||||
WHERE d.id=%s
|
||||
GROUP BY d.id,d.name,d.source_root
|
||||
""",
|
||||
(dataset_id,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
if not row:
|
||||
raise RuntimeError(f"dataset {dataset_id} not found")
|
||||
actual_root = str(Path(row["source_root"]).resolve())
|
||||
if actual_root.lower() != expected_root.lower():
|
||||
raise RuntimeError(f"source_root mismatch: {actual_root} != {expected_root}")
|
||||
print(json.dumps({
|
||||
"id": int(row["id"]),
|
||||
"name": row["name"],
|
||||
"source_root": actual_root,
|
||||
"cells": int(row["cells"] or 0),
|
||||
"done_cells": int(row["done_cells"] or 0),
|
||||
"failed_cells": int(row["failed_cells"] or 0),
|
||||
}, ensure_ascii=False))
|
||||
finally:
|
||||
conn.close()
|
||||
'@ | Set-Content $Helper -Encoding UTF8
|
||||
|
||||
$DatasetText = (& $Python $Helper $ProjectRoot $DatasetId $SourceRoot | Out-String).Trim()
|
||||
if ($LASTEXITCODE -ne 0) { throw "Dataset verification failed" }
|
||||
$Dataset = $DatasetText | ConvertFrom-Json
|
||||
if ([int]$Dataset.cells -ne $ExpectedCells -or
|
||||
[int]$Dataset.done_cells -ne $ExpectedCells -or
|
||||
[int]$Dataset.failed_cells -ne 0) {
|
||||
throw "Dataset state mismatch: $DatasetText"
|
||||
}
|
||||
Write-Host $DatasetText
|
||||
}
|
||||
finally {
|
||||
Remove-Item $Tmp -Recurse -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
|
||||
if (-not $SkipQueryEquivalence) {
|
||||
Run "STEP 2 - v1.4.5 vs v1.4.6 candidate query equivalence + SPATIAL plan" {
|
||||
& $Python ".\tools\cm95_v1_4_6_query_equivalence.py" `
|
||||
--dataset-id $DatasetId
|
||||
}
|
||||
}
|
||||
else {
|
||||
Step "STEP 2 - Query equivalence SKIPPED by user"
|
||||
}
|
||||
|
||||
Run "STEP 3A - Re-run mapping gap audit" {
|
||||
& $Python ".\tools\cm95_global_mapping_gap_audit_v1.py" `
|
||||
--dataset-id $DatasetId `
|
||||
--mapping $Mapping `
|
||||
--output $GapAudit
|
||||
}
|
||||
|
||||
Run "STEP 3B - Japan gap closure" {
|
||||
& $Python ".\tools\cm95_global_mapping_gap_closure_audit_v2.py" `
|
||||
--gap-audit $GapAudit `
|
||||
--mapping $Mapping `
|
||||
--disposition $Disposition `
|
||||
--output $Closure
|
||||
}
|
||||
|
||||
$C = Get-Content $Closure -Raw | ConvertFrom-Json
|
||||
if ([string]$C.overall -ne "PASS") { throw "Mapping gap closure != PASS" }
|
||||
if (@($C.unclassified_variants).Count -ne 0) { throw "Unclassified gap variants remain" }
|
||||
|
||||
Run "STEP 3C - Mapping / Style / Sprite contract" {
|
||||
& $Python ".\tools\cm95_mapping_contract_check_v1.py" `
|
||||
--mapping $Mapping `
|
||||
--style ".\config\style\style.navsea-delivery-full-semantic-unified-20260808-cm95-v1.json" `
|
||||
--sprite-json ".\config\sprite\semantic-sprite-20260808.json" `
|
||||
--output $MappingContract
|
||||
}
|
||||
|
||||
$MC = Get-Content $MappingContract -Raw | ConvertFrom-Json
|
||||
if ([string]$MC.overall -ne "PASS") { throw "Mapping/style/sprite contract != PASS" }
|
||||
|
||||
# FID manifest is independent from builder query implementation; reuse when valid.
|
||||
if (-not (Test-Path $Manifest)) {
|
||||
Run "STEP 4 - FID manifest missing; rebuild" {
|
||||
& $Cm95Pbf build-fid-manifest `
|
||||
--dataset-id $DatasetId `
|
||||
--manifest $Manifest `
|
||||
--rebuild-fid-manifest
|
||||
}
|
||||
}
|
||||
else {
|
||||
Step "STEP 4 - Reuse existing FID manifest"
|
||||
Write-Host $Manifest
|
||||
}
|
||||
|
||||
# v1.4.6 builder contract changes the immutable plan hash, so create a NEW plan/release.
|
||||
Run "STEP 5A - Create NEW v1.4.6 production plan" {
|
||||
& $Cm95Pbf plan-production `
|
||||
--dataset-id $DatasetId `
|
||||
--zooms 0-12 `
|
||||
--profile delivery `
|
||||
--mapping $Mapping `
|
||||
--policy $Policy `
|
||||
--manifest $Manifest `
|
||||
--output $Plan
|
||||
}
|
||||
|
||||
Run "STEP 5B - Audit v1.4.6 production plan" {
|
||||
& $Python ".\tools\cm95_production_plan_audit_v1.py" `
|
||||
--plan $Plan `
|
||||
--expected-zooms 0-12 `
|
||||
--output $PlanAudit
|
||||
}
|
||||
|
||||
$PA = Get-Content $PlanAudit -Raw | ConvertFrom-Json
|
||||
if ([string]$PA.overall -ne "PASS") { throw "Production plan audit != PASS" }
|
||||
|
||||
$PlanObj = Get-Content $Plan -Raw | ConvertFrom-Json
|
||||
if ([string]$PlanObj.builder_contract_version -ne "cm95pbf-builder/0.4.6-spatial-candidate-query") {
|
||||
throw "Wrong builder contract in plan: $($PlanObj.builder_contract_version)"
|
||||
}
|
||||
|
||||
$ReleaseId = [string]$PlanObj.release_id
|
||||
Write-Host ""
|
||||
Write-Host "Plan ID : $($PlanObj.plan_id)"
|
||||
Write-Host "Release ID : $ReleaseId"
|
||||
Write-Host "Jobs : $($PlanObj.job_count)"
|
||||
Write-Host "Workers : $Workers"
|
||||
|
||||
# Deliberately do NOT migrate PASS jobs from v1.4.5 partial release.
|
||||
# The new query must receive a clean production run for output equivalence/final gate.
|
||||
Run "STEP 6 - Full v1.4.6 PBF production build" {
|
||||
& $Cm95Pbf build-production `
|
||||
--plan $Plan `
|
||||
--release-root ".\build\releases" `
|
||||
--workers $Workers `
|
||||
--max-attempts 3
|
||||
}
|
||||
|
||||
Step "STEP 7 - Production status"
|
||||
$StatusText = (
|
||||
& $Cm95Pbf status-production `
|
||||
--plan $Plan `
|
||||
--release-root ".\build\releases" |
|
||||
Out-String
|
||||
).Trim()
|
||||
if ($LASTEXITCODE -ne 0) { throw "status-production failed" }
|
||||
Write-Host $StatusText
|
||||
$S = $StatusText | ConvertFrom-Json
|
||||
$Pass = StateCount $S.states "PASS"
|
||||
$Fail = StateCount $S.states "FAILED"
|
||||
$Pending = StateCount $S.states "PENDING"
|
||||
$Running = StateCount $S.states "RUNNING"
|
||||
if ($Pass -ne [int]$S.jobs -or $Fail -ne 0 -or $Pending -ne 0 -or $Running -ne 0) {
|
||||
throw "Production state gate failed"
|
||||
}
|
||||
|
||||
Run "STEP 8 - Finalize v1.4.6 release" {
|
||||
& $Cm95Pbf finalize-production `
|
||||
--plan $Plan `
|
||||
--release-root ".\build\releases" `
|
||||
--style ".\config\style\style.navsea-delivery-full-semantic-unified-20260808-cm95-v1.json" `
|
||||
--sprite-json ".\config\sprite\semantic-sprite-20260808.json" `
|
||||
--gap-disposition $Disposition
|
||||
}
|
||||
|
||||
$ReleaseManifest = ".\build\releases\$ReleaseId\release_manifest_v1.json"
|
||||
if (-not (Test-Path $ReleaseManifest)) { throw "Release manifest missing" }
|
||||
$Final = Get-Content $ReleaseManifest -Raw | ConvertFrom-Json
|
||||
if ([string]$Final.status -ne "PASS") { throw "Final release status != PASS" }
|
||||
|
||||
Step "CM95 JAPAN v1.4.6 COMPLETE / PASS"
|
||||
Write-Host "Dataset ID : $DatasetId"
|
||||
Write-Host "Release ID : $ReleaseId"
|
||||
Write-Host "Jobs : $($S.jobs)"
|
||||
Write-Host "PBF files : $($S.pbf_files)"
|
||||
Write-Host "Output features : $($S.output_features)"
|
||||
Write-Host "Final status : $($Final.status)"
|
||||
Write-Host "Semantic status : $($Final.semantic_completeness)"
|
||||
Write-Host ""
|
||||
Write-Host "Web test:"
|
||||
Write-Host " cd `"$ProjectRoot`""
|
||||
Write-Host " & `".\.venv\Scripts\python.exe`" .\web\cm95_web.py --release-id `"$ReleaseId`""
|
||||
153
scripts/cm95_japan_v1_4_7_finalize_existing.ps1
Normal file
153
scripts/cm95_japan_v1_4_7_finalize_existing.ps1
Normal file
@@ -0,0 +1,153 @@
|
||||
param(
|
||||
[string]$ProjectRoot = "D:\React\cm93-mysql-importer-starter"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function Step([string]$Text) {
|
||||
Write-Host ""
|
||||
Write-Host "======================================================================"
|
||||
Write-Host $Text
|
||||
Write-Host "======================================================================"
|
||||
}
|
||||
|
||||
function Run([string]$Label, [scriptblock]$Action) {
|
||||
Step $Label
|
||||
& $Action
|
||||
$Code = $LASTEXITCODE
|
||||
if ($null -ne $Code -and $Code -ne 0) {
|
||||
throw "$Label FAILED (exit=$Code)"
|
||||
}
|
||||
}
|
||||
|
||||
Set-Location $ProjectRoot
|
||||
|
||||
$Python = ".\.venv\Scripts\python.exe"
|
||||
$Cm95Pbf = ".\.venv\Scripts\cm95pbf.exe"
|
||||
$Plan = ".\build\production\cm95_production_plan_japan_full_detail_v1_4_6.json"
|
||||
$ReleaseRoot = ".\build\releases"
|
||||
$Disposition = ".\config\mappings\cm95_gap_disposition_japan_dataset_1_v1.yaml"
|
||||
$ExpectedPlanId = "6bc3c1352da347b77256c83d8ab3b771fcde151d70332e8d96c6e50605cb5d8f"
|
||||
$ExpectedReleaseId = "cm95-d1-62a85bd40d3a-6bc3c1352da3"
|
||||
$ExpectedBuilder = "cm95pbf-builder/0.4.6-spatial-candidate-query"
|
||||
|
||||
foreach ($Path in @($Python,$Cm95Pbf,$Plan,$Disposition)) {
|
||||
if (-not (Test-Path $Path)) {
|
||||
throw "Missing required path: $Path"
|
||||
}
|
||||
}
|
||||
|
||||
Run "STEP 0A - pip install v1.4.7 editable" {
|
||||
& $Python -m pip install -e .
|
||||
}
|
||||
|
||||
Run "STEP 0B - CM95 v1.4.7 preflight" {
|
||||
& $Python ".\tools\cm95_v1_4_7_preflight.py"
|
||||
}
|
||||
|
||||
Step "STEP 1 - Verify existing v1.4.6 plan / release identity"
|
||||
$PlanObj = Get-Content $Plan -Raw | ConvertFrom-Json
|
||||
if ([string]$PlanObj.plan_id -ne $ExpectedPlanId) {
|
||||
throw "Unexpected plan_id: $($PlanObj.plan_id)"
|
||||
}
|
||||
if ([string]$PlanObj.release_id -ne $ExpectedReleaseId) {
|
||||
throw "Unexpected release_id: $($PlanObj.release_id)"
|
||||
}
|
||||
if ([string]$PlanObj.builder_contract_version -ne $ExpectedBuilder) {
|
||||
throw "Unexpected builder contract: $($PlanObj.builder_contract_version)"
|
||||
}
|
||||
Write-Host "Plan ID : $($PlanObj.plan_id)"
|
||||
Write-Host "Release ID : $($PlanObj.release_id)"
|
||||
Write-Host "Builder : $($PlanObj.builder_contract_version)"
|
||||
Write-Host "PBF rebuild : NO"
|
||||
|
||||
Step "STEP 2 - Verify existing production is already 27038/27038 PASS"
|
||||
$StatusText = (
|
||||
& $Cm95Pbf status-production `
|
||||
--plan $Plan `
|
||||
--release-root $ReleaseRoot |
|
||||
Out-String
|
||||
).Trim()
|
||||
if ($LASTEXITCODE -ne 0) { throw "status-production failed" }
|
||||
Write-Host $StatusText
|
||||
$S = $StatusText | ConvertFrom-Json
|
||||
if ([int]$S.jobs -ne 27038 -or
|
||||
[int]$S.attempts -ne 27038 -or
|
||||
[int64]$S.output_features -ne 2974375 -or
|
||||
[int]$S.pbf_files -ne 27030 -or
|
||||
[int]$S.states.PASS -ne 27038) {
|
||||
throw "Existing production baseline does not match expected completed v1.4.6 release"
|
||||
}
|
||||
|
||||
$ReleaseDir = Join-Path $ReleaseRoot $ExpectedReleaseId
|
||||
$AuditJson = ".\build\production\cm95_nav_name_conflicts_v1_4_7_precheck.json"
|
||||
$AuditCsv = ".\build\production\cm95_nav_name_conflicts_v1_4_7_precheck.csv"
|
||||
|
||||
Run "STEP 3 - Re-audit the 10 trace occurrences / 4 unique ambiguous outputs" {
|
||||
& $Python ".\tools\cm95_nav_name_conflict_audit_v1.py" `
|
||||
--release-dir $ReleaseDir `
|
||||
--profile delivery `
|
||||
--output-json $AuditJson `
|
||||
--output-csv $AuditCsv
|
||||
}
|
||||
|
||||
$A = Get-Content $AuditJson -Raw | ConvertFrom-Json
|
||||
if ([int]$A.report_name_conflict_occurrences -ne 10 -or
|
||||
[int]$A.trace_name_conflict_occurrences -ne 10 -or
|
||||
[int]$A.unique_conflict_outputs -ne 4 -or
|
||||
[int]$A.affected_tiles -ne 3) {
|
||||
throw "Nav name conflict baseline changed unexpectedly"
|
||||
}
|
||||
|
||||
Run "STEP 4 - Finalize EXISTING PBF release with audited ambiguous-name omission gate" {
|
||||
& $Cm95Pbf finalize-production `
|
||||
--plan $Plan `
|
||||
--release-root $ReleaseRoot `
|
||||
--style ".\config\style\style.navsea-delivery-full-semantic-unified-20260808-cm95-v1.json" `
|
||||
--sprite-json ".\config\sprite\semantic-sprite-20260808.json" `
|
||||
--gap-disposition $Disposition
|
||||
}
|
||||
|
||||
$Manifest = Join-Path $ReleaseDir "release_manifest_v1.json"
|
||||
if (-not (Test-Path $Manifest)) {
|
||||
throw "release manifest missing: $Manifest"
|
||||
}
|
||||
|
||||
$F = Get-Content $Manifest -Raw | ConvertFrom-Json
|
||||
if ([string]$F.status -ne "PASS") {
|
||||
throw "release manifest status != PASS"
|
||||
}
|
||||
if ([string]$F.nav_name_conflict_audit.overall -ne "PASS") {
|
||||
throw "nav_name_conflict_audit != PASS"
|
||||
}
|
||||
if ([int]$F.nav_name_conflict_audit.report_occurrences -ne 10 -or
|
||||
[int]$F.nav_name_conflict_audit.trace_occurrences -ne 10 -or
|
||||
[int]$F.nav_name_conflict_audit.safe_omission_occurrences -ne 10 -or
|
||||
[int]$F.nav_name_conflict_audit.unique_conflict_outputs -ne 4 -or
|
||||
[int]$F.nav_name_conflict_audit.affected_tiles -ne 3 -or
|
||||
[int]$F.nav_name_conflict_audit.labeled_conflict_occurrences -ne 0 -or
|
||||
[int]$F.nav_name_conflict_audit.missing_pbf_occurrences -ne 0) {
|
||||
throw "audited ambiguous-name omission counts differ from expected baseline"
|
||||
}
|
||||
if ([int64]$F.planned_output_features -ne 2974375 -or
|
||||
[int64]$F.decoded_pbf_features -ne 2974375) {
|
||||
throw "feature count changed unexpectedly"
|
||||
}
|
||||
if ([string]$F.style_sprite_contract.overall -ne "PASS") {
|
||||
throw "style/sprite contract != PASS"
|
||||
}
|
||||
|
||||
Step "CM95 JAPAN v1.4.7 FINALIZE COMPLETE / PASS"
|
||||
Write-Host "Release ID : $ExpectedReleaseId"
|
||||
Write-Host "PBF rebuilt : NO"
|
||||
Write-Host "Jobs : 27038 / 27038 PASS"
|
||||
Write-Host "Decoded features : $($F.decoded_pbf_features)"
|
||||
Write-Host "Ambiguous-name occurrences: $($F.nav_name_conflict_audit.report_occurrences)"
|
||||
Write-Host "Unique ambiguous outputs : $($F.nav_name_conflict_audit.unique_conflict_outputs)"
|
||||
Write-Host "Labels safely omitted : $($F.nav_name_conflict_audit.safe_omission_occurrences)"
|
||||
Write-Host "Unsafe labels : $($F.nav_name_conflict_audit.labeled_conflict_occurrences)"
|
||||
Write-Host "Style/Sprite : $($F.style_sprite_contract.overall)"
|
||||
Write-Host "Semantic completeness : $($F.semantic_completeness)"
|
||||
Write-Host "FINAL STATUS : $($F.status)"
|
||||
Write-Host "Manifest : $Manifest"
|
||||
Reference in New Issue
Block a user