947 lines
28 KiB
PowerShell
947 lines
28 KiB
PowerShell
param(
|
||
[string]$ProjectRoot = "D:\React\cm93-mysql-importer-starter",
|
||
[string]$SourceRoot = "D:\Ship\CM93V2\CM93JAPAN",
|
||
[int]$ExpectedCells = 1492,
|
||
[int]$Workers = 4,
|
||
[switch]$Force
|
||
)
|
||
|
||
# CM95 Japan Full Rebuild Pipeline v2
|
||
#
|
||
# 从零重建：
|
||
# CM93 source -> fresh MySQL -> audit -> mapping gates
|
||
# -> fresh FID manifest -> fresh production plan
|
||
# -> full PBF build -> final release gate
|
||
#
|
||
# Windows 11 / PowerShell
|
||
#
|
||
# 破坏性范围：
|
||
# - DROP .env 中 CM93_DB_NAME 整个数据库
|
||
# - 删除旧 build\production
|
||
# - 删除旧 build\releases
|
||
# - 删除旧 build\pbf
|
||
# - 删除旧 build\pbf-reports
|
||
# - 删除旧 build\cm95_fid_manifest*.sqlite*
|
||
#
|
||
# 不会删除：
|
||
# - D:\Ship\CM93V2\CM93JAPAN
|
||
# - src\
|
||
# - config\
|
||
# - web\
|
||
# - logs\ 历史文件
|
||
#
|
||
# v2 修正：
|
||
# - 禁止 python.exe -c 多行代码，避免 Windows/PowerShell 引号损坏
|
||
# - 临时 Python helper 一律落盘执行
|
||
# - audit JSON gate 对齐当前 cm93db AuditResult
|
||
# - geometry_validity_issues 保持 informational
|
||
# - status JSON 使用 Out-String 后再 ConvertFrom-Json
|
||
# - 生成当前 Dataset 专属 semantic disposition summary
|
||
#
|
||
# 如果新 Dataset 出现未分类 mapping gap，会在 PBF build 前 STOP。
|
||
|
||
Set-StrictMode -Version Latest
|
||
$ErrorActionPreference = "Stop"
|
||
|
||
function Write-Step {
|
||
param([string]$Text)
|
||
Write-Host ""
|
||
Write-Host "======================================================================"
|
||
Write-Host $Text
|
||
Write-Host "======================================================================"
|
||
}
|
||
|
||
function Invoke-Checked {
|
||
param(
|
||
[string]$Label,
|
||
[scriptblock]$Action
|
||
)
|
||
|
||
Write-Step $Label
|
||
|
||
& $Action
|
||
$ExitCode = $LASTEXITCODE
|
||
|
||
if ($null -ne $ExitCode -and $ExitCode -ne 0) {
|
||
throw "$Label FAILED (exit code=$ExitCode)"
|
||
}
|
||
}
|
||
|
||
function Remove-PathIfExists {
|
||
param([string]$PathValue)
|
||
|
||
if (Test-Path $PathValue) {
|
||
Write-Host "DELETE: $PathValue"
|
||
Remove-Item $PathValue -Recurse -Force
|
||
}
|
||
}
|
||
|
||
function Get-StateCount {
|
||
param(
|
||
[object]$States,
|
||
[string]$Name
|
||
)
|
||
|
||
if ($null -eq $States) {
|
||
return 0
|
||
}
|
||
|
||
$Prop = $States.PSObject.Properties[$Name]
|
||
if ($null -eq $Prop) {
|
||
return 0
|
||
}
|
||
|
||
return [int]$Prop.Value
|
||
}
|
||
|
||
if (-not (Test-Path $ProjectRoot -PathType Container)) {
|
||
throw "ProjectRoot 不存在: $ProjectRoot"
|
||
}
|
||
|
||
if (-not (Test-Path $SourceRoot -PathType Container)) {
|
||
throw "SourceRoot 不存在: $SourceRoot"
|
||
}
|
||
|
||
Set-Location $ProjectRoot
|
||
|
||
$ProjectRoot = (Resolve-Path $ProjectRoot).Path
|
||
$SourceRoot = (Resolve-Path $SourceRoot).Path
|
||
|
||
$Python = Join-Path $ProjectRoot ".venv\Scripts\python.exe"
|
||
$Cm93Db = Join-Path $ProjectRoot ".venv\Scripts\cm93db.exe"
|
||
$Cm95Pbf = Join-Path $ProjectRoot ".venv\Scripts\cm95pbf.exe"
|
||
$EnvPath = Join-Path $ProjectRoot ".env"
|
||
|
||
foreach ($RequiredExe in @($Python, $Cm93Db, $Cm95Pbf)) {
|
||
if (-not (Test-Path $RequiredExe -PathType Leaf)) {
|
||
throw "缺少运行文件: $RequiredExe"
|
||
}
|
||
}
|
||
|
||
if (-not (Test-Path $EnvPath -PathType Leaf)) {
|
||
throw "缺少项目 .env: $EnvPath"
|
||
}
|
||
|
||
# 临时 helper 放系统 TEMP，不受 build 清理影响。
|
||
$TempRoot = Join-Path $env:TEMP ("cm95_japan_pipeline_v2_" + $PID)
|
||
|
||
Remove-Item $TempRoot -Recurse -Force -ErrorAction SilentlyContinue
|
||
New-Item -ItemType Directory -Path $TempRoot -Force | Out-Null
|
||
|
||
try {
|
||
|
||
# ----------------------------------------------------------------------
|
||
# STEP 0 - Source hard gate
|
||
# ----------------------------------------------------------------------
|
||
|
||
Write-Step "STEP 0 - Japan CM93 源目录检查"
|
||
|
||
$RequiredRegions = @(
|
||
"03300360",
|
||
"03300420",
|
||
"03900360",
|
||
"03900420"
|
||
)
|
||
|
||
$ActualRegions = @(
|
||
Get-ChildItem $SourceRoot -Directory -ErrorAction Stop |
|
||
Where-Object { $_.Name -match '^\d{8}$' } |
|
||
ForEach-Object { $_.Name } |
|
||
Sort-Object
|
||
)
|
||
|
||
$MissingRegions = @(
|
||
$RequiredRegions |
|
||
Where-Object { $_ -notin $ActualRegions }
|
||
)
|
||
|
||
$ExtraRegions = @(
|
||
$ActualRegions |
|
||
Where-Object { $_ -notin $RequiredRegions }
|
||
)
|
||
|
||
Write-Host "SourceRoot:"
|
||
Write-Host " $SourceRoot"
|
||
Write-Host ""
|
||
Write-Host "Numeric regions:"
|
||
$ActualRegions | ForEach-Object { Write-Host " $_" }
|
||
|
||
if ($MissingRegions.Count -gt 0) {
|
||
throw "缺少 Japan 必需区域: $($MissingRegions -join ', ')"
|
||
}
|
||
|
||
if ($ExtraRegions.Count -gt 0) {
|
||
throw "发现 Japan 工作集之外的数字区域，停止: $($ExtraRegions -join ', ')"
|
||
}
|
||
|
||
if (-not (Test-Path (Join-Path $SourceRoot "CM93OBJ.DIC"))) {
|
||
throw "缺少 CM93OBJ.DIC"
|
||
}
|
||
|
||
$HasAttrDic =
|
||
(Test-Path (Join-Path $SourceRoot "CM93ATTR.DIC")) -or
|
||
(Test-Path (Join-Path $SourceRoot "ATTRLUT.DIC"))
|
||
|
||
if (-not $HasAttrDic) {
|
||
throw "缺少 CM93ATTR.DIC / ATTRLUT.DIC"
|
||
}
|
||
|
||
Write-Host ""
|
||
Write-Host "Japan source region gate: PASS"
|
||
|
||
# ----------------------------------------------------------------------
|
||
# STEP 1 - Current code baseline + read-only checks
|
||
# ----------------------------------------------------------------------
|
||
|
||
Invoke-Checked "STEP 1A - pip install -e ." {
|
||
& $Python -m pip install -e .
|
||
}
|
||
|
||
if (-not (Test-Path ".\tools\cm95_v1_4_5_preflight.py")) {
|
||
throw "缺少 tools\cm95_v1_4_5_preflight.py；当前项目不是 v1.4.5 baseline"
|
||
}
|
||
|
||
Invoke-Checked "STEP 1B - CM95 v1.4.5 preflight" {
|
||
& $Python ".\tools\cm95_v1_4_5_preflight.py"
|
||
}
|
||
|
||
Invoke-Checked "STEP 1C - CM93 scan" {
|
||
& $Cm93Db scan $SourceRoot
|
||
}
|
||
|
||
Invoke-Checked "STEP 1D - CM93 dictionaries" {
|
||
& $Cm93Db inspect-dicts $SourceRoot
|
||
}
|
||
|
||
# ----------------------------------------------------------------------
|
||
# STEP 2 - Read DB config with a real helper .py file
|
||
# ----------------------------------------------------------------------
|
||
|
||
$ReadDbInfoPy = Join-Path $TempRoot "read_db_info.py"
|
||
|
||
@'
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import os
|
||
import sys
|
||
from pathlib import Path
|
||
|
||
from dotenv import load_dotenv
|
||
|
||
env_path = Path(sys.argv[1]).resolve()
|
||
load_dotenv(dotenv_path=env_path)
|
||
|
||
result = {
|
||
"host": os.getenv("CM93_DB_HOST", "127.0.0.1"),
|
||
"port": int(os.getenv("CM93_DB_PORT", "3306")),
|
||
"database": (os.getenv("CM93_DB_NAME", "cm93_chart") or "").strip(),
|
||
"user": os.getenv("CM93_DB_USER", "root"),
|
||
"charset": os.getenv("CM93_DB_CHARSET", "utf8mb4"),
|
||
}
|
||
|
||
if not result["database"]:
|
||
raise SystemExit("CM93_DB_NAME is empty")
|
||
|
||
print(json.dumps(result, ensure_ascii=False))
|
||
'@ | Set-Content $ReadDbInfoPy -Encoding UTF8
|
||
|
||
$DbInfoText = (& $Python $ReadDbInfoPy $EnvPath | Out-String).Trim()
|
||
|
||
if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($DbInfoText)) {
|
||
throw "无法读取数据库配置"
|
||
}
|
||
|
||
$DbInfo = $DbInfoText | ConvertFrom-Json
|
||
$DbName = [string]$DbInfo.database
|
||
|
||
$ForbiddenDbNames = @(
|
||
"mysql",
|
||
"information_schema",
|
||
"performance_schema",
|
||
"sys"
|
||
)
|
||
|
||
if ($DbName.ToLowerInvariant() -in $ForbiddenDbNames) {
|
||
throw "拒绝删除系统数据库: $DbName"
|
||
}
|
||
|
||
Write-Step "STEP 2 - DESTRUCTIVE RESET CONFIRMATION"
|
||
|
||
Write-Host "将永久删除整个 MySQL database:"
|
||
Write-Host " $DbName"
|
||
Write-Host ""
|
||
Write-Host "数据库服务器:"
|
||
Write-Host " $($DbInfo.host):$($DbInfo.port)"
|
||
Write-Host ""
|
||
Write-Host "也会删除旧 CM95 build/release 产物。"
|
||
Write-Host ""
|
||
Write-Host "不会删除 CM93 source:"
|
||
Write-Host " $SourceRoot"
|
||
|
||
if (-not $Force) {
|
||
$Expected = "DELETE $DbName"
|
||
$Answer = Read-Host "请输入 [$Expected] 继续"
|
||
|
||
if ($Answer -ne $Expected) {
|
||
throw "用户取消。"
|
||
}
|
||
}
|
||
else {
|
||
Write-Host "-Force 已指定，跳过交互确认。"
|
||
}
|
||
|
||
# ----------------------------------------------------------------------
|
||
# STEP 3 - Delete old generated artifacts
|
||
# ----------------------------------------------------------------------
|
||
|
||
Write-Step "STEP 3 - 删除旧 CM95 build/release 产物"
|
||
|
||
Remove-PathIfExists (Join-Path $ProjectRoot "build\production")
|
||
Remove-PathIfExists (Join-Path $ProjectRoot "build\releases")
|
||
Remove-PathIfExists (Join-Path $ProjectRoot "build\pbf")
|
||
Remove-PathIfExists (Join-Path $ProjectRoot "build\pbf-reports")
|
||
|
||
$BuildRoot = Join-Path $ProjectRoot "build"
|
||
|
||
if (Test-Path $BuildRoot) {
|
||
Get-ChildItem $BuildRoot -File -Filter "cm95_fid_manifest*.sqlite*" -ErrorAction SilentlyContinue |
|
||
ForEach-Object {
|
||
Write-Host "DELETE: $($_.FullName)"
|
||
Remove-Item $_.FullName -Force
|
||
}
|
||
}
|
||
|
||
New-Item -ItemType Directory -Force -Path ".\build\production" | Out-Null
|
||
New-Item -ItemType Directory -Force -Path ".\logs" | Out-Null
|
||
|
||
# ----------------------------------------------------------------------
|
||
# STEP 4 - DROP database using helper file, then let cm93db recreate schema
|
||
# ----------------------------------------------------------------------
|
||
|
||
$DropDbPy = Join-Path $TempRoot "drop_cm93_database.py"
|
||
|
||
@'
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
import re
|
||
import sys
|
||
from pathlib import Path
|
||
|
||
import pymysql
|
||
from dotenv import load_dotenv
|
||
|
||
env_path = Path(sys.argv[1]).resolve()
|
||
load_dotenv(dotenv_path=env_path)
|
||
|
||
host = os.getenv("CM93_DB_HOST", "127.0.0.1")
|
||
port = int(os.getenv("CM93_DB_PORT", "3306"))
|
||
user = os.getenv("CM93_DB_USER", "root")
|
||
password = os.getenv("CM93_DB_PASSWORD", "")
|
||
database = (os.getenv("CM93_DB_NAME", "cm93_chart") or "").strip()
|
||
charset = os.getenv("CM93_DB_CHARSET", "utf8mb4")
|
||
|
||
if not database:
|
||
raise RuntimeError("CM93_DB_NAME is empty")
|
||
|
||
if database.lower() in {
|
||
"mysql",
|
||
"information_schema",
|
||
"performance_schema",
|
||
"sys",
|
||
}:
|
||
raise RuntimeError(f"refuse to drop system database: {database}")
|
||
|
||
conn = pymysql.connect(
|
||
host=host,
|
||
port=port,
|
||
user=user,
|
||
password=password,
|
||
charset=charset,
|
||
autocommit=False,
|
||
cursorclass=pymysql.cursors.DictCursor,
|
||
)
|
||
|
||
try:
|
||
with conn.cursor() as cur:
|
||
cur.execute("SELECT VERSION() AS v")
|
||
version = str(cur.fetchone()["v"])
|
||
|
||
match = re.match(r"^(\d+)\.(\d+)\.(\d+)", version)
|
||
if not match or tuple(map(int, match.groups()[:2])) != (5, 7):
|
||
raise RuntimeError(
|
||
f"expected MySQL 5.7.x, got {version}; refusing destructive reset"
|
||
)
|
||
|
||
safe = database.replace("`", "``")
|
||
cur.execute(f"DROP DATABASE IF EXISTS `{safe}`")
|
||
|
||
conn.commit()
|
||
print(f"DROP_DATABASE_OK database={database} mysql={version}")
|
||
|
||
finally:
|
||
conn.close()
|
||
'@ | Set-Content $DropDbPy -Encoding UTF8
|
||
|
||
Invoke-Checked "STEP 4A - DROP old database" {
|
||
& $Python $DropDbPy $EnvPath
|
||
}
|
||
|
||
Invoke-Checked "STEP 4B - CREATE database + initialize schema" {
|
||
& $Cm93Db db-init --create-database
|
||
}
|
||
|
||
Invoke-Checked "STEP 4C - DB contract check" {
|
||
& $Cm93Db db-check
|
||
}
|
||
|
||
# ----------------------------------------------------------------------
|
||
# STEP 5 - Full import on fresh DB
|
||
# ----------------------------------------------------------------------
|
||
|
||
Invoke-Checked "STEP 5 - 全量 CM93 Japan import（fresh DB）" {
|
||
& $Cm93Db import $SourceRoot --fail-fast
|
||
}
|
||
|
||
# ----------------------------------------------------------------------
|
||
# STEP 6 - Full audit
|
||
# ----------------------------------------------------------------------
|
||
|
||
$Timestamp = Get-Date -Format "yyyyMMdd_HHmmss"
|
||
$AuditReport = ".\logs\cm93_audit_japan_full_rebuild_$Timestamp.json"
|
||
|
||
Invoke-Checked "STEP 6 - CM93 full audit + geometry validation" {
|
||
& $Cm93Db audit `
|
||
$SourceRoot `
|
||
--validate-geometry `
|
||
--report $AuditReport
|
||
}
|
||
|
||
$Audit = Get-Content $AuditReport -Raw | ConvertFrom-Json
|
||
|
||
if (-not [bool]$Audit.passed) {
|
||
throw "CM93 audit report passed=false；检查 $AuditReport"
|
||
}
|
||
|
||
$AuditHardFailures = @(
|
||
[int64]$Audit.source_parse_failed,
|
||
[int64]$Audit.db_missing_cells,
|
||
[int64]$Audit.db_not_done_cells,
|
||
[int64]$Audit.count_mismatches,
|
||
[int64]$Audit.hash_mismatches,
|
||
[int64]$Audit.raw_preservation_mismatches,
|
||
[int64]$Audit.unknown_objects,
|
||
[int64]$Audit.unknown_attributes,
|
||
[int64]$Audit.precision_failed
|
||
)
|
||
|
||
if (($AuditHardFailures | Measure-Object -Sum).Sum -ne 0) {
|
||
throw "CM93 audit hard gate failed；检查 $AuditReport"
|
||
}
|
||
|
||
if ($ExpectedCells -gt 0 -and [int]$Audit.source_cells -ne $ExpectedCells) {
|
||
throw "Source cell count 不符合当前 Japan baseline: $($Audit.source_cells) != $ExpectedCells"
|
||
}
|
||
|
||
Write-Host ""
|
||
Write-Host "CM93 AUDIT HARD GATE: PASS"
|
||
Write-Host "Source cells : $($Audit.source_cells)"
|
||
Write-Host "Geometry validity issues : $($Audit.geometry_validity_issues) (informational)"
|
||
|
||
# ----------------------------------------------------------------------
|
||
# STEP 7 - Resolve fresh Dataset
|
||
# ----------------------------------------------------------------------
|
||
|
||
$DatasetInfoPy = Join-Path $TempRoot "dataset_info.py"
|
||
|
||
@'
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import os
|
||
import sys
|
||
from pathlib import Path
|
||
|
||
import pymysql
|
||
from dotenv import load_dotenv
|
||
|
||
env_path = Path(sys.argv[1]).resolve()
|
||
source_root = str(Path(sys.argv[2]).resolve())
|
||
|
||
load_dotenv(dotenv_path=env_path)
|
||
|
||
conn = pymysql.connect(
|
||
host=os.getenv("CM93_DB_HOST", "127.0.0.1"),
|
||
port=int(os.getenv("CM93_DB_PORT", "3306")),
|
||
user=os.getenv("CM93_DB_USER", "root"),
|
||
password=os.getenv("CM93_DB_PASSWORD", ""),
|
||
database=os.getenv("CM93_DB_NAME", "cm93_chart"),
|
||
charset=os.getenv("CM93_DB_CHARSET", "utf8mb4"),
|
||
cursorclass=pymysql.cursors.DictCursor,
|
||
)
|
||
|
||
try:
|
||
with conn.cursor() as cur:
|
||
cur.execute("SELECT COUNT(*) AS n FROM chart_dataset")
|
||
dataset_count = int(cur.fetchone()["n"])
|
||
|
||
cur.execute(
|
||
"""
|
||
SELECT
|
||
d.id,
|
||
d.name,
|
||
d.source_root,
|
||
COUNT(c.id) AS cells,
|
||
SUM(c.import_status='DONE') AS done_cells,
|
||
SUM(c.import_status='FAILED') AS failed_cells
|
||
FROM chart_dataset d
|
||
LEFT JOIN chart_cell c
|
||
ON c.dataset_id=d.id
|
||
WHERE d.source_type='CM93'
|
||
AND d.source_root=%s
|
||
GROUP BY d.id,d.name,d.source_root
|
||
""",
|
||
(source_root,),
|
||
)
|
||
rows = cur.fetchall()
|
||
|
||
if dataset_count != 1:
|
||
raise RuntimeError(
|
||
f"fresh DB expected exactly 1 dataset, got {dataset_count}"
|
||
)
|
||
|
||
if len(rows) != 1:
|
||
raise RuntimeError(
|
||
f"expected one dataset for source_root={source_root}, got {len(rows)}"
|
||
)
|
||
|
||
row = rows[0]
|
||
|
||
print(
|
||
json.dumps(
|
||
{
|
||
"id": int(row["id"]),
|
||
"name": row["name"],
|
||
"source_root": row["source_root"],
|
||
"cells": int(row["cells"] or 0),
|
||
"done_cells": int(row["done_cells"] or 0),
|
||
"failed_cells": int(row["failed_cells"] or 0),
|
||
},
|
||
ensure_ascii=False,
|
||
)
|
||
)
|
||
|
||
finally:
|
||
conn.close()
|
||
'@ | Set-Content $DatasetInfoPy -Encoding UTF8
|
||
|
||
$DatasetText = (& $Python $DatasetInfoPy $EnvPath $SourceRoot | Out-String).Trim()
|
||
|
||
if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($DatasetText)) {
|
||
throw "无法读取 fresh Dataset 信息"
|
||
}
|
||
|
||
$Dataset = $DatasetText | ConvertFrom-Json
|
||
$DatasetId = [int]$Dataset.id
|
||
|
||
if ([int]$Dataset.cells -ne [int]$Audit.source_cells) {
|
||
throw "DB cell count != audit source_cells ($($Dataset.cells) != $($Audit.source_cells))"
|
||
}
|
||
|
||
if ([int]$Dataset.done_cells -ne [int]$Audit.source_cells) {
|
||
throw "不是所有 Cell 都 DONE"
|
||
}
|
||
|
||
if ([int]$Dataset.failed_cells -ne 0) {
|
||
throw "存在 FAILED Cell"
|
||
}
|
||
|
||
Write-Step "STEP 7 - Fresh Dataset confirmed"
|
||
|
||
Write-Host "Dataset ID : $DatasetId"
|
||
Write-Host "Name : $($Dataset.name)"
|
||
Write-Host "SourceRoot : $($Dataset.source_root)"
|
||
Write-Host "Cells : $($Dataset.cells)"
|
||
Write-Host "DONE : $($Dataset.done_cells)"
|
||
Write-Host "FAILED : $($Dataset.failed_cells)"
|
||
|
||
# ----------------------------------------------------------------------
|
||
# STEP 8 - Dataset-aware mapping gates
|
||
# ----------------------------------------------------------------------
|
||
|
||
$GapAudit = ".\build\production\cm95_global_mapping_gap_audit_dataset_${DatasetId}.json"
|
||
$GapClosure = ".\build\production\cm95_global_mapping_gap_closure_dataset_${DatasetId}.json"
|
||
$MappingContract = ".\build\production\cm95_mapping_contract_check_dataset_${DatasetId}.json"
|
||
|
||
Invoke-Checked "STEP 8A - Global mapping gap audit (fresh Dataset)" {
|
||
& $Python ".\tools\cm95_global_mapping_gap_audit_v1.py" `
|
||
--dataset-id $DatasetId `
|
||
--mapping ".\config\mappings\cm95_to_navsea_v1.yaml" `
|
||
--output $GapAudit
|
||
}
|
||
|
||
Invoke-Checked "STEP 8B - Mapping gap classification closure" {
|
||
& $Python ".\tools\cm95_global_mapping_gap_closure_audit_v2.py" `
|
||
--gap-audit $GapAudit `
|
||
--mapping ".\config\mappings\cm95_to_navsea_v1.yaml" `
|
||
--disposition ".\config\mappings\cm95_gap_disposition_v1.yaml" `
|
||
--output $GapClosure
|
||
}
|
||
|
||
$Closure = Get-Content $GapClosure -Raw | ConvertFrom-Json
|
||
|
||
if ([string]$Closure.overall -ne "PASS") {
|
||
Write-Host ""
|
||
Write-Host "NEW DATASET MAPPING GATE STOPPED PIPELINE."
|
||
Write-Host "新 Japan Dataset 有当前 disposition 尚未收口的 gap variant。"
|
||
Write-Host ""
|
||
Write-Host "检查:"
|
||
Write-Host " $GapAudit"
|
||
Write-Host " $GapClosure"
|
||
throw "Mapping gap closure FAIL；PBF generation intentionally stopped."
|
||
}
|
||
|
||
Invoke-Checked "STEP 8C - Mapping / Style / Sprite static contract" {
|
||
& $Python ".\tools\cm95_mapping_contract_check_v1.py" `
|
||
--mapping ".\config\mappings\cm95_to_navsea_v1.yaml" `
|
||
--style ".\config\style\style.navsea-delivery-full-semantic-unified-20260808-cm95-v1.json" `
|
||
--sprite-json ".\config\sprite\semantic-sprite-20260808.json" `
|
||
--output $MappingContract
|
||
}
|
||
|
||
$MapContract = Get-Content $MappingContract -Raw | ConvertFrom-Json
|
||
|
||
if ([string]$MapContract.overall -ne "PASS") {
|
||
throw "Mapping / Style / Sprite static contract FAIL"
|
||
}
|
||
|
||
# ----------------------------------------------------------------------
|
||
# STEP 8D - Build Dataset-specific effective gap disposition
|
||
# ----------------------------------------------------------------------
|
||
|
||
$EffectiveDisposition = ".\build\production\cm95_gap_disposition_dataset_${DatasetId}_effective.yaml"
|
||
$EffectiveDispositionPy = Join-Path $TempRoot "build_effective_disposition.py"
|
||
|
||
@'
|
||
from __future__ import annotations
|
||
|
||
import copy
|
||
import sys
|
||
from pathlib import Path
|
||
|
||
import yaml
|
||
|
||
template_path = Path(sys.argv[1])
|
||
gap_path = Path(sys.argv[2])
|
||
closure_path = Path(sys.argv[3])
|
||
dataset_id = int(sys.argv[4])
|
||
out_path = Path(sys.argv[5])
|
||
|
||
template = yaml.safe_load(template_path.read_text(encoding="utf-8")) or {}
|
||
gap = yaml.safe_load(gap_path.read_text(encoding="utf-8")) or {}
|
||
closure = yaml.safe_load(closure_path.read_text(encoding="utf-8")) or {}
|
||
|
||
template_by_key = {
|
||
(
|
||
str(v.get("object_class") or ""),
|
||
str(v.get("geometry_type") or "").upper(),
|
||
): v
|
||
for v in template.get("variants") or []
|
||
if isinstance(v, dict)
|
||
}
|
||
|
||
effective_variants = []
|
||
|
||
for item in gap.get("gap_variants") or []:
|
||
key = (
|
||
str(item.get("object_class") or ""),
|
||
str(item.get("geometry_type") or "").upper(),
|
||
)
|
||
|
||
src = template_by_key.get(key)
|
||
if src is None:
|
||
raise RuntimeError(
|
||
f"unclassified gap variant while building effective disposition: {key}"
|
||
)
|
||
|
||
row = copy.deepcopy(src)
|
||
row["object_class"] = key[0]
|
||
row["geometry_type"] = key[1]
|
||
row["feature_count"] = int(item.get("feature_count") or 0)
|
||
|
||
if item.get("object_class_id") is not None:
|
||
row["object_class_id"] = item.get("object_class_id")
|
||
|
||
if item.get("dictionary_description") is not None:
|
||
row["dictionary_description"] = item.get("dictionary_description")
|
||
|
||
if item.get("gap_reason") is not None:
|
||
row["gap_reason_v1"] = item.get("gap_reason")
|
||
|
||
effective_variants.append(row)
|
||
|
||
pending = [
|
||
v for v in effective_variants
|
||
if str(v.get("disposition") or "")
|
||
== "ENGINEERING_ONLY_PENDING_SEMANTIC_REVIEW"
|
||
]
|
||
|
||
delivery = [
|
||
v for v in effective_variants
|
||
if str(v.get("disposition") or "")
|
||
== "DELIVERY_MAPPED_V1_4_3"
|
||
]
|
||
|
||
existing = [
|
||
v for v in effective_variants
|
||
if str(v.get("disposition") or "").startswith("EXISTING_")
|
||
]
|
||
|
||
result = {
|
||
"schema": "navsea.cm95.gap-disposition/v1",
|
||
"version": "v1.4.5-dataset-effective",
|
||
"dataset_id": dataset_id,
|
||
"source_audit": str(gap_path),
|
||
"policy": (
|
||
"Dataset-specific effective disposition generated from "
|
||
"cm95_gap_disposition_v1.yaml after current-dataset closure PASS."
|
||
),
|
||
"summary": {
|
||
"observed_gap_variants": len(effective_variants),
|
||
"delivery_mapped_v1_4_3": len(delivery),
|
||
"existing_explicit_non_delivery": len(existing),
|
||
"pending_semantic_review": len(pending),
|
||
"delivery_mapped_v1_4_3_feature_rows": sum(
|
||
int(v.get("feature_count") or 0) for v in delivery
|
||
),
|
||
"existing_explicit_non_delivery_feature_rows": sum(
|
||
int(v.get("feature_count") or 0) for v in existing
|
||
),
|
||
"pending_semantic_review_feature_rows": sum(
|
||
int(v.get("feature_count") or 0) for v in pending
|
||
),
|
||
},
|
||
"variants": effective_variants,
|
||
}
|
||
|
||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||
out_path.write_text(
|
||
yaml.safe_dump(
|
||
result,
|
||
allow_unicode=True,
|
||
sort_keys=False,
|
||
width=120,
|
||
),
|
||
encoding="utf-8",
|
||
)
|
||
|
||
print(f"Effective disposition: {out_path}")
|
||
print(f"Observed gap variants : {len(effective_variants)}")
|
||
print(f"Pending review : {len(pending)}")
|
||
print(
|
||
"Pending rows : "
|
||
f"{result['summary']['pending_semantic_review_feature_rows']}"
|
||
)
|
||
'@ | Set-Content $EffectiveDispositionPy -Encoding UTF8
|
||
|
||
Invoke-Checked "STEP 8D - Dataset-specific semantic disposition summary" {
|
||
& $Python $EffectiveDispositionPy `
|
||
".\config\mappings\cm95_gap_disposition_v1.yaml" `
|
||
$GapAudit `
|
||
$GapClosure `
|
||
$DatasetId `
|
||
$EffectiveDisposition
|
||
}
|
||
|
||
# ----------------------------------------------------------------------
|
||
# STEP 9 - Fresh deterministic FID manifest
|
||
# ----------------------------------------------------------------------
|
||
|
||
$Manifest = ".\build\cm95_fid_manifest_dataset_${DatasetId}_v1.sqlite"
|
||
|
||
Invoke-Checked "STEP 9 - Rebuild FID manifest from zero" {
|
||
& $Cm95Pbf build-fid-manifest `
|
||
--dataset-id $DatasetId `
|
||
--manifest $Manifest `
|
||
--rebuild-fid-manifest
|
||
}
|
||
|
||
# ----------------------------------------------------------------------
|
||
# STEP 10 - Fresh immutable production plan
|
||
# ----------------------------------------------------------------------
|
||
|
||
$Plan = ".\build\production\cm95_production_plan_dataset_${DatasetId}_v1_4_5.json"
|
||
$PlanAudit = ".\build\production\cm95_production_plan_audit_dataset_${DatasetId}_v1_4_5.json"
|
||
|
||
Invoke-Checked "STEP 10A - plan-production z0-z12" {
|
||
& $Cm95Pbf plan-production `
|
||
--dataset-id $DatasetId `
|
||
--zooms 0-12 `
|
||
--profile delivery `
|
||
--manifest $Manifest `
|
||
--output $Plan
|
||
}
|
||
|
||
Invoke-Checked "STEP 10B - production plan audit" {
|
||
& $Python ".\tools\cm95_production_plan_audit_v1.py" `
|
||
--plan $Plan `
|
||
--expected-zooms 0-12 `
|
||
--output $PlanAudit
|
||
}
|
||
|
||
$PlanObj = Get-Content $Plan -Raw | ConvertFrom-Json
|
||
$PlanAuditObj = Get-Content $PlanAudit -Raw | ConvertFrom-Json
|
||
|
||
if ([string]$PlanAuditObj.overall -ne "PASS") {
|
||
throw "Production plan audit FAIL"
|
||
}
|
||
|
||
$ReleaseId = [string]$PlanObj.release_id
|
||
|
||
if ([string]::IsNullOrWhiteSpace($ReleaseId)) {
|
||
throw "Plan 没有 release_id"
|
||
}
|
||
|
||
Write-Host ""
|
||
Write-Host "Plan ID : $($PlanObj.plan_id)"
|
||
Write-Host "Release ID : $ReleaseId"
|
||
Write-Host "Jobs : $($PlanObj.job_count)"
|
||
|
||
# ----------------------------------------------------------------------
|
||
# STEP 11 - Full PBF build
|
||
# ----------------------------------------------------------------------
|
||
|
||
Invoke-Checked "STEP 11 - build-production FULL" {
|
||
& $Cm95Pbf build-production `
|
||
--plan $Plan `
|
||
--release-root ".\build\releases" `
|
||
--workers $Workers `
|
||
--max-attempts 3
|
||
}
|
||
|
||
# ----------------------------------------------------------------------
|
||
# STEP 12 - Production status
|
||
# ----------------------------------------------------------------------
|
||
|
||
Write-Step "STEP 12 - Production status"
|
||
|
||
$StatusText = (
|
||
& $Cm95Pbf status-production `
|
||
--plan $Plan `
|
||
--release-root ".\build\releases" |
|
||
Out-String
|
||
).Trim()
|
||
|
||
if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($StatusText)) {
|
||
throw "status-production FAILED"
|
||
}
|
||
|
||
Write-Host $StatusText
|
||
|
||
$Status = $StatusText | ConvertFrom-Json
|
||
$States = $Status.states
|
||
|
||
$PassCount = Get-StateCount $States "PASS"
|
||
$FailedCount = Get-StateCount $States "FAILED"
|
||
$PendingCount = Get-StateCount $States "PENDING"
|
||
$RunningCount = Get-StateCount $States "RUNNING"
|
||
|
||
if ($PassCount -ne [int]$Status.jobs -or
|
||
$FailedCount -ne 0 -or
|
||
$PendingCount -ne 0 -or
|
||
$RunningCount -ne 0) {
|
||
throw "Production state gate FAIL"
|
||
}
|
||
|
||
# ----------------------------------------------------------------------
|
||
# STEP 13 - Final release gate
|
||
# ----------------------------------------------------------------------
|
||
|
||
Invoke-Checked "STEP 13 - finalize-production" {
|
||
& $Cm95Pbf finalize-production `
|
||
--plan $Plan `
|
||
--release-root ".\build\releases" `
|
||
--style ".\config\style\style.navsea-delivery-full-semantic-unified-20260808-cm95-v1.json" `
|
||
--sprite-json ".\config\sprite\semantic-sprite-20260808.json" `
|
||
--gap-disposition $EffectiveDisposition
|
||
}
|
||
|
||
$ReleaseDir = Join-Path $ProjectRoot ("build\releases\" + $ReleaseId)
|
||
$ReleaseManifest = Join-Path $ReleaseDir "release_manifest_v1.json"
|
||
|
||
if (-not (Test-Path $ReleaseManifest -PathType Leaf)) {
|
||
throw "finalize 已运行，但 release manifest 不存在: $ReleaseManifest"
|
||
}
|
||
|
||
$Final = Get-Content $ReleaseManifest -Raw | ConvertFrom-Json
|
||
|
||
if ([string]$Final.status -ne "PASS") {
|
||
throw "Final release manifest status != PASS"
|
||
}
|
||
|
||
# ----------------------------------------------------------------------
|
||
# STEP 14 - Save compact baseline summary
|
||
# ----------------------------------------------------------------------
|
||
|
||
$SummaryPath = ".\build\production\CM95_JAPAN_FULL_REBUILD_CURRENT.json"
|
||
|
||
$Summary = [ordered]@{
|
||
schema = "navsea.cm95.japan-full-rebuild-pipeline/v2"
|
||
completed_at = (Get-Date).ToString("o")
|
||
project_root = $ProjectRoot
|
||
source_root = $SourceRoot
|
||
database = $DbName
|
||
dataset_id = $DatasetId
|
||
source_cells = [int]$Audit.source_cells
|
||
source_geometry_validity_issues = [int]$Audit.geometry_validity_issues
|
||
audit_report = $AuditReport
|
||
mapping_gap_audit = $GapAudit
|
||
mapping_gap_closure = [string]$Closure.overall
|
||
effective_gap_disposition = $EffectiveDisposition
|
||
semantic_completeness = [string]$Final.semantic_completeness
|
||
semantic_backlog = $Final.semantic_backlog
|
||
fid_manifest = $Manifest
|
||
plan = $Plan
|
||
plan_id = [string]$PlanObj.plan_id
|
||
release_id = $ReleaseId
|
||
release_dir = $ReleaseDir
|
||
release_manifest = $ReleaseManifest
|
||
jobs = [int]$Status.jobs
|
||
pbf_files = [int]$Status.pbf_files
|
||
output_features = [int64]$Status.output_features
|
||
final_status = [string]$Final.status
|
||
}
|
||
|
||
$Summary |
|
||
ConvertTo-Json -Depth 12 |
|
||
Set-Content $SummaryPath -Encoding UTF8
|
||
|
||
Write-Step "PIPELINE COMPLETE / PASS"
|
||
|
||
Write-Host "Database : $DbName"
|
||
Write-Host "Dataset ID : $DatasetId"
|
||
Write-Host "Source cells : $($Audit.source_cells)"
|
||
Write-Host "Plan ID : $($PlanObj.plan_id)"
|
||
Write-Host "Release ID : $ReleaseId"
|
||
Write-Host "Jobs : $($Status.jobs) PASS"
|
||
Write-Host "PBF files : $($Status.pbf_files)"
|
||
Write-Host "Output features: $($Status.output_features)"
|
||
Write-Host "Final status : $($Final.status)"
|
||
Write-Host "Semantic : $($Final.semantic_completeness)"
|
||
Write-Host ""
|
||
Write-Host "Summary:"
|
||
Write-Host " $SummaryPath"
|
||
Write-Host ""
|
||
Write-Host "Release:"
|
||
Write-Host " $ReleaseDir"
|
||
Write-Host ""
|
||
Write-Host "Web test:"
|
||
Write-Host " cd `"$ProjectRoot`""
|
||
Write-Host " & `".\.venv\Scripts\python.exe`" .\web\cm95_web.py --release-id `"$ReleaseId`""
|
||
|
||
}
|
||
finally {
|
||
Remove-Item $TempRoot -Recurse -Force -ErrorAction SilentlyContinue
|
||
}
|