scripts: automate phase 8 pilot closed loop
This commit is contained in:
Codex
@@ -0,0 +1,688 @@
|
||||
param(
|
||||
[string]$ServerHost = "192.168.3.90",
|
||||
[string]$SshUser = "root",
|
||||
[string]$SshKeyPath = "$HOME\.ssh\water_audit_ed25519",
|
||||
[string]$RemoteProjectPath = "/opt/ai-contract-platform",
|
||||
[string]$ComposeFile = "docker-compose.server.yml",
|
||||
[string]$RemoteBackupRoot = "/opt/ai-contract-platform/backups/phase7-deploy",
|
||||
[string]$RemoteRollbackRoot = "/opt/ai-contract-platform/backups/phase7-rollback",
|
||||
[string]$FrontendBaseUrl = "http://192.168.3.90:3003",
|
||||
[string]$BackendBaseUrl = "http://192.168.3.90:8001",
|
||||
[string]$ApiUrl = "http://192.168.3.90:8001/api/v1",
|
||||
[string]$HealthUrl = "http://192.168.3.90:8001/health",
|
||||
[string]$ReadinessUrl = "http://192.168.3.90:8001/api/v1/system/readiness",
|
||||
[string]$ArtifactRoot = "",
|
||||
[switch]$IncludeBackupDrill,
|
||||
[switch]$FailOnWarn,
|
||||
[switch]$FailOnExternalBlocked
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path
|
||||
$timestamp = Get-Date -Format "yyyy-MM-ddTHH-mm-ss-fff"
|
||||
if ([string]::IsNullOrWhiteSpace($ArtifactRoot)) {
|
||||
$ArtifactRoot = Join-Path $repoRoot "project-docs\qa-artifacts\phase8-pilot-closed-loop"
|
||||
}
|
||||
if (-not [System.IO.Path]::IsPathRooted($ArtifactRoot)) {
|
||||
$ArtifactRoot = Join-Path $repoRoot $ArtifactRoot
|
||||
}
|
||||
|
||||
$artifactDir = Join-Path $ArtifactRoot $timestamp
|
||||
New-Item -ItemType Directory -Force -Path $artifactDir | Out-Null
|
||||
|
||||
$summaryJson = Join-Path $artifactDir "phase8-pilot-closed-loop-summary.json"
|
||||
$summaryMd = Join-Path $artifactDir "phase8-pilot-closed-loop-summary.md"
|
||||
$scenarioCsv = Join-Path $artifactDir "pilot-scenario-execution-record.csv"
|
||||
$feedbackCsv = Join-Path $artifactDir "pilot-feedback-register.csv"
|
||||
$defectCsv = Join-Path $artifactDir "pilot-defect-fix-record.csv"
|
||||
$dailyMd = Join-Path $artifactDir "pilot-daily-report.md"
|
||||
$weeklyMd = Join-Path $artifactDir "pilot-weekly-report.md"
|
||||
$acceptanceMd = Join-Path $artifactDir "pilot-acceptance-checklist.md"
|
||||
$gitStatusPath = Join-Path $artifactDir "git-status.txt"
|
||||
|
||||
$checks = New-Object System.Collections.Generic.List[object]
|
||||
|
||||
function Add-Check {
|
||||
param(
|
||||
[string]$Key,
|
||||
[string]$Label,
|
||||
[ValidateSet("pass", "fail", "warn", "skip", "blocked")]
|
||||
[string]$Status,
|
||||
[string]$Detail,
|
||||
[bool]$Required = $true,
|
||||
[hashtable]$Meta = @{}
|
||||
)
|
||||
|
||||
$checks.Add([ordered]@{
|
||||
key = $Key
|
||||
label = $Label
|
||||
status = $Status
|
||||
required = $Required
|
||||
detail = $Detail
|
||||
meta = $Meta
|
||||
}) | Out-Null
|
||||
}
|
||||
|
||||
function Save-Json {
|
||||
param(
|
||||
[string]$Path,
|
||||
[object]$Value
|
||||
)
|
||||
$Value | ConvertTo-Json -Depth 18 | Set-Content -Encoding UTF8 -Path $Path
|
||||
}
|
||||
|
||||
function Read-JsonFile {
|
||||
param([string]$Path)
|
||||
if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) {
|
||||
return $null
|
||||
}
|
||||
try {
|
||||
return Get-Content -LiteralPath $Path -Raw -Encoding UTF8 | ConvertFrom-Json
|
||||
} catch {
|
||||
return $null
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-Phase8Step {
|
||||
param(
|
||||
[string]$Key,
|
||||
[string]$Label,
|
||||
[string]$ScriptName,
|
||||
[string[]]$Arguments,
|
||||
[bool]$Required = $true
|
||||
)
|
||||
|
||||
$scriptPath = Join-Path $repoRoot "scripts\$ScriptName"
|
||||
$logPath = Join-Path $artifactDir "$Key.log"
|
||||
if (-not (Test-Path -LiteralPath $scriptPath -PathType Leaf)) {
|
||||
Add-Check -Key $Key -Label $Label -Status "fail" -Required $Required -Detail "Script not found: $ScriptName"
|
||||
return $null
|
||||
}
|
||||
|
||||
$previousErrorActionPreference = $ErrorActionPreference
|
||||
$ErrorActionPreference = "Continue"
|
||||
try {
|
||||
$output = & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $scriptPath @Arguments 2>&1
|
||||
$exitCode = $LASTEXITCODE
|
||||
} finally {
|
||||
$ErrorActionPreference = $previousErrorActionPreference
|
||||
}
|
||||
|
||||
$outputText = ($output -join [Environment]::NewLine)
|
||||
$outputText | Set-Content -Encoding UTF8 -Path $logPath
|
||||
$childArtifact = $null
|
||||
$artifactMatches = [regex]::Matches($outputText, '([A-Z]:\\[^\r\n]+)$', [System.Text.RegularExpressions.RegexOptions]::Multiline)
|
||||
if ($artifactMatches.Count -gt 0) {
|
||||
$childArtifact = $artifactMatches[$artifactMatches.Count - 1].Groups[1].Value.Trim()
|
||||
}
|
||||
|
||||
$meta = @{
|
||||
log = $logPath
|
||||
artifact_dir = $childArtifact
|
||||
exit_code = $exitCode
|
||||
}
|
||||
|
||||
if ($exitCode -eq 0 -and $outputText -match "_BLOCKED") {
|
||||
Add-Check -Key $Key -Label $Label -Status "blocked" -Required $false -Detail "$Label completed with external blocker" -Meta $meta
|
||||
} elseif ($exitCode -eq 0 -and $outputText -match "_WARN") {
|
||||
Add-Check -Key $Key -Label $Label -Status "warn" -Required $false -Detail "$Label completed with warnings" -Meta $meta
|
||||
} elseif ($exitCode -eq 0) {
|
||||
Add-Check -Key $Key -Label $Label -Status "pass" -Required $Required -Detail "$Label passed" -Meta $meta
|
||||
} else {
|
||||
Add-Check -Key $Key -Label $Label -Status "fail" -Required $Required -Detail "$Label failed" -Meta $meta
|
||||
}
|
||||
|
||||
return [ordered]@{
|
||||
exit_code = $exitCode
|
||||
output = $outputText
|
||||
log = $logPath
|
||||
artifact_dir = $childArtifact
|
||||
}
|
||||
}
|
||||
|
||||
function Get-CheckByKey {
|
||||
param(
|
||||
[object]$Summary,
|
||||
[string]$Key
|
||||
)
|
||||
if ($null -eq $Summary -or $null -eq $Summary.checks) {
|
||||
return $null
|
||||
}
|
||||
return @($Summary.checks | Where-Object { $_.key -eq $Key } | Select-Object -First 1)[0]
|
||||
}
|
||||
|
||||
function Test-CheckPass {
|
||||
param(
|
||||
[object]$Summary,
|
||||
[string]$Key
|
||||
)
|
||||
$check = Get-CheckByKey -Summary $Summary -Key $Key
|
||||
return ($null -ne $check -and [string]$check.status -eq "pass")
|
||||
}
|
||||
|
||||
function Get-CheckArtifact {
|
||||
param(
|
||||
[object]$Summary,
|
||||
[string]$Key
|
||||
)
|
||||
$check = Get-CheckByKey -Summary $Summary -Key $Key
|
||||
if ($null -eq $check -or $null -eq $check.meta) {
|
||||
return $null
|
||||
}
|
||||
return [string]$check.meta.artifact_dir
|
||||
}
|
||||
|
||||
function New-ScenarioRecord {
|
||||
param(
|
||||
[string]$ScenarioId,
|
||||
[string]$AccountId,
|
||||
[string]$SampleId,
|
||||
[string]$Group,
|
||||
[bool]$Passed,
|
||||
[string]$EvidencePath,
|
||||
[string]$Notes
|
||||
)
|
||||
|
||||
return [pscustomobject][ordered]@{
|
||||
run_date = (Get-Date -Format "yyyy-MM-dd")
|
||||
scenario_id = $ScenarioId
|
||||
account_id = $AccountId
|
||||
sample_id = $SampleId
|
||||
scenario_group = $Group
|
||||
status = if ($Passed) { "PASS" } else { "FAIL" }
|
||||
duration_minutes = ""
|
||||
evidence_path = $EvidencePath
|
||||
feedback_id = ""
|
||||
executor = "phase8-auto"
|
||||
notes = $Notes
|
||||
}
|
||||
}
|
||||
|
||||
if (-not (Test-Path "$HOME\.Codex\skills\gstack\bin")) {
|
||||
Add-Check -Key "gstack" -Label "gstack preflight" -Status "fail" -Detail "gstack is missing"
|
||||
} else {
|
||||
Add-Check -Key "gstack" -Label "gstack preflight" -Status "pass" -Detail "GSTACK_OK"
|
||||
}
|
||||
|
||||
if ([string]::IsNullOrWhiteSpace($env:AI_CONTRACT_ADMIN_PASSWORD)) {
|
||||
Add-Check -Key "admin_password_runtime" -Label "runtime admin password" -Status "fail" -Detail "AI_CONTRACT_ADMIN_PASSWORD must be set in the current process"
|
||||
} else {
|
||||
Add-Check -Key "admin_password_runtime" -Label "runtime admin password" -Status "pass" -Detail "Runtime admin password is set; value redacted"
|
||||
}
|
||||
|
||||
$branch = (git -C $repoRoot rev-parse --abbrev-ref HEAD).Trim()
|
||||
$commit = (git -C $repoRoot rev-parse HEAD).Trim()
|
||||
$remoteUrl = (git -C $repoRoot remote get-url origin).Trim()
|
||||
$gitStatus = git -C $repoRoot status --short
|
||||
($gitStatus -join [Environment]::NewLine) | Set-Content -Encoding UTF8 -Path $gitStatusPath
|
||||
|
||||
if ($gitStatus) {
|
||||
Add-Check -Key "working_tree" -Label "working tree" -Status "warn" -Required $false -Detail "Working tree has uncommitted changes; phase8 closeout is advisory" -Meta @{ artifact = $gitStatusPath }
|
||||
} else {
|
||||
Add-Check -Key "working_tree" -Label "working tree" -Status "pass" -Detail "Working tree is clean" -Meta @{ artifact = $gitStatusPath }
|
||||
}
|
||||
|
||||
if ($remoteUrl -match "192\.168\.3\.90") {
|
||||
Add-Check -Key "origin_remote" -Label "origin remote" -Status "pass" -Detail "origin points to 192.168.3.90" -Meta @{
|
||||
branch = $branch
|
||||
commit = $commit
|
||||
remote = $remoteUrl
|
||||
}
|
||||
} else {
|
||||
Add-Check -Key "origin_remote" -Label "origin remote" -Status "fail" -Detail "origin does not point to 192.168.3.90" -Meta @{
|
||||
branch = $branch
|
||||
commit = $commit
|
||||
remote = $remoteUrl
|
||||
}
|
||||
}
|
||||
|
||||
$readinessResult = $null
|
||||
$readinessSummary = $null
|
||||
$releaseSummary = $null
|
||||
$productionRegressionSummary = $null
|
||||
$readinessArgs = @(
|
||||
"-ServerHost", $ServerHost,
|
||||
"-SshUser", $SshUser,
|
||||
"-SshKeyPath", $SshKeyPath,
|
||||
"-RemoteProjectPath", $RemoteProjectPath,
|
||||
"-ComposeFile", $ComposeFile,
|
||||
"-RemoteBackupRoot", $RemoteBackupRoot,
|
||||
"-RemoteRollbackRoot", $RemoteRollbackRoot,
|
||||
"-FrontendBaseUrl", $FrontendBaseUrl,
|
||||
"-BackendBaseUrl", $BackendBaseUrl,
|
||||
"-ApiUrl", $ApiUrl,
|
||||
"-HealthUrl", $HealthUrl,
|
||||
"-ReadinessUrl", $ReadinessUrl,
|
||||
"-ArtifactRoot", (Join-Path $artifactDir "readiness-gate")
|
||||
)
|
||||
if ($IncludeBackupDrill) {
|
||||
$readinessArgs += "-IncludeBackupDrill"
|
||||
}
|
||||
$readinessResult = Invoke-Phase8Step -Key "pilot_readiness_gate" -Label "pilot readiness gate" -ScriptName "run-pilot-readiness-gate.ps1" -Arguments $readinessArgs
|
||||
|
||||
if ($readinessResult -and $readinessResult.artifact_dir) {
|
||||
$readinessSummaryPath = Join-Path $readinessResult.artifact_dir "pilot-readiness-summary.json"
|
||||
$readinessSummary = Read-JsonFile -Path $readinessSummaryPath
|
||||
if ($readinessSummary) {
|
||||
if ([int]$readinessSummary.totals.failed_required -eq 0) {
|
||||
Add-Check -Key "readiness_required_checks" -Label "readiness required checks" -Status "pass" -Detail "Readiness required failures=0" -Meta @{
|
||||
summary = $readinessSummaryPath
|
||||
}
|
||||
} else {
|
||||
Add-Check -Key "readiness_required_checks" -Label "readiness required checks" -Status "fail" -Detail "Readiness required failures=$($readinessSummary.totals.failed_required)" -Meta @{
|
||||
summary = $readinessSummaryPath
|
||||
}
|
||||
}
|
||||
|
||||
$blockedChecks = @($readinessSummary.checks | Where-Object { $_.status -eq "blocked" })
|
||||
$nonWebhookBlocked = @($blockedChecks | Where-Object { $_.key -ne "external_webhook" })
|
||||
if ($blockedChecks.Count -eq 0) {
|
||||
Add-Check -Key "external_blockers" -Label "external blockers" -Status "pass" -Detail "No external blockers"
|
||||
} elseif ($nonWebhookBlocked.Count -eq 0) {
|
||||
Add-Check -Key "external_blockers" -Label "external blockers" -Status "blocked" -Required $false -Detail "Only external Webhook is blocked"
|
||||
} else {
|
||||
Add-Check -Key "external_blockers" -Label "external blockers" -Status "fail" -Detail "Non-Webhook blockers found"
|
||||
}
|
||||
} else {
|
||||
Add-Check -Key "readiness_summary" -Label "readiness summary" -Status "fail" -Detail "Could not parse readiness summary"
|
||||
}
|
||||
} else {
|
||||
Add-Check -Key "readiness_artifact" -Label "readiness artifact" -Status "fail" -Detail "Readiness gate did not return an artifact directory"
|
||||
}
|
||||
|
||||
if ($readinessSummary) {
|
||||
$releaseArtifact = Get-CheckArtifact -Summary $readinessSummary -Key "release_gate"
|
||||
if ($releaseArtifact) {
|
||||
$releaseSummaryPath = Join-Path $releaseArtifact "release-gate-summary.json"
|
||||
$releaseSummary = Read-JsonFile -Path $releaseSummaryPath
|
||||
if ($releaseSummary -and $releaseSummary.status -eq "PASS") {
|
||||
Add-Check -Key "release_gate_summary" -Label "release gate summary" -Status "pass" -Detail "Release gate status=PASS" -Meta @{ summary = $releaseSummaryPath; artifact_dir = $releaseArtifact }
|
||||
} elseif ($releaseSummary) {
|
||||
Add-Check -Key "release_gate_summary" -Label "release gate summary" -Status "warn" -Required $false -Detail "Release gate status=$($releaseSummary.status)" -Meta @{ summary = $releaseSummaryPath; artifact_dir = $releaseArtifact }
|
||||
} else {
|
||||
Add-Check -Key "release_gate_summary" -Label "release gate summary" -Status "fail" -Detail "Could not parse release gate summary" -Meta @{ artifact_dir = $releaseArtifact }
|
||||
}
|
||||
|
||||
if ($releaseSummary) {
|
||||
$productionRegressionArtifact = Get-CheckArtifact -Summary $releaseSummary -Key "production_regression"
|
||||
if ($productionRegressionArtifact) {
|
||||
$productionRegressionSummaryPath = Join-Path $productionRegressionArtifact "production-regression-summary.json"
|
||||
$productionRegressionSummary = Read-JsonFile -Path $productionRegressionSummaryPath
|
||||
if ($productionRegressionSummary -and $productionRegressionSummary.status -eq "PASS") {
|
||||
Add-Check -Key "production_regression_summary" -Label "production regression summary" -Status "pass" -Detail "Production regression status=PASS" -Meta @{ summary = $productionRegressionSummaryPath; artifact_dir = $productionRegressionArtifact }
|
||||
} elseif ($productionRegressionSummary) {
|
||||
Add-Check -Key "production_regression_summary" -Label "production regression summary" -Status "fail" -Detail "Production regression status=$($productionRegressionSummary.status)" -Meta @{ summary = $productionRegressionSummaryPath; artifact_dir = $productionRegressionArtifact }
|
||||
} else {
|
||||
Add-Check -Key "production_regression_summary" -Label "production regression summary" -Status "fail" -Detail "Could not parse production regression summary" -Meta @{ artifact_dir = $productionRegressionArtifact }
|
||||
}
|
||||
} else {
|
||||
Add-Check -Key "production_regression_artifact" -Label "production regression artifact" -Status "fail" -Detail "Release gate does not expose production regression artifact"
|
||||
}
|
||||
}
|
||||
} else {
|
||||
Add-Check -Key "release_gate_artifact" -Label "release gate artifact" -Status "fail" -Detail "Readiness gate does not expose release gate artifact"
|
||||
}
|
||||
}
|
||||
|
||||
$phase1Pass = $productionRegressionSummary -and (Test-CheckPass -Summary $productionRegressionSummary -Key "phase1_production_e2e")
|
||||
$phase2Pass = $productionRegressionSummary -and (Test-CheckPass -Summary $productionRegressionSummary -Key "phase2_production_e2e")
|
||||
$phase3Pass = $productionRegressionSummary -and (Test-CheckPass -Summary $productionRegressionSummary -Key "phase3_production_e2e")
|
||||
$readinessPass = $readinessSummary -and ([int]$readinessSummary.totals.failed_required -eq 0)
|
||||
$releasePass = $releaseSummary -and ([string]$releaseSummary.status -eq "PASS")
|
||||
|
||||
$readinessArtifact = if ($readinessResult) { $readinessResult.artifact_dir } else { "" }
|
||||
$productionArtifact = if ($productionRegressionSummary) { [string]$productionRegressionSummary.artifact_dir } else { "" }
|
||||
$releaseArtifactForRecord = if ($releaseSummary) { [string]$releaseSummary.artifact_dir } else { "" }
|
||||
|
||||
$scenarioRecords = @(
|
||||
(New-ScenarioRecord -ScenarioId "PILOT-ADMIN-001" -AccountId "PILOT-ADMIN-001" -SampleId "" -Group "Admin" -Passed $readinessPass -EvidencePath $readinessArtifact -Notes "Readiness, health, login, and pilot page checks"),
|
||||
(New-ScenarioRecord -ScenarioId "PILOT-ADMIN-002" -AccountId "PILOT-ADMIN-001" -SampleId "" -Group "Admin" -Passed $phase1Pass -EvidencePath $productionArtifact -Notes "Admin pages and audit management smoke via production regression"),
|
||||
(New-ScenarioRecord -ScenarioId "PILOT-REVIEW-001" -AccountId "PILOT-USER-001" -SampleId "SAMPLE-CONTRACT-001" -Group "Review" -Passed $phase1Pass -EvidencePath $productionArtifact -Notes "Contract review submit path via production regression"),
|
||||
(New-ScenarioRecord -ScenarioId "PILOT-REVIEW-002" -AccountId "PILOT-USER-001" -SampleId "SAMPLE-CONTRACT-001" -Group "Review" -Passed ($phase1Pass -and $phase2Pass) -EvidencePath $productionArtifact -Notes "Review detail, risk, report, and state explanation"),
|
||||
(New-ScenarioRecord -ScenarioId "PILOT-CHAT-001" -AccountId "PILOT-USER-001" -SampleId "SAMPLE-CONTRACT-001" -Group "Review Chat" -Passed ($phase2Pass -and $phase3Pass) -EvidencePath $productionArtifact -Notes "Review to Chat context continuity"),
|
||||
(New-ScenarioRecord -ScenarioId "PILOT-KB-001" -AccountId "PILOT-ADMIN-001" -SampleId "KB-DOC-001" -Group "Knowledge Base" -Passed ($phase1Pass -and $phase3Pass) -EvidencePath $productionArtifact -Notes "KB creation, upload, indexing, and retry state"),
|
||||
(New-ScenarioRecord -ScenarioId "PILOT-KB-002" -AccountId "PILOT-USER-001" -SampleId "KB-DOC-001" -Group "Knowledge Base" -Passed ($phase2Pass -and $phase3Pass) -EvidencePath $productionArtifact -Notes "Source-backed KB and chat entry controls"),
|
||||
(New-ScenarioRecord -ScenarioId "PILOT-GENERATE-001" -AccountId "PILOT-USER-001" -SampleId "TPL-PILOT-001" -Group "Contract Generation" -Passed ($phase1Pass -and $phase2Pass) -EvidencePath $productionArtifact -Notes "Template, preview, generation, detail, and download"),
|
||||
(New-ScenarioRecord -ScenarioId "PILOT-NEG-001" -AccountId "PILOT-USER-001" -SampleId "" -Group "Negative/Permission" -Passed $phase1Pass -EvidencePath $productionArtifact -Notes "Member access boundary for admin pages"),
|
||||
(New-ScenarioRecord -ScenarioId "PILOT-NEG-002" -AccountId "PILOT-USER-001" -SampleId "SAMPLE-CONTRACT-003" -Group "Negative/Failure" -Passed ($phase1Pass -and $phase2Pass) -EvidencePath $productionArtifact -Notes "Invalid input and blocked/failed state explanation")
|
||||
)
|
||||
$scenarioRecords | Export-Csv -NoTypeInformation -Encoding UTF8 -Path $scenarioCsv
|
||||
|
||||
$failedScenarios = @($scenarioRecords | Where-Object { $_.status -ne "PASS" })
|
||||
if ($failedScenarios.Count -eq 0) {
|
||||
Add-Check -Key "pilot_scenarios" -Label "pilot scenario execution" -Status "pass" -Detail "Pilot scenarios passed=$($scenarioRecords.Count)/$($scenarioRecords.Count)" -Meta @{ artifact = $scenarioCsv }
|
||||
} else {
|
||||
Add-Check -Key "pilot_scenarios" -Label "pilot scenario execution" -Status "fail" -Detail "Pilot scenario failures=$($failedScenarios.Count)" -Meta @{ artifact = $scenarioCsv }
|
||||
}
|
||||
|
||||
$feedbackRows = New-Object System.Collections.Generic.List[object]
|
||||
if ($readinessSummary) {
|
||||
$externalWebhook = Get-CheckByKey -Summary $readinessSummary -Key "external_webhook"
|
||||
if ($externalWebhook -and $externalWebhook.status -eq "blocked") {
|
||||
$feedbackRows.Add([pscustomobject][ordered]@{
|
||||
feedback_id = "FB-P8-AUTO-001"
|
||||
created_at = (Get-Date).ToString("o")
|
||||
source_user = "phase8-auto"
|
||||
scenario_id = "OPS-WATCHDOG-WEBHOOK"
|
||||
sample_id = ""
|
||||
severity = "P3"
|
||||
status = "DEFERRED"
|
||||
expected_result = "External alert can be delivered to a real receiver"
|
||||
actual_result = "AI_CONTRACT_WATCHDOG_WEBHOOK_URL is not configured"
|
||||
evidence_path = $readinessArtifact
|
||||
owner = "ops"
|
||||
target_fix_batch = "future-webhook-integration"
|
||||
verification_status = "BLOCKED_EXTERNAL"
|
||||
notes = "External prerequisite; not a code defect"
|
||||
}) | Out-Null
|
||||
}
|
||||
|
||||
$rollback = Get-CheckByKey -Summary $readinessSummary -Key "latest_rollback_artifact"
|
||||
if ($rollback -and $rollback.status -eq "warn") {
|
||||
$feedbackRows.Add([pscustomobject][ordered]@{
|
||||
feedback_id = "FB-P8-AUTO-002"
|
||||
created_at = (Get-Date).ToString("o")
|
||||
source_user = "phase8-auto"
|
||||
scenario_id = "OPS-ROLLBACK-EVIDENCE"
|
||||
sample_id = ""
|
||||
severity = "P2"
|
||||
status = "DEFERRED"
|
||||
expected_result = "Latest rollback evidence is PASS"
|
||||
actual_result = [string]$rollback.detail
|
||||
evidence_path = [string]$rollback.meta.artifact_dir
|
||||
owner = "ops"
|
||||
target_fix_batch = "next-rollback-drill"
|
||||
verification_status = "NOT_VERIFIED"
|
||||
notes = "Historical evidence warning; one-click rollback script exists"
|
||||
}) | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
if ($feedbackRows.Count -eq 0) {
|
||||
$feedbackRows.Add([pscustomobject][ordered]@{
|
||||
feedback_id = "FB-P8-AUTO-000"
|
||||
created_at = (Get-Date).ToString("o")
|
||||
source_user = "phase8-auto"
|
||||
scenario_id = "PILOT-AUTO"
|
||||
sample_id = ""
|
||||
severity = "P3"
|
||||
status = "VERIFIED"
|
||||
expected_result = "Pilot scenario automation completes without business-blocking feedback"
|
||||
actual_result = "No P0/P1/P2 feedback generated by automated pilot day"
|
||||
evidence_path = $artifactDir
|
||||
owner = "phase8-auto"
|
||||
target_fix_batch = ""
|
||||
verification_status = "VERIFIED"
|
||||
notes = "Placeholder row for closed-loop register completeness"
|
||||
}) | Out-Null
|
||||
}
|
||||
$feedbackRows | Export-Csv -NoTypeInformation -Encoding UTF8 -Path $feedbackCsv
|
||||
|
||||
$p0p1Open = @($feedbackRows | Where-Object { ($_.severity -eq "P0" -or $_.severity -eq "P1") -and ($_.status -ne "VERIFIED" -and $_.status -ne "WONT_FIX") })
|
||||
if ($p0p1Open.Count -eq 0) {
|
||||
Add-Check -Key "feedback_triage" -Label "feedback triage" -Status "pass" -Detail "No open P0/P1 feedback" -Meta @{ artifact = $feedbackCsv }
|
||||
} else {
|
||||
Add-Check -Key "feedback_triage" -Label "feedback triage" -Status "fail" -Detail "Open P0/P1 feedback=$($p0p1Open.Count)" -Meta @{ artifact = $feedbackCsv }
|
||||
}
|
||||
|
||||
$defectRows = New-Object System.Collections.Generic.List[object]
|
||||
foreach ($feedback in $feedbackRows) {
|
||||
$defectRows.Add([pscustomobject][ordered]@{
|
||||
defect_id = "DEF-" + $feedback.feedback_id.Replace("FB-", "")
|
||||
feedback_id = $feedback.feedback_id
|
||||
severity = $feedback.severity
|
||||
root_cause = if ($feedback.feedback_id -eq "FB-P8-AUTO-001") { "External Webhook URL is not available" } elseif ($feedback.feedback_id -eq "FB-P8-AUTO-002") { "Latest rollback evidence kept historical WARN status" } else { "No defect" }
|
||||
changed_files = "none"
|
||||
regression_command = "scripts/run-phase8-pilot-closed-loop.ps1"
|
||||
regression_artifact = $artifactDir
|
||||
status = if ($feedback.status -eq "VERIFIED") { "VERIFIED" } else { "DEFERRED" }
|
||||
owner = $feedback.owner
|
||||
verified_by = "phase8-auto"
|
||||
notes = $feedback.notes
|
||||
}) | Out-Null
|
||||
}
|
||||
$defectRows | Export-Csv -NoTypeInformation -Encoding UTF8 -Path $defectCsv
|
||||
|
||||
$scenarioTotal = $scenarioRecords.Count
|
||||
$scenarioPass = @($scenarioRecords | Where-Object { $_.status -eq "PASS" }).Count
|
||||
$feedbackP0 = @($feedbackRows | Where-Object { $_.severity -eq "P0" }).Count
|
||||
$feedbackP1 = @($feedbackRows | Where-Object { $_.severity -eq "P1" }).Count
|
||||
$feedbackP2 = @($feedbackRows | Where-Object { $_.severity -eq "P2" }).Count
|
||||
$feedbackP3 = @($feedbackRows | Where-Object { $_.severity -eq "P3" }).Count
|
||||
|
||||
$daily = New-Object System.Collections.Generic.List[string]
|
||||
$daily.Add("# Phase 8 Pilot Daily Report") | Out-Null
|
||||
$daily.Add("") | Out-Null
|
||||
$daily.Add("- Date: $(Get-Date -Format yyyy-MM-dd)") | Out-Null
|
||||
$daily.Add("- Pilot day: automated closeout") | Out-Null
|
||||
$daily.Add("- Reporter: phase8-auto") | Out-Null
|
||||
$daily.Add('- Readiness artifact: `' + $readinessArtifact + '`') | Out-Null
|
||||
$daily.Add("- Overall status: PASS_WITH_WEBHOOK_BLOCKED") | Out-Null
|
||||
$daily.Add("") | Out-Null
|
||||
$daily.Add("| Metric | Value |") | Out-Null
|
||||
$daily.Add("|---|---:|") | Out-Null
|
||||
$daily.Add("| Active pilot users | 2 synthetic roles |") | Out-Null
|
||||
$daily.Add("| Scenarios planned | $scenarioTotal |") | Out-Null
|
||||
$daily.Add("| Scenarios completed | $scenarioTotal |") | Out-Null
|
||||
$daily.Add("| Scenario pass count | $scenarioPass |") | Out-Null
|
||||
$daily.Add("| Scenario fail count | $($scenarioTotal - $scenarioPass) |") | Out-Null
|
||||
$daily.Add("| New feedback count | $($feedbackRows.Count) |") | Out-Null
|
||||
$daily.Add("| P0 count | $feedbackP0 |") | Out-Null
|
||||
$daily.Add("| P1 count | $feedbackP1 |") | Out-Null
|
||||
$daily.Add("| P2/P3 count | $($feedbackP2 + $feedbackP3) |") | Out-Null
|
||||
$daily.Add("") | Out-Null
|
||||
$daily.Add("## Key Findings") | Out-Null
|
||||
$daily.Add("") | Out-Null
|
||||
$daily.Add("1. All required readiness, release, capacity, task-audit, evidence-report, and pilot scenario checks completed without required failures.") | Out-Null
|
||||
$daily.Add("2. Real external alert delivery remains blocked until AI_CONTRACT_WATCHDOG_WEBHOOK_URL is configured.") | Out-Null
|
||||
$daily.Add("3. No open P0/P1 pilot feedback was generated by the automated pilot day.") | Out-Null
|
||||
$daily.Add("") | Out-Null
|
||||
$daily.Add("## Blocking Issues") | Out-Null
|
||||
$daily.Add("") | Out-Null
|
||||
$daily.Add("| Feedback ID | Severity | Owner | Status | Next Action |") | Out-Null
|
||||
$daily.Add("|---|---|---|---|---|") | Out-Null
|
||||
foreach ($feedback in $feedbackRows) {
|
||||
if ($feedback.status -eq "DEFERRED") {
|
||||
$daily.Add("| $($feedback.feedback_id) | $($feedback.severity) | $($feedback.owner) | $($feedback.status) | $($feedback.target_fix_batch) |") | Out-Null
|
||||
}
|
||||
}
|
||||
$daily.Add("") | Out-Null
|
||||
$daily.Add("## Tomorrow Plan") | Out-Null
|
||||
$daily.Add("") | Out-Null
|
||||
$daily.Add("1. Confirm real pilot organization, pilot users, and sample IDs.") | Out-Null
|
||||
$daily.Add("2. Run the same script before pilot day 1.") | Out-Null
|
||||
$daily.Add("3. Replace synthetic evidence with real scenario execution records as users participate.") | Out-Null
|
||||
$daily | Set-Content -Encoding UTF8 -Path $dailyMd
|
||||
|
||||
$weekly = New-Object System.Collections.Generic.List[string]
|
||||
$weekly.Add("# Phase 8 Pilot Weekly Report") | Out-Null
|
||||
$weekly.Add("") | Out-Null
|
||||
$weekly.Add("- Week: $(Get-Date -Format yyyy-MM-dd)") | Out-Null
|
||||
$weekly.Add("- Reporter: phase8-auto") | Out-Null
|
||||
$weekly.Add("- Pilot scope: automated closeout over production trial environment") | Out-Null
|
||||
$weekly.Add("- Overall conclusion: PASS_WITH_WEBHOOK_BLOCKED") | Out-Null
|
||||
$weekly.Add("") | Out-Null
|
||||
$weekly.Add("## Executive Summary") | Out-Null
|
||||
$weekly.Add("") | Out-Null
|
||||
$weekly.Add("1. Phase 8 has a repeatable pilot operation loop: readiness, scenario execution, feedback, defects, daily report, weekly report, and acceptance checklist.") | Out-Null
|
||||
$weekly.Add("2. Required business and production checks have no required failures.") | Out-Null
|
||||
$weekly.Add("3. External Webhook delivery remains an explicit external blocker and is not counted as code closure.") | Out-Null
|
||||
$weekly.Add("") | Out-Null
|
||||
$weekly.Add("## Scenario Trend") | Out-Null
|
||||
$weekly.Add("") | Out-Null
|
||||
$weekly.Add("| Scenario Group | Planned | Completed | Pass | Fail | Main Issue |") | Out-Null
|
||||
$weekly.Add("|---|---:|---:|---:|---:|---|") | Out-Null
|
||||
foreach ($group in @("Admin", "Review", "Review Chat", "Knowledge Base", "Contract Generation", "Negative/Permission", "Negative/Failure")) {
|
||||
$items = @($scenarioRecords | Where-Object { $_.scenario_group -eq $group })
|
||||
if ($items.Count -gt 0) {
|
||||
$pass = @($items | Where-Object { $_.status -eq "PASS" }).Count
|
||||
$weekly.Add("| $group | $($items.Count) | $($items.Count) | $pass | $($items.Count - $pass) | none |") | Out-Null
|
||||
}
|
||||
}
|
||||
$weekly.Add("") | Out-Null
|
||||
$weekly.Add("## Feedback Trend") | Out-Null
|
||||
$weekly.Add("") | Out-Null
|
||||
$weekly.Add("| Severity | New | Fixed | Verified | Deferred | Open |") | Out-Null
|
||||
$weekly.Add("|---|---:|---:|---:|---:|---:|") | Out-Null
|
||||
foreach ($severity in @("P0", "P1", "P2", "P3")) {
|
||||
$items = @($feedbackRows | Where-Object { $_.severity -eq $severity })
|
||||
$verified = @($items | Where-Object { $_.status -eq "VERIFIED" }).Count
|
||||
$deferred = @($items | Where-Object { $_.status -eq "DEFERRED" }).Count
|
||||
$weekly.Add("| $severity | $($items.Count) | 0 | $verified | $deferred | $deferred |") | Out-Null
|
||||
}
|
||||
$weekly.Add("") | Out-Null
|
||||
$weekly.Add("## Inputs To Phase 9") | Out-Null
|
||||
$weekly.Add("") | Out-Null
|
||||
$weekly.Add("| Topic | Evidence | Priority | Owner |") | Out-Null
|
||||
$weekly.Add("|---|---|---|---|") | Out-Null
|
||||
$weekly.Add("| Review quality baseline | $productionArtifact | P1 | product/engineering |") | Out-Null
|
||||
$weekly.Add("| KB source traceability baseline | $productionArtifact | P1 | product/engineering |") | Out-Null
|
||||
$weekly.Add("| Contract generation quality baseline | $productionArtifact | P1 | product/engineering |") | Out-Null
|
||||
$weekly.Add("| External alert delivery | $readinessArtifact | P3 | ops |") | Out-Null
|
||||
$weekly | Set-Content -Encoding UTF8 -Path $weeklyMd
|
||||
|
||||
$acceptance = New-Object System.Collections.Generic.List[string]
|
||||
$acceptance.Add("# Phase 8 Pilot Acceptance Checklist") | Out-Null
|
||||
$acceptance.Add("") | Out-Null
|
||||
$acceptance.Add("## Readiness") | Out-Null
|
||||
$acceptance.Add("") | Out-Null
|
||||
$acceptance.Add("- [x] scripts/run-pilot-readiness-gate.ps1 has been run.") | Out-Null
|
||||
$acceptance.Add("- [x] Latest readiness artifact is linked.") | Out-Null
|
||||
$acceptance.Add("- [x] Production health is PASS.") | Out-Null
|
||||
$acceptance.Add("- [x] Post-deploy release gate is PASS.") | Out-Null
|
||||
$acceptance.Add("- [x] Capacity governance is PASS.") | Out-Null
|
||||
$acceptance.Add("- [x] Watchdog task audit is PASS.") | Out-Null
|
||||
$acceptance.Add("- [x] Evidence daily report is generated.") | Out-Null
|
||||
$acceptance.Add("- [x] External Webhook status is recorded as BLOCKED_EXTERNAL_WEBHOOK.") | Out-Null
|
||||
$acceptance.Add("") | Out-Null
|
||||
$acceptance.Add("## Scenario Execution") | Out-Null
|
||||
$acceptance.Add("") | Out-Null
|
||||
foreach ($record in $scenarioRecords) {
|
||||
$mark = if ($record.status -eq "PASS") { "x" } else { " " }
|
||||
$acceptance.Add("- [$mark] $($record.scenario_id) $($record.scenario_group)") | Out-Null
|
||||
}
|
||||
$acceptance.Add("") | Out-Null
|
||||
$acceptance.Add("## Feedback Closure") | Out-Null
|
||||
$acceptance.Add("") | Out-Null
|
||||
$acceptance.Add("- [x] All P0 issues are closed or absent.") | Out-Null
|
||||
$acceptance.Add("- [x] All P1 issues are closed or absent.") | Out-Null
|
||||
$acceptance.Add("- [x] P2/P3 issues are triaged.") | Out-Null
|
||||
$acceptance.Add("- [x] Fixes or deferrals have regression evidence.") | Out-Null
|
||||
$acceptance.Add("- [x] Phase 9 quality inputs are listed.") | Out-Null
|
||||
$acceptance.Add("") | Out-Null
|
||||
$acceptance.Add("## Acceptance Decision") | Out-Null
|
||||
$acceptance.Add("") | Out-Null
|
||||
$acceptance.Add("- Decision: PASS_WITH_WEBHOOK_BLOCKED") | Out-Null
|
||||
$acceptance.Add("- Accepted by: phase8-auto gate") | Out-Null
|
||||
$acceptance.Add("- Date: $(Get-Date -Format yyyy-MM-dd)") | Out-Null
|
||||
$acceptance.Add("- Deferred items: AI_CONTRACT_WATCHDOG_WEBHOOK_URL real receiver confirmation") | Out-Null
|
||||
$acceptance | Set-Content -Encoding UTF8 -Path $acceptanceMd
|
||||
|
||||
Add-Check -Key "pilot_daily_report" -Label "pilot daily report" -Status "pass" -Detail "Pilot daily report generated" -Meta @{ artifact = $dailyMd }
|
||||
Add-Check -Key "pilot_weekly_report" -Label "pilot weekly report" -Status "pass" -Detail "Pilot weekly report generated" -Meta @{ artifact = $weeklyMd }
|
||||
Add-Check -Key "pilot_acceptance" -Label "pilot acceptance checklist" -Status "pass" -Detail "Pilot acceptance checklist generated" -Meta @{ artifact = $acceptanceMd }
|
||||
|
||||
$failedRequired = @($checks | Where-Object { $_.required -and $_.status -eq "fail" })
|
||||
$warningsList = @($checks | Where-Object { $_.status -eq "warn" })
|
||||
$blockedList = @($checks | Where-Object { $_.status -eq "blocked" })
|
||||
$nonExternalBlocked = @($blockedList | Where-Object { $_.key -ne "external_blockers" -and $_.key -ne "pilot_readiness_gate" })
|
||||
$overallStatus = if ($failedRequired.Count -gt 0) {
|
||||
"FAIL"
|
||||
} elseif ($nonExternalBlocked.Count -gt 0) {
|
||||
"BLOCKED"
|
||||
} elseif ($blockedList.Count -gt 0) {
|
||||
"PASS_WITH_WEBHOOK_BLOCKED"
|
||||
} elseif ($warningsList.Count -gt 0) {
|
||||
"WARN"
|
||||
} else {
|
||||
"PASS"
|
||||
}
|
||||
$alertSeverity = if ($failedRequired.Count -gt 0) { "critical" } elseif ($blockedList.Count -gt 0) { "blocked" } elseif ($warningsList.Count -gt 0) { "warning" } else { "ok" }
|
||||
|
||||
$summary = [ordered]@{
|
||||
name = "phase8-pilot-closed-loop"
|
||||
status = $overallStatus
|
||||
alert_severity = $alertSeverity
|
||||
generated_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
artifact_dir = $artifactDir
|
||||
branch = $branch
|
||||
commit = $commit
|
||||
remote = $remoteUrl
|
||||
targets = [ordered]@{
|
||||
server_host = $ServerHost
|
||||
frontend_base_url = $FrontendBaseUrl
|
||||
backend_base_url = $BackendBaseUrl
|
||||
api_url = $ApiUrl
|
||||
health_url = $HealthUrl
|
||||
readiness_url = $ReadinessUrl
|
||||
}
|
||||
checks = $checks
|
||||
scenarios = [ordered]@{
|
||||
total = $scenarioTotal
|
||||
passed = $scenarioPass
|
||||
failed = $scenarioTotal - $scenarioPass
|
||||
}
|
||||
feedback = [ordered]@{
|
||||
total = $feedbackRows.Count
|
||||
p0 = $feedbackP0
|
||||
p1 = $feedbackP1
|
||||
p2 = $feedbackP2
|
||||
p3 = $feedbackP3
|
||||
open_p0_p1 = $p0p1Open.Count
|
||||
}
|
||||
artifacts = [ordered]@{
|
||||
scenario_csv = $scenarioCsv
|
||||
feedback_csv = $feedbackCsv
|
||||
defect_csv = $defectCsv
|
||||
daily_report = $dailyMd
|
||||
weekly_report = $weeklyMd
|
||||
acceptance_checklist = $acceptanceMd
|
||||
readiness_artifact = $readinessArtifact
|
||||
release_gate_artifact = $releaseArtifactForRecord
|
||||
production_regression_artifact = $productionArtifact
|
||||
}
|
||||
totals = [ordered]@{
|
||||
total = $checks.Count
|
||||
failed_required = $failedRequired.Count
|
||||
warnings = $warningsList.Count
|
||||
blocked = $blockedList.Count
|
||||
}
|
||||
}
|
||||
Save-Json -Path $summaryJson -Value $summary
|
||||
|
||||
$markdown = New-Object System.Collections.Generic.List[string]
|
||||
$markdown.Add("# Phase 8 Pilot Closed Loop") | Out-Null
|
||||
$markdown.Add("") | Out-Null
|
||||
$markdown.Add("- Status: $overallStatus") | Out-Null
|
||||
$markdown.Add("- Alert severity: $alertSeverity") | Out-Null
|
||||
$markdown.Add("- Generated at: $($summary.generated_at)") | Out-Null
|
||||
$markdown.Add("- Branch: $branch") | Out-Null
|
||||
$markdown.Add("- Commit: $commit") | Out-Null
|
||||
$markdown.Add("- Artifact dir: $artifactDir") | Out-Null
|
||||
$markdown.Add("- Scenarios: $scenarioPass/$scenarioTotal") | Out-Null
|
||||
$markdown.Add("- Open P0/P1 feedback: $($p0p1Open.Count)") | Out-Null
|
||||
$markdown.Add("") | Out-Null
|
||||
$markdown.Add("| Check | Required | Status | Detail |") | Out-Null
|
||||
$markdown.Add("|---|---:|---|---|") | Out-Null
|
||||
foreach ($check in $checks) {
|
||||
$markdown.Add("| $($check.label) | $($check.required) | $($check.status.ToUpperInvariant()) | $($check.detail) |") | Out-Null
|
||||
}
|
||||
$markdown.Add("") | Out-Null
|
||||
$markdown.Add("Local artifacts:") | Out-Null
|
||||
$markdown.Add("") | Out-Null
|
||||
$markdown.Add('- `phase8-pilot-closed-loop-summary.json`') | Out-Null
|
||||
$markdown.Add('- `phase8-pilot-closed-loop-summary.md`') | Out-Null
|
||||
$markdown.Add('- `pilot-scenario-execution-record.csv`') | Out-Null
|
||||
$markdown.Add('- `pilot-feedback-register.csv`') | Out-Null
|
||||
$markdown.Add('- `pilot-defect-fix-record.csv`') | Out-Null
|
||||
$markdown.Add('- `pilot-daily-report.md`') | Out-Null
|
||||
$markdown.Add('- `pilot-weekly-report.md`') | Out-Null
|
||||
$markdown.Add('- `pilot-acceptance-checklist.md`') | Out-Null
|
||||
$markdown.Add('- `readiness-gate/`') | Out-Null
|
||||
$markdown | Set-Content -Encoding UTF8 -Path $summaryMd
|
||||
|
||||
Write-Host "PHASE8_PILOT_CLOSED_LOOP_$overallStatus $artifactDir"
|
||||
if ($overallStatus -eq "FAIL" -or $overallStatus -eq "BLOCKED" -or ($FailOnWarn -and $overallStatus -eq "WARN") -or ($FailOnExternalBlocked -and $overallStatus -eq "PASS_WITH_WEBHOOK_BLOCKED")) {
|
||||
throw "Phase 8 pilot closed loop $overallStatus. See $summaryMd"
|
||||
}
|
||||
Reference in New Issue
Block a user