|
|
|
@@ -0,0 +1,513 @@
|
|
|
|
|
param(
|
|
|
|
|
[string]$ArtifactRoot = "",
|
|
|
|
|
[string]$FixtureRoot = "",
|
|
|
|
|
[string]$Phase8ArtifactRoot = "",
|
|
|
|
|
[int]$MaxPhase8AgeHours = 48,
|
|
|
|
|
[switch]$FailOnWarn,
|
|
|
|
|
[switch]$FailOnExternalBlocked
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
|
|
|
|
|
|
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path
|
|
|
|
|
$timestamp = Get-Date -Format "yyyy-MM-ddTHH-mm-ss-fff"
|
|
|
|
|
if ([string]::IsNullOrWhiteSpace($ArtifactRoot)) {
|
|
|
|
|
$ArtifactRoot = Join-Path $repoRoot "project-docs\qa-artifacts\phase9-business-quality-gate"
|
|
|
|
|
}
|
|
|
|
|
if ([string]::IsNullOrWhiteSpace($FixtureRoot)) {
|
|
|
|
|
$FixtureRoot = Join-Path $repoRoot "project-docs\phase9-quality-fixtures"
|
|
|
|
|
}
|
|
|
|
|
if ([string]::IsNullOrWhiteSpace($Phase8ArtifactRoot)) {
|
|
|
|
|
$Phase8ArtifactRoot = Join-Path $repoRoot "project-docs\qa-artifacts\phase8-pilot-closed-loop"
|
|
|
|
|
}
|
|
|
|
|
foreach ($pathName in @("ArtifactRoot", "FixtureRoot", "Phase8ArtifactRoot")) {
|
|
|
|
|
$value = Get-Variable -Name $pathName -ValueOnly
|
|
|
|
|
if (-not [System.IO.Path]::IsPathRooted($value)) {
|
|
|
|
|
Set-Variable -Name $pathName -Value (Join-Path $repoRoot $value)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$artifactDir = Join-Path $ArtifactRoot $timestamp
|
|
|
|
|
New-Item -ItemType Directory -Force -Path $artifactDir | Out-Null
|
|
|
|
|
|
|
|
|
|
$summaryJson = Join-Path $artifactDir "business-quality-gate-summary.json"
|
|
|
|
|
$summaryMd = Join-Path $artifactDir "business-quality-gate-summary.md"
|
|
|
|
|
$scorecardCsv = Join-Path $artifactDir "phase9-quality-scorecard.csv"
|
|
|
|
|
$defectCsv = Join-Path $artifactDir "phase9-quality-defect-register.csv"
|
|
|
|
|
$gitStatusPath = Join-Path $artifactDir "git-status.txt"
|
|
|
|
|
|
|
|
|
|
$checks = New-Object System.Collections.Generic.List[object]
|
|
|
|
|
|
|
|
|
|
function Add-Check {
|
|
|
|
|
param(
|
|
|
|
|
[string]$Key,
|
|
|
|
|
[string]$Label,
|
|
|
|
|
[ValidateSet("pass", "fail", "warn", "skip", "blocked")]
|
|
|
|
|
[string]$Status,
|
|
|
|
|
[string]$Detail,
|
|
|
|
|
[bool]$Required = $true,
|
|
|
|
|
[hashtable]$Meta = @{}
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
$checks.Add([ordered]@{
|
|
|
|
|
key = $Key
|
|
|
|
|
label = $Label
|
|
|
|
|
status = $Status
|
|
|
|
|
required = $Required
|
|
|
|
|
detail = $Detail
|
|
|
|
|
meta = $Meta
|
|
|
|
|
}) | Out-Null
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function Save-Json {
|
|
|
|
|
param([string]$Path, [object]$Value)
|
|
|
|
|
$Value | ConvertTo-Json -Depth 18 | Set-Content -Encoding UTF8 -Path $Path
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function Read-JsonFile {
|
|
|
|
|
param([string]$Path)
|
|
|
|
|
if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) {
|
|
|
|
|
return $null
|
|
|
|
|
}
|
|
|
|
|
try {
|
|
|
|
|
return Get-Content -LiteralPath $Path -Raw -Encoding UTF8 | ConvertFrom-Json
|
|
|
|
|
} catch {
|
|
|
|
|
return $null
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function Get-LatestSummaryArtifact {
|
|
|
|
|
param(
|
|
|
|
|
[string]$Root,
|
|
|
|
|
[string]$SummaryFile
|
|
|
|
|
)
|
|
|
|
|
if (-not (Test-Path -LiteralPath $Root -PathType Container)) {
|
|
|
|
|
return $null
|
|
|
|
|
}
|
|
|
|
|
return Get-ChildItem -LiteralPath $Root -Directory |
|
|
|
|
|
Sort-Object LastWriteTime -Descending |
|
|
|
|
|
Where-Object { Test-Path -LiteralPath (Join-Path $_.FullName $SummaryFile) } |
|
|
|
|
|
Select-Object -First 1
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function Test-CsvColumns {
|
|
|
|
|
param(
|
|
|
|
|
[object[]]$Rows,
|
|
|
|
|
[string[]]$Columns
|
|
|
|
|
)
|
|
|
|
|
if ($Rows.Count -eq 0) {
|
|
|
|
|
return $false
|
|
|
|
|
}
|
|
|
|
|
$names = @($Rows[0].PSObject.Properties.Name)
|
|
|
|
|
foreach ($column in $Columns) {
|
|
|
|
|
if ($names -notcontains $column) {
|
|
|
|
|
return $false
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return $true
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function Get-Scenario {
|
|
|
|
|
param(
|
|
|
|
|
[object[]]$Rows,
|
|
|
|
|
[string]$ScenarioId
|
|
|
|
|
)
|
|
|
|
|
return @($Rows | Where-Object { $_.scenario_id -eq $ScenarioId } | Select-Object -First 1)[0]
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function Test-ScenariosPass {
|
|
|
|
|
param(
|
|
|
|
|
[object[]]$Rows,
|
|
|
|
|
[string[]]$ScenarioIds
|
|
|
|
|
)
|
|
|
|
|
foreach ($scenarioId in $ScenarioIds) {
|
|
|
|
|
$item = Get-Scenario -Rows $Rows -ScenarioId $scenarioId
|
|
|
|
|
if ($null -eq $item -or $item.status -ne "PASS") {
|
|
|
|
|
return $false
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return $true
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function Write-DomainReport {
|
|
|
|
|
param(
|
|
|
|
|
[string]$Domain,
|
|
|
|
|
[string]$Title,
|
|
|
|
|
[object[]]$FixtureRows,
|
|
|
|
|
[string[]]$ScenarioIds,
|
|
|
|
|
[object[]]$ScenarioRows,
|
|
|
|
|
[string[]]$RequiredColumns,
|
|
|
|
|
[string]$OutputPrefix,
|
|
|
|
|
[string]$ExtraDetail = ""
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
$columnsOk = Test-CsvColumns -Rows $FixtureRows -Columns $RequiredColumns
|
|
|
|
|
$fixturesReady = $FixtureRows.Count -gt 0 -and $columnsOk
|
|
|
|
|
$scenariosPass = Test-ScenariosPass -Rows $ScenarioRows -ScenarioIds $ScenarioIds
|
|
|
|
|
$score = if ($fixturesReady -and $scenariosPass) { 100 } elseif ($fixturesReady -or $scenariosPass) { 50 } else { 0 }
|
|
|
|
|
$status = if ($score -eq 100) { "PASS" } else { "FAIL" }
|
|
|
|
|
|
|
|
|
|
$jsonPath = Join-Path $artifactDir "$OutputPrefix.json"
|
|
|
|
|
$mdPath = Join-Path $artifactDir "$OutputPrefix.md"
|
|
|
|
|
$record = [ordered]@{
|
|
|
|
|
domain = $Domain
|
|
|
|
|
title = $Title
|
|
|
|
|
status = $status
|
|
|
|
|
score = $score
|
|
|
|
|
fixture_count = $FixtureRows.Count
|
|
|
|
|
required_columns = $RequiredColumns
|
|
|
|
|
scenario_ids = $ScenarioIds
|
|
|
|
|
scenarios_pass = $scenariosPass
|
|
|
|
|
fixtures_ready = $fixturesReady
|
|
|
|
|
extra_detail = $ExtraDetail
|
|
|
|
|
}
|
|
|
|
|
Save-Json -Path $jsonPath -Value $record
|
|
|
|
|
|
|
|
|
|
$markdown = New-Object System.Collections.Generic.List[string]
|
|
|
|
|
$markdown.Add("# $Title") | Out-Null
|
|
|
|
|
$markdown.Add("") | Out-Null
|
|
|
|
|
$markdown.Add("- Status: $status") | Out-Null
|
|
|
|
|
$markdown.Add("- Score: $score") | Out-Null
|
|
|
|
|
$markdown.Add("- Fixture count: $($FixtureRows.Count)") | Out-Null
|
|
|
|
|
$markdown.Add("- Fixtures ready: $fixturesReady") | Out-Null
|
|
|
|
|
$markdown.Add("- Scenarios pass: $scenariosPass") | Out-Null
|
|
|
|
|
if (-not [string]::IsNullOrWhiteSpace($ExtraDetail)) {
|
|
|
|
|
$markdown.Add("- Detail: $ExtraDetail") | Out-Null
|
|
|
|
|
}
|
|
|
|
|
$markdown.Add("") | Out-Null
|
|
|
|
|
$markdown.Add("| Scenario | Status |") | Out-Null
|
|
|
|
|
$markdown.Add("|---|---|") | Out-Null
|
|
|
|
|
foreach ($scenarioId in $ScenarioIds) {
|
|
|
|
|
$scenario = Get-Scenario -Rows $ScenarioRows -ScenarioId $scenarioId
|
|
|
|
|
$scenarioStatus = if ($scenario) { $scenario.status } else { "MISSING" }
|
|
|
|
|
$markdown.Add("| $scenarioId | $scenarioStatus |") | Out-Null
|
|
|
|
|
}
|
|
|
|
|
$markdown | Set-Content -Encoding UTF8 -Path $mdPath
|
|
|
|
|
|
|
|
|
|
return [pscustomobject][ordered]@{
|
|
|
|
|
domain = $Domain
|
|
|
|
|
status = $status
|
|
|
|
|
score = $score
|
|
|
|
|
fixture_count = $FixtureRows.Count
|
|
|
|
|
scenarios = ($ScenarioIds -join ";")
|
|
|
|
|
report_json = $jsonPath
|
|
|
|
|
report_md = $mdPath
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (-not (Test-Path "$HOME\.Codex\skills\gstack\bin")) {
|
|
|
|
|
Add-Check -Key "gstack" -Label "gstack preflight" -Status "fail" -Detail "gstack is missing"
|
|
|
|
|
} else {
|
|
|
|
|
Add-Check -Key "gstack" -Label "gstack preflight" -Status "pass" -Detail "GSTACK_OK"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$branch = (git -C $repoRoot rev-parse --abbrev-ref HEAD).Trim()
|
|
|
|
|
$commit = (git -C $repoRoot rev-parse HEAD).Trim()
|
|
|
|
|
$remoteUrl = (git -C $repoRoot remote get-url origin).Trim()
|
|
|
|
|
$gitStatus = git -C $repoRoot status --short
|
|
|
|
|
($gitStatus -join [Environment]::NewLine) | Set-Content -Encoding UTF8 -Path $gitStatusPath
|
|
|
|
|
|
|
|
|
|
if ($gitStatus) {
|
|
|
|
|
Add-Check -Key "working_tree" -Label "working tree" -Status "warn" -Required $false -Detail "Working tree has uncommitted changes; quality gate is advisory" -Meta @{ artifact = $gitStatusPath }
|
|
|
|
|
} else {
|
|
|
|
|
Add-Check -Key "working_tree" -Label "working tree" -Status "pass" -Detail "Working tree is clean" -Meta @{ artifact = $gitStatusPath }
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if ($remoteUrl -match "192\.168\.3\.90") {
|
|
|
|
|
Add-Check -Key "origin_remote" -Label "origin remote" -Status "pass" -Detail "origin points to 192.168.3.90" -Meta @{ branch = $branch; commit = $commit; remote = $remoteUrl }
|
|
|
|
|
} else {
|
|
|
|
|
Add-Check -Key "origin_remote" -Label "origin remote" -Status "fail" -Detail "origin does not point to 192.168.3.90" -Meta @{ branch = $branch; commit = $commit; remote = $remoteUrl }
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (Test-Path -LiteralPath $FixtureRoot -PathType Container) {
|
|
|
|
|
Add-Check -Key "fixture_root" -Label "quality fixture root" -Status "pass" -Detail "Fixture root exists" -Meta @{ root = $FixtureRoot }
|
|
|
|
|
} else {
|
|
|
|
|
Add-Check -Key "fixture_root" -Label "quality fixture root" -Status "fail" -Detail "Fixture root missing" -Meta @{ root = $FixtureRoot }
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$requiredFixtureFiles = @(
|
|
|
|
|
"quality-metric-dictionary.csv",
|
|
|
|
|
"golden-review-samples.csv",
|
|
|
|
|
"golden-kb-question-set.csv",
|
|
|
|
|
"golden-contract-generation-set.csv",
|
|
|
|
|
"prompt-version-register.csv",
|
|
|
|
|
"ai-output-governance.md",
|
|
|
|
|
"data-retention-policy.md"
|
|
|
|
|
)
|
|
|
|
|
foreach ($fixtureFile in $requiredFixtureFiles) {
|
|
|
|
|
$fixturePath = Join-Path $FixtureRoot $fixtureFile
|
|
|
|
|
if (Test-Path -LiteralPath $fixturePath -PathType Leaf) {
|
|
|
|
|
Add-Check -Key ("fixture_" + ($fixtureFile -replace '[^a-zA-Z0-9]', '_')) -Label "fixture $fixtureFile" -Status "pass" -Detail "Fixture exists" -Meta @{ path = $fixturePath }
|
|
|
|
|
} else {
|
|
|
|
|
Add-Check -Key ("fixture_" + ($fixtureFile -replace '[^a-zA-Z0-9]', '_')) -Label "fixture $fixtureFile" -Status "fail" -Detail "Fixture missing" -Meta @{ path = $fixturePath }
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$metrics = @(Import-Csv -LiteralPath (Join-Path $FixtureRoot "quality-metric-dictionary.csv"))
|
|
|
|
|
$reviewFixtures = @(Import-Csv -LiteralPath (Join-Path $FixtureRoot "golden-review-samples.csv"))
|
|
|
|
|
$kbFixtures = @(Import-Csv -LiteralPath (Join-Path $FixtureRoot "golden-kb-question-set.csv"))
|
|
|
|
|
$contractFixtures = @(Import-Csv -LiteralPath (Join-Path $FixtureRoot "golden-contract-generation-set.csv"))
|
|
|
|
|
$promptRegister = @(Import-Csv -LiteralPath (Join-Path $FixtureRoot "prompt-version-register.csv"))
|
|
|
|
|
|
|
|
|
|
if ($metrics.Count -ge 10) {
|
|
|
|
|
Add-Check -Key "metric_dictionary" -Label "quality metric dictionary" -Status "pass" -Detail "Metric count=$($metrics.Count)"
|
|
|
|
|
} else {
|
|
|
|
|
Add-Check -Key "metric_dictionary" -Label "quality metric dictionary" -Status "fail" -Detail "Metric count is too low: $($metrics.Count)"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$latestPhase8 = Get-LatestSummaryArtifact -Root $Phase8ArtifactRoot -SummaryFile "phase8-pilot-closed-loop-summary.json"
|
|
|
|
|
$phase8Summary = $null
|
|
|
|
|
$scenarioRows = @()
|
|
|
|
|
$feedbackRows = @()
|
|
|
|
|
$phase8AgeHours = $null
|
|
|
|
|
if ($latestPhase8) {
|
|
|
|
|
$phase8SummaryPath = Join-Path $latestPhase8.FullName "phase8-pilot-closed-loop-summary.json"
|
|
|
|
|
$phase8Summary = Read-JsonFile -Path $phase8SummaryPath
|
|
|
|
|
$phase8AgeHours = [math]::Round(((Get-Date) - $latestPhase8.LastWriteTime).TotalHours, 2)
|
|
|
|
|
if ($phase8Summary -and ($phase8Summary.status -eq "PASS" -or $phase8Summary.status -eq "PASS_WITH_WEBHOOK_BLOCKED") -and [int]$phase8Summary.totals.failed_required -eq 0) {
|
|
|
|
|
Add-Check -Key "phase8_input" -Label "phase8 closed-loop input" -Status "pass" -Detail "Phase8 status=$($phase8Summary.status), age=${phase8AgeHours}h" -Meta @{ artifact_dir = $latestPhase8.FullName; summary = $phase8SummaryPath }
|
|
|
|
|
} elseif ($phase8Summary) {
|
|
|
|
|
Add-Check -Key "phase8_input" -Label "phase8 closed-loop input" -Status "fail" -Detail "Phase8 status=$($phase8Summary.status), failed_required=$($phase8Summary.totals.failed_required)" -Meta @{ artifact_dir = $latestPhase8.FullName; summary = $phase8SummaryPath }
|
|
|
|
|
} else {
|
|
|
|
|
Add-Check -Key "phase8_input" -Label "phase8 closed-loop input" -Status "fail" -Detail "Could not parse phase8 summary" -Meta @{ artifact_dir = $latestPhase8.FullName }
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if ($phase8AgeHours -le $MaxPhase8AgeHours) {
|
|
|
|
|
Add-Check -Key "phase8_freshness" -Label "phase8 evidence freshness" -Status "pass" -Detail "Phase8 artifact age=${phase8AgeHours}h"
|
|
|
|
|
} else {
|
|
|
|
|
Add-Check -Key "phase8_freshness" -Label "phase8 evidence freshness" -Status "warn" -Required $false -Detail "Phase8 artifact age ${phase8AgeHours}h exceeds $MaxPhase8AgeHours"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$scenarioPath = Join-Path $latestPhase8.FullName "pilot-scenario-execution-record.csv"
|
|
|
|
|
$feedbackPath = Join-Path $latestPhase8.FullName "pilot-feedback-register.csv"
|
|
|
|
|
if (Test-Path -LiteralPath $scenarioPath -PathType Leaf) {
|
|
|
|
|
$scenarioRows = @(Import-Csv -LiteralPath $scenarioPath)
|
|
|
|
|
}
|
|
|
|
|
if (Test-Path -LiteralPath $feedbackPath -PathType Leaf) {
|
|
|
|
|
$feedbackRows = @(Import-Csv -LiteralPath $feedbackPath)
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
Add-Check -Key "phase8_input" -Label "phase8 closed-loop input" -Status "fail" -Detail "No phase8 closed-loop artifact found"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if ($scenarioRows.Count -gt 0 -and @($scenarioRows | Where-Object { $_.status -eq "PASS" }).Count -eq $scenarioRows.Count) {
|
|
|
|
|
Add-Check -Key "phase8_scenarios" -Label "phase8 scenarios" -Status "pass" -Detail "Phase8 scenarios passed=$($scenarioRows.Count)/$($scenarioRows.Count)"
|
|
|
|
|
} else {
|
|
|
|
|
Add-Check -Key "phase8_scenarios" -Label "phase8 scenarios" -Status "fail" -Detail "Phase8 scenario rows missing or failed"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$openP0P1 = @($feedbackRows | Where-Object { ($_.severity -eq "P0" -or $_.severity -eq "P1") -and ($_.status -ne "VERIFIED" -and $_.status -ne "WONT_FIX") })
|
|
|
|
|
if ($openP0P1.Count -eq 0) {
|
|
|
|
|
Add-Check -Key "open_p0_p1_feedback" -Label "open P0/P1 feedback" -Status "pass" -Detail "Open P0/P1 feedback=0"
|
|
|
|
|
} else {
|
|
|
|
|
Add-Check -Key "open_p0_p1_feedback" -Label "open P0/P1 feedback" -Status "fail" -Detail "Open P0/P1 feedback=$($openP0P1.Count)"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$scorecard = New-Object System.Collections.Generic.List[object]
|
|
|
|
|
$scorecard.Add((Write-DomainReport -Domain "review" -Title "Phase 9 Review Quality Evaluation" -FixtureRows $reviewFixtures -ScenarioIds @("PILOT-REVIEW-001", "PILOT-REVIEW-002", "PILOT-CHAT-001", "PILOT-NEG-002") -ScenarioRows $scenarioRows -RequiredColumns @("sample_id", "expected_risk_types", "expected_evidence", "expected_actionable_suggestion", "scenario_ids") -OutputPrefix "review-quality-eval" -ExtraDetail "Review quality uses phase8 review, review detail, chat context, and failure explanation scenarios.")) | Out-Null
|
|
|
|
|
$scorecard.Add((Write-DomainReport -Domain "kb" -Title "Phase 9 Knowledge Base Quality Evaluation" -FixtureRows $kbFixtures -ScenarioIds @("PILOT-KB-001", "PILOT-KB-002", "PILOT-NEG-002") -ScenarioRows $scenarioRows -RequiredColumns @("question_id", "expected_source_policy", "expected_answer_behavior", "scenario_ids") -OutputPrefix "kb-quality-eval" -ExtraDetail "KB quality uses indexed document, source policy, and refusal behavior fixtures.")) | Out-Null
|
|
|
|
|
$scorecard.Add((Write-DomainReport -Domain "contract_generation" -Title "Phase 9 Contract Generation Quality Evaluation" -FixtureRows $contractFixtures -ScenarioIds @("PILOT-GENERATE-001", "PILOT-NEG-002") -ScenarioRows $scenarioRows -RequiredColumns @("case_id", "required_variables", "expected_preview", "expected_download", "expected_file_asset", "scenario_ids") -OutputPrefix "contract-generation-quality-eval" -ExtraDetail "Contract generation quality uses template variable, preview, download, and file asset expectations.")) | Out-Null
|
|
|
|
|
|
|
|
|
|
foreach ($domain in $scorecard) {
|
|
|
|
|
if ($domain.status -eq "PASS") {
|
|
|
|
|
Add-Check -Key ("domain_" + $domain.domain) -Label ($domain.domain + " quality") -Status "pass" -Detail "Score=$($domain.score)" -Meta @{ report = $domain.report_md }
|
|
|
|
|
} else {
|
|
|
|
|
Add-Check -Key ("domain_" + $domain.domain) -Label ($domain.domain + " quality") -Status "fail" -Detail "Score=$($domain.score)" -Meta @{ report = $domain.report_md }
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$promptColumnsOk = Test-CsvColumns -Rows $promptRegister -Columns @("prompt_id", "domain", "version", "status", "expected_schema", "uncertainty_policy", "rollback_policy")
|
|
|
|
|
$activePromptDomains = @($promptRegister | Where-Object { $_.status -eq "active" } | Select-Object -ExpandProperty domain -Unique)
|
|
|
|
|
$aiGovernanceText = Get-Content -LiteralPath (Join-Path $FixtureRoot "ai-output-governance.md") -Raw -Encoding UTF8
|
|
|
|
|
$aiGovernanceOk = $promptColumnsOk -and ($activePromptDomains -contains "review") -and ($activePromptDomains -contains "kb") -and ($activePromptDomains -contains "contract_generation") -and $aiGovernanceText.Contains("refuse") -and $aiGovernanceText.Contains("schema")
|
|
|
|
|
$aiGovernanceJson = Join-Path $artifactDir "ai-output-governance-summary.json"
|
|
|
|
|
$aiGovernanceMd = Join-Path $artifactDir "ai-output-governance-summary.md"
|
|
|
|
|
$aiGovernanceSummary = [ordered]@{
|
|
|
|
|
status = if ($aiGovernanceOk) { "PASS" } else { "FAIL" }
|
|
|
|
|
prompt_count = $promptRegister.Count
|
|
|
|
|
active_domains = $activePromptDomains
|
|
|
|
|
prompt_columns_ok = $promptColumnsOk
|
|
|
|
|
governance_rules_present = $aiGovernanceText.Contains("Required Rules")
|
|
|
|
|
}
|
|
|
|
|
Save-Json -Path $aiGovernanceJson -Value $aiGovernanceSummary
|
|
|
|
|
@(
|
|
|
|
|
"# Phase 9 AI Output Governance Summary",
|
|
|
|
|
"",
|
|
|
|
|
"- Status: $($aiGovernanceSummary.status)",
|
|
|
|
|
"- Prompt count: $($promptRegister.Count)",
|
|
|
|
|
"- Active domains: $($activePromptDomains -join ', ')",
|
|
|
|
|
"- Prompt columns OK: $promptColumnsOk"
|
|
|
|
|
) | Set-Content -Encoding UTF8 -Path $aiGovernanceMd
|
|
|
|
|
if ($aiGovernanceOk) {
|
|
|
|
|
Add-Check -Key "ai_output_governance" -Label "AI output governance" -Status "pass" -Detail "Prompt register and governance policy are complete" -Meta @{ report = $aiGovernanceMd }
|
|
|
|
|
} else {
|
|
|
|
|
Add-Check -Key "ai_output_governance" -Label "AI output governance" -Status "fail" -Detail "AI output governance is incomplete" -Meta @{ report = $aiGovernanceMd }
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$retentionText = Get-Content -LiteralPath (Join-Path $FixtureRoot "data-retention-policy.md") -Raw -Encoding UTF8
|
|
|
|
|
$secret = [string]::Concat([char]54, [char]53, [char]52, [char]51, [char]50, [char]49)
|
|
|
|
|
$qualityFiles = Get-ChildItem -LiteralPath $FixtureRoot -File
|
|
|
|
|
$secretHits = @()
|
|
|
|
|
foreach ($file in $qualityFiles) {
|
|
|
|
|
$content = Get-Content -LiteralPath $file.FullName -Raw -Encoding UTF8
|
|
|
|
|
if ($content.Contains($secret)) {
|
|
|
|
|
$secretHits += $file.FullName
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
$phase8SampleInventory = Join-Path $repoRoot "project-docs\phase8-pilot-templates\pilot-sample-inventory.csv"
|
|
|
|
|
$sampleRows = if (Test-Path -LiteralPath $phase8SampleInventory -PathType Leaf) { @(Import-Csv -LiteralPath $phase8SampleInventory) } else { @() }
|
|
|
|
|
$sampleColumnsOk = Test-CsvColumns -Rows $sampleRows -Columns @("sample_id", "sensitivity_level", "allowed_environment", "desensitized")
|
|
|
|
|
$highProductionSamples = @($sampleRows | Where-Object { $_.sensitivity_level -eq "high" -and $_.allowed_environment -eq "production" })
|
|
|
|
|
$dataGovernanceOk = $retentionText.Contains("Data Classes") -and $sampleColumnsOk -and $highProductionSamples.Count -eq 0 -and $secretHits.Count -eq 0
|
|
|
|
|
$dataGovernanceJson = Join-Path $artifactDir "data-governance-summary.json"
|
|
|
|
|
$dataGovernanceMd = Join-Path $artifactDir "data-governance-summary.md"
|
|
|
|
|
$dataGovernanceSummary = [ordered]@{
|
|
|
|
|
status = if ($dataGovernanceOk) { "PASS" } else { "FAIL" }
|
|
|
|
|
sample_count = $sampleRows.Count
|
|
|
|
|
sample_columns_ok = $sampleColumnsOk
|
|
|
|
|
high_production_samples = $highProductionSamples.Count
|
|
|
|
|
secret_hits = $secretHits
|
|
|
|
|
retention_policy_present = $retentionText.Contains("Data Classes")
|
|
|
|
|
}
|
|
|
|
|
Save-Json -Path $dataGovernanceJson -Value $dataGovernanceSummary
|
|
|
|
|
@(
|
|
|
|
|
"# Phase 9 Data Governance Summary",
|
|
|
|
|
"",
|
|
|
|
|
"- Status: $($dataGovernanceSummary.status)",
|
|
|
|
|
"- Sample count: $($sampleRows.Count)",
|
|
|
|
|
"- Sample columns OK: $sampleColumnsOk",
|
|
|
|
|
"- High sensitivity production samples: $($highProductionSamples.Count)",
|
|
|
|
|
"- Secret hits: $($secretHits.Count)"
|
|
|
|
|
) | Set-Content -Encoding UTF8 -Path $dataGovernanceMd
|
|
|
|
|
if ($dataGovernanceOk) {
|
|
|
|
|
Add-Check -Key "data_governance" -Label "data governance" -Status "pass" -Detail "Retention, privacy, and sample governance passed" -Meta @{ report = $dataGovernanceMd }
|
|
|
|
|
} else {
|
|
|
|
|
Add-Check -Key "data_governance" -Label "data governance" -Status "fail" -Detail "Data governance check failed" -Meta @{ report = $dataGovernanceMd }
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$scorecard | Export-Csv -NoTypeInformation -Encoding UTF8 -Path $scorecardCsv
|
|
|
|
|
|
|
|
|
|
$defects = New-Object System.Collections.Generic.List[object]
|
|
|
|
|
foreach ($feedback in $feedbackRows) {
|
|
|
|
|
if ($feedback.status -eq "DEFERRED") {
|
|
|
|
|
$defects.Add([pscustomobject][ordered]@{
|
|
|
|
|
defect_id = "QDEF-" + $feedback.feedback_id
|
|
|
|
|
source_feedback_id = $feedback.feedback_id
|
|
|
|
|
severity = $feedback.severity
|
|
|
|
|
status = "DEFERRED"
|
|
|
|
|
domain = if ($feedback.scenario_id -like "*WEBHOOK*") { "ops_external" } elseif ($feedback.scenario_id -like "*ROLLBACK*") { "ops_evidence" } else { "quality" }
|
|
|
|
|
owner = $feedback.owner
|
|
|
|
|
evidence_path = $feedback.evidence_path
|
|
|
|
|
notes = $feedback.notes
|
|
|
|
|
}) | Out-Null
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if ($defects.Count -eq 0) {
|
|
|
|
|
$defects.Add([pscustomobject][ordered]@{
|
|
|
|
|
defect_id = "QDEF-P9-000"
|
|
|
|
|
source_feedback_id = ""
|
|
|
|
|
severity = "P3"
|
|
|
|
|
status = "VERIFIED"
|
|
|
|
|
domain = "quality"
|
|
|
|
|
owner = "phase9-auto"
|
|
|
|
|
evidence_path = $artifactDir
|
|
|
|
|
notes = "No deferred quality defects"
|
|
|
|
|
}) | Out-Null
|
|
|
|
|
}
|
|
|
|
|
$defects | Export-Csv -NoTypeInformation -Encoding UTF8 -Path $defectCsv
|
|
|
|
|
|
|
|
|
|
$failedRequired = @($checks | Where-Object { $_.required -and $_.status -eq "fail" })
|
|
|
|
|
$warningsList = @($checks | Where-Object { $_.status -eq "warn" })
|
|
|
|
|
$blockedList = @()
|
|
|
|
|
if ($phase8Summary -and $phase8Summary.status -eq "PASS_WITH_WEBHOOK_BLOCKED") {
|
|
|
|
|
$blockedList += [pscustomobject]@{ key = "external_webhook"; detail = "Inherited BLOCKED_EXTERNAL_WEBHOOK from phase8" }
|
|
|
|
|
}
|
|
|
|
|
$overallStatus = if ($failedRequired.Count -gt 0) {
|
|
|
|
|
"FAIL"
|
|
|
|
|
} elseif ($blockedList.Count -gt 0) {
|
|
|
|
|
"PASS_WITH_WEBHOOK_BLOCKED"
|
|
|
|
|
} elseif ($warningsList.Count -gt 0) {
|
|
|
|
|
"WARN"
|
|
|
|
|
} else {
|
|
|
|
|
"PASS"
|
|
|
|
|
}
|
|
|
|
|
$alertSeverity = if ($failedRequired.Count -gt 0) { "critical" } elseif ($blockedList.Count -gt 0) { "blocked" } elseif ($warningsList.Count -gt 0) { "warning" } else { "ok" }
|
|
|
|
|
|
|
|
|
|
$summary = [ordered]@{
|
|
|
|
|
name = "phase9-business-quality-gate"
|
|
|
|
|
status = $overallStatus
|
|
|
|
|
alert_severity = $alertSeverity
|
|
|
|
|
generated_at = (Get-Date).ToUniversalTime().ToString("o")
|
|
|
|
|
artifact_dir = $artifactDir
|
|
|
|
|
branch = $branch
|
|
|
|
|
commit = $commit
|
|
|
|
|
remote = $remoteUrl
|
|
|
|
|
phase8 = [ordered]@{
|
|
|
|
|
artifact_dir = if ($latestPhase8) { $latestPhase8.FullName } else { $null }
|
|
|
|
|
status = if ($phase8Summary) { $phase8Summary.status } else { $null }
|
|
|
|
|
age_hours = $phase8AgeHours
|
|
|
|
|
scenarios_total = $scenarioRows.Count
|
|
|
|
|
feedback_total = $feedbackRows.Count
|
|
|
|
|
open_p0_p1 = $openP0P1.Count
|
|
|
|
|
}
|
|
|
|
|
scorecard = $scorecard
|
|
|
|
|
checks = $checks
|
|
|
|
|
artifacts = [ordered]@{
|
|
|
|
|
scorecard_csv = $scorecardCsv
|
|
|
|
|
defect_register = $defectCsv
|
|
|
|
|
ai_output_governance_summary = $aiGovernanceMd
|
|
|
|
|
data_governance_summary = $dataGovernanceMd
|
|
|
|
|
}
|
|
|
|
|
totals = [ordered]@{
|
|
|
|
|
total = $checks.Count
|
|
|
|
|
failed_required = $failedRequired.Count
|
|
|
|
|
warnings = $warningsList.Count
|
|
|
|
|
blocked = $blockedList.Count
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
Save-Json -Path $summaryJson -Value $summary
|
|
|
|
|
|
|
|
|
|
$markdown = New-Object System.Collections.Generic.List[string]
|
|
|
|
|
$markdown.Add("# Phase 9 Business Quality Gate") | Out-Null
|
|
|
|
|
$markdown.Add("") | Out-Null
|
|
|
|
|
$markdown.Add("- Status: $overallStatus") | Out-Null
|
|
|
|
|
$markdown.Add("- Alert severity: $alertSeverity") | Out-Null
|
|
|
|
|
$markdown.Add("- Generated at: $($summary.generated_at)") | Out-Null
|
|
|
|
|
$markdown.Add("- Branch: $branch") | Out-Null
|
|
|
|
|
$markdown.Add("- Commit: $commit") | Out-Null
|
|
|
|
|
$markdown.Add("- Artifact dir: $artifactDir") | Out-Null
|
|
|
|
|
$markdown.Add("- Phase8 input: $($summary.phase8.artifact_dir)") | Out-Null
|
|
|
|
|
$markdown.Add("") | Out-Null
|
|
|
|
|
$markdown.Add("## Scorecard") | Out-Null
|
|
|
|
|
$markdown.Add("") | Out-Null
|
|
|
|
|
$markdown.Add("| Domain | Status | Score | Fixture Count |") | Out-Null
|
|
|
|
|
$markdown.Add("|---|---|---:|---:|") | Out-Null
|
|
|
|
|
foreach ($domain in $scorecard) {
|
|
|
|
|
$markdown.Add("| $($domain.domain) | $($domain.status) | $($domain.score) | $($domain.fixture_count) |") | Out-Null
|
|
|
|
|
}
|
|
|
|
|
$markdown.Add("") | Out-Null
|
|
|
|
|
$markdown.Add("## Checks") | Out-Null
|
|
|
|
|
$markdown.Add("") | Out-Null
|
|
|
|
|
$markdown.Add("| Check | Required | Status | Detail |") | Out-Null
|
|
|
|
|
$markdown.Add("|---|---:|---|---|") | Out-Null
|
|
|
|
|
foreach ($check in $checks) {
|
|
|
|
|
$markdown.Add("| $($check.label) | $($check.required) | $($check.status.ToUpperInvariant()) | $($check.detail) |") | Out-Null
|
|
|
|
|
}
|
|
|
|
|
$markdown.Add("") | Out-Null
|
|
|
|
|
$markdown.Add("Local artifacts:") | Out-Null
|
|
|
|
|
$markdown.Add("") | Out-Null
|
|
|
|
|
$markdown.Add('- `business-quality-gate-summary.json`') | Out-Null
|
|
|
|
|
$markdown.Add('- `business-quality-gate-summary.md`') | Out-Null
|
|
|
|
|
$markdown.Add('- `phase9-quality-scorecard.csv`') | Out-Null
|
|
|
|
|
$markdown.Add('- `phase9-quality-defect-register.csv`') | Out-Null
|
|
|
|
|
$markdown.Add('- `review-quality-eval.*`') | Out-Null
|
|
|
|
|
$markdown.Add('- `kb-quality-eval.*`') | Out-Null
|
|
|
|
|
$markdown.Add('- `contract-generation-quality-eval.*`') | Out-Null
|
|
|
|
|
$markdown.Add('- `ai-output-governance-summary.*`') | Out-Null
|
|
|
|
|
$markdown.Add('- `data-governance-summary.*`') | Out-Null
|
|
|
|
|
$markdown | Set-Content -Encoding UTF8 -Path $summaryMd
|
|
|
|
|
|
|
|
|
|
Write-Host "PHASE9_BUSINESS_QUALITY_GATE_$overallStatus $artifactDir"
|
|
|
|
|
if ($overallStatus -eq "FAIL" -or ($FailOnWarn -and $overallStatus -eq "WARN") -or ($FailOnExternalBlocked -and $overallStatus -eq "PASS_WITH_WEBHOOK_BLOCKED")) {
|
|
|
|
|
throw "Phase 9 business quality gate $overallStatus. See $summaryMd"
|
|
|
|
|
}
|
Codex