628 lines
28 KiB
PowerShell
628 lines
28 KiB
PowerShell
param(
|
|
[string]$ApiBaseUrl = "http://192.168.3.90:8001/api/v1",
|
|
[string]$ManifestPath = "",
|
|
[string]$ArtifactRoot = "",
|
|
[string]$AdminEmail = "admin@example.com",
|
|
[string]$AdminPassword = "",
|
|
[string]$AdminPasswordFile = "",
|
|
[string]$KnowledgeBaseName = "AI合同平台法条全文库（Phase14）",
|
|
[string]$KnowledgeBaseId = "",
|
|
[switch]$ForceReupload,
|
|
[switch]$SkipKnowledgeBasePublish
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
Add-Type -AssemblyName System.Net.Http
|
|
|
|
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path
|
|
$timestamp = Get-Date -Format "yyyy-MM-ddTHH-mm-ss-fff"
|
|
|
|
if ([string]::IsNullOrWhiteSpace($ManifestPath)) {
|
|
$ManifestPath = Join-Path $repoRoot "project-docs\phase14-legal-fulltext-library\legal-article-manifest.json"
|
|
}
|
|
if (-not [System.IO.Path]::IsPathRooted($ManifestPath)) {
|
|
$ManifestPath = Join-Path $repoRoot $ManifestPath
|
|
}
|
|
|
|
if ([string]::IsNullOrWhiteSpace($ArtifactRoot)) {
|
|
$ArtifactRoot = Join-Path $repoRoot "project-docs\qa-artifacts\phase14-legal-fulltext-import"
|
|
}
|
|
if (-not [System.IO.Path]::IsPathRooted($ArtifactRoot)) {
|
|
$ArtifactRoot = Join-Path $repoRoot $ArtifactRoot
|
|
}
|
|
|
|
if ([string]::IsNullOrWhiteSpace($AdminPasswordFile)) {
|
|
$AdminPasswordFile = Join-Path $HOME ".ai-contract-platform\production-admin-password.txt"
|
|
}
|
|
|
|
$artifactDir = Join-Path $ArtifactRoot $timestamp
|
|
$downloadDir = Join-Path $artifactDir "official-downloads"
|
|
New-Item -ItemType Directory -Force -Path $downloadDir | Out-Null
|
|
|
|
$summaryJson = Join-Path $artifactDir "phase14-legal-fulltext-import-summary.json"
|
|
$summaryMd = Join-Path $artifactDir "phase14-legal-fulltext-import-summary.md"
|
|
$ApiBaseUrl = $ApiBaseUrl.TrimEnd("/")
|
|
|
|
$checks = New-Object System.Collections.Generic.List[object]
|
|
$imports = New-Object System.Collections.Generic.List[object]
|
|
$verification = New-Object System.Collections.Generic.List[object]
|
|
$articlePayloads = New-Object System.Collections.Generic.List[object]
|
|
$articlePrefix = [string][char]0x7B2C
|
|
$articleSuffix = [string][char]0x6761
|
|
|
|
function Add-Check {
|
|
param([string]$Key, [string]$Label, [string]$Status, [string]$Detail, [bool]$Required = $true, [hashtable]$Meta = @{})
|
|
$checks.Add([ordered]@{ key = $Key; label = $Label; status = $Status; required = $Required; detail = $Detail; meta = $Meta }) | Out-Null
|
|
}
|
|
|
|
function Add-Import {
|
|
param([string]$SourceId, [string]$ArticleNo, [string]$Action, [string]$Status, [string]$TargetId, [string]$Detail, [hashtable]$Meta = @{})
|
|
$imports.Add([ordered]@{ source_id = $SourceId; article_no = $ArticleNo; action = $Action; status = $Status; target_id = $TargetId; detail = $Detail; meta = $Meta }) | Out-Null
|
|
}
|
|
|
|
function Add-Verification {
|
|
param([string]$Key, [string]$Status, [string]$Detail, [hashtable]$Meta = @{})
|
|
$verification.Add([ordered]@{ key = $Key; status = $Status; detail = $Detail; meta = $Meta }) | Out-Null
|
|
}
|
|
|
|
function Save-Json {
|
|
param([string]$Path, [object]$Value)
|
|
$Value | ConvertTo-Json -Depth 60 | Set-Content -Encoding UTF8 -Path $Path
|
|
}
|
|
|
|
function Get-ResponseData {
|
|
param([object]$Response)
|
|
if ($null -eq $Response) { return $null }
|
|
if ($Response.PSObject.Properties.Name -contains "data") { return $Response.data }
|
|
return $Response
|
|
}
|
|
|
|
function Invoke-ApiJson {
|
|
param([string]$Method, [string]$Url, [hashtable]$Headers = @{}, [object]$Body = $null)
|
|
$args = @{ Method = $Method; Uri = $Url; Headers = $Headers; ErrorAction = "Stop" }
|
|
if ($null -ne $Body) {
|
|
$args["ContentType"] = "application/json; charset=utf-8"
|
|
$args["Body"] = ($Body | ConvertTo-Json -Depth 60)
|
|
}
|
|
return Invoke-RestMethod @args
|
|
}
|
|
|
|
function Invoke-FormRequest {
|
|
param(
|
|
[string]$Method,
|
|
[string]$Url,
|
|
[hashtable]$Headers = @{},
|
|
[hashtable]$Fields = @{},
|
|
[string]$FileField = "",
|
|
[string]$FileName = "",
|
|
[byte[]]$FileContent = $null,
|
|
[string]$FileContentType = "text/markdown; charset=utf-8"
|
|
)
|
|
|
|
$client = [System.Net.Http.HttpClient]::new()
|
|
try {
|
|
foreach ($headerName in $Headers.Keys) {
|
|
$client.DefaultRequestHeaders.Remove($headerName) | Out-Null
|
|
$client.DefaultRequestHeaders.Add($headerName, [string]$Headers[$headerName])
|
|
}
|
|
|
|
$form = [System.Net.Http.MultipartFormDataContent]::new()
|
|
try {
|
|
foreach ($fieldName in $Fields.Keys) {
|
|
$value = [string]$Fields[$fieldName]
|
|
$stringContent = [System.Net.Http.StringContent]::new($value, [System.Text.Encoding]::UTF8)
|
|
$form.Add($stringContent, $fieldName)
|
|
}
|
|
if ($FileContent -ne $null -and -not [string]::IsNullOrWhiteSpace($FileField)) {
|
|
$byteContent = [System.Net.Http.ByteArrayContent]::new($FileContent)
|
|
$byteContent.Headers.ContentType = [System.Net.Http.Headers.MediaTypeHeaderValue]::Parse($FileContentType)
|
|
$form.Add($byteContent, $FileField, $FileName)
|
|
}
|
|
|
|
$request = [System.Net.Http.HttpRequestMessage]::new([System.Net.Http.HttpMethod]::new($Method), $Url)
|
|
$request.Content = $form
|
|
$response = $client.SendAsync($request).GetAwaiter().GetResult()
|
|
$body = $response.Content.ReadAsStringAsync().GetAwaiter().GetResult()
|
|
if (-not $response.IsSuccessStatusCode) {
|
|
throw "API request failed: $Method $Url, status=$([int]$response.StatusCode), response=$body"
|
|
}
|
|
if ([string]::IsNullOrWhiteSpace($body)) { return $null }
|
|
return $body | ConvertFrom-Json
|
|
} finally {
|
|
if ($null -ne $form) { $form.Dispose() }
|
|
}
|
|
} finally {
|
|
$client.Dispose()
|
|
}
|
|
}
|
|
|
|
function Get-PagedApiItems {
|
|
param([string]$Url, [hashtable]$Headers)
|
|
$items = New-Object System.Collections.Generic.List[object]
|
|
$page = 1
|
|
$pageSize = 200
|
|
while ($true) {
|
|
$separator = if ($Url.Contains("?")) { "&" } else { "?" }
|
|
$response = Invoke-ApiJson -Method "GET" -Url "$Url${separator}page=$page&page_size=$pageSize" -Headers $Headers
|
|
$data = Get-ResponseData -Response $response
|
|
if ($null -eq $data -or $null -eq $data.items) { break }
|
|
foreach ($item in @($data.items)) { $items.Add($item) | Out-Null }
|
|
if ($items.Count -ge [int]$data.total -or @($data.items).Count -eq 0) { break }
|
|
$page += 1
|
|
}
|
|
return $items.ToArray()
|
|
}
|
|
|
|
function Get-LegalArticlesBySourceIds {
|
|
param(
|
|
[string[]]$SourceIds,
|
|
[hashtable]$Headers,
|
|
[string]$PublishedStatus = "",
|
|
[string]$IndexStatus = ""
|
|
)
|
|
$items = New-Object System.Collections.Generic.List[object]
|
|
foreach ($sourceId in @($SourceIds | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } | Select-Object -Unique)) {
|
|
$query = "source_id=$([uri]::EscapeDataString($sourceId))"
|
|
if (-not [string]::IsNullOrWhiteSpace($PublishedStatus)) {
|
|
$query += "&published_status=$([uri]::EscapeDataString($PublishedStatus))"
|
|
}
|
|
if (-not [string]::IsNullOrWhiteSpace($IndexStatus)) {
|
|
$query += "&index_status=$([uri]::EscapeDataString($IndexStatus))"
|
|
}
|
|
$sourceItems = Get-PagedApiItems -Url "$ApiBaseUrl/legal-articles?$query" -Headers $Headers
|
|
foreach ($item in @($sourceItems)) {
|
|
$items.Add($item) | Out-Null
|
|
}
|
|
}
|
|
return $items.ToArray()
|
|
}
|
|
|
|
function Get-AccessToken {
|
|
$password = $AdminPassword
|
|
if ([string]::IsNullOrWhiteSpace($password) -and -not [string]::IsNullOrWhiteSpace($env:AICP_ADMIN_PASSWORD)) {
|
|
$password = $env:AICP_ADMIN_PASSWORD
|
|
}
|
|
if ([string]::IsNullOrWhiteSpace($password) -and (Test-Path -LiteralPath $AdminPasswordFile -PathType Leaf)) {
|
|
$password = (Get-Content -LiteralPath $AdminPasswordFile -Raw).Trim()
|
|
}
|
|
if ([string]::IsNullOrWhiteSpace($password)) {
|
|
throw "Admin password is required. Use -AdminPassword, AICP_ADMIN_PASSWORD, or $AdminPasswordFile."
|
|
}
|
|
|
|
$response = Invoke-ApiJson -Method "POST" -Url "$ApiBaseUrl/auth/login" -Body @{ account = $AdminEmail; password = $password }
|
|
$data = Get-ResponseData -Response $response
|
|
if ($null -eq $data -or [string]::IsNullOrWhiteSpace($data.access_token)) {
|
|
throw "Login response has no access_token."
|
|
}
|
|
return [string]$data.access_token
|
|
}
|
|
|
|
function Get-AuthHeaders {
|
|
param([string]$Token)
|
|
return @{ Authorization = "Bearer $Token" }
|
|
}
|
|
|
|
function Get-TextChecksum {
|
|
param([string]$Text)
|
|
$bytes = [System.Text.Encoding]::UTF8.GetBytes($Text)
|
|
$sha = [System.Security.Cryptography.SHA256]::Create()
|
|
try {
|
|
return ([System.BitConverter]::ToString($sha.ComputeHash($bytes))).Replace("-", "").ToLowerInvariant()
|
|
} finally {
|
|
$sha.Dispose()
|
|
}
|
|
}
|
|
|
|
function Get-Base64Url {
|
|
param([string]$Value)
|
|
return [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($Value))
|
|
}
|
|
|
|
function Get-OfficialDetailUrl {
|
|
param([string]$Bbbs)
|
|
return "https://flk.npc.gov.cn/detail2.html?$(Get-Base64Url -Value $Bbbs)"
|
|
}
|
|
|
|
function Invoke-OfficialJson {
|
|
param([string]$Url)
|
|
$client = [System.Net.Http.HttpClient]::new()
|
|
try {
|
|
$client.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0")
|
|
$client.DefaultRequestHeaders.Add("Referer", "https://flk.npc.gov.cn/")
|
|
$bytes = $client.GetByteArrayAsync($Url).GetAwaiter().GetResult()
|
|
$text = [System.Text.Encoding]::UTF8.GetString($bytes)
|
|
return $text | ConvertFrom-Json
|
|
} finally {
|
|
$client.Dispose()
|
|
}
|
|
}
|
|
|
|
function Get-OfficialLawDetail {
|
|
param([string]$Bbbs)
|
|
$encoded = [uri]::EscapeDataString($Bbbs)
|
|
$response = Invoke-OfficialJson -Url "https://flk.npc.gov.cn/law-search/search/flfgDetails?bbbs=$encoded"
|
|
if ($response.code -ne 200 -or $null -eq $response.data) {
|
|
throw "Official detail request failed for $Bbbs."
|
|
}
|
|
return $response.data
|
|
}
|
|
|
|
function Get-OfficialDocxUrl {
|
|
param([string]$Bbbs)
|
|
$encoded = [uri]::EscapeDataString($Bbbs)
|
|
$response = Invoke-OfficialJson -Url "https://flk.npc.gov.cn/law-search/download/pc?format=docx&bbbs=$encoded&fileId="
|
|
if ($response.code -ne 200 -or $null -eq $response.data -or [string]::IsNullOrWhiteSpace([string]$response.data.url)) {
|
|
throw "Official docx download URL request failed for $Bbbs."
|
|
}
|
|
return [string]$response.data.url
|
|
}
|
|
|
|
function Save-OfficialDocx {
|
|
param([string]$Url, [string]$Path)
|
|
Invoke-WebRequest -Uri $Url -OutFile $Path -UseBasicParsing -Headers @{ "User-Agent" = "Mozilla/5.0"; "Referer" = "https://flk.npc.gov.cn/" } -ErrorAction Stop
|
|
}
|
|
|
|
function Convert-DocxToParagraphs {
|
|
param([string]$DocxPath, [string]$WorkDir)
|
|
$zipPath = Join-Path $WorkDir "document.zip"
|
|
$unzipDir = Join-Path $WorkDir "document"
|
|
Copy-Item -LiteralPath $DocxPath -Destination $zipPath -Force
|
|
Expand-Archive -LiteralPath $zipPath -DestinationPath $unzipDir -Force
|
|
$documentXmlPath = Join-Path $unzipDir "word\document.xml"
|
|
if (-not (Test-Path -LiteralPath $documentXmlPath -PathType Leaf)) {
|
|
throw "word/document.xml not found in official docx."
|
|
}
|
|
$xml = Get-Content -LiteralPath $documentXmlPath -Raw -Encoding UTF8
|
|
$paragraphs = New-Object System.Collections.Generic.List[string]
|
|
foreach ($paragraphMatch in [regex]::Matches($xml, "<w:p[\s\S]*?</w:p>")) {
|
|
$parts = New-Object System.Collections.Generic.List[string]
|
|
foreach ($textMatch in [regex]::Matches($paragraphMatch.Value, "<w:t[^>]*>([\s\S]*?)</w:t>")) {
|
|
$parts.Add([System.Net.WebUtility]::HtmlDecode($textMatch.Groups[1].Value)) | Out-Null
|
|
}
|
|
$text = ($parts.ToArray() -join "").Trim()
|
|
if (-not [string]::IsNullOrWhiteSpace($text)) {
|
|
$paragraphs.Add($text) | Out-Null
|
|
}
|
|
}
|
|
return $paragraphs.ToArray()
|
|
}
|
|
|
|
function Test-ArticleHeader {
|
|
param([string]$Text)
|
|
if ([string]::IsNullOrWhiteSpace($Text)) { return $false }
|
|
if (-not $Text.StartsWith($articlePrefix)) { return $false }
|
|
$suffixIndex = $Text.IndexOf($articleSuffix)
|
|
return $suffixIndex -gt 0 -and $suffixIndex -le 16
|
|
}
|
|
|
|
function Get-ArticleText {
|
|
param([string[]]$Paragraphs, [string]$ArticleNo)
|
|
$start = -1
|
|
for ($i = 0; $i -lt $Paragraphs.Count; $i++) {
|
|
if ($Paragraphs[$i].StartsWith($ArticleNo)) {
|
|
$start = $i
|
|
break
|
|
}
|
|
}
|
|
if ($start -lt 0) {
|
|
throw "Article not found: $ArticleNo"
|
|
}
|
|
$selected = New-Object System.Collections.Generic.List[string]
|
|
for ($i = $start; $i -lt $Paragraphs.Count; $i++) {
|
|
if ($i -gt $start -and (Test-ArticleHeader -Text $Paragraphs[$i])) {
|
|
break
|
|
}
|
|
$selected.Add($Paragraphs[$i]) | Out-Null
|
|
}
|
|
return ($selected.ToArray() -join "`n").Trim()
|
|
}
|
|
|
|
function ConvertTo-TagsJson {
|
|
param([object[]]$Values)
|
|
$tags = New-Object System.Collections.Generic.List[string]
|
|
foreach ($value in $Values) {
|
|
$text = ([string]$value).Trim()
|
|
if (-not [string]::IsNullOrWhiteSpace($text) -and -not $tags.Contains($text)) {
|
|
$tags.Add($text) | Out-Null
|
|
}
|
|
}
|
|
if ($tags.Count -eq 0) { return $null }
|
|
return ($tags.ToArray() | ConvertTo-Json -Compress)
|
|
}
|
|
|
|
function Ensure-KnowledgeBase {
|
|
param([hashtable]$Headers)
|
|
$items = Get-PagedApiItems -Url "$ApiBaseUrl/knowledge-bases" -Headers $Headers
|
|
if (-not [string]::IsNullOrWhiteSpace($KnowledgeBaseId)) {
|
|
$byId = @($items | Where-Object { $_.id -eq $KnowledgeBaseId }) | Select-Object -First 1
|
|
if ($byId) { return $byId }
|
|
}
|
|
$existing = @(
|
|
$items |
|
|
Where-Object { $_.name -eq $KnowledgeBaseName } |
|
|
Sort-Object `
|
|
@{ Expression = { [int]($_.document_count -as [int]) }; Descending = $true }, `
|
|
@{ Expression = { if ($_.updated_at) { [datetime]$_.updated_at } else { [datetime]::MinValue } }; Descending = $true } |
|
|
Select-Object -First 1
|
|
) | Select-Object -First 1
|
|
if ($existing) { return $existing }
|
|
$response = Invoke-FormRequest -Method "POST" -Url "$ApiBaseUrl/knowledge-bases" -Headers $Headers -Fields @{
|
|
name = $KnowledgeBaseName
|
|
description = "Phase14 legal article fulltext library imported from official law sources."
|
|
}
|
|
return Get-ResponseData -Response $response
|
|
}
|
|
|
|
function New-ArticleMarkdown {
|
|
param([object]$Source, [object]$Article, [string]$ArticleText, [string]$OfficialUrl)
|
|
return @"
|
|
# $($Source.title) $($Article.article_no)
|
|
|
|
## Source metadata
|
|
|
|
- source_id: $($Source.source_id)
|
|
- source_title: $($Source.title)
|
|
- source_type: $($Source.source_type)
|
|
- authority: $($Source.authority)
|
|
- published_date: $($Source.published_date)
|
|
- effective_date: $($Source.effective_date)
|
|
- validity_status: $($Source.validity_status)
|
|
- region: $($Source.region)
|
|
- official_url: $OfficialUrl
|
|
- chapter_title: $($Article.chapter_title)
|
|
- section_title: $($Article.section_title)
|
|
|
|
## Article full text
|
|
|
|
$ArticleText
|
|
"@
|
|
}
|
|
|
|
function Import-ArticleDocument {
|
|
param(
|
|
[hashtable]$Headers,
|
|
[string]$KnowledgeBaseId,
|
|
[object]$Source,
|
|
[object]$Article,
|
|
[string]$ArticleText,
|
|
[string]$OfficialUrl,
|
|
[object[]]$ExistingDocuments
|
|
)
|
|
$fileArticleNo = (Get-TextChecksum -Text $Article.article_no).Substring(0, 8)
|
|
$fileName = "$($Source.source_id)-$fileArticleNo.md"
|
|
$existing = @($ExistingDocuments | Where-Object { $_.file_name -eq $fileName }) | Select-Object -First 1
|
|
$tagsJson = ConvertTo-TagsJson -Values @($Source.source_id, $Source.knowledge_domain, "phase14", "legal-fulltext", $Article.tags)
|
|
$fields = @{
|
|
note = "Phase14 legal article fulltext import."
|
|
source_type = $Source.source_type
|
|
authority = $Source.authority
|
|
effective_date = $Source.effective_date
|
|
region = $Source.region
|
|
}
|
|
if ($tagsJson) { $fields["tags_json"] = $tagsJson }
|
|
$content = New-ArticleMarkdown -Source $Source -Article $Article -ArticleText $ArticleText -OfficialUrl $OfficialUrl
|
|
$bytes = [System.Text.Encoding]::UTF8.GetBytes($content)
|
|
|
|
if ($existing -and -not $ForceReupload) {
|
|
Add-Import -SourceId $Source.source_id -ArticleNo $Article.article_no -Action "skip_existing" -Status "skipped" -TargetId $existing.id -Detail "Existing knowledge document reused." -Meta @{ file_name = $fileName; status = $existing.status }
|
|
return $existing
|
|
}
|
|
if ($existing -and $ForceReupload) {
|
|
$response = Invoke-FormRequest -Method "POST" -Url "$ApiBaseUrl/knowledge-bases/$KnowledgeBaseId/documents/$($existing.id)/retry" -Headers $Headers -Fields $fields -FileField "file" -FileName $fileName -FileContent $bytes
|
|
$data = Get-ResponseData -Response $response
|
|
Add-Import -SourceId $Source.source_id -ArticleNo $Article.article_no -Action "retry_existing" -Status $data.status -TargetId $data.id -Detail $data.status_detail -Meta @{ file_name = $fileName }
|
|
return $data
|
|
}
|
|
|
|
$response = Invoke-FormRequest -Method "POST" -Url "$ApiBaseUrl/knowledge-bases/$KnowledgeBaseId/documents" -Headers $Headers -Fields $fields -FileField "file" -FileName $fileName -FileContent $bytes
|
|
$created = Get-ResponseData -Response $response
|
|
Add-Import -SourceId $Source.source_id -ArticleNo $Article.article_no -Action "upload" -Status $created.status -TargetId $created.id -Detail $created.status_detail -Meta @{ file_name = $fileName }
|
|
return $created
|
|
}
|
|
|
|
if (-not (Test-Path -LiteralPath $ManifestPath -PathType Leaf)) {
|
|
Add-Check -Key "manifest_exists" -Label "Manifest exists" -Status "fail" -Detail "Manifest missing: $ManifestPath"
|
|
} else {
|
|
Add-Check -Key "manifest_exists" -Label "Manifest exists" -Status "pass" -Detail $ManifestPath
|
|
}
|
|
|
|
$manifest = $null
|
|
if ((@($checks | Where-Object { $_.required -and $_.status -eq "fail" })).Count -eq 0) {
|
|
$manifest = Get-Content -LiteralPath $ManifestPath -Raw -Encoding UTF8 | ConvertFrom-Json
|
|
$sourceCount = @($manifest.sources).Count
|
|
$articleCount = 0
|
|
foreach ($source in @($manifest.sources)) { $articleCount += @($source.articles).Count }
|
|
if ($sourceCount -gt 0 -and $articleCount -gt 0) {
|
|
Add-Check -Key "manifest_shape" -Label "Manifest shape" -Status "pass" -Detail "sources=$sourceCount, articles=$articleCount" -Meta @{ sources = $sourceCount; articles = $articleCount }
|
|
} else {
|
|
Add-Check -Key "manifest_shape" -Label "Manifest shape" -Status "fail" -Detail "Manifest has no sources or articles." -Meta @{ sources = $sourceCount; articles = $articleCount }
|
|
}
|
|
}
|
|
|
|
$token = ""
|
|
$headers = @{}
|
|
$targetKnowledgeBase = $null
|
|
if ((@($checks | Where-Object { $_.required -and $_.status -eq "fail" })).Count -eq 0) {
|
|
$token = Get-AccessToken
|
|
$headers = Get-AuthHeaders -Token $token
|
|
Add-Check -Key "api_login" -Label "API login" -Status "pass" -Detail "Admin API login succeeded."
|
|
$targetKnowledgeBase = Ensure-KnowledgeBase -Headers $headers
|
|
Add-Check -Key "target_knowledge_base" -Label "Target knowledge base" -Status "pass" -Detail $targetKnowledgeBase.name -Meta @{ knowledge_base_id = $targetKnowledgeBase.id }
|
|
}
|
|
|
|
if ((@($checks | Where-Object { $_.required -and $_.status -eq "fail" })).Count -eq 0) {
|
|
$existingDocs = Get-PagedApiItems -Url "$ApiBaseUrl/knowledge-bases/$($targetKnowledgeBase.id)/documents" -Headers $headers
|
|
foreach ($source in @($manifest.sources)) {
|
|
try {
|
|
$officialDetail = Get-OfficialLawDetail -Bbbs $source.bbbs
|
|
$officialUrl = Get-OfficialDetailUrl -Bbbs $source.bbbs
|
|
if ([string]$officialDetail.title -ne [string]$source.title) {
|
|
Add-Check -Key "official_title_$($source.source_id)" -Label "Official title check $($source.source_id)" -Status "fail" -Detail "Expected=$($source.title), actual=$($officialDetail.title)"
|
|
continue
|
|
}
|
|
$docxUrl = Get-OfficialDocxUrl -Bbbs $source.bbbs
|
|
$sourceDir = Join-Path $downloadDir $source.source_id
|
|
New-Item -ItemType Directory -Force -Path $sourceDir | Out-Null
|
|
$docxPath = Join-Path $sourceDir "official.docx"
|
|
Save-OfficialDocx -Url $docxUrl -Path $docxPath
|
|
$docxBytes = [System.IO.File]::ReadAllBytes($docxPath)
|
|
$sourceChecksum = Get-TextChecksum -Text ([Convert]::ToBase64String($docxBytes))
|
|
$paragraphs = Convert-DocxToParagraphs -DocxPath $docxPath -WorkDir $sourceDir
|
|
Add-Check -Key "official_download_$($source.source_id)" -Label "Official download $($source.source_id)" -Status "pass" -Detail "paragraphs=$($paragraphs.Count)" -Meta @{ title = $source.title; bbbs = $source.bbbs; checksum = $sourceChecksum }
|
|
|
|
foreach ($article in @($source.articles)) {
|
|
try {
|
|
$articleText = Get-ArticleText -Paragraphs $paragraphs -ArticleNo $article.article_no
|
|
$knowledgeDocument = Import-ArticleDocument -Headers $headers -KnowledgeBaseId $targetKnowledgeBase.id -Source $source -Article $article -ArticleText $articleText -OfficialUrl $officialUrl -ExistingDocuments $existingDocs
|
|
$articlePayloads.Add([ordered]@{
|
|
source_id = $source.source_id
|
|
source_title = $source.title
|
|
source_type = $source.source_type
|
|
authority = $source.authority
|
|
effective_date = $source.effective_date
|
|
validity_status = $source.validity_status
|
|
region = $source.region
|
|
official_url = $officialUrl
|
|
article_no = $article.article_no
|
|
chapter_title = $article.chapter_title
|
|
section_title = $article.section_title
|
|
article_text = $articleText
|
|
tags = @($source.source_id, $source.knowledge_domain, "phase14", "legal-fulltext") + @($article.tags)
|
|
source_checksum = $sourceChecksum
|
|
text_checksum = Get-TextChecksum -Text $articleText
|
|
knowledge_base_id = $targetKnowledgeBase.id
|
|
knowledge_document_id = $knowledgeDocument.id
|
|
index_status = if ($knowledgeDocument.status -eq "indexed") { "indexed" } else { [string]$knowledgeDocument.status }
|
|
published_status = "published"
|
|
published_by = "CODEX_PHASE14_IMPORT"
|
|
}) | Out-Null
|
|
} catch {
|
|
Add-Import -SourceId $source.source_id -ArticleNo $article.article_no -Action "extract_upload" -Status "failed" -TargetId "" -Detail $_.Exception.Message
|
|
}
|
|
}
|
|
} catch {
|
|
Add-Check -Key "official_download_$($source.source_id)" -Label "Official download $($source.source_id)" -Status "fail" -Detail $_.Exception.Message
|
|
}
|
|
}
|
|
}
|
|
|
|
$importFailures = @($imports | Where-Object { $_.status -eq "failed" }).Count
|
|
if ((@($checks | Where-Object { $_.required -and $_.status -eq "fail" })).Count -eq 0 -and $importFailures -eq 0) {
|
|
$bulkResponse = Invoke-ApiJson -Method "POST" -Url "$ApiBaseUrl/legal-articles/bulk-upsert" -Headers $headers -Body @{ articles = $articlePayloads.ToArray() }
|
|
$bulkData = Get-ResponseData -Response $bulkResponse
|
|
Add-Check -Key "legal_articles_bulk_upsert" -Label "Legal articles bulk upsert" -Status "pass" -Detail "total=$($bulkData.total), created=$($bulkData.created_count), updated=$($bulkData.updated_count)" -Meta @{ total = $bulkData.total; created = $bulkData.created_count; updated = $bulkData.updated_count }
|
|
|
|
if ($SkipKnowledgeBasePublish) {
|
|
Add-Import -SourceId "knowledge_base" -ArticleNo "" -Action "publish" -Status "skipped" -TargetId $targetKnowledgeBase.id -Detail "Skipped by parameter."
|
|
} else {
|
|
$publishResponse = Invoke-ApiJson -Method "POST" -Url "$ApiBaseUrl/knowledge-bases/$($targetKnowledgeBase.id)/publish" -Headers $headers
|
|
$publishedKb = Get-ResponseData -Response $publishResponse
|
|
Add-Import -SourceId "knowledge_base" -ArticleNo "" -Action "publish" -Status "published" -TargetId $publishedKb.id -Detail "Knowledge base published." -Meta @{ current_version_label = $publishedKb.current_version_label; published_at = $publishedKb.published_at }
|
|
}
|
|
}
|
|
|
|
if ((@($checks | Where-Object { $_.required -and $_.status -eq "fail" })).Count -eq 0 -and $importFailures -eq 0) {
|
|
$expectedArticleCount = $articlePayloads.Count
|
|
$documentsAfter = Get-PagedApiItems -Url "$ApiBaseUrl/knowledge-bases/$($targetKnowledgeBase.id)/documents" -Headers $headers
|
|
$documentIds = @($articlePayloads | ForEach-Object { $_.knowledge_document_id })
|
|
$targetDocs = @($documentsAfter | Where-Object { $documentIds -contains $_.id })
|
|
$indexedDocs = @($targetDocs | Where-Object { $_.status -eq "indexed" })
|
|
if ($targetDocs.Count -eq $expectedArticleCount -and $indexedDocs.Count -eq $expectedArticleCount) {
|
|
Add-Verification -Key "knowledge_documents_indexed" -Status "pass" -Detail "All legal article documents are indexed." -Meta @{ expected = $expectedArticleCount; target_documents = $targetDocs.Count; indexed = $indexedDocs.Count }
|
|
} else {
|
|
Add-Verification -Key "knowledge_documents_indexed" -Status "fail" -Detail "Legal article document indexing is incomplete." -Meta @{ expected = $expectedArticleCount; target_documents = $targetDocs.Count; indexed = $indexedDocs.Count }
|
|
}
|
|
|
|
$batchSourceIds = @($articlePayloads | ForEach-Object { [string]$_.source_id } | Select-Object -Unique)
|
|
$apiArticles = Get-LegalArticlesBySourceIds -SourceIds $batchSourceIds -Headers $headers -PublishedStatus "published" -IndexStatus "indexed"
|
|
$expectedChecksums = @($articlePayloads | ForEach-Object { $_.text_checksum })
|
|
$matchedArticles = @($apiArticles | Where-Object { $expectedChecksums -contains $_.text_checksum })
|
|
if ($matchedArticles.Count -eq $expectedArticleCount) {
|
|
Add-Verification -Key "legal_articles_runtime_table" -Status "pass" -Detail "All legal articles are in runtime table." -Meta @{ expected = $expectedArticleCount; matched = $matchedArticles.Count }
|
|
} else {
|
|
Add-Verification -Key "legal_articles_runtime_table" -Status "fail" -Detail "Runtime legal article table is incomplete." -Meta @{ expected = $expectedArticleCount; matched = $matchedArticles.Count }
|
|
}
|
|
}
|
|
|
|
$failedRequired = @($checks | Where-Object { $_.required -and $_.status -eq "fail" }).Count
|
|
$failedVerification = @($verification | Where-Object { $_.status -eq "fail" }).Count
|
|
$warnings = @($checks | Where-Object { $_.status -eq "warn" }).Count + @($verification | Where-Object { $_.status -eq "warn" }).Count
|
|
$overallStatus = if ($failedRequired -gt 0 -or $failedVerification -gt 0 -or $importFailures -gt 0) { "FAIL" } elseif ($warnings -gt 0) { "WARN" } else { "PASS" }
|
|
|
|
$summary = [ordered]@{
|
|
name = "phase14-legal-fulltext-import"
|
|
status = $overallStatus
|
|
generated_at = (Get-Date).ToUniversalTime().ToString("o")
|
|
api_base_url = $ApiBaseUrl
|
|
manifest_path = $ManifestPath
|
|
artifact_dir = $artifactDir
|
|
knowledge_base_name = $KnowledgeBaseName
|
|
knowledge_base_id = if ($targetKnowledgeBase) { $targetKnowledgeBase.id } else { "" }
|
|
checks = $checks.ToArray()
|
|
imports = $imports.ToArray()
|
|
verification = $verification.ToArray()
|
|
totals = [ordered]@{
|
|
sources = if ($manifest) { @($manifest.sources).Count } else { 0 }
|
|
articles_extracted = $articlePayloads.Count
|
|
import_failures = $importFailures
|
|
failed_required_checks = $failedRequired
|
|
failed_verifications = $failedVerification
|
|
}
|
|
}
|
|
Save-Json -Path $summaryJson -Value $summary
|
|
|
|
$md = New-Object System.Collections.Generic.List[string]
|
|
$md.Add("# Phase14 legal fulltext import report") | Out-Null
|
|
$md.Add("") | Out-Null
|
|
$md.Add("- status: $overallStatus") | Out-Null
|
|
$md.Add("- generated_at: $($summary.generated_at)") | Out-Null
|
|
$md.Add("- api_base_url: $ApiBaseUrl") | Out-Null
|
|
$md.Add("- manifest_path: $ManifestPath") | Out-Null
|
|
$md.Add("- knowledge_base_name: $KnowledgeBaseName") | Out-Null
|
|
$md.Add("- knowledge_base_id: $($summary.knowledge_base_id)") | Out-Null
|
|
$md.Add("") | Out-Null
|
|
$md.Add("## Totals") | Out-Null
|
|
$md.Add("") | Out-Null
|
|
$md.Add("| item | value |") | Out-Null
|
|
$md.Add("|---|---:|") | Out-Null
|
|
$md.Add("| sources | $($summary.totals.sources) |") | Out-Null
|
|
$md.Add("| articles_extracted | $($summary.totals.articles_extracted) |") | Out-Null
|
|
$md.Add("| import_failures | $($summary.totals.import_failures) |") | Out-Null
|
|
$md.Add("| failed_required_checks | $($summary.totals.failed_required_checks) |") | Out-Null
|
|
$md.Add("| failed_verifications | $($summary.totals.failed_verifications) |") | Out-Null
|
|
$md.Add("") | Out-Null
|
|
$md.Add("## Checks") | Out-Null
|
|
$md.Add("") | Out-Null
|
|
$md.Add("| key | required | status | detail |") | Out-Null
|
|
$md.Add("|---|---:|---|---|") | Out-Null
|
|
foreach ($check in $checks) {
|
|
$detail = ([string]$check.detail).Replace("|", "/")
|
|
$requiredText = if ($check.required) { "yes" } else { "no" }
|
|
$md.Add("| $($check.key) | $requiredText | $($check.status) | $detail |") | Out-Null
|
|
}
|
|
$md.Add("") | Out-Null
|
|
$md.Add("## Imports") | Out-Null
|
|
$md.Add("") | Out-Null
|
|
$md.Add("| source_id | article_no | action | status | target_id | detail |") | Out-Null
|
|
$md.Add("|---|---|---|---|---|---|") | Out-Null
|
|
foreach ($item in $imports) {
|
|
$detail = ([string]$item.detail).Replace("|", "/")
|
|
$md.Add("| $($item.source_id) | $($item.article_no) | $($item.action) | $($item.status) | $($item.target_id) | $detail |") | Out-Null
|
|
}
|
|
$md.Add("") | Out-Null
|
|
$md.Add("## Verification") | Out-Null
|
|
$md.Add("") | Out-Null
|
|
$md.Add("| key | status | detail |") | Out-Null
|
|
$md.Add("|---|---|---|") | Out-Null
|
|
foreach ($item in $verification) {
|
|
$detail = ([string]$item.detail).Replace("|", "/")
|
|
$md.Add("| $($item.key) | $($item.status) | $detail |") | Out-Null
|
|
}
|
|
$md | Set-Content -Encoding UTF8 -Path $summaryMd
|
|
|
|
Write-Host "PHASE14_LEGAL_FULLTEXT_IMPORT_$overallStatus $artifactDir"
|
|
if ($overallStatus -eq "FAIL") {
|
|
throw "Phase14 legal fulltext import failed. See: $summaryMd"
|
|
}
|