fix: harden RC-04 cold install and UAT recovery

This commit is contained in:
2026-09-20 17:59:33 +08:00
parent 9eed54f816
commit be828e81e5
7 changed files with 698 additions and 39 deletions
@@ -70,8 +70,8 @@ def upgrade() -> None:
sa.CheckConstraint("status in ('pending', 'approved', 'rejected')", name="ck_reliable_audit_change_status"),
)
for name, columns in (
("ix_reliable_evidence_audit_client_change_requests_owner_tenant_id", ["owner_tenant_id"]),
("ix_reliable_evidence_audit_client_change_requests_audit_client_id", ["audit_client_id"]),
("ix_reliable_audit_change_owner_tenant_id", ["owner_tenant_id"]),
("ix_reliable_audit_change_audit_client_id", ["audit_client_id"]),
("ix_reliable_evidence_audit_client_change_requests_action", ["action"]),
("ix_reliable_evidence_audit_client_change_requests_status", ["status"]),
("ix_reliable_evidence_audit_client_change_requests_proposed_by", ["proposed_by"]),
+151 -2
View File
@@ -12,9 +12,10 @@ import shutil
# Docker commands use fixed argv lists and never invoke a host shell.
import subprocess # nosec B404
import sys
import tarfile
import tempfile
from datetime import datetime, timezone
from pathlib import Path
from pathlib import Path, PurePosixPath
from typing import BinaryIO
from urllib.parse import quote
@@ -26,7 +27,19 @@ IDENTIFIER_PATTERN = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$")
DOCKER_PROJECT_PATTERN = re.compile(r"^[a-z0-9][a-z0-9_-]{0,62}$")
S3_BUCKET_PATTERN = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9._-]{0,62}$")
BACKUP_RUN_PATTERN = re.compile(r"^backup-[0-9]{8}T[0-9]{6}Z$")
WRITER_SERVICES = ("contract-api", "contract-review-worker", "ai-platform-api", "worker")
WRITER_SERVICES = (
"contract-api",
"contract-review-worker",
"contract-playbook-governance-worker",
"ai-platform-api",
"worker",
)
REQUIRED_BACKUP_ARTIFACTS = {
"postgres-contract.dump",
"postgres-ai_platform.dump",
"postgres-operations.dump",
"contract-files.tar.gz",
}
def parse_env_file(path: Path) -> dict[str, str]:
@@ -132,6 +145,7 @@ def create_backup(
ops_profile=True,
)
)
_verify_contract_archive(partial_dir / "contract-files.tar.gz")
minio_env_path = _write_minio_alias_env(values)
try:
_run(
@@ -145,6 +159,11 @@ def create_backup(
minio_env_path.unlink(missing_ok=True)
artifacts = _artifact_metadata(partial_dir)
minio_mirror = _minio_mirror_metadata(
partial_dir,
artifacts=artifacts,
bucket=values.get("AI_PLATFORM_OBJECT_STORAGE_BUCKET", "ai-platform-uploads"),
)
manifest = {
"schema_version": "yuqei-production-backup.v1",
"backup_id": run_id,
@@ -158,6 +177,7 @@ def create_backup(
"ai_platform_api": values.get("AI_PLATFORM_API_IMAGE"),
"ai_worker": values.get("AI_WORKER_IMAGE"),
},
"minio_mirror": minio_mirror,
"artifacts": artifacts,
}
(partial_dir / "manifest.json").write_text(
@@ -183,10 +203,15 @@ def verify_manifest(backup_dir: Path) -> dict[str, object]:
artifacts = payload.get("artifacts")
if not isinstance(artifacts, list) or not artifacts:
raise RuntimeError("Backup manifest has no artifacts.")
artifact_paths: set[str] = set()
minio_records: list[dict[str, object]] = []
for item in artifacts:
if not isinstance(item, dict):
raise RuntimeError("Backup manifest artifact is invalid.")
relative_path = str(item.get("path", ""))
if relative_path in artifact_paths:
raise RuntimeError(f"Duplicate backup artifact path: {relative_path}")
artifact_paths.add(relative_path)
target = (backup_dir / relative_path).resolve()
target.relative_to(backup_dir.resolve())
if not target.is_file():
@@ -195,6 +220,19 @@ def verify_manifest(backup_dir: Path) -> dict[str, object]:
raise RuntimeError(f"Backup artifact size mismatch: {relative_path}")
if _sha256(target) != item.get("sha256"):
raise RuntimeError(f"Backup artifact checksum mismatch: {relative_path}")
if relative_path.startswith("minio/"):
minio_records.append(
{
"path": relative_path,
"size_bytes": target.stat().st_size,
"sha256": item.get("sha256"),
}
)
missing = sorted(REQUIRED_BACKUP_ARTIFACTS - artifact_paths)
if missing:
raise RuntimeError(f"Backup manifest is missing required artifacts: {missing}")
_verify_minio_mirror(backup_dir, payload, minio_records)
_verify_contract_archive(backup_dir / "contract-files.tar.gz")
return payload
@@ -224,6 +262,117 @@ def _artifact_metadata(root: Path) -> list[dict[str, object]]:
]
def _minio_content_index(records: list[dict[str, object]]) -> str:
canonical = json.dumps(
sorted(records, key=lambda item: str(item["path"]).encode("utf-8")),
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
).encode("utf-8")
return hashlib.sha256(canonical).hexdigest()
def _verify_contract_archive(path: Path) -> None:
with tarfile.open(path, mode="r:gz") as archive:
seen: set[str] = set()
seen_folded: set[str] = set()
for member in archive.getmembers():
raw_name = member.name
if "\\" in raw_name or "\x00" in raw_name:
raise RuntimeError("Contract file archive contains an unsafe path.")
member_path = PurePosixPath(raw_name)
if member_path.is_absolute() or ".." in member_path.parts:
raise RuntimeError("Contract file archive contains an unsafe path.")
if not (member.isfile() or member.isdir()):
raise RuntimeError("Contract file archive contains a link or special entry.")
normalized = member_path.as_posix().removeprefix("./")
if normalized not in {"", "."}:
folded = normalized.casefold()
if normalized in seen or folded in seen_folded:
raise RuntimeError("Contract file archive contains a duplicate path.")
seen.add(normalized)
seen_folded.add(folded)
def _minio_mirror_metadata(
root: Path,
*,
artifacts: list[dict[str, object]],
bucket: str,
) -> dict[str, object]:
minio_root = root / "minio"
if minio_root.is_symlink() or not minio_root.is_dir():
raise RuntimeError("MinIO mirror directory is missing or unsafe.")
records = [
{
"path": str(item["path"]),
"size_bytes": int(item["size_bytes"]),
"sha256": str(item["sha256"]),
}
for item in artifacts
if str(item.get("path", "")).startswith("minio/")
]
return {
"bucket": bucket,
"path": "minio",
"object_count": len(records),
"total_size_bytes": sum(int(item["size_bytes"]) for item in records),
"content_index_sha256": _minio_content_index(records),
}
def _verify_minio_mirror(
backup_dir: Path,
manifest: dict[str, object],
records: list[dict[str, object]],
) -> None:
minio_root = backup_dir / "minio"
if minio_root.is_symlink() or not minio_root.is_dir():
raise RuntimeError("MinIO mirror directory is missing or unsafe.")
actual_files: set[str] = set()
for path in minio_root.rglob("*"):
relative = path.relative_to(backup_dir).as_posix()
if path.is_symlink():
raise RuntimeError(f"MinIO mirror contains a symbolic link: {relative}")
if path.is_file():
actual_files.add(relative)
elif not path.is_dir():
raise RuntimeError(f"MinIO mirror contains a special entry: {relative}")
recorded_files = {str(item["path"]) for item in records}
if actual_files != recorded_files:
raise RuntimeError("MinIO mirror content differs from the backup manifest.")
metadata = manifest.get("minio_mirror")
if metadata is None:
if not records:
raise RuntimeError(
"Legacy backup has no MinIO mirror metadata or recorded MinIO content."
)
return
if not isinstance(metadata, dict):
raise RuntimeError("MinIO mirror metadata is invalid.")
expected = {
"bucket",
"path",
"object_count",
"total_size_bytes",
"content_index_sha256",
}
if set(metadata) != expected or metadata.get("path") != "minio":
raise RuntimeError("MinIO mirror metadata fields are invalid.")
if (
not isinstance(metadata.get("bucket"), str)
or not S3_BUCKET_PATTERN.fullmatch(metadata["bucket"])
):
raise RuntimeError("MinIO mirror bucket metadata is invalid.")
if metadata.get("object_count") != len(records):
raise RuntimeError("MinIO mirror object count mismatch.")
if metadata.get("total_size_bytes") != sum(int(item["size_bytes"]) for item in records):
raise RuntimeError("MinIO mirror total size mismatch.")
if metadata.get("content_index_sha256") != _minio_content_index(records):
raise RuntimeError("MinIO mirror content index mismatch.")
def minio_mirror_command(
*,
values: dict[str, str],
+103 -7
View File
@@ -13,7 +13,7 @@ import subprocess # nosec B404
import sys
import tarfile
from datetime import datetime, timezone
from pathlib import Path
from pathlib import Path, PurePosixPath
REPO_ROOT = Path(__file__).resolve().parents[2]
@@ -21,6 +21,13 @@ BASE_COMPOSE = REPO_ROOT / "yuqei-infra" / "compose" / "docker-compose.yml"
PRODUCTION_COMPOSE = REPO_ROOT / "yuqei-infra" / "compose" / "docker-compose.production.yml"
DEFAULT_ARTIFACT_ROOT = REPO_ROOT / "project-docs" / "qa-artifacts" / "v2-production-restore-check"
SAFE_DATABASE_PATTERN = re.compile(r"^yuqei_restore_check_(contract|ai_platform|operations)_[0-9]{14}$")
S3_BUCKET_PATTERN = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9._-]{0,62}$")
REQUIRED_BACKUP_ARTIFACTS = {
"postgres-contract.dump",
"postgres-ai_platform.dump",
"postgres-operations.dump",
"contract-files.tar.gz",
}
def read_env(path: Path) -> dict[str, str]:
@@ -46,10 +53,15 @@ def validate_backup(backup_dir: Path) -> tuple[dict[str, object], list[dict[str,
raise RuntimeError("Backup manifest has no artifacts.")
verified: list[dict[str, object]] = []
root = backup_dir.resolve()
artifact_paths: set[str] = set()
minio_records: list[dict[str, object]] = []
for item in artifacts:
if not isinstance(item, dict):
raise RuntimeError("Invalid artifact entry.")
relative_path = str(item.get("path", ""))
if relative_path in artifact_paths:
raise RuntimeError(f"Duplicate backup artifact path: {relative_path}")
artifact_paths.add(relative_path)
path = (root / relative_path).resolve()
path.relative_to(root)
if not path.is_file():
@@ -59,17 +71,101 @@ def validate_backup(backup_dir: Path) -> tuple[dict[str, object], list[dict[str,
if actual_size != int(item.get("size_bytes", -1)) or actual_sha256 != item.get("sha256"):
raise RuntimeError(f"Backup artifact integrity mismatch: {relative_path}")
verified.append({"path": relative_path, "size_bytes": actual_size, "sha256_match": True})
if relative_path.startswith("minio/"):
minio_records.append(
{"path": relative_path, "size_bytes": actual_size, "sha256": actual_sha256}
)
missing = sorted(REQUIRED_BACKUP_ARTIFACTS - artifact_paths)
if missing:
raise RuntimeError(f"Backup manifest is missing required artifacts: {missing}")
_verify_minio_mirror(backup_dir, manifest, minio_records)
contract_archive = backup_dir / "contract-files.tar.gz"
if contract_archive.is_file():
with tarfile.open(contract_archive, mode="r:gz") as archive:
for member in archive.getmembers():
member_path = Path(member.name)
if member_path.is_absolute() or ".." in member_path.parts:
raise RuntimeError("Contract file archive contains an unsafe path.")
with tarfile.open(contract_archive, mode="r:gz") as archive:
seen: set[str] = set()
seen_folded: set[str] = set()
for member in archive.getmembers():
raw_name = member.name
if "\\" in raw_name or "\x00" in raw_name:
raise RuntimeError("Contract file archive contains an unsafe path.")
member_path = PurePosixPath(raw_name)
if member_path.is_absolute() or ".." in member_path.parts:
raise RuntimeError("Contract file archive contains an unsafe path.")
if not (member.isfile() or member.isdir()):
raise RuntimeError("Contract file archive contains a link or special entry.")
normalized = member_path.as_posix().removeprefix("./")
if normalized not in {"", "."}:
folded = normalized.casefold()
if normalized in seen or folded in seen_folded:
raise RuntimeError("Contract file archive contains a duplicate path.")
seen.add(normalized)
seen_folded.add(folded)
return manifest, verified
def _minio_content_index(records: list[dict[str, object]]) -> str:
canonical = json.dumps(
sorted(records, key=lambda item: str(item["path"]).encode("utf-8")),
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
).encode("utf-8")
return hashlib.sha256(canonical).hexdigest()
def _verify_minio_mirror(
backup_dir: Path,
manifest: dict[str, object],
records: list[dict[str, object]],
) -> None:
minio_root = backup_dir / "minio"
if minio_root.is_symlink() or not minio_root.is_dir():
raise RuntimeError("MinIO mirror directory is missing or unsafe.")
actual_files: set[str] = set()
for path in minio_root.rglob("*"):
relative = path.relative_to(backup_dir).as_posix()
if path.is_symlink():
raise RuntimeError(f"MinIO mirror contains a symbolic link: {relative}")
if path.is_file():
actual_files.add(relative)
elif not path.is_dir():
raise RuntimeError(f"MinIO mirror contains a special entry: {relative}")
recorded_files = {str(item["path"]) for item in records}
if actual_files != recorded_files:
raise RuntimeError("MinIO mirror content differs from the backup manifest.")
metadata = manifest.get("minio_mirror")
if metadata is None:
if not records:
raise RuntimeError(
"Legacy backup has no MinIO mirror metadata or recorded MinIO content."
)
return
if not isinstance(metadata, dict):
raise RuntimeError("MinIO mirror metadata is invalid.")
expected = {
"bucket",
"path",
"object_count",
"total_size_bytes",
"content_index_sha256",
}
if set(metadata) != expected or metadata.get("path") != "minio":
raise RuntimeError("MinIO mirror metadata fields are invalid.")
if (
not isinstance(metadata.get("bucket"), str)
or not S3_BUCKET_PATTERN.fullmatch(metadata["bucket"])
):
raise RuntimeError("MinIO mirror bucket metadata is invalid.")
if metadata.get("object_count") != len(records):
raise RuntimeError("MinIO mirror object count mismatch.")
if metadata.get("total_size_bytes") != sum(int(item["size_bytes"]) for item in records):
raise RuntimeError("MinIO mirror total size mismatch.")
if metadata.get("content_index_sha256") != _minio_content_index(records):
raise RuntimeError("MinIO mirror content index mismatch.")
def restore_database_checks(
*,
backup_dir: Path,
+151 -13
View File
@@ -123,6 +123,105 @@ function safeFilename(value) {
return String(value).replace(/[^\w-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 90) || "artifact";
}
function safeRequestPath(value) {
try {
return new URL(String(value)).pathname || "/";
} catch {
return "<unparseable>";
}
}
function sanitizeFailure(value) {
const normalized = String(value || "Unknown failure")
.replace(/[\u0000-\u001f\u007f]+/g, " ")
.replace(/\b(?:Bearer|Basic)\s+[A-Za-z0-9._~+/=-]+/gi, (match) => `${match.split(/\s+/, 1)[0]} [REDACTED]`)
.replace(/https?:\/\/[^\s"'<>]+/gi, (url) => safeRequestPath(url))
.replace(
/\b(password|passwd|client[-_]?secret|secret|access[-_]?token|refresh[-_]?token|id[-_]?token|session[-_]?token|token|api[-_]?key|proxy[-_]?key|private[-_]?key|credential|authorization|cookie|session(?:[-_]?id)?)\b(\s*[:=]\s*)(?:"[^"]*"|'[^']*'|[^\s,;}]*)/gi,
(_match, label, separator) => `${label}${separator}[REDACTED]`,
)
.trim();
return (normalized || "Unknown failure").slice(0, 500);
}
function requestMetadata(request) {
return {
method: String(request.method() || "UNKNOWN").toUpperCase(),
path: safeRequestPath(request.url()),
resource: String(request.resourceType() || "unknown"),
};
}
function isExpectedHttpError(entry) {
return entry.method === "GET" && entry.path === "/favicon.ico" && entry.status === 404;
}
function isExpectedConsoleError(message) {
const location = typeof message.location === "function" ? message.location() : null;
return safeRequestPath(location?.url || "") === "/favicon.ico";
}
function observeBrowserErrors(page) {
const diagnostics = {
consoleErrors: [],
pageErrors: [],
requestFailures: [],
unexpectedHttpResponses: [],
};
page.on("console", (message) => {
if (message.type() === "error" && !isExpectedConsoleError(message)) {
diagnostics.consoleErrors.push({
resource: "console",
failure: sanitizeFailure(message.text()),
});
}
});
page.on("pageerror", (error) => {
diagnostics.pageErrors.push({
resource: "page",
failure: sanitizeFailure(error?.message || error),
});
});
page.on("requestfailed", (request) => {
diagnostics.requestFailures.push({
...requestMetadata(request),
failure: sanitizeFailure(request.failure()?.errorText),
});
});
page.on("response", (response) => {
const status = response.status();
if (status < 400) {
return;
}
const entry = {
...requestMetadata(response.request()),
status,
failure: `HTTP ${status}`,
};
if (!isExpectedHttpError(entry)) {
diagnostics.unexpectedHttpResponses.push(entry);
}
});
return diagnostics;
}
function assertBrowserErrorsAbsent(diagnostics) {
const counts = {
console: diagnostics.consoleErrors.length,
page: diagnostics.pageErrors.length,
request: diagnostics.requestFailures.length,
http: diagnostics.unexpectedHttpResponses.length,
};
const total = Object.values(counts).reduce((sum, count) => sum + count, 0);
if (total > 0) {
throw new Error(
`Unexpected browser errors detected: console=${counts.console}, page=${counts.page}, request=${counts.request}, http=${counts.http}.`,
);
}
}
async function waitForContract(apiUrl, predicate, timeoutMs) {
const deadline = Date.now() + Math.min(timeoutMs, 10000);
let contract = null;
@@ -156,6 +255,37 @@ async function login(page, args) {
await page.waitForURL(/\/workspace(?:\?.*)?$/, { timeout: args.timeoutMs });
}
async function verifyAuthenticatedSession(page, args) {
const endpoint = `${args.baseUrl}/api/auth/session`;
const result = await page.evaluate(async (sessionEndpoint) => {
const response = await fetch(sessionEndpoint, {
cache: "no-store",
credentials: "same-origin",
});
const payload = await response.json().catch(() => null);
const session = payload && typeof payload.session === "object" ? payload.session : null;
return {
ok: response.ok,
status: response.status,
tenantId: typeof session?.tenant_id === "string" ? session.tenant_id : null,
userId: typeof session?.user_id === "string" ? session.user_id : null,
};
}, endpoint);
if (!result.ok) {
throw new Error(`Authenticated session verification failed: HTTP ${result.status}.`);
}
if (!result.tenantId || !result.userId) {
throw new Error("Authenticated session verification failed: the session endpoint returned no tenant/user identity.");
}
if (result.tenantId !== args.tenantId || result.userId !== args.userId) {
throw new Error(
`Authenticated session identity mismatch: expected tenant=${args.tenantId}, user=${args.userId}; got tenant=${result.tenantId}, user=${result.userId}.`,
);
}
return { tenantId: result.tenantId, userId: result.userId };
}
async function createContractFromTemplate(page, args) {
await page.goto(`${args.baseUrl}/legal-documents?template=TPL-COOPERATION-FRAMEWORK-001`, {
waitUntil: "domcontentloaded",
@@ -580,12 +710,7 @@ async function main() {
const browser = await chromium.launch({ headless: !args.headed, channel: "chrome" });
const page = await browser.newPage({ viewport: { width: 1440, height: 1050 } });
page.setDefaultTimeout(args.timeoutMs);
const consoleErrors = [];
page.on("console", (message) => {
if (message.type() === "error" && !message.text().includes("favicon.ico")) {
consoleErrors.push(message.text());
}
});
const browserErrors = observeBrowserErrors(page);
const summary = {
status: "fail",
@@ -595,16 +720,18 @@ async function main() {
outputDir: args.outputDir,
contractId: args.contractId || null,
login: false,
authenticatedSession: null,
templateDraft: false,
wordDownload: null,
review: null,
fileRepository: null,
lifecycle: null,
consoleErrors,
...browserErrors,
};
try {
await login(page, args);
summary.authenticatedSession = await verifyAuthenticatedSession(page, args);
summary.login = true;
if (!summary.contractId) {
@@ -625,13 +752,13 @@ async function main() {
summary.review = await startAndRunReview(page, args, summary.contractId);
summary.fileRepository = await uploadAndVerifyFile(page, args, summary.contractId);
summary.lifecycle = await runContractLifecycle(page, args, summary.contractId);
summary.consoleErrors = consoleErrors;
await page.waitForTimeout(250);
assertBrowserErrorsAbsent(browserErrors);
summary.status = "pass";
} catch (error) {
summary.error = error instanceof Error ? error.message : String(error);
await page.screenshot({ path: path.join(args.outputDir, "core-regression-failure.png"), fullPage: true }).catch(() => null);
} finally {
summary.consoleErrors = consoleErrors;
fs.writeFileSync(path.join(args.outputDir, "core-user-flow-summary.json"), JSON.stringify(summary, null, 2), "utf8");
await browser.close();
}
@@ -643,7 +770,18 @@ async function main() {
}
}
main().catch((error) => {
console.error(error);
process.exit(1);
});
if (require.main === module) {
main().catch((error) => {
console.error(error);
process.exit(1);
});
}
module.exports = {
assertBrowserErrorsAbsent,
isExpectedHttpError,
observeBrowserErrors,
safeRequestPath,
sanitizeFailure,
verifyAuthenticatedSession,
};
@@ -1,6 +1,7 @@
from __future__ import annotations
import ast
import re
from pathlib import Path
@@ -12,6 +13,8 @@ CONTRACT_MIGRATIONS = (
AI_MIGRATIONS = (
REPO_ROOT / "yuqei-ai-platform" / "services" / "ai-platform-api" / "migrations" / "versions"
)
POSTGRES_IDENTIFIER_MAX_BYTES = 63
MIGRATION_IDENTIFIER_PATTERN = re.compile(r"(?<![A-Za-z0-9_])(?:ix|ux|ck|fk|pk)_[A-Za-z0-9_]+")
def _literal_assignment(module: ast.Module, name: str) -> object:
@@ -43,6 +46,18 @@ def _migration_heads(directory: Path) -> set[str]:
return revisions - parents
def _explicit_identifier_literals(directory: Path) -> list[tuple[Path, int, str]]:
identifiers: list[tuple[Path, int, str]] = []
for path in directory.glob("*.py"):
module = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
for node in ast.walk(module):
if not isinstance(node, ast.Constant) or not isinstance(node.value, str):
continue
for match in MIGRATION_IDENTIFIER_PATTERN.finditer(node.value):
identifiers.append((path, node.lineno, match.group(0)))
return identifiers
def _workflow_step(workflow: str, name: str) -> str:
marker = f" - name: {name}"
start = workflow.index(marker)
@@ -83,3 +98,17 @@ def test_ci_migration_steps_track_real_unique_heads_and_use_empty_sqlite() -> No
]
assert '20260826_0073 (head)' not in workflow
def test_explicit_migration_identifiers_fit_postgresql_limit() -> None:
offenders = [
f"{path.relative_to(REPO_ROOT)}:{line}: {identifier!r} "
f"({len(identifier.encode('utf-8'))} bytes)"
for directory in (CONTRACT_MIGRATIONS, AI_MIGRATIONS)
for path, line, identifier in _explicit_identifier_literals(directory)
if len(identifier.encode("utf-8")) > POSTGRES_IDENTIFIER_MAX_BYTES
]
assert offenders == [], (
"Explicit migration identifiers exceed PostgreSQL's 63-byte limit:\n"
+ "\n".join(offenders)
)
@@ -5,6 +5,7 @@ import hashlib
import importlib.util
import io
import json
import shutil
import sys
import tarfile
from datetime import datetime, timezone
@@ -132,6 +133,16 @@ def test_minio_backup_invokes_pinned_mc_directly_without_a_shell(tmp_path: Path)
assert all("secret" not in token for token in command)
def test_backup_quiesces_every_long_running_application_writer() -> None:
assert backup_run.WRITER_SERVICES == (
"contract-api",
"contract-review-worker",
"contract-playbook-governance-worker",
"ai-platform-api",
"worker",
)
@pytest.mark.parametrize(
("label", "validation_name", "table_name"),
[
@@ -674,42 +685,150 @@ def test_production_preflight_rejects_smoke_database_public_bind_and_fallback()
assert {"production_modes", "internal_port_binding", "database_boundaries"} <= failed
def test_backup_manifest_and_contract_archive_are_verified(tmp_path: Path) -> None:
def _complete_backup_fixture(tmp_path: Path) -> tuple[Path, dict[str, object]]:
backup_dir = tmp_path / "backup-20260716T120000Z"
backup_dir.mkdir()
backup_dir.mkdir(parents=True)
archive_path = backup_dir / "contract-files.tar.gz"
with tarfile.open(archive_path, mode="w:gz") as archive:
content = b"contract"
info = tarfile.TarInfo("contract-1/document.txt")
info.size = len(content)
archive.addfile(info, io.BytesIO(content))
dump_path = backup_dir / "postgres-contract.dump"
dump_path.write_bytes(b"postgres-dump")
for label in ("contract", "ai_platform", "operations"):
(backup_dir / f"postgres-{label}.dump").write_bytes(
f"postgres-{label}-dump".encode("utf-8")
)
minio_object = backup_dir / "minio" / "tenant-a" / "object.bin"
minio_object.parent.mkdir(parents=True)
minio_object.write_bytes(b"minio-object")
artifacts = []
for path in (archive_path, dump_path):
for path in sorted(backup_dir.rglob("*")):
if not path.is_file():
continue
artifacts.append(
{
"path": path.name,
"path": path.relative_to(backup_dir).as_posix(),
"size_bytes": path.stat().st_size,
"sha256": hashlib.sha256(path.read_bytes()).hexdigest(),
}
)
minio_records = [
item for item in artifacts if str(item["path"]).startswith("minio/")
]
manifest: dict[str, object] = {
"schema_version": "yuqei-production-backup.v1",
"backup_id": backup_dir.name,
"created_at": "2026-07-16T12:00:00+00:00",
"minio_mirror": {
"bucket": "ai-platform-uploads",
"path": "minio",
"object_count": len(minio_records),
"total_size_bytes": sum(int(item["size_bytes"]) for item in minio_records),
"content_index_sha256": backup_run._minio_content_index(minio_records),
},
"artifacts": artifacts,
}
(backup_dir / "manifest.json").write_text(
json.dumps(
{
"schema_version": "yuqei-production-backup.v1",
"backup_id": backup_dir.name,
"created_at": "2026-07-16T12:00:00+00:00",
"artifacts": artifacts,
}
),
json.dumps(manifest),
encoding="utf-8",
)
return backup_dir, manifest
def test_backup_manifest_and_contract_archive_are_verified(tmp_path: Path) -> None:
backup_dir, _ = _complete_backup_fixture(tmp_path)
generated_manifest = backup_run.verify_manifest(backup_dir)
manifest, verified = backup_verify.validate_backup(backup_dir)
assert generated_manifest["backup_id"] == backup_dir.name
assert manifest["backup_id"] == backup_dir.name
assert len(verified) == 2
assert len(verified) == 5
def test_backup_manifest_requires_all_databases_contract_files_and_minio(
tmp_path: Path,
) -> None:
backup_dir, manifest = _complete_backup_fixture(tmp_path)
manifest["artifacts"] = [
item
for item in manifest["artifacts"]
if item["path"] != "postgres-operations.dump"
]
(backup_dir / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8")
with pytest.raises(RuntimeError, match="missing required artifacts"):
backup_verify.validate_backup(backup_dir)
legacy_dir, legacy_manifest = _complete_backup_fixture(tmp_path / "legacy")
legacy_manifest.pop("minio_mirror")
legacy_manifest["artifacts"] = [
item
for item in legacy_manifest["artifacts"]
if not str(item["path"]).startswith("minio/")
]
shutil.rmtree(legacy_dir / "minio")
(legacy_dir / "minio").mkdir()
(legacy_dir / "manifest.json").write_text(
json.dumps(legacy_manifest), encoding="utf-8"
)
with pytest.raises(RuntimeError, match="no MinIO mirror metadata or recorded MinIO content"):
backup_verify.validate_backup(legacy_dir)
def test_backup_manifest_supports_explicit_empty_minio_and_legacy_content(
tmp_path: Path,
) -> None:
empty_dir, empty_manifest = _complete_backup_fixture(tmp_path / "empty")
minio_path = empty_dir / "minio" / "tenant-a" / "object.bin"
minio_path.unlink()
empty_manifest["artifacts"] = [
item
for item in empty_manifest["artifacts"]
if not str(item["path"]).startswith("minio/")
]
empty_manifest["minio_mirror"] = {
"bucket": "ai-platform-uploads",
"path": "minio",
"object_count": 0,
"total_size_bytes": 0,
"content_index_sha256": backup_run._minio_content_index([]),
}
(empty_dir / "manifest.json").write_text(
json.dumps(empty_manifest), encoding="utf-8"
)
backup_verify.validate_backup(empty_dir)
legacy_dir, legacy_manifest = _complete_backup_fixture(tmp_path / "legacy-content")
legacy_manifest.pop("minio_mirror")
(legacy_dir / "manifest.json").write_text(
json.dumps(legacy_manifest), encoding="utf-8"
)
backup_verify.validate_backup(legacy_dir)
@pytest.mark.parametrize("member_type", (tarfile.SYMTYPE, tarfile.LNKTYPE, tarfile.CHRTYPE))
def test_contract_archive_rejects_links_and_special_entries(
tmp_path: Path,
member_type: bytes,
) -> None:
backup_dir, manifest = _complete_backup_fixture(tmp_path)
archive_path = backup_dir / "contract-files.tar.gz"
with tarfile.open(archive_path, mode="w:gz") as archive:
info = tarfile.TarInfo("unsafe-entry")
info.type = member_type
if member_type in {tarfile.SYMTYPE, tarfile.LNKTYPE}:
info.linkname = "outside"
archive.addfile(info)
for item in manifest["artifacts"]:
if item["path"] == "contract-files.tar.gz":
item["size_bytes"] = archive_path.stat().st_size
item["sha256"] = hashlib.sha256(archive_path.read_bytes()).hexdigest()
(backup_dir / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8")
with pytest.raises(RuntimeError, match="link or special entry"):
backup_run.verify_manifest(backup_dir)
with pytest.raises(RuntimeError, match="link or special entry"):
backup_verify.validate_backup(backup_dir)
def test_provider_acceptance_separates_technical_success_from_legal_approval() -> None:
@@ -0,0 +1,128 @@
const assert = require("node:assert/strict");
const { EventEmitter } = require("node:events");
const test = require("node:test");
const {
assertBrowserErrorsAbsent,
observeBrowserErrors,
safeRequestPath,
sanitizeFailure,
verifyAuthenticatedSession,
} = require("../scripts/verify-core-user-flows.cjs");
test("safeRequestPath removes origins, queries, and fragments", () => {
assert.equal(safeRequestPath("https://example.test/contracts/123?token=secret#fragment"), "/contracts/123");
assert.equal(safeRequestPath("not a URL"), "<unparseable>");
});
test("sanitizeFailure redacts credentials and URL queries", () => {
const sanitized = sanitizeFailure(
"Bearer top-secret-token failed at https://example.test/api/jobs?access_token=secret password=hidden access_token=also-hidden",
);
assert.equal(sanitized.includes("top-secret-token"), false);
assert.equal(sanitized.includes("access_token=secret"), false);
assert.equal(sanitized.includes("access_token=also-hidden"), false);
assert.equal(sanitized.includes("password=hidden"), false);
assert.match(sanitized, /Bearer \[REDACTED\]/);
assert.match(sanitized, /\/api\/jobs/);
assert.match(sanitized, /password=\[REDACTED\]/);
});
test("browser diagnostics retain only safe metadata and gate every unexpected class", () => {
const page = new EventEmitter();
const diagnostics = observeBrowserErrors(page);
const request = {
method: () => "get",
url: () => "https://example.test/api/contracts?token=secret",
resourceType: () => "fetch",
failure: () => ({ errorText: "net::ERR_FAILED token=secret" }),
};
page.emit("console", {
type: () => "error",
text: () => "Bearer secret-value was rejected",
location: () => ({ url: "https://example.test/app" }),
});
page.emit("pageerror", new Error("password=secret"));
page.emit("requestfailed", request);
page.emit("response", {
status: () => 503,
request: () => request,
});
page.emit("response", {
status: () => 404,
request: () => ({
method: () => "GET",
url: () => "https://example.test/favicon.ico?cache=1",
resourceType: () => "image",
}),
});
assert.deepEqual(diagnostics.consoleErrors, [{ resource: "console", failure: "Bearer [REDACTED] was rejected" }]);
assert.deepEqual(diagnostics.pageErrors, [{ resource: "page", failure: "password=[REDACTED]" }]);
assert.deepEqual(diagnostics.requestFailures, [{
method: "GET",
path: "/api/contracts",
resource: "fetch",
failure: "net::ERR_FAILED token=[REDACTED]",
}]);
assert.deepEqual(diagnostics.unexpectedHttpResponses, [{
method: "GET",
path: "/api/contracts",
resource: "fetch",
status: 503,
failure: "HTTP 503",
}]);
assert.throws(() => assertBrowserErrorsAbsent(diagnostics), /console=1, page=1, request=1, http=1/);
});
test("browser diagnostics accept an entirely clean run", () => {
assert.doesNotThrow(() => assertBrowserErrorsAbsent({
consoleErrors: [],
pageErrors: [],
requestFailures: [],
unexpectedHttpResponses: [],
}));
});
test("authenticated session must match the supplied tenant and user", async () => {
const page = {
evaluate: async (_callback, endpoint) => {
assert.equal(endpoint, "https://example.test/api/auth/session");
return { ok: true, status: 200, tenantId: "tenant-a", userId: "user-a" };
},
};
const args = { baseUrl: "https://example.test", tenantId: "tenant-a", userId: "user-a" };
assert.deepEqual(await verifyAuthenticatedSession(page, args), { tenantId: "tenant-a", userId: "user-a" });
});
test("authenticated session mismatch fails before the journey proceeds", async () => {
const page = {
evaluate: async () => ({ ok: true, status: 200, tenantId: "tenant-a", userId: "other-user" }),
};
const args = { baseUrl: "https://example.test", tenantId: "tenant-a", userId: "user-a" };
await assert.rejects(
verifyAuthenticatedSession(page, args),
/Authenticated session identity mismatch/,
);
});
test("unavailable or malformed session endpoint fails clearly", async () => {
const args = { baseUrl: "https://example.test", tenantId: "tenant-a", userId: "user-a" };
await assert.rejects(
verifyAuthenticatedSession({
evaluate: async () => ({ ok: false, status: 404, tenantId: null, userId: null }),
}, args),
/Authenticated session verification failed: HTTP 404/,
);
await assert.rejects(
verifyAuthenticatedSession({
evaluate: async () => ({ ok: true, status: 200, tenantId: null, userId: null }),
}, args),
/session endpoint returned no tenant\/user identity/,
);
});