diff --git a/yuqei-contract-platform/services/contract-api/migrations/versions/20260825_0072_add_evidence_external_trust_and_handoff.py b/yuqei-contract-platform/services/contract-api/migrations/versions/20260825_0072_add_evidence_external_trust_and_handoff.py index 47a01c3c..2f0ec83e 100644 --- a/yuqei-contract-platform/services/contract-api/migrations/versions/20260825_0072_add_evidence_external_trust_and_handoff.py +++ b/yuqei-contract-platform/services/contract-api/migrations/versions/20260825_0072_add_evidence_external_trust_and_handoff.py @@ -70,8 +70,8 @@ def upgrade() -> None: sa.CheckConstraint("status in ('pending', 'approved', 'rejected')", name="ck_reliable_audit_change_status"), ) for name, columns in ( - ("ix_reliable_evidence_audit_client_change_requests_owner_tenant_id", ["owner_tenant_id"]), - ("ix_reliable_evidence_audit_client_change_requests_audit_client_id", ["audit_client_id"]), + ("ix_reliable_audit_change_owner_tenant_id", ["owner_tenant_id"]), + ("ix_reliable_audit_change_audit_client_id", ["audit_client_id"]), ("ix_reliable_evidence_audit_client_change_requests_action", ["action"]), ("ix_reliable_evidence_audit_client_change_requests_status", ["status"]), ("ix_reliable_evidence_audit_client_change_requests_proposed_by", ["proposed_by"]), diff --git a/yuqei-infra/backup/run-production-backup.py b/yuqei-infra/backup/run-production-backup.py index b88e5e31..71e3fa25 100644 --- a/yuqei-infra/backup/run-production-backup.py +++ b/yuqei-infra/backup/run-production-backup.py @@ -12,9 +12,10 @@ import shutil # Docker commands use fixed argv lists and never invoke a host shell. import subprocess # nosec B404 import sys +import tarfile import tempfile from datetime import datetime, timezone -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import BinaryIO from urllib.parse import quote @@ -26,7 +27,19 @@ IDENTIFIER_PATTERN = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$") DOCKER_PROJECT_PATTERN = re.compile(r"^[a-z0-9][a-z0-9_-]{0,62}$") S3_BUCKET_PATTERN = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9._-]{0,62}$") BACKUP_RUN_PATTERN = re.compile(r"^backup-[0-9]{8}T[0-9]{6}Z$") -WRITER_SERVICES = ("contract-api", "contract-review-worker", "ai-platform-api", "worker") +WRITER_SERVICES = ( + "contract-api", + "contract-review-worker", + "contract-playbook-governance-worker", + "ai-platform-api", + "worker", +) +REQUIRED_BACKUP_ARTIFACTS = { + "postgres-contract.dump", + "postgres-ai_platform.dump", + "postgres-operations.dump", + "contract-files.tar.gz", +} def parse_env_file(path: Path) -> dict[str, str]: @@ -132,6 +145,7 @@ def create_backup( ops_profile=True, ) ) + _verify_contract_archive(partial_dir / "contract-files.tar.gz") minio_env_path = _write_minio_alias_env(values) try: _run( @@ -145,6 +159,11 @@ def create_backup( minio_env_path.unlink(missing_ok=True) artifacts = _artifact_metadata(partial_dir) + minio_mirror = _minio_mirror_metadata( + partial_dir, + artifacts=artifacts, + bucket=values.get("AI_PLATFORM_OBJECT_STORAGE_BUCKET", "ai-platform-uploads"), + ) manifest = { "schema_version": "yuqei-production-backup.v1", "backup_id": run_id, @@ -158,6 +177,7 @@ def create_backup( "ai_platform_api": values.get("AI_PLATFORM_API_IMAGE"), "ai_worker": values.get("AI_WORKER_IMAGE"), }, + "minio_mirror": minio_mirror, "artifacts": artifacts, } (partial_dir / "manifest.json").write_text( @@ -183,10 +203,15 @@ def verify_manifest(backup_dir: Path) -> dict[str, object]: artifacts = payload.get("artifacts") if not isinstance(artifacts, list) or not artifacts: raise RuntimeError("Backup manifest has no artifacts.") + artifact_paths: set[str] = set() + minio_records: list[dict[str, object]] = [] for item in artifacts: if not isinstance(item, dict): raise RuntimeError("Backup manifest artifact is invalid.") relative_path = str(item.get("path", "")) + if relative_path in artifact_paths: + raise RuntimeError(f"Duplicate backup artifact path: {relative_path}") + artifact_paths.add(relative_path) target = (backup_dir / relative_path).resolve() target.relative_to(backup_dir.resolve()) if not target.is_file(): @@ -195,6 +220,19 @@ def verify_manifest(backup_dir: Path) -> dict[str, object]: raise RuntimeError(f"Backup artifact size mismatch: {relative_path}") if _sha256(target) != item.get("sha256"): raise RuntimeError(f"Backup artifact checksum mismatch: {relative_path}") + if relative_path.startswith("minio/"): + minio_records.append( + { + "path": relative_path, + "size_bytes": target.stat().st_size, + "sha256": item.get("sha256"), + } + ) + missing = sorted(REQUIRED_BACKUP_ARTIFACTS - artifact_paths) + if missing: + raise RuntimeError(f"Backup manifest is missing required artifacts: {missing}") + _verify_minio_mirror(backup_dir, payload, minio_records) + _verify_contract_archive(backup_dir / "contract-files.tar.gz") return payload @@ -224,6 +262,117 @@ def _artifact_metadata(root: Path) -> list[dict[str, object]]: ] +def _minio_content_index(records: list[dict[str, object]]) -> str: + canonical = json.dumps( + sorted(records, key=lambda item: str(item["path"]).encode("utf-8")), + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return hashlib.sha256(canonical).hexdigest() + + +def _verify_contract_archive(path: Path) -> None: + with tarfile.open(path, mode="r:gz") as archive: + seen: set[str] = set() + seen_folded: set[str] = set() + for member in archive.getmembers(): + raw_name = member.name + if "\\" in raw_name or "\x00" in raw_name: + raise RuntimeError("Contract file archive contains an unsafe path.") + member_path = PurePosixPath(raw_name) + if member_path.is_absolute() or ".." in member_path.parts: + raise RuntimeError("Contract file archive contains an unsafe path.") + if not (member.isfile() or member.isdir()): + raise RuntimeError("Contract file archive contains a link or special entry.") + normalized = member_path.as_posix().removeprefix("./") + if normalized not in {"", "."}: + folded = normalized.casefold() + if normalized in seen or folded in seen_folded: + raise RuntimeError("Contract file archive contains a duplicate path.") + seen.add(normalized) + seen_folded.add(folded) + + +def _minio_mirror_metadata( + root: Path, + *, + artifacts: list[dict[str, object]], + bucket: str, +) -> dict[str, object]: + minio_root = root / "minio" + if minio_root.is_symlink() or not minio_root.is_dir(): + raise RuntimeError("MinIO mirror directory is missing or unsafe.") + records = [ + { + "path": str(item["path"]), + "size_bytes": int(item["size_bytes"]), + "sha256": str(item["sha256"]), + } + for item in artifacts + if str(item.get("path", "")).startswith("minio/") + ] + return { + "bucket": bucket, + "path": "minio", + "object_count": len(records), + "total_size_bytes": sum(int(item["size_bytes"]) for item in records), + "content_index_sha256": _minio_content_index(records), + } + + +def _verify_minio_mirror( + backup_dir: Path, + manifest: dict[str, object], + records: list[dict[str, object]], +) -> None: + minio_root = backup_dir / "minio" + if minio_root.is_symlink() or not minio_root.is_dir(): + raise RuntimeError("MinIO mirror directory is missing or unsafe.") + actual_files: set[str] = set() + for path in minio_root.rglob("*"): + relative = path.relative_to(backup_dir).as_posix() + if path.is_symlink(): + raise RuntimeError(f"MinIO mirror contains a symbolic link: {relative}") + if path.is_file(): + actual_files.add(relative) + elif not path.is_dir(): + raise RuntimeError(f"MinIO mirror contains a special entry: {relative}") + recorded_files = {str(item["path"]) for item in records} + if actual_files != recorded_files: + raise RuntimeError("MinIO mirror content differs from the backup manifest.") + + metadata = manifest.get("minio_mirror") + if metadata is None: + if not records: + raise RuntimeError( + "Legacy backup has no MinIO mirror metadata or recorded MinIO content." + ) + return + if not isinstance(metadata, dict): + raise RuntimeError("MinIO mirror metadata is invalid.") + expected = { + "bucket", + "path", + "object_count", + "total_size_bytes", + "content_index_sha256", + } + if set(metadata) != expected or metadata.get("path") != "minio": + raise RuntimeError("MinIO mirror metadata fields are invalid.") + if ( + not isinstance(metadata.get("bucket"), str) + or not S3_BUCKET_PATTERN.fullmatch(metadata["bucket"]) + ): + raise RuntimeError("MinIO mirror bucket metadata is invalid.") + if metadata.get("object_count") != len(records): + raise RuntimeError("MinIO mirror object count mismatch.") + if metadata.get("total_size_bytes") != sum(int(item["size_bytes"]) for item in records): + raise RuntimeError("MinIO mirror total size mismatch.") + if metadata.get("content_index_sha256") != _minio_content_index(records): + raise RuntimeError("MinIO mirror content index mismatch.") + + def minio_mirror_command( *, values: dict[str, str], diff --git a/yuqei-infra/backup/verify-production-backup.py b/yuqei-infra/backup/verify-production-backup.py index 91e75255..9d04d1d3 100644 --- a/yuqei-infra/backup/verify-production-backup.py +++ b/yuqei-infra/backup/verify-production-backup.py @@ -13,7 +13,7 @@ import subprocess # nosec B404 import sys import tarfile from datetime import datetime, timezone -from pathlib import Path +from pathlib import Path, PurePosixPath REPO_ROOT = Path(__file__).resolve().parents[2] @@ -21,6 +21,13 @@ BASE_COMPOSE = REPO_ROOT / "yuqei-infra" / "compose" / "docker-compose.yml" PRODUCTION_COMPOSE = REPO_ROOT / "yuqei-infra" / "compose" / "docker-compose.production.yml" DEFAULT_ARTIFACT_ROOT = REPO_ROOT / "project-docs" / "qa-artifacts" / "v2-production-restore-check" SAFE_DATABASE_PATTERN = re.compile(r"^yuqei_restore_check_(contract|ai_platform|operations)_[0-9]{14}$") +S3_BUCKET_PATTERN = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9._-]{0,62}$") +REQUIRED_BACKUP_ARTIFACTS = { + "postgres-contract.dump", + "postgres-ai_platform.dump", + "postgres-operations.dump", + "contract-files.tar.gz", +} def read_env(path: Path) -> dict[str, str]: @@ -46,10 +53,15 @@ def validate_backup(backup_dir: Path) -> tuple[dict[str, object], list[dict[str, raise RuntimeError("Backup manifest has no artifacts.") verified: list[dict[str, object]] = [] root = backup_dir.resolve() + artifact_paths: set[str] = set() + minio_records: list[dict[str, object]] = [] for item in artifacts: if not isinstance(item, dict): raise RuntimeError("Invalid artifact entry.") relative_path = str(item.get("path", "")) + if relative_path in artifact_paths: + raise RuntimeError(f"Duplicate backup artifact path: {relative_path}") + artifact_paths.add(relative_path) path = (root / relative_path).resolve() path.relative_to(root) if not path.is_file(): @@ -59,17 +71,101 @@ def validate_backup(backup_dir: Path) -> tuple[dict[str, object], list[dict[str, if actual_size != int(item.get("size_bytes", -1)) or actual_sha256 != item.get("sha256"): raise RuntimeError(f"Backup artifact integrity mismatch: {relative_path}") verified.append({"path": relative_path, "size_bytes": actual_size, "sha256_match": True}) + if relative_path.startswith("minio/"): + minio_records.append( + {"path": relative_path, "size_bytes": actual_size, "sha256": actual_sha256} + ) + + missing = sorted(REQUIRED_BACKUP_ARTIFACTS - artifact_paths) + if missing: + raise RuntimeError(f"Backup manifest is missing required artifacts: {missing}") + _verify_minio_mirror(backup_dir, manifest, minio_records) contract_archive = backup_dir / "contract-files.tar.gz" - if contract_archive.is_file(): - with tarfile.open(contract_archive, mode="r:gz") as archive: - for member in archive.getmembers(): - member_path = Path(member.name) - if member_path.is_absolute() or ".." in member_path.parts: - raise RuntimeError("Contract file archive contains an unsafe path.") + with tarfile.open(contract_archive, mode="r:gz") as archive: + seen: set[str] = set() + seen_folded: set[str] = set() + for member in archive.getmembers(): + raw_name = member.name + if "\\" in raw_name or "\x00" in raw_name: + raise RuntimeError("Contract file archive contains an unsafe path.") + member_path = PurePosixPath(raw_name) + if member_path.is_absolute() or ".." in member_path.parts: + raise RuntimeError("Contract file archive contains an unsafe path.") + if not (member.isfile() or member.isdir()): + raise RuntimeError("Contract file archive contains a link or special entry.") + normalized = member_path.as_posix().removeprefix("./") + if normalized not in {"", "."}: + folded = normalized.casefold() + if normalized in seen or folded in seen_folded: + raise RuntimeError("Contract file archive contains a duplicate path.") + seen.add(normalized) + seen_folded.add(folded) return manifest, verified +def _minio_content_index(records: list[dict[str, object]]) -> str: + canonical = json.dumps( + sorted(records, key=lambda item: str(item["path"]).encode("utf-8")), + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return hashlib.sha256(canonical).hexdigest() + + +def _verify_minio_mirror( + backup_dir: Path, + manifest: dict[str, object], + records: list[dict[str, object]], +) -> None: + minio_root = backup_dir / "minio" + if minio_root.is_symlink() or not minio_root.is_dir(): + raise RuntimeError("MinIO mirror directory is missing or unsafe.") + actual_files: set[str] = set() + for path in minio_root.rglob("*"): + relative = path.relative_to(backup_dir).as_posix() + if path.is_symlink(): + raise RuntimeError(f"MinIO mirror contains a symbolic link: {relative}") + if path.is_file(): + actual_files.add(relative) + elif not path.is_dir(): + raise RuntimeError(f"MinIO mirror contains a special entry: {relative}") + recorded_files = {str(item["path"]) for item in records} + if actual_files != recorded_files: + raise RuntimeError("MinIO mirror content differs from the backup manifest.") + + metadata = manifest.get("minio_mirror") + if metadata is None: + if not records: + raise RuntimeError( + "Legacy backup has no MinIO mirror metadata or recorded MinIO content." + ) + return + if not isinstance(metadata, dict): + raise RuntimeError("MinIO mirror metadata is invalid.") + expected = { + "bucket", + "path", + "object_count", + "total_size_bytes", + "content_index_sha256", + } + if set(metadata) != expected or metadata.get("path") != "minio": + raise RuntimeError("MinIO mirror metadata fields are invalid.") + if ( + not isinstance(metadata.get("bucket"), str) + or not S3_BUCKET_PATTERN.fullmatch(metadata["bucket"]) + ): + raise RuntimeError("MinIO mirror bucket metadata is invalid.") + if metadata.get("object_count") != len(records): + raise RuntimeError("MinIO mirror object count mismatch.") + if metadata.get("total_size_bytes") != sum(int(item["size_bytes"]) for item in records): + raise RuntimeError("MinIO mirror total size mismatch.") + if metadata.get("content_index_sha256") != _minio_content_index(records): + raise RuntimeError("MinIO mirror content index mismatch.") + + def restore_database_checks( *, backup_dir: Path, diff --git a/yuqei-infra/scripts/verify-core-user-flows.cjs b/yuqei-infra/scripts/verify-core-user-flows.cjs index 07b03f68..b9886bae 100644 --- a/yuqei-infra/scripts/verify-core-user-flows.cjs +++ b/yuqei-infra/scripts/verify-core-user-flows.cjs @@ -123,6 +123,105 @@ function safeFilename(value) { return String(value).replace(/[^\w-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 90) || "artifact"; } +function safeRequestPath(value) { + try { + return new URL(String(value)).pathname || "/"; + } catch { + return ""; + } +} + +function sanitizeFailure(value) { + const normalized = String(value || "Unknown failure") + .replace(/[\u0000-\u001f\u007f]+/g, " ") + .replace(/\b(?:Bearer|Basic)\s+[A-Za-z0-9._~+/=-]+/gi, (match) => `${match.split(/\s+/, 1)[0]} [REDACTED]`) + .replace(/https?:\/\/[^\s"'<>]+/gi, (url) => safeRequestPath(url)) + .replace( + /\b(password|passwd|client[-_]?secret|secret|access[-_]?token|refresh[-_]?token|id[-_]?token|session[-_]?token|token|api[-_]?key|proxy[-_]?key|private[-_]?key|credential|authorization|cookie|session(?:[-_]?id)?)\b(\s*[:=]\s*)(?:"[^"]*"|'[^']*'|[^\s,;}]*)/gi, + (_match, label, separator) => `${label}${separator}[REDACTED]`, + ) + .trim(); + return (normalized || "Unknown failure").slice(0, 500); +} + +function requestMetadata(request) { + return { + method: String(request.method() || "UNKNOWN").toUpperCase(), + path: safeRequestPath(request.url()), + resource: String(request.resourceType() || "unknown"), + }; +} + +function isExpectedHttpError(entry) { + return entry.method === "GET" && entry.path === "/favicon.ico" && entry.status === 404; +} + +function isExpectedConsoleError(message) { + const location = typeof message.location === "function" ? message.location() : null; + return safeRequestPath(location?.url || "") === "/favicon.ico"; +} + +function observeBrowserErrors(page) { + const diagnostics = { + consoleErrors: [], + pageErrors: [], + requestFailures: [], + unexpectedHttpResponses: [], + }; + + page.on("console", (message) => { + if (message.type() === "error" && !isExpectedConsoleError(message)) { + diagnostics.consoleErrors.push({ + resource: "console", + failure: sanitizeFailure(message.text()), + }); + } + }); + page.on("pageerror", (error) => { + diagnostics.pageErrors.push({ + resource: "page", + failure: sanitizeFailure(error?.message || error), + }); + }); + page.on("requestfailed", (request) => { + diagnostics.requestFailures.push({ + ...requestMetadata(request), + failure: sanitizeFailure(request.failure()?.errorText), + }); + }); + page.on("response", (response) => { + const status = response.status(); + if (status < 400) { + return; + } + const entry = { + ...requestMetadata(response.request()), + status, + failure: `HTTP ${status}`, + }; + if (!isExpectedHttpError(entry)) { + diagnostics.unexpectedHttpResponses.push(entry); + } + }); + + return diagnostics; +} + +function assertBrowserErrorsAbsent(diagnostics) { + const counts = { + console: diagnostics.consoleErrors.length, + page: diagnostics.pageErrors.length, + request: diagnostics.requestFailures.length, + http: diagnostics.unexpectedHttpResponses.length, + }; + const total = Object.values(counts).reduce((sum, count) => sum + count, 0); + if (total > 0) { + throw new Error( + `Unexpected browser errors detected: console=${counts.console}, page=${counts.page}, request=${counts.request}, http=${counts.http}.`, + ); + } +} + async function waitForContract(apiUrl, predicate, timeoutMs) { const deadline = Date.now() + Math.min(timeoutMs, 10000); let contract = null; @@ -156,6 +255,37 @@ async function login(page, args) { await page.waitForURL(/\/workspace(?:\?.*)?$/, { timeout: args.timeoutMs }); } +async function verifyAuthenticatedSession(page, args) { + const endpoint = `${args.baseUrl}/api/auth/session`; + const result = await page.evaluate(async (sessionEndpoint) => { + const response = await fetch(sessionEndpoint, { + cache: "no-store", + credentials: "same-origin", + }); + const payload = await response.json().catch(() => null); + const session = payload && typeof payload.session === "object" ? payload.session : null; + return { + ok: response.ok, + status: response.status, + tenantId: typeof session?.tenant_id === "string" ? session.tenant_id : null, + userId: typeof session?.user_id === "string" ? session.user_id : null, + }; + }, endpoint); + + if (!result.ok) { + throw new Error(`Authenticated session verification failed: HTTP ${result.status}.`); + } + if (!result.tenantId || !result.userId) { + throw new Error("Authenticated session verification failed: the session endpoint returned no tenant/user identity."); + } + if (result.tenantId !== args.tenantId || result.userId !== args.userId) { + throw new Error( + `Authenticated session identity mismatch: expected tenant=${args.tenantId}, user=${args.userId}; got tenant=${result.tenantId}, user=${result.userId}.`, + ); + } + return { tenantId: result.tenantId, userId: result.userId }; +} + async function createContractFromTemplate(page, args) { await page.goto(`${args.baseUrl}/legal-documents?template=TPL-COOPERATION-FRAMEWORK-001`, { waitUntil: "domcontentloaded", @@ -580,12 +710,7 @@ async function main() { const browser = await chromium.launch({ headless: !args.headed, channel: "chrome" }); const page = await browser.newPage({ viewport: { width: 1440, height: 1050 } }); page.setDefaultTimeout(args.timeoutMs); - const consoleErrors = []; - page.on("console", (message) => { - if (message.type() === "error" && !message.text().includes("favicon.ico")) { - consoleErrors.push(message.text()); - } - }); + const browserErrors = observeBrowserErrors(page); const summary = { status: "fail", @@ -595,16 +720,18 @@ async function main() { outputDir: args.outputDir, contractId: args.contractId || null, login: false, + authenticatedSession: null, templateDraft: false, wordDownload: null, review: null, fileRepository: null, lifecycle: null, - consoleErrors, + ...browserErrors, }; try { await login(page, args); + summary.authenticatedSession = await verifyAuthenticatedSession(page, args); summary.login = true; if (!summary.contractId) { @@ -625,13 +752,13 @@ async function main() { summary.review = await startAndRunReview(page, args, summary.contractId); summary.fileRepository = await uploadAndVerifyFile(page, args, summary.contractId); summary.lifecycle = await runContractLifecycle(page, args, summary.contractId); - summary.consoleErrors = consoleErrors; + await page.waitForTimeout(250); + assertBrowserErrorsAbsent(browserErrors); summary.status = "pass"; } catch (error) { summary.error = error instanceof Error ? error.message : String(error); await page.screenshot({ path: path.join(args.outputDir, "core-regression-failure.png"), fullPage: true }).catch(() => null); } finally { - summary.consoleErrors = consoleErrors; fs.writeFileSync(path.join(args.outputDir, "core-user-flow-summary.json"), JSON.stringify(summary, null, 2), "utf8"); await browser.close(); } @@ -643,7 +770,18 @@ async function main() { } } -main().catch((error) => { - console.error(error); - process.exit(1); -}); +if (require.main === module) { + main().catch((error) => { + console.error(error); + process.exit(1); + }); +} + +module.exports = { + assertBrowserErrorsAbsent, + isExpectedHttpError, + observeBrowserErrors, + safeRequestPath, + sanitizeFailure, + verifyAuthenticatedSession, +}; diff --git a/yuqei-infra/tests/test_ci_migration_quality_gate.py b/yuqei-infra/tests/test_ci_migration_quality_gate.py index 712c4533..00825d3d 100644 --- a/yuqei-infra/tests/test_ci_migration_quality_gate.py +++ b/yuqei-infra/tests/test_ci_migration_quality_gate.py @@ -1,6 +1,7 @@ from __future__ import annotations import ast +import re from pathlib import Path @@ -12,6 +13,8 @@ CONTRACT_MIGRATIONS = ( AI_MIGRATIONS = ( REPO_ROOT / "yuqei-ai-platform" / "services" / "ai-platform-api" / "migrations" / "versions" ) +POSTGRES_IDENTIFIER_MAX_BYTES = 63 +MIGRATION_IDENTIFIER_PATTERN = re.compile(r"(? object: @@ -43,6 +46,18 @@ def _migration_heads(directory: Path) -> set[str]: return revisions - parents +def _explicit_identifier_literals(directory: Path) -> list[tuple[Path, int, str]]: + identifiers: list[tuple[Path, int, str]] = [] + for path in directory.glob("*.py"): + module = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + for node in ast.walk(module): + if not isinstance(node, ast.Constant) or not isinstance(node.value, str): + continue + for match in MIGRATION_IDENTIFIER_PATTERN.finditer(node.value): + identifiers.append((path, node.lineno, match.group(0))) + return identifiers + + def _workflow_step(workflow: str, name: str) -> str: marker = f" - name: {name}" start = workflow.index(marker) @@ -83,3 +98,17 @@ def test_ci_migration_steps_track_real_unique_heads_and_use_empty_sqlite() -> No ] assert '20260826_0073 (head)' not in workflow + + +def test_explicit_migration_identifiers_fit_postgresql_limit() -> None: + offenders = [ + f"{path.relative_to(REPO_ROOT)}:{line}: {identifier!r} " + f"({len(identifier.encode('utf-8'))} bytes)" + for directory in (CONTRACT_MIGRATIONS, AI_MIGRATIONS) + for path, line, identifier in _explicit_identifier_literals(directory) + if len(identifier.encode("utf-8")) > POSTGRES_IDENTIFIER_MAX_BYTES + ] + assert offenders == [], ( + "Explicit migration identifiers exceed PostgreSQL's 63-byte limit:\n" + + "\n".join(offenders) + ) diff --git a/yuqei-infra/tests/test_production_release_tools.py b/yuqei-infra/tests/test_production_release_tools.py index 3789c1a9..d056a971 100644 --- a/yuqei-infra/tests/test_production_release_tools.py +++ b/yuqei-infra/tests/test_production_release_tools.py @@ -5,6 +5,7 @@ import hashlib import importlib.util import io import json +import shutil import sys import tarfile from datetime import datetime, timezone @@ -132,6 +133,16 @@ def test_minio_backup_invokes_pinned_mc_directly_without_a_shell(tmp_path: Path) assert all("secret" not in token for token in command) +def test_backup_quiesces_every_long_running_application_writer() -> None: + assert backup_run.WRITER_SERVICES == ( + "contract-api", + "contract-review-worker", + "contract-playbook-governance-worker", + "ai-platform-api", + "worker", + ) + + @pytest.mark.parametrize( ("label", "validation_name", "table_name"), [ @@ -674,42 +685,150 @@ def test_production_preflight_rejects_smoke_database_public_bind_and_fallback() assert {"production_modes", "internal_port_binding", "database_boundaries"} <= failed -def test_backup_manifest_and_contract_archive_are_verified(tmp_path: Path) -> None: +def _complete_backup_fixture(tmp_path: Path) -> tuple[Path, dict[str, object]]: backup_dir = tmp_path / "backup-20260716T120000Z" - backup_dir.mkdir() + backup_dir.mkdir(parents=True) archive_path = backup_dir / "contract-files.tar.gz" with tarfile.open(archive_path, mode="w:gz") as archive: content = b"contract" info = tarfile.TarInfo("contract-1/document.txt") info.size = len(content) archive.addfile(info, io.BytesIO(content)) - dump_path = backup_dir / "postgres-contract.dump" - dump_path.write_bytes(b"postgres-dump") + for label in ("contract", "ai_platform", "operations"): + (backup_dir / f"postgres-{label}.dump").write_bytes( + f"postgres-{label}-dump".encode("utf-8") + ) + minio_object = backup_dir / "minio" / "tenant-a" / "object.bin" + minio_object.parent.mkdir(parents=True) + minio_object.write_bytes(b"minio-object") artifacts = [] - for path in (archive_path, dump_path): + for path in sorted(backup_dir.rglob("*")): + if not path.is_file(): + continue artifacts.append( { - "path": path.name, + "path": path.relative_to(backup_dir).as_posix(), "size_bytes": path.stat().st_size, "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), } ) + minio_records = [ + item for item in artifacts if str(item["path"]).startswith("minio/") + ] + manifest: dict[str, object] = { + "schema_version": "yuqei-production-backup.v1", + "backup_id": backup_dir.name, + "created_at": "2026-07-16T12:00:00+00:00", + "minio_mirror": { + "bucket": "ai-platform-uploads", + "path": "minio", + "object_count": len(minio_records), + "total_size_bytes": sum(int(item["size_bytes"]) for item in minio_records), + "content_index_sha256": backup_run._minio_content_index(minio_records), + }, + "artifacts": artifacts, + } (backup_dir / "manifest.json").write_text( - json.dumps( - { - "schema_version": "yuqei-production-backup.v1", - "backup_id": backup_dir.name, - "created_at": "2026-07-16T12:00:00+00:00", - "artifacts": artifacts, - } - ), + json.dumps(manifest), encoding="utf-8", ) + return backup_dir, manifest + +def test_backup_manifest_and_contract_archive_are_verified(tmp_path: Path) -> None: + backup_dir, _ = _complete_backup_fixture(tmp_path) + + generated_manifest = backup_run.verify_manifest(backup_dir) manifest, verified = backup_verify.validate_backup(backup_dir) + assert generated_manifest["backup_id"] == backup_dir.name assert manifest["backup_id"] == backup_dir.name - assert len(verified) == 2 + assert len(verified) == 5 + + +def test_backup_manifest_requires_all_databases_contract_files_and_minio( + tmp_path: Path, +) -> None: + backup_dir, manifest = _complete_backup_fixture(tmp_path) + manifest["artifacts"] = [ + item + for item in manifest["artifacts"] + if item["path"] != "postgres-operations.dump" + ] + (backup_dir / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8") + with pytest.raises(RuntimeError, match="missing required artifacts"): + backup_verify.validate_backup(backup_dir) + + legacy_dir, legacy_manifest = _complete_backup_fixture(tmp_path / "legacy") + legacy_manifest.pop("minio_mirror") + legacy_manifest["artifacts"] = [ + item + for item in legacy_manifest["artifacts"] + if not str(item["path"]).startswith("minio/") + ] + shutil.rmtree(legacy_dir / "minio") + (legacy_dir / "minio").mkdir() + (legacy_dir / "manifest.json").write_text( + json.dumps(legacy_manifest), encoding="utf-8" + ) + with pytest.raises(RuntimeError, match="no MinIO mirror metadata or recorded MinIO content"): + backup_verify.validate_backup(legacy_dir) + + +def test_backup_manifest_supports_explicit_empty_minio_and_legacy_content( + tmp_path: Path, +) -> None: + empty_dir, empty_manifest = _complete_backup_fixture(tmp_path / "empty") + minio_path = empty_dir / "minio" / "tenant-a" / "object.bin" + minio_path.unlink() + empty_manifest["artifacts"] = [ + item + for item in empty_manifest["artifacts"] + if not str(item["path"]).startswith("minio/") + ] + empty_manifest["minio_mirror"] = { + "bucket": "ai-platform-uploads", + "path": "minio", + "object_count": 0, + "total_size_bytes": 0, + "content_index_sha256": backup_run._minio_content_index([]), + } + (empty_dir / "manifest.json").write_text( + json.dumps(empty_manifest), encoding="utf-8" + ) + backup_verify.validate_backup(empty_dir) + + legacy_dir, legacy_manifest = _complete_backup_fixture(tmp_path / "legacy-content") + legacy_manifest.pop("minio_mirror") + (legacy_dir / "manifest.json").write_text( + json.dumps(legacy_manifest), encoding="utf-8" + ) + backup_verify.validate_backup(legacy_dir) + + +@pytest.mark.parametrize("member_type", (tarfile.SYMTYPE, tarfile.LNKTYPE, tarfile.CHRTYPE)) +def test_contract_archive_rejects_links_and_special_entries( + tmp_path: Path, + member_type: bytes, +) -> None: + backup_dir, manifest = _complete_backup_fixture(tmp_path) + archive_path = backup_dir / "contract-files.tar.gz" + with tarfile.open(archive_path, mode="w:gz") as archive: + info = tarfile.TarInfo("unsafe-entry") + info.type = member_type + if member_type in {tarfile.SYMTYPE, tarfile.LNKTYPE}: + info.linkname = "outside" + archive.addfile(info) + for item in manifest["artifacts"]: + if item["path"] == "contract-files.tar.gz": + item["size_bytes"] = archive_path.stat().st_size + item["sha256"] = hashlib.sha256(archive_path.read_bytes()).hexdigest() + (backup_dir / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8") + + with pytest.raises(RuntimeError, match="link or special entry"): + backup_run.verify_manifest(backup_dir) + with pytest.raises(RuntimeError, match="link or special entry"): + backup_verify.validate_backup(backup_dir) def test_provider_acceptance_separates_technical_success_from_legal_approval() -> None: diff --git a/yuqei-infra/tests/test_verify_core_user_flows.cjs b/yuqei-infra/tests/test_verify_core_user_flows.cjs new file mode 100644 index 00000000..5db27115 --- /dev/null +++ b/yuqei-infra/tests/test_verify_core_user_flows.cjs @@ -0,0 +1,128 @@ +const assert = require("node:assert/strict"); +const { EventEmitter } = require("node:events"); +const test = require("node:test"); + +const { + assertBrowserErrorsAbsent, + observeBrowserErrors, + safeRequestPath, + sanitizeFailure, + verifyAuthenticatedSession, +} = require("../scripts/verify-core-user-flows.cjs"); + +test("safeRequestPath removes origins, queries, and fragments", () => { + assert.equal(safeRequestPath("https://example.test/contracts/123?token=secret#fragment"), "/contracts/123"); + assert.equal(safeRequestPath("not a URL"), ""); +}); + +test("sanitizeFailure redacts credentials and URL queries", () => { + const sanitized = sanitizeFailure( + "Bearer top-secret-token failed at https://example.test/api/jobs?access_token=secret password=hidden access_token=also-hidden", + ); + + assert.equal(sanitized.includes("top-secret-token"), false); + assert.equal(sanitized.includes("access_token=secret"), false); + assert.equal(sanitized.includes("access_token=also-hidden"), false); + assert.equal(sanitized.includes("password=hidden"), false); + assert.match(sanitized, /Bearer \[REDACTED\]/); + assert.match(sanitized, /\/api\/jobs/); + assert.match(sanitized, /password=\[REDACTED\]/); +}); + +test("browser diagnostics retain only safe metadata and gate every unexpected class", () => { + const page = new EventEmitter(); + const diagnostics = observeBrowserErrors(page); + const request = { + method: () => "get", + url: () => "https://example.test/api/contracts?token=secret", + resourceType: () => "fetch", + failure: () => ({ errorText: "net::ERR_FAILED token=secret" }), + }; + + page.emit("console", { + type: () => "error", + text: () => "Bearer secret-value was rejected", + location: () => ({ url: "https://example.test/app" }), + }); + page.emit("pageerror", new Error("password=secret")); + page.emit("requestfailed", request); + page.emit("response", { + status: () => 503, + request: () => request, + }); + page.emit("response", { + status: () => 404, + request: () => ({ + method: () => "GET", + url: () => "https://example.test/favicon.ico?cache=1", + resourceType: () => "image", + }), + }); + + assert.deepEqual(diagnostics.consoleErrors, [{ resource: "console", failure: "Bearer [REDACTED] was rejected" }]); + assert.deepEqual(diagnostics.pageErrors, [{ resource: "page", failure: "password=[REDACTED]" }]); + assert.deepEqual(diagnostics.requestFailures, [{ + method: "GET", + path: "/api/contracts", + resource: "fetch", + failure: "net::ERR_FAILED token=[REDACTED]", + }]); + assert.deepEqual(diagnostics.unexpectedHttpResponses, [{ + method: "GET", + path: "/api/contracts", + resource: "fetch", + status: 503, + failure: "HTTP 503", + }]); + assert.throws(() => assertBrowserErrorsAbsent(diagnostics), /console=1, page=1, request=1, http=1/); +}); + +test("browser diagnostics accept an entirely clean run", () => { + assert.doesNotThrow(() => assertBrowserErrorsAbsent({ + consoleErrors: [], + pageErrors: [], + requestFailures: [], + unexpectedHttpResponses: [], + })); +}); + +test("authenticated session must match the supplied tenant and user", async () => { + const page = { + evaluate: async (_callback, endpoint) => { + assert.equal(endpoint, "https://example.test/api/auth/session"); + return { ok: true, status: 200, tenantId: "tenant-a", userId: "user-a" }; + }, + }; + const args = { baseUrl: "https://example.test", tenantId: "tenant-a", userId: "user-a" }; + + assert.deepEqual(await verifyAuthenticatedSession(page, args), { tenantId: "tenant-a", userId: "user-a" }); +}); + +test("authenticated session mismatch fails before the journey proceeds", async () => { + const page = { + evaluate: async () => ({ ok: true, status: 200, tenantId: "tenant-a", userId: "other-user" }), + }; + const args = { baseUrl: "https://example.test", tenantId: "tenant-a", userId: "user-a" }; + + await assert.rejects( + verifyAuthenticatedSession(page, args), + /Authenticated session identity mismatch/, + ); +}); + +test("unavailable or malformed session endpoint fails clearly", async () => { + const args = { baseUrl: "https://example.test", tenantId: "tenant-a", userId: "user-a" }; + + await assert.rejects( + verifyAuthenticatedSession({ + evaluate: async () => ({ ok: false, status: 404, tenantId: null, userId: null }), + }, args), + /Authenticated session verification failed: HTTP 404/, + ); + await assert.rejects( + verifyAuthenticatedSession({ + evaluate: async () => ({ ok: true, status: 200, tenantId: null, userId: null }), + }, args), + /session endpoint returned no tenant\/user identity/, + ); +});