From 7c77174f2a5fdd044bfe927657869ee97dc43916 Mon Sep 17 00:00:00 2001 From: yuqei Date: Sun, 27 Sep 2026 11:03:35 +0800 Subject: [PATCH] release: assemble deterministic BUILD-05 package v2 --- yuqei-infra/deploy/README.md | 1 + .../scripts/assemble-build05-v2-package.py | 998 ++++++++++++++++++ .../test_build05_v2_package_orchestrator.py | 276 +++++ 3 files changed, 1275 insertions(+) create mode 100644 yuqei-infra/scripts/assemble-build05-v2-package.py create mode 100644 yuqei-infra/tests/test_build05_v2_package_orchestrator.py diff --git a/yuqei-infra/deploy/README.md b/yuqei-infra/deploy/README.md index 138f1ce4..04c077f9 100644 --- a/yuqei-infra/deploy/README.md +++ b/yuqei-infra/deploy/README.md @@ -26,6 +26,7 @@ The current RC-05 through RC-08 release must follow the repository [mandatory de - `plan-rollback-rehearsal.py`: verifies the frozen package and source-bound release tools before emitting a non-writing rollback rehearsal plan. - `../scripts/build-infrastructure-image-bundle.py`: file-only validator for the four application and seven infrastructure image archives, security reports, content-addressed layers, and offline import plan. - `../scripts/build-third-party-materials.py`: deterministic third-party notice, license-text, and corresponding-source material builder/verifier. +- `../scripts/assemble-build05-v2-package.py`: read-only-by-default BUILD-05 package preflight and atomic assembler. It only composes already validated application, infrastructure, third-party, recovery, and external-evidence inputs; `--assemble` is required to create an output outside the source repository. - `../release-trust/third-party-materials-baseline.json`: source-frozen technical trust baseline. The shipped baseline intentionally remains `NOT_PROVISIONED` until reviewed real catalog, license, and source identities are supplied. ## Prepare without changing traffic diff --git a/yuqei-infra/scripts/assemble-build05-v2-package.py b/yuqei-infra/scripts/assemble-build05-v2-package.py new file mode 100644 index 00000000..578b772e --- /dev/null +++ b/yuqei-infra/scripts/assemble-build05-v2-package.py @@ -0,0 +1,998 @@ +#!/usr/bin/env python3 +"""Preflight and atomically assemble one self-contained BUILD-05 package v2. + +This command composes the existing strict validators. It does not build or +pull images, invoke Docker, contact a registry or database, sign evidence, or +apply a release. The default mode is read-only preflight. ``--assemble`` is +required before any output directory is created. +""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import os +import re +import shutil +import stat +import subprocess +import sys +import tempfile +from pathlib import Path, PurePosixPath +from typing import Any, Iterable, Mapping, Sequence + + +sys.dont_write_bytecode = True + +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parents[1] +INFRA_ROOT = REPO_ROOT / "yuqei-infra" +if str(SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPT_DIR)) +INSTALLABLE_TOOL = SCRIPT_DIR / "build-installable-release.py" +INFRASTRUCTURE_TOOL = SCRIPT_DIR / "build-infrastructure-image-bundle.py" +THIRD_PARTY_TOOL = SCRIPT_DIR / "build-third-party-materials.py" +PACKAGE_TOOL = INFRA_ROOT / "deploy" / "release-package.py" + +RESULT_SCHEMA = "yuqei.build05-v2-package-assembly-result.v1" +HEX64_RE = re.compile(r"^[0-9a-f]{64}$") +COMMIT_RE = re.compile(r"^[0-9a-f]{40}(?:[0-9a-f]{24})?$") +SAFE_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") +PROJECT_RE = re.compile(r"^[a-z0-9][a-z0-9_-]{0,62}$") +MAX_JSON_BYTES = 16 * 1024 * 1024 + + +class AssemblyError(RuntimeError): + """Raised when package input or publication cannot be proven safe.""" + + +def _canonical_json_bytes(value: object) -> bytes: + try: + return json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + except (TypeError, ValueError) as exc: + raise AssemblyError("value cannot be encoded as canonical JSON") from exc + + +def _canonical_file_bytes(value: object) -> bytes: + return _canonical_json_bytes(value) + b"\n" + + +def _canonical_sha256(value: object) -> str: + return hashlib.sha256(_canonical_json_bytes(value)).hexdigest() + + +def _with_hash(payload: Mapping[str, Any], field: str) -> dict[str, Any]: + result = dict(payload) + result[field] = _canonical_sha256(result) + return result + + +def _file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _reject_constant(value: str) -> None: + raise AssemblyError(f"non-finite JSON value is forbidden: {value}") + + +def _reject_duplicate_pairs(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise AssemblyError(f"duplicate JSON key is forbidden: {key}") + result[key] = value + return result + + +def _read_json(path: Path, *, label: str) -> dict[str, Any]: + target = _regular_file(path, label=label) + try: + size = target.stat().st_size + if size <= 0 or size > MAX_JSON_BYTES: + raise AssemblyError(f"{label} has an invalid size") + payload = json.loads( + target.read_text(encoding="utf-8"), + object_pairs_hook=_reject_duplicate_pairs, + parse_constant=_reject_constant, + ) + except AssemblyError: + raise + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise AssemblyError(f"{label} is not strict UTF-8 JSON") from exc + if not isinstance(payload, dict): + raise AssemblyError(f"{label} must be a JSON object") + return payload + + +def _is_linklike(path: Path, info: os.stat_result | None = None) -> bool: + if path.is_symlink(): + return True + if hasattr(path, "is_junction") and path.is_junction(): + return True + details = info if info is not None else path.lstat() + attributes = getattr(details, "st_file_attributes", 0) + reparse_flag = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0) + return bool(reparse_flag and attributes & reparse_flag) + + +def _absolute(path: Path) -> Path: + return Path(os.path.abspath(os.fspath(path))) + + +def _reject_link_components(path: Path, *, label: str, require_leaf: bool = True) -> None: + absolute = _absolute(path) + current = Path(absolute.anchor) + parts = absolute.parts[1:] if absolute.anchor else absolute.parts + for index, part in enumerate(parts): + current /= part + if not current.exists() and not current.is_symlink(): + if require_leaf or index < len(parts) - 1: + raise AssemblyError(f"{label} path component is missing: {current}") + continue + try: + info = current.lstat() + except OSError as exc: + raise AssemblyError(f"cannot inspect {label} path") from exc + if _is_linklike(current, info): + raise AssemblyError(f"{label} contains a symbolic link or reparse point") + + +def _directory(path: Path, *, label: str) -> Path: + if not path.is_absolute(): + raise AssemblyError(f"{label} must be an absolute path") + lexical = _absolute(path) + _reject_link_components(lexical, label=label) + try: + info = lexical.stat() + except OSError as exc: + raise AssemblyError(f"{label} is missing") from exc + if not stat.S_ISDIR(info.st_mode): + raise AssemblyError(f"{label} must be a real directory") + return lexical.resolve(strict=True) + + +def _regular_file(path: Path, *, label: str, root: Path | None = None) -> Path: + lexical = _absolute(path) + _reject_link_components(lexical, label=label) + try: + resolved = lexical.resolve(strict=True) + info = resolved.stat() + except OSError as exc: + raise AssemblyError(f"{label} is missing") from exc + if root is not None: + try: + resolved.relative_to(root) + except ValueError as exc: + raise AssemblyError(f"{label} escapes its declared root") from exc + if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1: + raise AssemblyError(f"{label} must be a single-link regular file") + return resolved + + +def _safe_relative(value: object, *, label: str) -> str: + if not isinstance(value, str) or not value or "\\" in value or "\x00" in value: + raise AssemblyError(f"unsafe {label}") + path = PurePosixPath(value) + if path.is_absolute() or any(part in {"", ".", ".."} for part in path.parts): + raise AssemblyError(f"unsafe {label}: {value!r}") + canonical = path.as_posix() + if canonical != value or re.match(r"^[A-Za-z]:", value): + raise AssemblyError(f"non-canonical {label}: {value!r}") + return canonical + + +def _relative_file(root: Path, relative: object, *, label: str) -> Path: + safe = _safe_relative(relative, label=label) + return _regular_file(root.joinpath(*PurePosixPath(safe).parts), label=label, root=root) + + +def _assert_outside_repo(path: Path, repo_root: Path, *, label: str) -> None: + lexical = _absolute(path) + try: + lexical.relative_to(repo_root) + except ValueError: + return + raise AssemblyError(f"{label} must remain outside the source repository") + + +def _load_module(path: Path, name: str) -> Any: + source = _regular_file(path, label=f"reviewed module {path.name}") + spec = importlib.util.spec_from_file_location(name, source) + if spec is None or spec.loader is None: + raise AssemblyError(f"cannot load reviewed module: {path.name}") + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + try: + spec.loader.exec_module(module) + except Exception: + sys.modules.pop(name, None) + raise + return module + + +def _git(repo: Path, *args: str) -> str: + environment = { + "PATH": os.environ.get("PATH", ""), + "SYSTEMROOT": os.environ.get("SYSTEMROOT", ""), + "WINDIR": os.environ.get("WINDIR", ""), + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_OPTIONAL_LOCKS": "0", + "LC_ALL": "C", + "LANG": "C", + } + try: + completed = subprocess.run( + ["git", "-c", "core.autocrlf=false", "-c", "core.eol=lf", "-C", str(repo), *args], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=environment, + ) + except (OSError, subprocess.CalledProcessError) as exc: + raise AssemblyError(f"Git candidate check failed: {' '.join(args)}") from exc + return completed.stdout.decode("utf-8", errors="strict").strip() + + +def _verify_repo(repo: Path, expected_commit: str, expected_tag: str) -> None: + if _git(repo, "rev-parse", "HEAD") != expected_commit: + raise AssemblyError("repository HEAD does not match the expected candidate commit") + if _git(repo, "status", "--porcelain=v1", "--untracked-files=all"): + raise AssemblyError("repository checkout is not clean") + if _git(repo, "rev-parse", f"refs/tags/{expected_tag}^{{commit}}") != expected_commit: + raise AssemblyError("candidate tag does not resolve to the expected commit") + if _git(repo, "for-each-ref", "--format=%(refname)", "refs/replace"): + raise AssemblyError("Git replacement refs are forbidden") + + +def _walk_file_records(value: object) -> Iterable[Mapping[str, Any]]: + if isinstance(value, Mapping): + if {"path", "sha256", "size_bytes"}.issubset(value): + yield value + for nested in value.values(): + yield from _walk_file_records(nested) + elif isinstance(value, list): + for nested in value: + yield from _walk_file_records(nested) + + +def _record_paths(payload: Mapping[str, Any], *, label: str) -> set[str]: + paths: dict[str, str] = {} + for record in _walk_file_records(payload): + relative = _safe_relative(record.get("path"), label=f"{label} file path") + digest = record.get("sha256") + size = record.get("size_bytes") + if not isinstance(digest, str) or not HEX64_RE.fullmatch(digest): + raise AssemblyError(f"{label} file record has an invalid SHA-256: {relative}") + if not isinstance(size, int) or isinstance(size, bool) or size <= 0: + raise AssemblyError(f"{label} file record has an invalid size: {relative}") + prior = paths.get(relative.casefold()) + if prior is not None and prior != relative: + raise AssemblyError(f"{label} file paths collide by case") + paths[relative.casefold()] = relative + return set(paths.values()) + + +def _validate_record_files(root: Path, payload: Mapping[str, Any], *, label: str) -> None: + for record in _walk_file_records(payload): + relative = _safe_relative(record.get("path"), label=f"{label} file path") + path = _relative_file(root, relative, label=f"{label} file {relative}") + if path.stat().st_size != record.get("size_bytes") or _file_sha256(path) != record.get("sha256"): + raise AssemblyError(f"{label} file record drifted: {relative}") + + +def _copy_file_exclusive(source: Path, destination: Path) -> None: + destination.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + descriptor = -1 + try: + descriptor = os.open( + destination, + os.O_CREAT | os.O_EXCL | os.O_WRONLY | getattr(os, "O_BINARY", 0), + 0o600, + ) + with source.open("rb") as reader: + while chunk := reader.read(1024 * 1024): + view = memoryview(chunk) + while view: + written = os.write(descriptor, view) + view = view[written:] + os.fsync(descriptor) + except OSError as exc: + raise AssemblyError(f"cannot create package file: {destination}") from exc + finally: + if descriptor >= 0: + os.close(descriptor) + if os.name != "nt": + destination.chmod(0o600) + + +def _copy_paths(source_root: Path, destination_root: Path, paths: Iterable[str], *, label: str) -> None: + ordered = sorted(set(paths), key=lambda item: item.encode("utf-8")) + if len({item.casefold() for item in ordered}) != len(ordered): + raise AssemblyError(f"{label} paths collide by case") + for relative in ordered: + source = _relative_file(source_root, relative, label=f"{label} source {relative}") + destination = destination_root.joinpath(*PurePosixPath(relative).parts) + _copy_file_exclusive(source, destination) + if source.stat().st_size != destination.stat().st_size or _file_sha256(source) != _file_sha256(destination): + raise AssemblyError(f"copied {label} file drifted: {relative}") + + +def _copy_materials_tree(source_root: Path, destination_root: Path) -> None: + paths: list[str] = [] + for path in source_root.rglob("*"): + if path.is_dir() and not _is_linklike(path): + continue + relative = path.relative_to(source_root).as_posix() + _regular_file(path, label=f"third-party material {relative}", root=source_root) + paths.append(relative) + _copy_paths(source_root, destination_root, paths, label="third-party material") + + +def _write_exclusive(path: Path, payload: bytes, *, label: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + descriptor = -1 + try: + descriptor = os.open( + path, + os.O_CREAT | os.O_EXCL | os.O_WRONLY | getattr(os, "O_BINARY", 0), + 0o600, + ) + view = memoryview(payload) + while view: + written = os.write(descriptor, view) + view = view[written:] + os.fsync(descriptor) + except OSError as exc: + raise AssemblyError(f"cannot create {label}") from exc + finally: + if descriptor >= 0: + os.close(descriptor) + if os.name != "nt": + path.chmod(0o600) + + +def _files_below(root: Path, *, exclude: set[str]) -> list[dict[str, Any]]: + records: list[dict[str, Any]] = [] + for path in root.rglob("*"): + if path.is_dir() and not _is_linklike(path): + continue + relative = path.relative_to(root).as_posix() + _regular_file(path, label=f"staged package file {relative}", root=root) + if relative in exclude: + continue + records.append( + { + "path": relative, + "sha256": _file_sha256(path), + "size_bytes": path.stat().st_size, + } + ) + return sorted(records, key=lambda item: item["path"].encode("utf-8")) + + +def _image_maps( + installable: Mapping[str, Any], infrastructure_bundle: Mapping[str, Any] +) -> tuple[dict[str, dict[str, str]], dict[str, dict[str, str]]]: + application = { + str(item["environment_key"]): { + "reference": str(item["registry_reference"]), + "digest": str(item["manifest_digest"]), + } + for item in installable.get("images", []) + if isinstance(item, Mapping) + } + infrastructure = { + str(item["environment_key"]): { + "reference": str(item["canonical_reference"]), + "digest": str(item["manifest_digest"]), + } + for item in infrastructure_bundle.get("infrastructure_images", []) + if isinstance(item, Mapping) + } + return application, infrastructure + + +def _validate_cross_identity( + *, + expected_commit: str, + expected_tag: str, + source_manifest: Mapping[str, Any], + source_manifest_sha256: str, + installable: Mapping[str, Any], + infrastructure_bundle: Mapping[str, Any], + third_party_manifest: Mapping[str, Any], +) -> None: + identities = [ + source_manifest.get("candidate"), + installable.get("candidate"), + infrastructure_bundle.get("candidate"), + third_party_manifest.get("candidate"), + ] + for identity in identities: + if not isinstance(identity, Mapping): + raise AssemblyError("a package input has no candidate identity") + if identity.get("commit") != expected_commit: + raise AssemblyError("package inputs bind different candidate commits") + if "tag" in identity and identity.get("tag") != expected_tag: + raise AssemblyError("package inputs bind different candidate tags") + if third_party_manifest["candidate"].get("source_manifest_sha256") != source_manifest_sha256: + raise AssemblyError("third-party materials bind a different source manifest") + + expected_images: dict[str, tuple[str, str]] = {} + for group in (installable.get("images"), infrastructure_bundle.get("infrastructure_images")): + if not isinstance(group, list): + raise AssemblyError("formal image inventory is missing") + for item in group: + if not isinstance(item, Mapping) or not isinstance(item.get("role"), str): + raise AssemblyError("formal image inventory is invalid") + expected_images[item["role"]] = ( + str(item.get("manifest_digest")), + str(item.get("config_digest")), + ) + observed_images: dict[str, tuple[str, str]] = {} + raw_images = third_party_manifest["candidate"].get("images") + if not isinstance(raw_images, list): + raise AssemblyError("third-party image inventory is missing") + for item in raw_images: + if not isinstance(item, Mapping) or not isinstance(item.get("role"), str): + raise AssemblyError("third-party image inventory is invalid") + observed_images[item["role"]] = ( + str(item.get("manifest_digest")), + str(item.get("config_digest")), + ) + if len(expected_images) != 11 or observed_images != expected_images: + raise AssemblyError("third-party materials bind a different eleven-image inventory") + + +def _preflight(args: argparse.Namespace) -> dict[str, Any]: + repo = _directory(args.repo_root, label="repository root") + if not COMMIT_RE.fullmatch(args.expected_candidate_commit): + raise AssemblyError("expected candidate commit is invalid") + if not args.expected_candidate_tag or len(args.expected_candidate_tag) > 255: + raise AssemblyError("expected candidate tag is invalid") + if not SAFE_ID_RE.fullmatch(args.release_id): + raise AssemblyError("release_id is invalid") + if not PROJECT_RE.fullmatch(args.compose_project_name): + raise AssemblyError("compose project name is invalid") + if not HEX64_RE.fullmatch(args.expected_build_receipt_sha256): + raise AssemblyError("expected build receipt SHA-256 is invalid") + + output = args.output_dir + if not output.is_absolute(): + raise AssemblyError("output directory must be an absolute path") + output = _absolute(output) + parent = _directory(output.parent, label="output parent") + _assert_outside_repo(output, repo, label="output directory") + if output.exists() or output.is_symlink(): + raise AssemblyError("refusing to overwrite an existing output path") + + application_root = _directory(args.application_root, label="application artifact root") + infrastructure_root = _directory(args.infrastructure_root, label="infrastructure artifact root") + third_party_input_root = _directory( + args.third_party_input_manifest.parent, label="third-party input root" + ) + materials_root = _directory(args.third_party_materials_dir, label="third-party materials root") + for label, root in ( + ("application artifact root", application_root), + ("infrastructure artifact root", infrastructure_root), + ("third-party input root", third_party_input_root), + ("third-party materials root", materials_root), + ): + _assert_outside_repo(root, repo, label=label) + try: + output.relative_to(root) + except ValueError: + pass + else: + raise AssemblyError("output directory must not be inside an input artifact root") + try: + root.relative_to(output) + except ValueError: + pass + else: + raise AssemblyError("input artifact roots must not be inside the output directory") + + installable_path = _relative_file( + application_root, + args.installable_manifest, + label="installable release manifest", + ) + source_set_path = _relative_file( + infrastructure_root, + args.infrastructure_source_set, + label="infrastructure source set", + ) + infrastructure_bundle_path = _relative_file( + infrastructure_root, + args.infrastructure_bundle, + label="infrastructure image bundle", + ) + input_manifest_path = _regular_file( + args.third_party_input_manifest, + label="third-party input manifest", + root=third_party_input_root, + ) + receipt_path = _regular_file( + args.third_party_verification_receipt, + label="third-party verification receipt", + ) + _assert_outside_repo(receipt_path, repo, label="third-party verification receipt") + + installable_module = _load_module(INSTALLABLE_TOOL, "yuqei_build05_installable_validator") + infrastructure_module = _load_module(INFRASTRUCTURE_TOOL, "yuqei_build05_infrastructure_validator") + third_party_module = _load_module(THIRD_PARTY_TOOL, "yuqei_build05_third_party_validator") + package_module = _load_module(PACKAGE_TOOL, "yuqei_build05_release_package_validator") + + try: + installable = installable_module.validate_installable_release_manifest( + artifact_root=application_root, + manifest_path=installable_path, + expected_receipt_sha256=args.expected_build_receipt_sha256, + repo=repo, + ) + except Exception as exc: + raise AssemblyError(f"installable release validation failed: {exc}") from exc + + source_set = _read_json(source_set_path, label="infrastructure source set") + included_bundle = _read_json( + infrastructure_bundle_path, label="infrastructure image bundle" + ) + try: + recomputed_bundle = infrastructure_module.build_bundle( + artifact_root=infrastructure_root, + source_set=source_set, + application_manifest=installable, + ) + except Exception as exc: + raise AssemblyError(f"infrastructure bundle validation failed: {exc}") from exc + if recomputed_bundle != included_bundle: + raise AssemblyError("infrastructure bundle differs from deterministic recomputation") + + input_manifest = _read_json(input_manifest_path, label="third-party input manifest") + profile = input_manifest.get("profile") + if profile not in {"internal-hosted", "redistributable"}: + raise AssemblyError("third-party profile is invalid") + try: + recomputed_receipt = third_party_module.verify( + input_manifest_path, materials_root, profile + ) + except Exception as exc: + raise AssemblyError(f"third-party materials validation failed: {exc}") from exc + receipt = _read_json(receipt_path, label="third-party verification receipt") + if receipt != recomputed_receipt: + raise AssemblyError("third-party receipt differs from independent recomputation") + materials_manifest_path = _relative_file( + materials_root, "materials-manifest.json", label="third-party materials manifest" + ) + materials_manifest = _read_json( + materials_manifest_path, label="third-party materials manifest" + ) + + receipt_record = installable.get("build_receipt") + if not isinstance(receipt_record, Mapping): + raise AssemblyError("installable manifest has no build receipt") + build_receipt_path = _relative_file( + application_root, + receipt_record.get("path"), + label="trusted build receipt", + ) + if _file_sha256(build_receipt_path) != args.expected_build_receipt_sha256: + raise AssemblyError("trusted build receipt raw SHA-256 drifted") + build_receipt = _read_json(build_receipt_path, label="trusted build receipt") + receipt_candidate = build_receipt.get("candidate") + source_record = ( + receipt_candidate.get("source_manifest") + if isinstance(receipt_candidate, Mapping) + else None + ) + if not isinstance(source_record, Mapping): + raise AssemblyError("trusted build receipt has no source manifest record") + source_manifest_path = _relative_file( + application_root, + source_record.get("path"), + label="release source manifest", + ) + source_manifest = _read_json(source_manifest_path, label="release source manifest") + source_manifest_sha256 = _file_sha256(source_manifest_path) + if source_record.get("sha256") != source_manifest_sha256: + raise AssemblyError("build receipt source manifest hash drifted") + + _verify_repo(repo, args.expected_candidate_commit, args.expected_candidate_tag) + _validate_cross_identity( + expected_commit=args.expected_candidate_commit, + expected_tag=args.expected_candidate_tag, + source_manifest=source_manifest, + source_manifest_sha256=source_manifest_sha256, + installable=installable, + infrastructure_bundle=included_bundle, + third_party_manifest=materials_manifest, + ) + _validate_record_files(infrastructure_root, source_set, label="infrastructure source set") + _validate_record_files(third_party_input_root, input_manifest, label="third-party input") + + application_paths = { + _safe_relative(args.installable_manifest, label="installable manifest path") + } + raw_installable_files = installable.get("files") + if not isinstance(raw_installable_files, list): + raise AssemblyError("installable manifest file inventory is missing") + for record in raw_installable_files: + if not isinstance(record, Mapping): + raise AssemblyError("installable manifest file record is invalid") + application_paths.add( + _safe_relative(record.get("path"), label="installable file path") + ) + infrastructure_paths = _record_paths(source_set, label="infrastructure source set") + infrastructure_paths.update( + _record_paths( + {"application_images": included_bundle.get("application_images")}, + label="infrastructure application-image closure", + ) + ) + infrastructure_paths.update( + { + _safe_relative(args.infrastructure_source_set, label="infrastructure source-set path"), + _safe_relative(args.infrastructure_bundle, label="infrastructure bundle path"), + } + ) + input_manifest_relative = input_manifest_path.relative_to(third_party_input_root).as_posix() + third_party_input_paths = _record_paths(input_manifest, label="third-party input") + third_party_input_paths.add(input_manifest_relative) + + return { + "repo": repo, + "output": output, + "output_parent": parent, + "application_root": application_root, + "infrastructure_root": infrastructure_root, + "third_party_input_root": third_party_input_root, + "materials_root": materials_root, + "receipt_path": receipt_path, + "installable_path": installable_path, + "source_manifest_path": source_manifest_path, + "source_manifest_sha256": source_manifest_sha256, + "source_manifest": source_manifest, + "installable": installable, + "infrastructure_bundle": included_bundle, + "source_set_path": source_set_path, + "infrastructure_bundle_path": infrastructure_bundle_path, + "input_manifest_path": input_manifest_path, + "materials_manifest_path": materials_manifest_path, + "package_module": package_module, + "application_paths": application_paths, + "infrastructure_paths": infrastructure_paths, + "third_party_input_paths": third_party_input_paths, + "profile": profile, + } + + +def _identity_payload( + state: Mapping[str, Any], *, release_id: str, package_module: Any +) -> dict[str, Any]: + application, infrastructure = _image_maps( + state["installable"], state["infrastructure_bundle"] + ) + expected_application = set(package_module.APPLICATION_IMAGE_KEYS) + expected_infrastructure = set(package_module.INFRASTRUCTURE_IMAGE_KEYS) + if set(application) != expected_application or set(infrastructure) != expected_infrastructure: + raise AssemblyError("image identity does not cover exactly four application and seven infrastructure images") + return _with_hash( + { + "schema_version": package_module.IMAGE_IDENTITY_SCHEMA_V2, + "release_id": release_id, + "source_commit": state["source_manifest"]["candidate"]["commit"], + "source_manifest_sha256": state["source_manifest_sha256"], + "application_images": application, + "infrastructure_images": infrastructure, + "recovery": { + "strategy": package_module.RECOVERY_STRATEGY_FIRST_RELEASE, + "evidence_status": "runtime_signed_evidence_required", + "first_release_only": True, + "predecessor_package_required": False, + "database_downgrade_allowed": False, + "required_restore_scopes": list( + package_module.FIRST_RELEASE_REQUIRED_RESTORE_SCOPES + ), + "required_postgres_databases": list( + package_module.FIRST_RELEASE_REQUIRED_POSTGRES_DATABASES + ), + "required_storage_targets": list( + package_module.FIRST_RELEASE_REQUIRED_STORAGE_TARGETS + ), + }, + }, + "identity_payload_sha256", + ) + + +def _rename_noreplace(source: Path, destination: Path) -> None: + if destination.exists() or destination.is_symlink(): + raise AssemblyError("refusing to overwrite an existing output path") + if sys.platform.startswith("linux"): + import ctypes + + libc = ctypes.CDLL(None, use_errno=True) + renameat2 = getattr(libc, "renameat2", None) + if renameat2 is None: + raise AssemblyError("atomic no-replace directory publication is unavailable") + renameat2.argtypes = [ctypes.c_int, ctypes.c_char_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint] + renameat2.restype = ctypes.c_int + result = renameat2( + -100, + os.fsencode(source), + -100, + os.fsencode(destination), + 1, + ) + if result != 0: + errno_value = ctypes.get_errno() + raise AssemblyError( + f"atomic no-replace publication failed: errno={errno_value}" + ) + return + try: + os.rename(source, destination) + except OSError as exc: + raise AssemblyError("atomic no-replace publication failed") from exc + + +def _assemble(state: Mapping[str, Any], args: argparse.Namespace) -> dict[str, Any]: + output: Path = state["output"] + parent: Path = state["output_parent"] + temporary = Path(tempfile.mkdtemp(prefix=f".{output.name}.partial-", dir=parent)) + published = False + try: + application_destination = temporary / "application" + infrastructure_destination = temporary / "infrastructure" + third_party_input_destination = temporary / "third-party/input" + third_party_materials_destination = temporary / "third-party/materials" + + _copy_paths( + state["application_root"], + application_destination, + state["application_paths"], + label="application artifact", + ) + _copy_paths( + state["infrastructure_root"], + infrastructure_destination, + state["infrastructure_paths"], + label="infrastructure artifact", + ) + _copy_paths( + state["third_party_input_root"], + third_party_input_destination, + state["third_party_input_paths"], + label="third-party input", + ) + _copy_materials_tree( + state["materials_root"], third_party_materials_destination + ) + receipt_destination = temporary / "third-party/verification-receipt.json" + _copy_file_exclusive(state["receipt_path"], receipt_destination) + + package_module = state["package_module"] + identity_path = temporary / "image-identity.json" + identity = _identity_payload( + state, release_id=args.release_id, package_module=package_module + ) + _write_exclusive( + identity_path, + _canonical_file_bytes(identity), + label="image identity", + ) + + checksum_path = temporary / "checksums.json" + package_path = temporary / "release-package.json" + records = _files_below( + temporary, + exclude={"checksums.json", "release-package.json"}, + ) + checksums = _with_hash( + { + "schema_version": package_module.CHECKSUM_SCHEMA, + "release_id": args.release_id, + "files": records, + }, + "manifest_payload_sha256", + ) + _write_exclusive( + checksum_path, + _canonical_file_bytes(checksums), + label="checksum manifest", + ) + + installable_relative = ( + PurePosixPath("application") + / PurePosixPath(_safe_relative(args.installable_manifest, label="installable manifest path")) + ).as_posix() + source_relative = ( + PurePosixPath("application") + / PurePosixPath(state["source_manifest_path"].relative_to(state["application_root"]).as_posix()) + ).as_posix() + source_set_relative = ( + PurePosixPath("infrastructure") + / PurePosixPath(_safe_relative(args.infrastructure_source_set, label="infrastructure source-set path")) + ).as_posix() + infrastructure_bundle_relative = ( + PurePosixPath("infrastructure") + / PurePosixPath(_safe_relative(args.infrastructure_bundle, label="infrastructure bundle path")) + ).as_posix() + input_relative = ( + PurePosixPath("third-party/input") + / PurePosixPath(state["input_manifest_path"].relative_to(state["third_party_input_root"]).as_posix()) + ).as_posix() + material_manifest_relative = "third-party/materials/materials-manifest.json" + receipt_relative = "third-party/verification-receipt.json" + + artifact_paths = { + "release_manifest": source_relative, + "checksum_manifest": "checksums.json", + "image_identity": "image-identity.json", + "installable_manifest": installable_relative, + "infrastructure_source_set": source_set_relative, + "infrastructure_image_bundle": infrastructure_bundle_relative, + "third_party_materials_input": input_relative, + "third_party_materials_manifest": material_manifest_relative, + "third_party_materials_verification_receipt": receipt_relative, + } + artifacts = { + name: { + "path": relative, + "sha256": _file_sha256( + temporary.joinpath(*PurePosixPath(relative).parts) + ), + } + for name, relative in artifact_paths.items() + } + package = _with_hash( + { + "schema_version": package_module.PACKAGE_SCHEMA_V2, + "release_id": args.release_id, + "compose_project_name": args.compose_project_name, + "deployment_scope": package_module.DEPLOYMENT_SCOPE, + "deployed_service_set": list(package_module.INSTALL_SERVICES), + "separately_deployed_artifacts": list( + package_module.SEPARATELY_DEPLOYED_IMAGE_KEYS + ), + "trusted_build_receipt_sha256": args.expected_build_receipt_sha256, + "artifacts": artifacts, + }, + "package_payload_sha256", + ) + _write_exclusive( + package_path, + _canonical_file_bytes(package), + label="release package manifest", + ) + package_sha256 = _file_sha256(package_path) + + try: + validated = package_module.validate_release_package( + temporary, + expected_package_sha256=package_sha256, + repo_root=state["repo"], + ) + except Exception as exc: + raise AssemblyError(f"assembled package failed strict revalidation: {exc}") from exc + if ( + validated.package_schema_version != package_module.PACKAGE_SCHEMA_V2 + or validated.package_sha256 != package_sha256 + or len(validated.docker_import_plan) != 11 + or validated.recovery_strategy + != package_module.RECOVERY_STRATEGY_FIRST_RELEASE + ): + raise AssemblyError("assembled package strict revalidation returned an unsafe state") + + _rename_noreplace(temporary, output) + published = True + return { + "schema_version": RESULT_SCHEMA, + "result": "PASS_ASSEMBLED_AND_REVALIDATED", + "mode": "assemble", + "candidate_commit": args.expected_candidate_commit, + "candidate_tag": args.expected_candidate_tag, + "release_id": args.release_id, + "package_schema_version": package_module.PACKAGE_SCHEMA_V2, + "package_sha256": package_sha256, + "file_count": len(records) + 2, + "image_count": len(validated.docker_import_plan), + "recovery_strategy": validated.recovery_strategy, + "output_dir": str(output), + "apply_performed": False, + "remote_connection_attempted": False, + "claims_not_made": [ + "installation", + "migration", + "snapshot capture", + "restore", + "UAT", + "pilot authorization", + "production authorization", + "public release authorization", + ], + } + finally: + if not published and temporary.exists(): + shutil.rmtree(temporary, ignore_errors=True) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo-root", type=Path, required=True) + parser.add_argument("--expected-candidate-commit", required=True) + parser.add_argument("--expected-candidate-tag", required=True) + parser.add_argument("--release-id", required=True) + parser.add_argument("--compose-project-name", required=True) + parser.add_argument("--expected-build-receipt-sha256", required=True) + parser.add_argument("--application-root", type=Path, required=True) + parser.add_argument( + "--installable-manifest", default="installable-release-manifest.json" + ) + parser.add_argument("--infrastructure-root", type=Path, required=True) + parser.add_argument( + "--infrastructure-source-set", default="infrastructure-source-set.json" + ) + parser.add_argument( + "--infrastructure-bundle", default="infrastructure-image-bundle.json" + ) + parser.add_argument("--third-party-input-manifest", type=Path, required=True) + parser.add_argument("--third-party-materials-dir", type=Path, required=True) + parser.add_argument( + "--third-party-verification-receipt", type=Path, required=True + ) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument( + "--assemble", + action="store_true", + help="create the package; omission performs read-only preflight only", + ) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = _parser().parse_args(argv) + try: + state = _preflight(args) + if args.assemble: + result = _assemble(state, args) + else: + result = { + "schema_version": RESULT_SCHEMA, + "result": "PASS_READ_ONLY_PREFLIGHT", + "mode": "preflight", + "candidate_commit": args.expected_candidate_commit, + "candidate_tag": args.expected_candidate_tag, + "release_id": args.release_id, + "package_schema_version": state["package_module"].PACKAGE_SCHEMA_V2, + "planned_output_dir": str(state["output"]), + "application_file_count": len(state["application_paths"]), + "infrastructure_file_count": len(state["infrastructure_paths"]), + "third_party_input_file_count": len(state["third_party_input_paths"]), + "third_party_profile": state["profile"], + "apply_performed": False, + "filesystem_output_created": False, + "remote_connection_attempted": False, + } + sys.stdout.buffer.write(_canonical_file_bytes(result)) + return 0 + except (AssemblyError, OSError, UnicodeError, ValueError) as exc: + sys.stderr.write(f"FAILED_CLOSED: {exc}\n") + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/yuqei-infra/tests/test_build05_v2_package_orchestrator.py b/yuqei-infra/tests/test_build05_v2_package_orchestrator.py new file mode 100644 index 00000000..9a6ed649 --- /dev/null +++ b/yuqei-infra/tests/test_build05_v2_package_orchestrator.py @@ -0,0 +1,276 @@ +from __future__ import annotations + +import hashlib +import importlib.util +import json +import sys +from pathlib import Path +from types import ModuleType, SimpleNamespace + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT_PATH = ( + REPO_ROOT / "yuqei-infra/scripts/assemble-build05-v2-package.py" +) +HELPER_PATH = REPO_ROOT / "yuqei-infra/tests/test_release_package_executor.py" + + +def _load(path: Path, name: str) -> ModuleType: + existing = sys.modules.get(name) + if existing is not None: + return existing + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +orchestrator = _load(SCRIPT_PATH, "yuqei_build05_v2_package_orchestrator_tests") +helpers = _load(HELPER_PATH, "yuqei_build05_v2_package_fixture_helpers") + + +@pytest.fixture(scope="module") +def release_inputs(tmp_path_factory: pytest.TempPathFactory) -> dict[str, object]: + root = tmp_path_factory.mktemp("build05-v2-orchestrator") + template = helpers._strict_build_template(root) + installable = json.loads( + (template["bundle_root"] / "installable-release-manifest.json").read_text( + encoding="utf-8" + ) + ) + return { + "root": root, + "application_root": template["bundle_root"], + "infrastructure_root": template["v2_root"], + "repo": template["repo"], + "receipt_sha256": template["receipt_sha256"], + "commit": installable["candidate"]["commit"], + "tag": installable["candidate"]["tag"], + } + + +def _reviewed_module_loader(inputs: dict[str, object]): + base_infrastructure = helpers.release_package._load_infrastructure_bundle_module() + original_infrastructure_root = Path(inputs["infrastructure_root"]) + + def fixture_bundle_adapter(*, artifact_root, source_set, application_manifest): + # The shared release-package fixture intentionally validates a modern + # synthetic Docker archive and substitutes the separately validated + # application records. Re-run that adapter against its immutable raw + # fixture root when the assembled copy is independently revalidated. + root = Path(artifact_root) + if root != original_infrastructure_root: + root = original_infrastructure_root + return base_infrastructure.build_bundle( + artifact_root=root, + source_set=source_set, + application_manifest=application_manifest, + ) + + infrastructure = SimpleNamespace( + __file__=base_infrastructure.__file__, + build_bundle=fixture_bundle_adapter, + ) + + def load(path: Path, _name: str): + if path.name == "build-installable-release.py": + return helpers.release_package._load_installable_validator_module() + if path.name == "build-infrastructure-image-bundle.py": + return infrastructure + if path.name == "build-third-party-materials.py": + return helpers.release_package._load_third_party_materials_module() + if path.name == "release-package.py": + helpers.release_package._INFRASTRUCTURE_BUNDLE_MODULE = infrastructure + return helpers.release_package + raise AssertionError(f"unexpected reviewed module: {path}") + + return load + + +def _argv(inputs: dict[str, object], output: Path, *, assemble: bool = False) -> list[str]: + infrastructure_root = Path(inputs["infrastructure_root"]) + args = [ + "--repo-root", + str(inputs["repo"]), + "--expected-candidate-commit", + str(inputs["commit"]), + "--expected-candidate-tag", + str(inputs["tag"]), + "--release-id", + "release-2", + "--compose-project-name", + "yuqei-prod", + "--expected-build-receipt-sha256", + str(inputs["receipt_sha256"]), + "--application-root", + str(inputs["application_root"]), + "--infrastructure-root", + str(infrastructure_root), + "--third-party-input-manifest", + str(infrastructure_root / "third-party-input/input-manifest.json"), + "--third-party-materials-dir", + str(infrastructure_root / "third-party-materials"), + "--third-party-verification-receipt", + str(infrastructure_root / "third-party-verification.json"), + "--output-dir", + str(output), + ] + if assemble: + args.append("--assemble") + return args + + +def _tree_hash(root: Path) -> str: + digest = hashlib.sha256() + for path in sorted( + (item for item in root.rglob("*") if item.is_file()), + key=lambda item: item.relative_to(root).as_posix().encode("utf-8"), + ): + relative = path.relative_to(root).as_posix().encode("utf-8") + digest.update(len(relative).to_bytes(8, "big")) + digest.update(relative) + digest.update(hashlib.sha256(path.read_bytes()).digest()) + return digest.hexdigest() + + +def test_default_preflight_is_read_only( + release_inputs: dict[str, object], + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsysbinary: pytest.CaptureFixture[bytes], +) -> None: + monkeypatch.setattr( + orchestrator, "_load_module", _reviewed_module_loader(release_inputs) + ) + output = tmp_path / "package" + + assert orchestrator.main(_argv(release_inputs, output)) == 0 + + captured = capsysbinary.readouterr() + result = json.loads(captured.out) + assert result["result"] == "PASS_READ_ONLY_PREFLIGHT" + assert result["filesystem_output_created"] is False + assert result["apply_performed"] is False + assert result["remote_connection_attempted"] is False + assert not output.exists() + + +def test_assemble_creates_one_strict_first_release_package_v2( + release_inputs: dict[str, object], + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsysbinary: pytest.CaptureFixture[bytes], +) -> None: + monkeypatch.setattr( + orchestrator, "_load_module", _reviewed_module_loader(release_inputs) + ) + output = tmp_path / "package" + + assert orchestrator.main(_argv(release_inputs, output, assemble=True)) == 0 + + result = json.loads(capsysbinary.readouterr().out) + assert result["result"] == "PASS_ASSEMBLED_AND_REVALIDATED" + assert result["package_schema_version"] == "yuqei.release-package.v2" + assert result["image_count"] == 11 + assert result["recovery_strategy"] == "first_release_snapshot_restore" + assert result["apply_performed"] is False + assert output.is_dir() + package_path = output / "release-package.json" + assert result["package_sha256"] == helpers._sha(package_path) + validated = helpers.release_package.validate_release_package( + output, + expected_package_sha256=result["package_sha256"], + repo_root=Path(release_inputs["repo"]), + ) + assert validated.package_schema_version == helpers.release_package.PACKAGE_SCHEMA_V2 + assert validated.recovery_strategy == helpers.release_package.RECOVERY_STRATEGY_FIRST_RELEASE + assert len(validated.docker_import_plan) == 11 + + +def test_existing_output_is_never_overwritten( + release_inputs: dict[str, object], + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsysbinary: pytest.CaptureFixture[bytes], +) -> None: + monkeypatch.setattr( + orchestrator, "_load_module", _reviewed_module_loader(release_inputs) + ) + output = tmp_path / "package" + assert orchestrator.main(_argv(release_inputs, output, assemble=True)) == 0 + capsysbinary.readouterr() + before = _tree_hash(output) + + assert orchestrator.main(_argv(release_inputs, output, assemble=True)) == 2 + + captured = capsysbinary.readouterr() + assert b"refusing to overwrite" in captured.err + assert _tree_hash(output) == before + + +def test_candidate_mismatch_fails_without_output( + release_inputs: dict[str, object], + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsysbinary: pytest.CaptureFixture[bytes], +) -> None: + monkeypatch.setattr( + orchestrator, "_load_module", _reviewed_module_loader(release_inputs) + ) + output = tmp_path / "package" + args = _argv(release_inputs, output, assemble=True) + index = args.index("--expected-candidate-commit") + 1 + args[index] = "f" * 40 + + assert orchestrator.main(args) == 2 + + captured = capsysbinary.readouterr() + assert b"repository HEAD does not match" in captured.err + assert not output.exists() + + +def test_output_inside_source_repository_is_rejected( + release_inputs: dict[str, object], + monkeypatch: pytest.MonkeyPatch, + capsysbinary: pytest.CaptureFixture[bytes], +) -> None: + monkeypatch.setattr( + orchestrator, "_load_module", _reviewed_module_loader(release_inputs) + ) + output = Path(release_inputs["repo"]) / "forbidden-package" + + assert orchestrator.main(_argv(release_inputs, output, assemble=True)) == 2 + + captured = capsysbinary.readouterr() + assert b"outside the source repository" in captured.err + assert not output.exists() + + +def test_failed_final_revalidation_removes_partial_output( + release_inputs: dict[str, object], + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsysbinary: pytest.CaptureFixture[bytes], +) -> None: + monkeypatch.setattr( + orchestrator, "_load_module", _reviewed_module_loader(release_inputs) + ) + output = tmp_path / "package" + + def reject_package(*_args, **_kwargs): + raise helpers.release_package.ReleasePackageError("simulated final rejection") + + monkeypatch.setattr( + helpers.release_package, "validate_release_package", reject_package + ) + + assert orchestrator.main(_argv(release_inputs, output, assemble=True)) == 2 + + captured = capsysbinary.readouterr() + assert b"assembled package failed strict revalidation" in captured.err + assert not output.exists() + assert list(tmp_path.glob(".package.partial-*")) == []