fix(release): bind scene prompts and holdout plan
This commit is contained in:
+26
@@ -3,6 +3,7 @@ from datetime import UTC, datetime
|
||||
from types import SimpleNamespace
|
||||
|
||||
from yuqei_sdk.contract_drafting import draft_digest
|
||||
from yuqei_sdk.prompt_source_baseline import scene_prompt_source_identity
|
||||
from yuqei_sdk.review_routing import (
|
||||
SCENE_REVIEW_ROUTES,
|
||||
REVIEW_ROUTING_VERSION,
|
||||
@@ -28,6 +29,13 @@ def _identity(snapshot):
|
||||
"resolution": "platform_shared_published" if status == "published" else status}
|
||||
|
||||
|
||||
def _matches_source_identity(actual, expected):
|
||||
return bool(actual and expected and all(
|
||||
actual.get(key) == expected.get(key)
|
||||
for key in ("name", "version", "content_sha256")
|
||||
))
|
||||
|
||||
|
||||
def build_scene_runtime_manifest(store, settings, *, template_id, contract_type):
|
||||
route = SCENE_REVIEW_ROUTES.get(template_id)
|
||||
if route is None:
|
||||
@@ -44,9 +52,16 @@ def build_scene_runtime_manifest(store, settings, *, template_id, contract_type)
|
||||
request.review_profile = "standard"
|
||||
ordinary = _identity(capture_review_prompt_snapshot(store, request))
|
||||
drafting = _identity(capture_drafting_prompt_snapshot(store))
|
||||
source_identity = scene_prompt_source_identity(template_id)
|
||||
issues = []
|
||||
if not source_identity:
|
||||
issues.append("prompt_source_identity_unresolved")
|
||||
expected_draft = (source_identity or {}).get("draft_prompt")
|
||||
expected_review = (source_identity or {}).get("review_prompt")
|
||||
if not drafting or drafting["status"] != "published" or not drafting["enabled"]:
|
||||
issues.append("draft_prompt_not_published")
|
||||
if not _matches_source_identity(drafting, expected_draft):
|
||||
issues.append("draft_runtime_source_mismatch")
|
||||
if not ordinary:
|
||||
issues.append("review_prompt_missing")
|
||||
else:
|
||||
@@ -54,6 +69,8 @@ def build_scene_runtime_manifest(store, settings, *, template_id, contract_type)
|
||||
issues.append("review_runtime_fallback")
|
||||
if ordinary["status"] != "published" or not ordinary["enabled"]:
|
||||
issues.append("review_prompt_not_published")
|
||||
if not _matches_source_identity(ordinary, expected_review):
|
||||
issues.append("review_runtime_source_mismatch")
|
||||
if any(ordinary[key] != trial[key] for key in ("name", "version", "content_sha256")):
|
||||
issues.append("review_trial_runtime_mismatch")
|
||||
if settings.provider_call_mode != "codex_cli" or settings.codex_cli_model != "gpt-5.6-sol":
|
||||
@@ -62,6 +79,15 @@ def build_scene_runtime_manifest(store, settings, *, template_id, contract_type)
|
||||
payload = {"schema_version": "scene-runtime-manifest.v1", "tenant_id": current_tenant_id(),
|
||||
"template_id": template_id, "contract_type": contract_type, "routing_version": REVIEW_ROUTING_VERSION,
|
||||
"requested_review_route": requested_review_route, "draft_prompt": drafting,
|
||||
"source_prompt_identity": source_identity,
|
||||
"source_prompt_identity_resolved": source_identity is not None,
|
||||
"runtime_source_alignment": {
|
||||
"draft_identity_match": _matches_source_identity(drafting, expected_draft),
|
||||
"review_identity_match": _matches_source_identity(ordinary, expected_review),
|
||||
"draft_published": bool(drafting and drafting["status"] == "published" and drafting["enabled"]),
|
||||
"review_published": bool(ordinary and ordinary["status"] == "published" and ordinary["enabled"]),
|
||||
"trial_review_matches_runtime": _matches_source_identity(trial, ordinary),
|
||||
},
|
||||
"prompt_storage_scope": "platform_shared",
|
||||
"trial_review_prompt": trial, "runtime_review_prompt": ordinary,
|
||||
"provider_mode": settings.provider_call_mode,
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
"""Unit route identities only; these tests do not call or approve a model."""
|
||||
from datetime import UTC, datetime
|
||||
from types import SimpleNamespace
|
||||
|
||||
import httpx
|
||||
@@ -7,11 +8,31 @@ from fastapi.testclient import TestClient
|
||||
|
||||
from yuqei_sdk import AiPlatformClient, RequestContext
|
||||
from yuqei_sdk.contract_drafting import draft_digest
|
||||
from yuqei_sdk.prompt_source_baseline import (
|
||||
DOMAIN_REVIEW_RUNTIME_IDENTITIES,
|
||||
DRAFTING_MODULES,
|
||||
DRAFT_RUNTIME_IDENTITY,
|
||||
NEW_REVIEW_RUNTIME_IDENTITIES,
|
||||
REVIEW_MODULES,
|
||||
scene_prompt_source_identity,
|
||||
)
|
||||
from yuqei_sdk.review_routing import SCENE_REVIEW_ROUTES, scene_review_evaluation_profile
|
||||
from yuqei_ai_platform_api.config import AiPlatformSettings
|
||||
from yuqei_ai_platform_api.contract_drafting import capture_drafting_prompt_snapshot
|
||||
from yuqei_ai_platform_api.contract_drafting import (
|
||||
DRAFT_PROMPT_CONTENT,
|
||||
DRAFT_PROMPT_NAMES,
|
||||
DRAFT_PROMPT_VERSION,
|
||||
capture_drafting_prompt_snapshot,
|
||||
)
|
||||
from yuqei_ai_platform_api.drafting_prompt_modules import drafting_module_evidence
|
||||
from yuqei_ai_platform_api.main import create_app
|
||||
from yuqei_ai_platform_api.review_scene_prompts import scene_review_template
|
||||
from yuqei_ai_platform_api.packaged_prompt_assets_import import (
|
||||
DEFAULT_SOURCE_FILE,
|
||||
DEFAULT_VERSION,
|
||||
load_packaged_prompt_assets,
|
||||
)
|
||||
from yuqei_ai_platform_api.repository import PromptTemplate
|
||||
from yuqei_ai_platform_api.review_scene_prompts import review_modules, scene_review_template
|
||||
from yuqei_ai_platform_api.scene_runtime_manifest import build_scene_runtime_manifest
|
||||
from yuqei_contract_api.contract_templates import get_contract_template
|
||||
from yuqei_ai_platform_api.scene_evaluation_gate import (
|
||||
@@ -38,7 +59,6 @@ def fixture_store(template_id=SCENE):
|
||||
review_profile=scene_review_evaluation_profile(template_id),
|
||||
)
|
||||
review = scene_review_template(request).model_copy(update={"status": "published"})
|
||||
from yuqei_ai_platform_api.repository import PromptTemplate
|
||||
empty = SimpleNamespace(get_prompt_template=lambda _name: None)
|
||||
drafting = PromptTemplate.model_validate(capture_drafting_prompt_snapshot(empty)["template"]).model_copy(update={"status": "published"})
|
||||
prompts = {review.name: review, drafting.name: drafting}
|
||||
@@ -51,8 +71,22 @@ def test_manifest_matches_actual_routes_without_returning_prompt_content(templat
|
||||
template = get_contract_template(template_id)
|
||||
manifest = build_scene_runtime_manifest(store, AiPlatformSettings(provider_call_mode="codex_cli"),
|
||||
template_id=template_id, contract_type=template.contract_type)
|
||||
assert manifest["alignment_errors"] == []
|
||||
assert manifest["published_prompt_alignment_verified"] is True
|
||||
expected = scene_prompt_source_identity(template_id)
|
||||
review_source_match = all(
|
||||
manifest["runtime_review_prompt"].get(key) == expected["review_prompt"][key]
|
||||
for key in ("name", "version", "content_sha256")
|
||||
)
|
||||
assert manifest["alignment_errors"] == ([] if review_source_match else ["review_runtime_source_mismatch"])
|
||||
assert manifest["published_prompt_alignment_verified"] is review_source_match
|
||||
assert manifest["source_prompt_identity"] == expected
|
||||
assert manifest["source_prompt_identity_resolved"] is True
|
||||
assert manifest["runtime_source_alignment"] == {
|
||||
"draft_identity_match": True,
|
||||
"review_identity_match": review_source_match,
|
||||
"draft_published": True,
|
||||
"review_published": True,
|
||||
"trial_review_matches_runtime": True,
|
||||
}
|
||||
assert manifest["model"] == "gpt-5.6-sol" and manifest["model_calls"] == 0
|
||||
assert manifest["authorizes_publication"] is False
|
||||
assert manifest["evaluation_gate"]["development_gate_passed"] is True
|
||||
@@ -143,7 +177,12 @@ def test_builtin_trial_fallback_and_unpublished_runtime_are_distinct():
|
||||
review = prompts.pop(SCENE_REVIEW_ROUTES[SCENE][1])
|
||||
prompts["contract_review_default"] = review.model_copy(update={"name": "contract_review_default", "version": "legacy"})
|
||||
manifest = build_scene_runtime_manifest(store, AiPlatformSettings(provider_call_mode="codex_cli"), template_id=SCENE, contract_type="transport")
|
||||
assert set(manifest["alignment_errors"]) == {"draft_prompt_not_published", "review_runtime_fallback", "review_trial_runtime_mismatch"}
|
||||
assert set(manifest["alignment_errors"]) == {
|
||||
"draft_prompt_not_published",
|
||||
"review_runtime_fallback",
|
||||
"review_runtime_source_mismatch",
|
||||
"review_trial_runtime_mismatch",
|
||||
}
|
||||
assert manifest["draft_prompt"]["status"] == "builtin_development"
|
||||
assert manifest["draft_prompt"]["resolution"] == "builtin_development"
|
||||
|
||||
@@ -154,6 +193,7 @@ def test_same_content_with_new_version_requires_reevaluation():
|
||||
prompts[name] = prompts[name].model_copy(update={"version": "different-version"})
|
||||
manifest = build_scene_runtime_manifest(store, AiPlatformSettings(provider_call_mode="codex_cli"), template_id=SCENE, contract_type="transport")
|
||||
assert "review_trial_runtime_mismatch" in manifest["alignment_errors"]
|
||||
assert "review_runtime_source_mismatch" in manifest["alignment_errors"]
|
||||
with pytest.raises(ValueError):
|
||||
build_scene_runtime_manifest(store, AiPlatformSettings(), template_id=SCENE, contract_type="labor")
|
||||
|
||||
@@ -168,6 +208,78 @@ def test_catalog_classification_is_not_source_approval():
|
||||
assert manifest["legal_correctness_verified"] is False
|
||||
|
||||
|
||||
def test_prompt_source_catalog_is_recomputed_from_the_actual_module_composers_and_packaged_assets():
|
||||
assert drafting_module_evidence(DRAFT_PROMPT_CONTENT) == list(DRAFTING_MODULES)
|
||||
draft = PromptTemplate(
|
||||
id="source-check",
|
||||
name=DRAFT_PROMPT_NAMES[0],
|
||||
version=DRAFT_PROMPT_VERSION,
|
||||
content=DRAFT_PROMPT_CONTENT,
|
||||
status="draft",
|
||||
updated_at=datetime(2026, 10, 6, tzinfo=UTC),
|
||||
)
|
||||
assert {
|
||||
"name": draft.name,
|
||||
"version": draft.version,
|
||||
"content_sha256": draft.content_hash,
|
||||
"source_kind": "composed_drafting_modules",
|
||||
} == DRAFT_RUNTIME_IDENTITY
|
||||
|
||||
actual_review_modules = review_modules()
|
||||
assert [
|
||||
{key: actual_review_modules[item["key"]][key] for key in ("key", "version", "content_sha256")}
|
||||
for item in REVIEW_MODULES
|
||||
] == list(REVIEW_MODULES)
|
||||
|
||||
route_inputs = {
|
||||
"contract_review_trade_procurement": ("purchase", "scene_evaluation_v3"),
|
||||
"contract_review_services": ("service", "scene_evaluation_v3"),
|
||||
"contract_review_transport_warehousing": ("transport", "scene_evaluation_v4"),
|
||||
"contract_review_manufacturing": ("manufacturing", "scene_evaluation_v4"),
|
||||
"contract_review_construction": ("construction", "scene_evaluation_v3"),
|
||||
"contract_review_marketing": ("marketing", "scene_evaluation_v4"),
|
||||
"contract_review_consumer_services": ("consumer_service", "scene_evaluation_v3"),
|
||||
"contract_review_notices_amendments": ("amendment", "scene_evaluation_v4"),
|
||||
}
|
||||
recomputed_new_review = {}
|
||||
for route, (contract_type, profile) in route_inputs.items():
|
||||
template = scene_review_template(SimpleNamespace(
|
||||
contract_type=contract_type,
|
||||
scene_template_id=None,
|
||||
review_profile=profile,
|
||||
))
|
||||
assert template.name == route
|
||||
recomputed_new_review[route] = {
|
||||
"name": template.name,
|
||||
"version": template.version,
|
||||
"content_sha256": template.content_hash,
|
||||
"source_kind": "new_review_module_composite",
|
||||
}
|
||||
assert recomputed_new_review == NEW_REVIEW_RUNTIME_IDENTITIES
|
||||
|
||||
assets, _, duplicate_keys = load_packaged_prompt_assets(DEFAULT_SOURCE_FILE)
|
||||
assert duplicate_keys == []
|
||||
assets_by_key = {asset.prompt_key: asset for asset in assets}
|
||||
recomputed_domain = {}
|
||||
for route in DOMAIN_REVIEW_RUNTIME_IDENTITIES:
|
||||
asset = assets_by_key[route]
|
||||
template = PromptTemplate(
|
||||
id=f"source-check-{route}",
|
||||
name=asset.prompt_key,
|
||||
version=DEFAULT_VERSION,
|
||||
content=asset.content,
|
||||
status="draft",
|
||||
updated_at=datetime(2026, 10, 6, tzinfo=UTC),
|
||||
)
|
||||
recomputed_domain[route] = {
|
||||
"name": template.name,
|
||||
"version": template.version,
|
||||
"content_sha256": template.content_hash,
|
||||
"source_kind": "existing_domain_prompt_asset",
|
||||
}
|
||||
assert recomputed_domain == DOMAIN_REVIEW_RUNTIME_IDENTITIES
|
||||
|
||||
|
||||
def test_sdk_uses_configured_prefix_and_scope_headers():
|
||||
seen = []
|
||||
def transport(request):
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
"""Current source Prompt identities for the multi-scene release baseline.
|
||||
|
||||
This catalog describes code-owned source bytes. It is deliberately separate
|
||||
from tenant Prompt rows: resolving an identity here never means that the row is
|
||||
published, enabled, evaluated by a real model, or approved for release.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from copy import deepcopy
|
||||
|
||||
from .contract_drafting import draft_digest
|
||||
from .review_routing import REVIEW_MODULE_KEYS, SCENE_REVIEW_ROUTES
|
||||
|
||||
|
||||
BASELINE_SCHEMA_VERSION = "prompt-source-baseline.v1"
|
||||
BASELINE_VERSION = "rc05-current-source-20261006"
|
||||
|
||||
DRAFTING_MODULES = (
|
||||
{"key": "draft_intake", "version": "tp06-22-20260909-1", "content_sha256": "64519f6e5e95d9f1408e8b3db4feb818dec7cb0386de7983f591947f1cb53983"},
|
||||
{"key": "template_match", "version": "tp05-20260828-1", "content_sha256": "68a7e57f142da85e4e1878f018f34e95a778b72995f1452a4e5778b753e0e560"},
|
||||
{"key": "missing_facts_questions", "version": "tp06-18-20260909-1", "content_sha256": "5643ab97bf661d546a99a3e609db390d574cb0e2940a6ace984e2a08e4ab83aa"},
|
||||
{"key": "clause_assembly_plan", "version": "tp06-22-20260909-1", "content_sha256": "eddb6c1685b33031d1f781220bbcc631e756f6ed8b8b30a706fc9af482a8c827"},
|
||||
{"key": "draft_consistency_check", "version": "tp06-20260830-2", "content_sha256": "005e2308d6d041968a9c1c5f2bbc20afa91feadea9f0ad0bdb48d8ed97fb5b52"},
|
||||
{"key": "candidate_revision_explanation", "version": "tp06-23-20260909-1", "content_sha256": "a204d4be6de9fde497a7b4fd0dbad81758a6981220480eb46a82753b53c79f3a"},
|
||||
)
|
||||
|
||||
REVIEW_MODULES = (
|
||||
{"key": "contract_review_trade_procurement", "version": "tp05-2-20260828-1", "content_sha256": "412b23ab4860866c05281e721d0c77090ffa302d1ecfe049ab5da627b2f92d6a"},
|
||||
{"key": "contract_review_services", "version": "tp05-2-20260828-1", "content_sha256": "a1472d3819dd19993872ff8b9d5e1206ba0c8a60a4a31a07fbe5c3957a73dc5e"},
|
||||
{"key": "contract_review_transport_warehousing", "version": "tp05-2-20260828-1", "content_sha256": "96d74ec8b5ac8239d63fba0bb46ea59afa25774d6f04620f89d321b6d5acf91a"},
|
||||
{"key": "contract_review_manufacturing", "version": "tp05-2-20260828-1", "content_sha256": "dc45ec97b7a924dd9b0b59da8b142399e390f2d69ffc20dce248a2f56576b5cb"},
|
||||
{"key": "contract_review_construction", "version": "tp05-2-20260828-1", "content_sha256": "c4772d3bcc4de00d0d932cda5c55e4a22478838fb20c1e6cf846ed0963ec9eba"},
|
||||
{"key": "contract_review_marketing", "version": "tp05-2-20260828-1", "content_sha256": "0bffc870ff1f60f0f0d4efbff34771e62481b3a38e65fecabd4e69f80f6c8ed6"},
|
||||
{"key": "contract_review_consumer_services", "version": "tp05-2-20260828-1", "content_sha256": "687976935fd5bf237deb3da1a5e6ba05226226ce54fec5079641088efd9692c1"},
|
||||
{"key": "contract_review_notices_amendments", "version": "tp05-2-20260828-1", "content_sha256": "c919316f36cc36c84b0e559a5b80587ae141d84074f7daa403e0eb1173fd0bf5"},
|
||||
)
|
||||
|
||||
DRAFT_RUNTIME_IDENTITY = {
|
||||
"name": "contract_drafting_v1",
|
||||
"version": "tp06-23-20260909-1",
|
||||
"content_sha256": "ab823d6efcf6cdb9cbc7741620bca7cb09f4c462b3a33295308d8cd7b0b82255",
|
||||
"source_kind": "composed_drafting_modules",
|
||||
}
|
||||
|
||||
# The eight newly authored review modules resolve to composed Prompt assets.
|
||||
# A route has one release-target identity even when older evaluation evidence is
|
||||
# still bound to an earlier profile. That evidence is not rewritten here.
|
||||
NEW_REVIEW_RUNTIME_IDENTITIES = {
|
||||
"contract_review_trade_procurement": {"name": "contract_review_trade_procurement", "version": "tp06-2-20260831-1", "content_sha256": "71956dde40698f3a7488da8f8b8214db8ca7bc8ebeddda6c4b965d74185bb457", "source_kind": "new_review_module_composite"},
|
||||
"contract_review_services": {"name": "contract_review_services", "version": "tp06-2-20260831-1", "content_sha256": "21ce149263a42f8cdd76f11800aedb66f5ee8e4b43cd5e41696092d96b31114d", "source_kind": "new_review_module_composite"},
|
||||
"contract_review_transport_warehousing": {"name": "contract_review_transport_warehousing", "version": "tp06-8-20260909-1", "content_sha256": "950d6a02476bf37e6c2a733710bdfdfefe474b44743bd9d7448bbf95c01c6b14", "source_kind": "new_review_module_composite"},
|
||||
"contract_review_manufacturing": {"name": "contract_review_manufacturing", "version": "tp06-8-20260909-1", "content_sha256": "97ba767d8d47166697d7c0790fa4e17cb7b7c72950e149d6bbf4bda9bfb3d5c4", "source_kind": "new_review_module_composite"},
|
||||
"contract_review_construction": {"name": "contract_review_construction", "version": "tp06-2-20260831-1", "content_sha256": "444552139eeec9b210f652c132ffb1ea42697e28be0c8fe33e2b46a663238cbc", "source_kind": "new_review_module_composite"},
|
||||
"contract_review_marketing": {"name": "contract_review_marketing", "version": "tp06-8-20260909-1", "content_sha256": "261fb010dcab3d20f6f0751df941b3be11a1db06be5e4f2e43122ea2e538fcf5", "source_kind": "new_review_module_composite"},
|
||||
"contract_review_consumer_services": {"name": "contract_review_consumer_services", "version": "tp06-2-20260831-1", "content_sha256": "c42da047fae64cf91cf544f3c694ef3ff99226dd8af0d4465b71f272d3b3f603", "source_kind": "new_review_module_composite"},
|
||||
"contract_review_notices_amendments": {"name": "contract_review_notices_amendments", "version": "tp06-8-20260909-1", "content_sha256": "19fbfc866d6a67c5da9aa18e55fcd239b3d8d9915a5fa1076e8befb64c64d6e3", "source_kind": "new_review_module_composite"},
|
||||
}
|
||||
|
||||
# These five routes predate the eight new review modules. Their release-target
|
||||
# identities come from the packaged phase-15 assets and remain separate runtime
|
||||
# dependencies; they are not counted as extra modules in the 14-module batch.
|
||||
DOMAIN_REVIEW_RUNTIME_IDENTITIES = {
|
||||
"contract_review_tech_saas_data_compliance": {"name": "contract_review_tech_saas_data_compliance", "version": "v1-phase15", "content_sha256": "0075ab909b0622fd0e50817b2f4cfc6498e4ddea740c6f7a185cbc498f05b5c1", "source_kind": "existing_domain_prompt_asset"},
|
||||
"contract_review_real_estate_property": {"name": "contract_review_real_estate_property", "version": "v1-phase15", "content_sha256": "87cdcc6ed723b6a1f1574a15da84e2e0b61d51e9fde47b190ecd1ad4aeb4eabb", "source_kind": "existing_domain_prompt_asset"},
|
||||
"contract_review_channel_franchise": {"name": "contract_review_channel_franchise", "version": "v1-phase15", "content_sha256": "523aba6a76a6a3ec0e5f1c38d92287462f4286ceb5d3a588bcd7496c37022859", "source_kind": "existing_domain_prompt_asset"},
|
||||
"contract_review_employment_hr": {"name": "contract_review_employment_hr", "version": "v1-phase15", "content_sha256": "c30e1424723565d76b1d4b5708593efc0bd67d74bf6c7ae56940d9af4dd0d1d7", "source_kind": "existing_domain_prompt_asset"},
|
||||
"contract_review_investment_finance": {"name": "contract_review_investment_finance", "version": "v1-phase15", "content_sha256": "37bb23f09cada7b04746b77a5cc6898719fe257a97b6de4a6e2ca22ca2f69bc6", "source_kind": "existing_domain_prompt_asset"},
|
||||
}
|
||||
|
||||
REVIEW_RUNTIME_IDENTITIES = {
|
||||
**NEW_REVIEW_RUNTIME_IDENTITIES,
|
||||
**DOMAIN_REVIEW_RUNTIME_IDENTITIES,
|
||||
}
|
||||
|
||||
|
||||
def _payload() -> dict:
|
||||
scenes = {}
|
||||
drafting_keys = [item["key"] for item in DRAFTING_MODULES]
|
||||
review_modules = {item["key"]: item for item in REVIEW_MODULES}
|
||||
for template_id, (contract_type, route) in sorted(SCENE_REVIEW_ROUTES.items()):
|
||||
target = REVIEW_RUNTIME_IDENTITIES[route]
|
||||
scenes[template_id] = {
|
||||
"template_id": template_id,
|
||||
"contract_type": contract_type,
|
||||
"review_route": route,
|
||||
"review_route_kind": (
|
||||
"new_review_module"
|
||||
if route in NEW_REVIEW_RUNTIME_IDENTITIES
|
||||
else "existing_domain_prompt"
|
||||
),
|
||||
"draft_prompt": deepcopy(DRAFT_RUNTIME_IDENTITY),
|
||||
"review_prompt": deepcopy(target),
|
||||
"drafting_module_keys": list(drafting_keys),
|
||||
"review_module_keys": [route] if route in review_modules else [],
|
||||
"source_identity_resolved": True,
|
||||
"technical_drafting_coverage_represented": True,
|
||||
"technical_review_coverage_represented": True,
|
||||
"published_runtime_verified": False,
|
||||
"real_model_result": False,
|
||||
"authorizes_publication": False,
|
||||
}
|
||||
return {
|
||||
"schema_version": BASELINE_SCHEMA_VERSION,
|
||||
"baseline_version": BASELINE_VERSION,
|
||||
"module_baseline": {
|
||||
"drafting": deepcopy(list(DRAFTING_MODULES)),
|
||||
"review": deepcopy(list(REVIEW_MODULES)),
|
||||
"module_count": len(DRAFTING_MODULES) + len(REVIEW_MODULES),
|
||||
},
|
||||
"runtime_targets": {
|
||||
"drafting": deepcopy(DRAFT_RUNTIME_IDENTITY),
|
||||
"new_review_modules": deepcopy(NEW_REVIEW_RUNTIME_IDENTITIES),
|
||||
"existing_domain_prompts": deepcopy(DOMAIN_REVIEW_RUNTIME_IDENTITIES),
|
||||
"identity_count": 1 + len(REVIEW_RUNTIME_IDENTITIES),
|
||||
},
|
||||
"scenes": scenes,
|
||||
"summary": {
|
||||
"scenes": len(scenes),
|
||||
"source_identities_resolved": sum(item["source_identity_resolved"] for item in scenes.values()),
|
||||
"technical_drafting_coverage_represented": sum(item["technical_drafting_coverage_represented"] for item in scenes.values()),
|
||||
"technical_review_coverage_represented": sum(item["technical_review_coverage_represented"] for item in scenes.values()),
|
||||
"published_runtime_verified": 0,
|
||||
"real_model_results": 0,
|
||||
"authorizes_publication": 0,
|
||||
},
|
||||
"not_runtime_publication_evidence": True,
|
||||
"not_legal_approval": True,
|
||||
"authorizes_publication": False,
|
||||
}
|
||||
|
||||
|
||||
def _validated_payload() -> dict:
|
||||
payload = _payload()
|
||||
module_baseline = payload["module_baseline"]
|
||||
if (module_baseline["module_count"] != 14
|
||||
or tuple(item["key"] for item in module_baseline["drafting"]) != tuple(
|
||||
item["key"] for item in DRAFTING_MODULES)
|
||||
or tuple(item["key"] for item in module_baseline["review"]) != REVIEW_MODULE_KEYS
|
||||
or len(payload["runtime_targets"]["new_review_modules"]) != 8
|
||||
or len(payload["runtime_targets"]["existing_domain_prompts"]) != 5
|
||||
or payload["runtime_targets"]["identity_count"] != 14
|
||||
or set(payload["scenes"]) != set(SCENE_REVIEW_ROUTES)
|
||||
or payload["summary"]["source_identities_resolved"] != 30
|
||||
or payload["summary"]["technical_drafting_coverage_represented"] != 30
|
||||
or payload["summary"]["technical_review_coverage_represented"] != 30):
|
||||
raise RuntimeError("Prompt source baseline is incomplete or inconsistent")
|
||||
return payload
|
||||
|
||||
|
||||
def prompt_source_baseline() -> dict:
|
||||
"""Return a defensive copy of the 14-module/14-runtime source baseline."""
|
||||
payload = _validated_payload()
|
||||
return {**deepcopy(payload), "manifest_sha256": draft_digest(payload)}
|
||||
|
||||
|
||||
def scene_prompt_source_identity(template_id: str) -> dict | None:
|
||||
"""Resolve one scene without consulting or mutating a runtime Prompt store."""
|
||||
if not isinstance(template_id, str):
|
||||
return None
|
||||
baseline = prompt_source_baseline()
|
||||
scene = baseline["scenes"].get(template_id)
|
||||
if scene is None:
|
||||
return None
|
||||
return {
|
||||
**deepcopy(scene),
|
||||
"baseline_version": baseline["baseline_version"],
|
||||
"baseline_manifest_sha256": baseline["manifest_sha256"],
|
||||
"not_runtime_publication_evidence": True,
|
||||
"not_legal_approval": True,
|
||||
"authorizes_publication": False,
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
from copy import deepcopy
|
||||
|
||||
from yuqei_sdk.prompt_source_baseline import (
|
||||
DOMAIN_REVIEW_RUNTIME_IDENTITIES,
|
||||
NEW_REVIEW_RUNTIME_IDENTITIES,
|
||||
prompt_source_baseline,
|
||||
scene_prompt_source_identity,
|
||||
)
|
||||
from yuqei_sdk.review_routing import SCENE_REVIEW_ROUTES
|
||||
|
||||
|
||||
def test_current_source_baseline_resolves_all_scene_identities_without_claiming_publication():
|
||||
baseline = prompt_source_baseline()
|
||||
|
||||
assert baseline["module_baseline"]["module_count"] == 14
|
||||
assert len(baseline["module_baseline"]["drafting"]) == 6
|
||||
assert len(baseline["module_baseline"]["review"]) == 8
|
||||
assert len(NEW_REVIEW_RUNTIME_IDENTITIES) == 8
|
||||
assert len(DOMAIN_REVIEW_RUNTIME_IDENTITIES) == 5
|
||||
assert baseline["runtime_targets"]["identity_count"] == 14
|
||||
assert set(baseline["scenes"]) == set(SCENE_REVIEW_ROUTES)
|
||||
assert baseline["summary"] == {
|
||||
"scenes": 30,
|
||||
"source_identities_resolved": 30,
|
||||
"technical_drafting_coverage_represented": 30,
|
||||
"technical_review_coverage_represented": 30,
|
||||
"published_runtime_verified": 0,
|
||||
"real_model_results": 0,
|
||||
"authorizes_publication": 0,
|
||||
}
|
||||
assert baseline["not_runtime_publication_evidence"] is True
|
||||
assert baseline["not_legal_approval"] is True
|
||||
assert baseline["authorizes_publication"] is False
|
||||
|
||||
|
||||
def test_scene_source_identity_keeps_new_modules_and_existing_domain_dependencies_distinct():
|
||||
new_module = scene_prompt_source_identity("TPL-CONSTRUCTION-PROJECT-001")
|
||||
domain = scene_prompt_source_identity("TPL-TECHNICAL-CONSULTING-001")
|
||||
|
||||
assert new_module["review_route_kind"] == "new_review_module"
|
||||
assert new_module["review_module_keys"] == ["contract_review_construction"]
|
||||
assert domain["review_route_kind"] == "existing_domain_prompt"
|
||||
assert domain["review_module_keys"] == []
|
||||
assert len(new_module["drafting_module_keys"]) == len(domain["drafting_module_keys"]) == 6
|
||||
assert new_module["published_runtime_verified"] is False
|
||||
assert domain["published_runtime_verified"] is False
|
||||
|
||||
|
||||
def test_source_identity_lookups_are_defensive_and_unknown_scenes_do_not_resolve():
|
||||
first = scene_prompt_source_identity("TPL-TRANSPORT-ORDINARY-001")
|
||||
original = deepcopy(first)
|
||||
first["drafting_module_keys"].clear()
|
||||
second = scene_prompt_source_identity("TPL-TRANSPORT-ORDINARY-001")
|
||||
|
||||
assert second == original
|
||||
assert scene_prompt_source_identity("unknown") is None
|
||||
@@ -105,6 +105,23 @@ def register_drafting_routes(app: FastAPI, *, settings, repository, evaluation=F
|
||||
template, _ = store.evaluation_source(item["template_id"])
|
||||
holdout_scenes.append(scene_holdout_evidence(
|
||||
item["template_id"], current_template_content_sha256=template.content_sha256))
|
||||
from yuqei_contract_api.current_source_holdout_plan import (
|
||||
load_current_source_holdout_plan, scene_current_source_holdout_binding,
|
||||
)
|
||||
source_plan = load_current_source_holdout_plan()
|
||||
source_bindings = []
|
||||
for item in source_plan["scenes"]:
|
||||
template, _ = store.evaluation_source(item["template_id"])
|
||||
source_bindings.append(scene_current_source_holdout_binding(
|
||||
item["template_id"], current_template_content_sha256=template.content_sha256))
|
||||
source_plan_summary = {
|
||||
**source_plan["summary"],
|
||||
"source_bindings_current": sum(
|
||||
item is not None and item["source_binding_current"] for item in source_bindings),
|
||||
"source_bindings_missing": sum(item is None for item in source_bindings),
|
||||
"source_bindings_stale": sum(
|
||||
item is not None and item["source_binding_stale"] for item in source_bindings),
|
||||
}
|
||||
return {
|
||||
"schema_version": registry["schema_version"],
|
||||
"registry_id": registry["registry_id"],
|
||||
@@ -128,6 +145,24 @@ def register_drafting_routes(app: FastAPI, *, settings, repository, evaluation=F
|
||||
"not_legal_approval": True,
|
||||
"authorizes_publication": False,
|
||||
},
|
||||
"current_source_holdout_plan": {
|
||||
"schema_version": source_plan["schema_version"],
|
||||
"plan_id": source_plan["plan_id"],
|
||||
"manifest_sha256": source_plan["manifest_sha256"],
|
||||
"prompt_source_baseline_manifest_sha256": source_plan[
|
||||
"prompt_source_baseline_manifest_sha256"],
|
||||
"historical_holdout_registry_manifest_sha256": source_plan[
|
||||
"historical_holdout_registry_manifest_sha256"],
|
||||
"summary": source_plan_summary,
|
||||
"scenes": source_bindings,
|
||||
"fixture_plan_only": True,
|
||||
"model_calls": 0,
|
||||
"live_database_writes": 0,
|
||||
"not_model_evidence": True,
|
||||
"not_runtime_publication_evidence": True,
|
||||
"not_legal_approval": True,
|
||||
"authorizes_publication": False,
|
||||
},
|
||||
"model_calls": 0,
|
||||
"live_database_writes": 0,
|
||||
"not_legal_approval": True,
|
||||
|
||||
+148
@@ -0,0 +1,148 @@
|
||||
"""Current-source holdout bindings, distinct from historical model results.
|
||||
|
||||
The plan closes fixture/identity registry gaps for every scene. It records no
|
||||
model outcome and cannot replace the immutable independent holdout registry.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from copy import deepcopy
|
||||
from functools import lru_cache
|
||||
|
||||
from yuqei_sdk.contract_drafting import draft_digest
|
||||
from yuqei_sdk.prompt_source_baseline import prompt_source_baseline
|
||||
from yuqei_sdk.review_routing import SCENE_REVIEW_ROUTES
|
||||
from yuqei_sdk.scene_review_expectations import REVIEW_EXPECTATION_POLICY
|
||||
|
||||
from .contract_templates import get_contract_template
|
||||
from .holdout_scene_evidence_registry import load_code_owned_holdout_registry
|
||||
from .template_evaluation_evidence import VARIANTS, scene_model_cases
|
||||
|
||||
|
||||
PLAN_SCHEMA_VERSION = "current-source-holdout-plan.v1"
|
||||
PLAN_ID = "rc05-current-source-scene-holdout-20261006"
|
||||
|
||||
|
||||
def _plan_payload() -> dict:
|
||||
baseline = prompt_source_baseline()
|
||||
historical = load_code_owned_holdout_registry()
|
||||
historical_by_template = {item["template_id"]: item for item in historical["scenes"]}
|
||||
scenes = []
|
||||
for template_id in sorted(SCENE_REVIEW_ROUTES):
|
||||
template = get_contract_template(template_id)
|
||||
identity = baseline["scenes"][template_id]
|
||||
prior = historical_by_template.get(template_id)
|
||||
cases = scene_model_cases(template_id)
|
||||
scenes.append({
|
||||
"template_id": template_id,
|
||||
"contract_type": template.contract_type,
|
||||
"template_content_sha256": template.content_sha256,
|
||||
"prompt_source_baseline_manifest_sha256": baseline["manifest_sha256"],
|
||||
"draft_prompt": deepcopy(identity["draft_prompt"]),
|
||||
"review_prompt": deepcopy(identity["review_prompt"]),
|
||||
"fixtures": [{
|
||||
"case_id": case["id"],
|
||||
"variant": case["variant"],
|
||||
"input_sha256": draft_digest(case["values"]),
|
||||
"expected_status": case["expected_status"],
|
||||
"expected_missing": list(case["expected_missing"]),
|
||||
"review_expectation_policy": REVIEW_EXPECTATION_POLICY,
|
||||
"review_expected_behavior": (
|
||||
"identify_missing_fact"
|
||||
if case["variant"] == "missing-key-fact"
|
||||
else "complete_full_review"
|
||||
),
|
||||
} for case in cases],
|
||||
"technical_drafting_coverage_represented": len(cases) == len(VARIANTS),
|
||||
"technical_review_coverage_represented": len(cases) == len(VARIANTS),
|
||||
"historical_result_reference": ({
|
||||
"registry_manifest_sha256": historical["manifest_sha256"],
|
||||
"source_registry_sha256": historical["source_registry_sha256"],
|
||||
"evaluated_template_content_sha256": prior["evaluated_template_content_sha256"],
|
||||
} if prior else None),
|
||||
"historical_results_rebound": False,
|
||||
"real_model_result": False,
|
||||
"published_runtime_verified": False,
|
||||
"legal_source_approval": False,
|
||||
"authorizes_publication": False,
|
||||
})
|
||||
return {
|
||||
"schema_version": PLAN_SCHEMA_VERSION,
|
||||
"plan_id": PLAN_ID,
|
||||
"prompt_source_baseline_manifest_sha256": baseline["manifest_sha256"],
|
||||
"historical_holdout_registry_manifest_sha256": historical["manifest_sha256"],
|
||||
"variants": list(VARIANTS),
|
||||
"scenes": scenes,
|
||||
"summary": {
|
||||
"scenes": len(scenes),
|
||||
"fixture_cases": sum(len(item["fixtures"]) for item in scenes),
|
||||
"source_bindings_current": len(scenes),
|
||||
"source_bindings_missing": 0,
|
||||
"source_bindings_stale": 0,
|
||||
"technical_drafting_coverage_represented": sum(
|
||||
item["technical_drafting_coverage_represented"] for item in scenes),
|
||||
"technical_review_coverage_represented": sum(
|
||||
item["technical_review_coverage_represented"] for item in scenes),
|
||||
"historical_model_results_rebound": 0,
|
||||
"real_model_results": 0,
|
||||
"published_runtime_verified": 0,
|
||||
"authorizes_publication": 0,
|
||||
},
|
||||
"fixture_plan_only": True,
|
||||
"not_model_evidence": True,
|
||||
"not_runtime_publication_evidence": True,
|
||||
"not_legal_approval": True,
|
||||
"authorizes_publication": False,
|
||||
}
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _cached_current_source_holdout_plan() -> dict:
|
||||
payload = _plan_payload()
|
||||
summary = payload["summary"]
|
||||
if (summary["scenes"] != 30
|
||||
or summary["fixture_cases"] != 30 * len(VARIANTS)
|
||||
or summary["source_bindings_current"] != 30
|
||||
or summary["technical_drafting_coverage_represented"] != 30
|
||||
or summary["technical_review_coverage_represented"] != 30
|
||||
or any(tuple(item["variant"] for item in scene["fixtures"]) != VARIANTS
|
||||
for scene in payload["scenes"])):
|
||||
raise RuntimeError("Current-source holdout plan is incomplete or inconsistent")
|
||||
return {**payload, "manifest_sha256": draft_digest(payload)}
|
||||
|
||||
|
||||
def load_current_source_holdout_plan() -> dict:
|
||||
"""Build a deterministic plan from the current template and Prompt sources."""
|
||||
return deepcopy(_cached_current_source_holdout_plan())
|
||||
|
||||
|
||||
def scene_current_source_holdout_binding(
|
||||
template_id: str, *, current_template_content_sha256: str,
|
||||
) -> dict | None:
|
||||
plan = load_current_source_holdout_plan()
|
||||
source = next((item for item in plan["scenes"] if item["template_id"] == template_id), None)
|
||||
if source is None:
|
||||
return None
|
||||
current_match = source["template_content_sha256"] == current_template_content_sha256
|
||||
payload = {
|
||||
**deepcopy(source),
|
||||
"plan_id": plan["plan_id"],
|
||||
"plan_manifest_sha256": plan["manifest_sha256"],
|
||||
"current_template_content_sha256": current_template_content_sha256,
|
||||
"current_template_match": current_match,
|
||||
"source_binding_current": current_match,
|
||||
"source_binding_missing": False,
|
||||
"source_binding_stale": not current_match,
|
||||
}
|
||||
return {**payload, "binding_sha256": draft_digest(payload)}
|
||||
|
||||
|
||||
def verify_scene_current_source_holdout_binding(
|
||||
binding: dict, *, current_template_content_sha256: str,
|
||||
) -> bool:
|
||||
if not isinstance(binding, dict) or not isinstance(binding.get("template_id"), str):
|
||||
return False
|
||||
expected = scene_current_source_holdout_binding(
|
||||
binding["template_id"],
|
||||
current_template_content_sha256=current_template_content_sha256,
|
||||
)
|
||||
return expected is not None and binding == expected
|
||||
+3
@@ -674,6 +674,9 @@ class DraftingWorkspaceStore:
|
||||
from .holdout_scene_evidence_registry import merge_holdout_scene_evidence, scene_holdout_evidence
|
||||
summary = merge_holdout_scene_evidence(summary, scene_holdout_evidence(
|
||||
template_id, current_template_content_sha256=current.content_sha256))
|
||||
from .current_source_holdout_plan import scene_current_source_holdout_binding
|
||||
summary["current_source_holdout_binding"] = scene_current_source_holdout_binding(
|
||||
template_id, current_template_content_sha256=current.content_sha256)
|
||||
runtime_reader = getattr(self.repository, "template_runtime_manifest_reader", None)
|
||||
runtime = runtime_reader(current, tenant) if runtime_reader else None
|
||||
from .template_release_readiness import assess_release_readiness
|
||||
|
||||
+45
@@ -3,6 +3,7 @@ from datetime import UTC, datetime
|
||||
import re
|
||||
|
||||
from yuqei_sdk.contract_drafting import draft_digest
|
||||
from yuqei_sdk.prompt_source_baseline import scene_prompt_source_identity
|
||||
from yuqei_sdk.review_routing import REVIEW_ROUTING_VERSION, scene_review_evaluation_profile, scene_review_route
|
||||
from yuqei_sdk.scene_review_expectations import REVIEW_EXPECTATION_POLICY
|
||||
|
||||
@@ -51,6 +52,12 @@ def assess_release_readiness(template, summary, runtime, *, tenant, now=None):
|
||||
issue("SCENE_HOLDOUT_TEMPLATE_STALE", "独立留出评测未绑定当前模板正文哈希。")
|
||||
elif holdout.get("missing_or_failed_variants"):
|
||||
issue("SCENE_HOLDOUT_DRAFT_COVERAGE", "当前模板仍有独立留出起草变体未通过；开发样本不能覆盖该失败。")
|
||||
source_holdout = summary.get("current_source_holdout_binding")
|
||||
source_holdout_valid = False
|
||||
if source_holdout is not None:
|
||||
from .current_source_holdout_plan import verify_scene_current_source_holdout_binding
|
||||
source_holdout_valid = verify_scene_current_source_holdout_binding(
|
||||
source_holdout, current_template_content_sha256=template.content_sha256)
|
||||
runtime_valid = isinstance(runtime, dict)
|
||||
if runtime_valid:
|
||||
try:
|
||||
@@ -85,11 +92,29 @@ def assess_release_readiness(template, summary, runtime, *, tenant, now=None):
|
||||
if runtime and not gate_verification["valid"]:
|
||||
issue("SCENE_REVIEW_EVALUATION_GATE_INVALID", "代码托管的专项审查证据与当前场景、Prompt 或固定登记簿不一致。")
|
||||
prompts_aligned = bool(runtime)
|
||||
expected_source_identity = scene_prompt_source_identity(template.id)
|
||||
runtime_source_identity_present = bool(runtime and "source_prompt_identity" in runtime)
|
||||
runtime_source_identity_valid = False
|
||||
if runtime:
|
||||
if runtime_source_identity_present:
|
||||
runtime_source_identity_valid = bool(
|
||||
runtime.get("source_prompt_identity_resolved") is True
|
||||
and runtime.get("source_prompt_identity") == expected_source_identity
|
||||
and all(draft_prompt.get(key) == expected_source_identity["draft_prompt"].get(key)
|
||||
for key in ("name", "version", "content_sha256"))
|
||||
and all(review_prompt.get(key) == expected_source_identity["review_prompt"].get(key)
|
||||
for key in ("name", "version", "content_sha256"))
|
||||
and runtime.get("published_prompt_alignment_verified") is True
|
||||
)
|
||||
if not runtime_source_identity_present:
|
||||
issue("SCENE_RUNTIME_SOURCE_IDENTITY_MISSING", "运行快照未携带当前源码 Prompt 身份，不能沿用旧版对齐规则放行。")
|
||||
elif not runtime_source_identity_valid:
|
||||
issue("SCENE_RUNTIME_SOURCE_IDENTITY_MISMATCH", "运行快照中的源码 Prompt 身份或实际发布内容与当前源码基线不一致。")
|
||||
prompts_aligned = (runtime.get("provider_mode") == "codex_cli" and runtime.get("model") == "gpt-5.6-sol"
|
||||
and draft_prompt.get("name") == "contract_drafting_v1" and draft_prompt.get("status") == "published"
|
||||
and draft_prompt.get("enabled") is True and review_prompt.get("status") == "published"
|
||||
and review_prompt.get("enabled") is True and review_prompt.get("name") == runtime["requested_review_route"]
|
||||
and runtime_source_identity_valid
|
||||
and all(review_prompt.get(key) and review_prompt[key] == trial_prompt.get(key)
|
||||
for key in ("name", "version", "content_sha256")))
|
||||
if not prompts_aligned:
|
||||
@@ -161,9 +186,29 @@ def assess_release_readiness(template, summary, runtime, *, tenant, now=None):
|
||||
return {"schema_version": "scene-release-readiness.v1", "issues": issues,
|
||||
"source_applicability_verified": source_verified, "source_references": references,
|
||||
"published_prompt_alignment_verified": prompts_aligned,
|
||||
"source_prompt_identity_alignment": {
|
||||
"manifest_field_present": runtime_source_identity_present,
|
||||
"source_identity_resolved": expected_source_identity is not None,
|
||||
"runtime_manifest_matches_current_source": runtime_source_identity_valid,
|
||||
"expected": expected_source_identity,
|
||||
"authorizes_publication": False,
|
||||
},
|
||||
"missing_draft_variants": missing_draft, "missing_review_variants": missing_review,
|
||||
"offline_registry_evidence": offline if offline_valid else None,
|
||||
"holdout_registry_evidence": holdout if holdout_valid else None,
|
||||
"current_source_holdout_binding": {
|
||||
"registry_present": source_holdout is not None,
|
||||
"registry_valid": source_holdout_valid,
|
||||
"current_template_match": bool(
|
||||
source_holdout_valid and source_holdout.get("current_template_match")),
|
||||
"source_binding_current": bool(
|
||||
source_holdout_valid and source_holdout.get("source_binding_current")),
|
||||
"fixture_plan_only": True,
|
||||
"historical_model_results_rebound": False,
|
||||
"real_model_result": False,
|
||||
"published_runtime_verified": False,
|
||||
"authorizes_publication": False,
|
||||
},
|
||||
"missing_holdout_draft_variants": (holdout.get("missing_or_failed_variants")
|
||||
if holdout_valid and holdout else list(VARIANTS)),
|
||||
"offline_prompt_alignment": {
|
||||
|
||||
+133
@@ -0,0 +1,133 @@
|
||||
from copy import deepcopy
|
||||
|
||||
from yuqei_sdk.prompt_source_baseline import prompt_source_baseline
|
||||
from yuqei_sdk.review_routing import SCENE_REVIEW_ROUTES, scene_review_route
|
||||
from yuqei_sdk.scene_review_expectations import REVIEW_EXPECTATION_POLICY
|
||||
|
||||
from yuqei_contract_api.contract_templates import get_contract_template
|
||||
from yuqei_contract_api.current_source_holdout_plan import (
|
||||
load_current_source_holdout_plan,
|
||||
scene_current_source_holdout_binding,
|
||||
verify_scene_current_source_holdout_binding,
|
||||
)
|
||||
from yuqei_contract_api.drafting_workspace import DraftingWorkspaceStore
|
||||
from yuqei_contract_api.holdout_scene_evidence_registry import load_code_owned_holdout_registry
|
||||
from yuqei_contract_api.in_memory_repository import InMemoryContractRepository
|
||||
|
||||
|
||||
def test_current_source_holdout_plan_has_all_scene_fixtures_without_rebinding_results():
|
||||
baseline = prompt_source_baseline()
|
||||
historical = load_code_owned_holdout_registry()
|
||||
plan = load_current_source_holdout_plan()
|
||||
|
||||
assert plan["prompt_source_baseline_manifest_sha256"] == baseline["manifest_sha256"]
|
||||
assert plan["historical_holdout_registry_manifest_sha256"] == historical["manifest_sha256"]
|
||||
assert {item["template_id"] for item in plan["scenes"]} == set(SCENE_REVIEW_ROUTES)
|
||||
assert plan["summary"] == {
|
||||
"scenes": 30,
|
||||
"fixture_cases": 90,
|
||||
"source_bindings_current": 30,
|
||||
"source_bindings_missing": 0,
|
||||
"source_bindings_stale": 0,
|
||||
"technical_drafting_coverage_represented": 30,
|
||||
"technical_review_coverage_represented": 30,
|
||||
"historical_model_results_rebound": 0,
|
||||
"real_model_results": 0,
|
||||
"published_runtime_verified": 0,
|
||||
"authorizes_publication": 0,
|
||||
}
|
||||
assert plan["fixture_plan_only"] is True
|
||||
assert plan["not_model_evidence"] is True
|
||||
assert plan["not_runtime_publication_evidence"] is True
|
||||
assert plan["not_legal_approval"] is True
|
||||
assert plan["authorizes_publication"] is False
|
||||
|
||||
historical_by_template = {item["template_id"]: item for item in historical["scenes"]}
|
||||
for scene in plan["scenes"]:
|
||||
source_template = get_contract_template(scene["template_id"])
|
||||
expected = baseline["scenes"][scene["template_id"]]
|
||||
assert scene["contract_type"] == source_template.contract_type
|
||||
assert expected["review_route"] == scene_review_route(
|
||||
source_template.contract_type, source_template.id)
|
||||
assert scene["prompt_source_baseline_manifest_sha256"] == baseline["manifest_sha256"]
|
||||
assert scene["draft_prompt"] == expected["draft_prompt"]
|
||||
assert scene["review_prompt"] == expected["review_prompt"]
|
||||
assert [item["variant"] for item in scene["fixtures"]] == list(plan["variants"])
|
||||
assert {item["review_expectation_policy"] for item in scene["fixtures"]} == {
|
||||
REVIEW_EXPECTATION_POLICY}
|
||||
assert {item["review_expected_behavior"] for item in scene["fixtures"]} == {
|
||||
"identify_missing_fact", "complete_full_review"}
|
||||
assert scene["historical_results_rebound"] is False
|
||||
if scene["template_id"] in historical_by_template:
|
||||
prior = historical_by_template[scene["template_id"]]
|
||||
assert scene["historical_result_reference"] == {
|
||||
"registry_manifest_sha256": historical["manifest_sha256"],
|
||||
"source_registry_sha256": historical["source_registry_sha256"],
|
||||
"evaluated_template_content_sha256": prior["evaluated_template_content_sha256"],
|
||||
}
|
||||
else:
|
||||
assert scene["historical_result_reference"] is None
|
||||
|
||||
|
||||
def test_scene_current_source_binding_detects_template_or_registry_tampering():
|
||||
template_id = "TPL-COMMISSION-MARKETING-001"
|
||||
current_hash = get_contract_template(template_id).content_sha256
|
||||
binding = scene_current_source_holdout_binding(
|
||||
template_id,
|
||||
current_template_content_sha256=current_hash,
|
||||
)
|
||||
|
||||
assert binding["current_template_match"] is True
|
||||
assert binding["source_binding_current"] is True
|
||||
assert binding["source_binding_missing"] is False
|
||||
assert binding["source_binding_stale"] is False
|
||||
assert verify_scene_current_source_holdout_binding(
|
||||
binding,
|
||||
current_template_content_sha256=current_hash,
|
||||
) is True
|
||||
|
||||
stale = scene_current_source_holdout_binding(
|
||||
template_id,
|
||||
current_template_content_sha256="0" * 64,
|
||||
)
|
||||
assert stale["source_binding_current"] is False
|
||||
assert stale["source_binding_stale"] is True
|
||||
assert verify_scene_current_source_holdout_binding(
|
||||
stale,
|
||||
current_template_content_sha256="0" * 64,
|
||||
) is True
|
||||
|
||||
tampered = deepcopy(binding)
|
||||
tampered["review_prompt"]["content_sha256"] = "0" * 64
|
||||
assert verify_scene_current_source_holdout_binding(
|
||||
tampered,
|
||||
current_template_content_sha256=current_hash,
|
||||
) is False
|
||||
|
||||
|
||||
def test_unknown_scene_has_no_current_source_binding():
|
||||
assert scene_current_source_holdout_binding(
|
||||
"unknown",
|
||||
current_template_content_sha256="0" * 64,
|
||||
) is None
|
||||
|
||||
|
||||
def test_evaluation_summary_exposes_current_source_plan_without_upgrading_historical_results():
|
||||
store = DraftingWorkspaceStore(
|
||||
InMemoryContractRepository(contracts=[]), mode="scene_evaluation")
|
||||
summary = store.template_evaluation_summary(
|
||||
"TPL-COMMISSION-MARKETING-001", tenant="default")
|
||||
|
||||
current = summary["current_source_holdout_binding"]
|
||||
assert current["source_binding_current"] is True
|
||||
assert current["real_model_result"] is False
|
||||
assert current["published_runtime_verified"] is False
|
||||
assert current["authorizes_publication"] is False
|
||||
historical = summary["holdout_registry_evidence"]
|
||||
assert historical["current_template_match"] is False
|
||||
assert historical["evaluated_template_content_sha256"] == (
|
||||
current["historical_result_reference"]["evaluated_template_content_sha256"])
|
||||
readiness = summary["release_readiness"]
|
||||
assert readiness["current_source_holdout_binding"]["source_binding_current"] is True
|
||||
assert readiness["current_source_holdout_binding"]["real_model_result"] is False
|
||||
assert readiness["prerequisites_satisfied"] is False
|
||||
+11
@@ -161,6 +161,17 @@ def test_authenticated_registry_endpoint_is_read_only_and_role_guarded(tmp_path)
|
||||
assert len(holdout["scenes"]) == 12
|
||||
assert holdout["model_calls"] == holdout["live_database_writes"] == 0
|
||||
assert not holdout["authorizes_publication"]
|
||||
source_plan = payload["current_source_holdout_plan"]
|
||||
assert source_plan["summary"]["scenes"] == 30
|
||||
assert source_plan["summary"]["fixture_cases"] == 90
|
||||
assert source_plan["summary"]["source_bindings_missing"] == 0
|
||||
assert source_plan["summary"]["source_bindings_stale"] == 0
|
||||
assert source_plan["summary"]["real_model_results"] == 0
|
||||
assert len(source_plan["scenes"]) == 30
|
||||
assert source_plan["fixture_plan_only"] is True
|
||||
assert source_plan["not_model_evidence"] is True
|
||||
assert source_plan["model_calls"] == source_plan["live_database_writes"] == 0
|
||||
assert not source_plan["authorizes_publication"]
|
||||
assert payload["model_calls"] == payload["live_database_writes"] == 0
|
||||
assert not payload["authorizes_publication"]
|
||||
repository = app.state.template_evaluation_store.repository
|
||||
|
||||
+39
-16
@@ -3,6 +3,7 @@ from datetime import UTC, datetime, timedelta
|
||||
|
||||
import pytest
|
||||
from yuqei_sdk.contract_drafting import draft_digest
|
||||
from yuqei_sdk.prompt_source_baseline import scene_prompt_source_identity
|
||||
from yuqei_sdk.review_routing import REVIEW_ROUTING_VERSION, SCENE_REVIEW_ROUTES, scene_review_route
|
||||
from yuqei_sdk.scene_review_expectations import REVIEW_EXPECTATION_POLICY
|
||||
from yuqei_sdk.scene_review_coverage import scene_review_coverage
|
||||
@@ -44,15 +45,16 @@ def evidence():
|
||||
"""Synthetic identities for policy tests, never a recorded model run."""
|
||||
trial = DraftingWorkspaceStore(InMemoryContractRepository(contracts=[]), mode="scene_evaluation")
|
||||
template, _ = trial.evaluation_source(SCENE)
|
||||
review_name = scene_review_route(template.contract_type, SCENE)
|
||||
review = {"name": review_name, "version": PROMPT_VERSION,
|
||||
"content_sha256": PROMPT_CONTENT_SHA256[review_name], "status": "published", "enabled": True}
|
||||
draft = {**scene_review_coverage(SCENE)["cases"][0]["draft_prompt"], "status": "published", "enabled": True}
|
||||
source_identity = scene_prompt_source_identity(SCENE)
|
||||
review = {**source_identity["review_prompt"], "status": "published", "enabled": True}
|
||||
draft = {**source_identity["draft_prompt"], "status": "published", "enabled": True}
|
||||
runtime = {"schema_version": "scene-runtime-manifest.v1", "tenant_id": "default", "template_id": SCENE,
|
||||
"contract_type": template.contract_type, "routing_version": REVIEW_ROUTING_VERSION,
|
||||
"requested_review_route": review["name"], "draft_prompt": draft, "runtime_review_prompt": review,
|
||||
"trial_review_prompt": review, "provider_mode": "codex_cli", "model": "gpt-5.6-sol",
|
||||
"evaluation_gate": evaluation_gate(SCENE, review)}
|
||||
"evaluation_gate": evaluation_gate(SCENE, review),
|
||||
"source_prompt_identity": source_identity, "source_prompt_identity_resolved": True,
|
||||
"published_prompt_alignment_verified": True}
|
||||
runtime.update(manifest_sha256=draft_digest(runtime), checked_at=datetime.now(UTC).isoformat())
|
||||
draft_fingerprint = {"prompt_name": draft["name"], "prompt_version": draft["version"], "prompt_sha256": draft["content_sha256"]}
|
||||
summary = {"template_content_sha256": template.content_sha256, "template_version": template.version,
|
||||
@@ -127,6 +129,29 @@ def test_summary_top_level_release_flags_reject_cached_claims_and_bad_runtime_ha
|
||||
assert "SCENE_RUNTIME_UNAVAILABLE" in codes(summary["release_readiness"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("change", ["missing", "forged"])
|
||||
def test_runtime_source_identity_is_required_and_cannot_be_forged(change):
|
||||
template, summary, runtime = evidence()
|
||||
if change == "missing":
|
||||
runtime.pop("source_prompt_identity")
|
||||
runtime.pop("source_prompt_identity_resolved")
|
||||
expected_code = "SCENE_RUNTIME_SOURCE_IDENTITY_MISSING"
|
||||
else:
|
||||
runtime["source_prompt_identity"]["draft_prompt"]["content_sha256"] = "f" * 64
|
||||
expected_code = "SCENE_RUNTIME_SOURCE_IDENTITY_MISMATCH"
|
||||
runtime["manifest_sha256"] = draft_digest({
|
||||
key: value for key, value in runtime.items()
|
||||
if key not in {"checked_at", "manifest_sha256"}
|
||||
})
|
||||
|
||||
result = assess_release_readiness(template, summary, runtime, tenant="default")
|
||||
|
||||
assert expected_code in codes(result)
|
||||
assert "SCENE_RUNTIME_PROMPT_MISMATCH" in codes(result)
|
||||
assert result["published_prompt_alignment_verified"] is False
|
||||
assert result["prerequisites_satisfied"] is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("change", ["tenant", "hash", "time", "naive_time", "shape", "route", "missing"])
|
||||
def test_invalid_runtime_never_reuses_cached_alignment(change):
|
||||
template, summary, runtime = evidence()
|
||||
@@ -191,13 +216,13 @@ def test_missing_fact_detection_counts_without_claiming_completed_review():
|
||||
assert not after["authorizes_publication"] and not after["legal_correctness_verified"]
|
||||
|
||||
|
||||
def test_code_owned_review_gate_fills_review_coverage_only_when_hash_and_prompt_match():
|
||||
def test_code_owned_review_gate_cannot_fill_coverage_for_an_older_draft_identity():
|
||||
template, summary, runtime = evidence()
|
||||
summary["review_coverage_by_prompt"] = []
|
||||
accepted = assess_release_readiness(template, summary, runtime, tenant="default")
|
||||
assert accepted["missing_review_variants"] == ["missing-key-fact", "opposite-adversarial"]
|
||||
assert accepted["scene_review_evaluation_gate"]["coverage_eligible"] is True
|
||||
assert accepted["scene_review_evaluation_gate"]["counted_variants"] == ["normal"]
|
||||
assert set(accepted["missing_review_variants"]) == set(VARIANTS)
|
||||
assert accepted["scene_review_evaluation_gate"]["coverage_eligible"] is False
|
||||
assert accepted["scene_review_evaluation_gate"]["counted_variants"] == []
|
||||
|
||||
runtime["evaluation_gate"]["provider_availability_evidence"]["technical_passed"] = 11
|
||||
runtime["evaluation_gate"]["gate_sha256"] = draft_digest({
|
||||
@@ -276,7 +301,7 @@ def test_stale_or_truncated_dynamic_summary_cannot_count_coverage(change):
|
||||
|
||||
|
||||
@pytest.mark.parametrize("template_id", sorted(SCENE_REVIEW_ROUTES))
|
||||
def test_every_scene_counts_exact_current_variants_and_stale_evidence_fails_closed(template_id):
|
||||
def test_every_scene_rejects_legacy_runtime_without_current_source_identity(template_id):
|
||||
trial = DraftingWorkspaceStore(InMemoryContractRepository(contracts=[]), mode="scene_evaluation")
|
||||
template, _ = trial.evaluation_source(template_id)
|
||||
registered = scene_review_coverage(template_id)
|
||||
@@ -295,12 +320,10 @@ def test_every_scene_counts_exact_current_variants_and_stale_evidence_fails_clos
|
||||
result = assess_release_readiness(template, summary, runtime, tenant="default")
|
||||
assert result["scene_review_evaluation_gate"]["registry_valid"]
|
||||
assert result["scene_review_evaluation_gate"]["current_template_match"] is expected_match
|
||||
expected = ({case["variant"] for case in registered["cases"]
|
||||
if case["draft_prompt"] == registered["cases"][0]["draft_prompt"]}
|
||||
if expected_match else set())
|
||||
assert set(result["scene_review_evaluation_gate"]["counted_variants"]) == expected
|
||||
assert set(result["scene_review_evaluation_gate"]["missing_variants"]) == set(VARIANTS) - expected
|
||||
assert set(result["missing_review_variants"]) == set(VARIANTS) - expected
|
||||
assert "SCENE_RUNTIME_SOURCE_IDENTITY_MISSING" in codes(result)
|
||||
assert result["scene_review_evaluation_gate"]["counted_variants"] == []
|
||||
assert set(result["scene_review_evaluation_gate"]["missing_variants"]) == set(VARIANTS)
|
||||
assert set(result["missing_review_variants"]) == set(VARIANTS)
|
||||
assert not result["authorizes_publication"]
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user