yuqei/ai-exchange
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test: lock training docs test inventory

Browse Source
This commit is contained in:
yuqei
2026-04-30 23:29:10 -04:00
parent 947fe6cd58
commit 072f6f6260
2 changed files with 134 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/test_documentation_consistency.py
+75 -1
View File
@@ -1,3 +1,4 @@
import ast
import re
import unittest
from pathlib import Path
@@ -7,10 +8,35 @@ ROOT = Path(__file__).resolve().parents[2]
PROJECT_README = ROOT / "README.md"
TRAINING_PRODUCT_DOC = ROOT / "docs" / "training_case_engine_m1_m6_product_readme.md"
TRAINING_PACKAGE_SMOKE_DOC = ROOT / "docs" / "examples" / "training_case_package_smoke_m6.md"
TRAINING_DOCS_CLI_TEST = ROOT / "tests" / "unit" / "test_training_case_docs_cli_m10.py"
TRAINING_DASHBOARD_HTML = ROOT / "src" / "web" / "index.html"
TRAINING_DASHBOARD_APP = ROOT / "src" / "web" / "app.js"
API_SERVER = ROOT / "src" / "api" / "server.py"
OFFLINE_CHECKS_SCRIPT = ROOT / "scripts" / "run_offline_checks.py"
CURRENT_TRAINING_DOC_MATURITY_END = 28
CURRENT_TRAINING_DOC_MATURITY_MARKER = f"M1-M{CURRENT_TRAINING_DOC_MATURITY_END}"
CURRENT_TRAINING_DOC_MATURITY_SPAN = f"through M{CURRENT_TRAINING_DOC_MATURITY_END}"
TRAINING_M24_M29_DOCUMENTATION_TEST_INVENTORY = {
"m1_m28_maturity_wording": (
"test_training_case_product_doc_reflects_current_m28_maturity",
),
"route_surface": (
"test_training_case_product_doc_api_routes_match_server_route_request",
"test_training_case_public_route_registry_is_documented_and_supported",
"test_training_case_product_doc_api_surface_wording_matches_public_routes",
"test_training_dashboard_routes_are_documented_public_api_routes",
),
"dashboard_public_docs_privacy": (
"test_training_readmes_name_dashboard_privacy_and_offline_registry_coverage",
"test_training_dashboard_and_public_docs_keep_privacy_wording_exclusionary",
),
"offline_registry_coverage": (
"test_training_readmes_name_dashboard_privacy_and_offline_registry_coverage",
),
}
TRAINING_M24_M29_PACKAGE_DOCS_INVENTORY_TEST = (
"test_m24_m29_package_docs_privacy_test_inventory_is_current"
)
TRAINING_PUBLIC_API_ROUTE_REGISTRY = (
("GET", "/training/cases"),
("POST", "/training/cases/import"),
@@ -181,6 +207,18 @@ def _training_public_privacy_lines(text: str) -> tuple[str, ...]:
return tuple(lines)
def _test_method_names_from_file(path: Path, class_name: str) -> tuple[str, ...]:
tree = ast.parse(path.read_text(encoding="utf-8"))
for node in tree.body:
if isinstance(node, ast.ClassDef) and node.name == class_name:
return tuple(
item.name
for item in node.body
if isinstance(item, ast.FunctionDef) and item.name.startswith("test_")
)
raise AssertionError(f"{path} is missing {class_name}")
class DocumentationConsistencyTests(unittest.TestCase):
def test_readme_lists_script_output_json_environment_variables(self) -> None:
script_env_vars = set()
@@ -217,7 +255,7 @@ class DocumentationConsistencyTests(unittest.TestCase):
text = TRAINING_PRODUCT_DOC.read_text(encoding="utf-8")
for snippet in (
"M1-M28",
CURRENT_TRAINING_DOC_MATURITY_MARKER,
"API contract coverage",
"error contract coverage",
"fixture/schema contract",
@@ -232,6 +270,42 @@ class DocumentationConsistencyTests(unittest.TestCase):
with self.subTest(stale_snippet=stale_snippet):
self.assertNotIn(stale_snippet, text)
def test_training_case_m24_m29_documentation_test_inventory_is_current(self) -> None:
test_names = set(
_test_method_names_from_file(Path(__file__), "DocumentationConsistencyTests")
)
package_docs_test_names = set(
_test_method_names_from_file(TRAINING_DOCS_CLI_TEST, "TrainingCaseDocsCliM10Tests")
)
inventory_names = tuple(
name
for names in TRAINING_M24_M29_DOCUMENTATION_TEST_INVENTORY.values()
for name in names
)
unique_inventory_names = set(inventory_names)
text = TRAINING_PRODUCT_DOC.read_text(encoding="utf-8")
self.assertEqual(len(inventory_names), 8)
self.assertEqual(len(unique_inventory_names), 7)
self.assertEqual(sorted(TRAINING_M24_M29_DOCUMENTATION_TEST_INVENTORY), [
"dashboard_public_docs_privacy",
"m1_m28_maturity_wording",
"offline_registry_coverage",
"route_surface",
])
for surface, names in TRAINING_M24_M29_DOCUMENTATION_TEST_INVENTORY.items():
with self.subTest(surface=surface):
self.assertTrue(names)
for name in names:
self.assertIn(name, test_names)
self.assertIn(
TRAINING_M24_M29_PACKAGE_DOCS_INVENTORY_TEST,
package_docs_test_names,
)
self.assertIn(CURRENT_TRAINING_DOC_MATURITY_MARKER, text)
self.assertIn(CURRENT_TRAINING_DOC_MATURITY_SPAN, text)
def test_training_readmes_name_dashboard_privacy_and_offline_registry_coverage(self) -> None:
expected_doc_snippets = (
"dashboard and public docs privacy smoke coverage",
tests/unit/test_training_case_docs_cli_m10.py
+59 -2
View File
@@ -25,6 +25,9 @@ OFFLINE_CHECK_NAME = "training_case_package"
OFFLINE_CHECK_DOC_NAME = f"`{OFFLINE_CHECK_NAME}`"
PACKAGE_SMOKE_SCRIPT = "scripts/run_training_case_package.py"
PACKAGE_SMOKE_COMMAND = f".venv/bin/python {PACKAGE_SMOKE_SCRIPT}"
CURRENT_TRAINING_DOC_MATURITY_END = 28
CURRENT_TRAINING_DOC_MATURITY_MARKER = f"M1-M{CURRENT_TRAINING_DOC_MATURITY_END}"
CURRENT_TRAINING_DOC_MATURITY_SPAN = f"through M{CURRENT_TRAINING_DOC_MATURITY_END}"
PUBLIC_FIELD = "answer" + "_key"
PRIVATE_MATERIAL_PHRASES = ("private solution material", "internal solution details")
PUBLIC_PACKAGE_DOCS = (DOC_EXAMPLE, README)
@@ -48,7 +51,7 @@ PACKAGE_IMPORT_EXPORT_SURFACE_TEXT = (
"public output",
)
PRODUCT_PACKAGE_WORDING = (
"M1-M28",
CURRENT_TRAINING_DOC_MATURITY_MARKER,
"offline checks registry smoke",
"public package",
"public export",
@@ -69,6 +72,29 @@ RESTRICTED_PUBLIC_TEXT = (
"sec" + "ret",
"api" + "_key",
)
TRAINING_M24_M29_PACKAGE_DOCS_TEST_INVENTORY = {
"package_smoke_privacy": (
"test_public_privacy_guard_rejects_solution_wording_drift",
"test_public_fixture_schema_contract_is_minimal_and_demo_safe",
"test_fixture_is_compatible_with_public_service_and_cli_export",
"test_smoke_docs_and_product_readme_share_fixture_and_public_export_contract",
"test_example_command_uses_supported_cli_flags_and_fixture",
"test_example_documents_cli_summary_keys_and_output_semantics",
"test_readme_and_product_docs_advertise_public_offline_smoke_contract",
"test_product_readme_uses_current_maturity_and_public_private_package_language",
"test_public_package_smoke_docs_lock_import_export_privacy_language",
"test_m1_m28_package_smoke_docs_cover_static_import_export_surface",
),
"offline_registry_coverage": (
"test_offline_check_registry_name_matches_package_smoke_docs",
"test_offline_checks_registry_runs_training_package_smoke_entry",
"test_training_docs_name_broader_offline_check_path",
),
"m1_m28_maturity_wording": (
"test_product_readme_uses_current_maturity_and_public_private_package_language",
"test_m1_m28_package_smoke_docs_cover_static_import_export_surface",
),
}
def _example_command_parts() -> list[str]:
@@ -370,6 +396,37 @@ class TrainingCaseDocsCliM10Tests(unittest.TestCase):
self.assertNotIn("M1-M21", text)
self.assertNotIn(PUBLIC_FIELD, text)
def test_m24_m29_package_docs_privacy_test_inventory_is_current(self) -> None:
test_names = set(unittest.TestLoader().getTestCaseNames(type(self)))
inventory_names = tuple(
name
for names in TRAINING_M24_M29_PACKAGE_DOCS_TEST_INVENTORY.values()
for name in names
)
unique_inventory_names = set(inventory_names)
product_text = PRODUCT_README.read_text(encoding="utf-8")
self.assertEqual(len(test_names), 14)
self.assertEqual(len(inventory_names), 15)
self.assertEqual(len(unique_inventory_names), 13)
self.assertEqual(sorted(TRAINING_M24_M29_PACKAGE_DOCS_TEST_INVENTORY), [
"m1_m28_maturity_wording",
"offline_registry_coverage",
"package_smoke_privacy",
])
self.assertEqual(
test_names - unique_inventory_names,
{"test_m24_m29_package_docs_privacy_test_inventory_is_current"},
)
for surface, names in TRAINING_M24_M29_PACKAGE_DOCS_TEST_INVENTORY.items():
with self.subTest(surface=surface):
self.assertTrue(names)
for name in names:
self.assertIn(name, test_names)
self.assertIn(CURRENT_TRAINING_DOC_MATURITY_MARKER, product_text)
self.assertIn(CURRENT_TRAINING_DOC_MATURITY_SPAN, product_text)
def test_public_package_smoke_docs_lock_import_export_privacy_language(self) -> None:
help_text = training_cli._build_parser().format_help()
@@ -394,7 +451,7 @@ class TrainingCaseDocsCliM10Tests(unittest.TestCase):
product_text = PRODUCT_README.read_text(encoding="utf-8")
cli_flags = _package_smoke_flags_from_source()
self.assertIn("M1-M28", product_text)
self.assertIn(CURRENT_TRAINING_DOC_MATURITY_MARKER, product_text)
self.assertEqual(cli_flags, ("--input", "--output"))
for path in PACKAGE_SMOKE_SURFACE_DOCS: