feat: enhance ai quality effect review

This commit is contained in:
2026-06-23 11:31:45 +08:00
parent f1fcf90797
commit 396259ebe9
7 changed files with 341 additions and 25 deletions
@@ -1759,6 +1759,20 @@ components:
current_quality_score:
type: number
format: float
post_change_total_runs:
type: integer
post_change_affected_runs:
type: integer
post_change_pending_labels:
type: integer
post_change_quality_score:
type: number
format: float
post_change_affected_rate:
type: number
format: float
post_change_observation_days:
type: integer
affected_rate_delta:
type: number
format: float
@@ -1767,6 +1781,16 @@ components:
quality_score_delta:
type: number
format: float
post_change_affected_rate_delta:
type: number
format: float
post_change_pending_labels_delta:
type: integer
post_change_quality_score_delta:
type: number
format: float
related_change_actions_count:
type: integer
effect_status:
type: string
enum: [pending_data, improved, unchanged, regressed]
@@ -9,7 +9,7 @@ from typing import Iterable, Protocol
from uuid import uuid4
from pydantic import BaseModel, Field
from sqlalchemy import create_engine, delete, select, text
from sqlalchemy import create_engine, delete, or_, select, text
from sqlalchemy.engine import Engine
from sqlalchemy.orm import Session, sessionmaker
from sqlalchemy.pool import StaticPool
@@ -365,9 +365,19 @@ class AiQualityImprovementAction(AiQualityImprovementActionCreate):
current_affected_runs: int = 0
current_pending_labels: int = 0
current_quality_score: float = 0.0
post_change_total_runs: int = 0
post_change_affected_runs: int = 0
post_change_pending_labels: int = 0
post_change_quality_score: float = 0.0
post_change_affected_rate: float = 0.0
post_change_observation_days: int = 0
affected_rate_delta: float = 0.0
pending_labels_delta: int = 0
quality_score_delta: float = 0.0
post_change_affected_rate_delta: float = 0.0
post_change_pending_labels_delta: int = 0
post_change_quality_score_delta: float = 0.0
related_change_actions_count: int = 0
effect_status: str = "pending_data"
effect_summary: str | None = None
resolution_note: str | None = None
@@ -896,7 +906,11 @@ class InMemoryAiPlatformRepository:
**payload.model_dump(exclude={"title"}),
)
self.quality_improvement_actions[action.id] = action
return _materialize_improvement_action(action, self.ai_run_audits)
return _materialize_improvement_action(
action,
self.ai_run_audits,
all_actions=self.quality_improvement_actions.values(),
)
def update_quality_improvement_action(
self,
@@ -939,7 +953,11 @@ class InMemoryAiPlatformRepository:
update["linked_import_batch_id"] = payload.linked_import_batch_id
updated = action.model_copy(update=update)
self.quality_improvement_actions[action_id] = updated
return _materialize_improvement_action(updated, self.ai_run_audits)
return _materialize_improvement_action(
updated,
self.ai_run_audits,
all_actions=self.quality_improvement_actions.values(),
)
def update_ai_run_quality_label(
self,
@@ -1557,9 +1575,21 @@ class SqlAlchemyAiPlatformRepository:
.order_by(AiQualityImprovementActionModel.updated_at.desc())
.limit(20)
).all()
all_action_rows = session.scalars(
select(AiQualityImprovementActionModel).order_by(
AiQualityImprovementActionModel.updated_at.desc()
)
).all()
action_days = [row.evaluation_days for row in action_rows]
resolved_since = [
_ensure_utc(row.resolved_at)
for row in action_rows
if row.resolved_at is not None
]
stats_cutoff = datetime.now(UTC) - timedelta(days=max([days, *action_days], default=days) - 1)
stats_cutoff = stats_cutoff.replace(hour=0, minute=0, second=0, microsecond=0)
if resolved_since:
stats_cutoff = min(stats_cutoff, min(resolved_since))
rows = session.scalars(
select(AiRunAuditModel)
.where(AiRunAuditModel.created_at >= stats_cutoff)
@@ -1572,6 +1602,7 @@ class SqlAlchemyAiPlatformRepository:
actions = _materialize_improvement_actions(
[_quality_improvement_action_from_model(row) for row in action_rows],
all_runs,
all_actions=[_quality_improvement_action_from_model(row) for row in all_action_rows],
)
return _build_quality_dashboard(
dashboard_runs,
@@ -1608,15 +1639,57 @@ class SqlAlchemyAiPlatformRepository:
max_days = max((row.evaluation_days for row in rows), default=14)
cutoff = datetime.now(UTC) - timedelta(days=max(max_days - 1, 0))
cutoff = cutoff.replace(hour=0, minute=0, second=0, microsecond=0)
resolved_since = [
_ensure_utc(row.resolved_at)
for row in rows
if row.resolved_at is not None
]
if resolved_since:
cutoff = min(cutoff, min(resolved_since))
audit_rows = session.scalars(
select(AiRunAuditModel)
.where(AiRunAuditModel.created_at >= cutoff)
.order_by(AiRunAuditModel.created_at.desc())
).all()
runs = [_ai_run_audit_from_model(row) for row in audit_rows]
related_statement = select(AiQualityImprovementActionModel)
if rows:
prompt_keys = [
(row.linked_prompt_template_id, row.linked_prompt_template_version)
for row in rows
if row.linked_prompt_template_id
]
import_batch_ids = [row.linked_import_batch_id for row in rows if row.linked_import_batch_id]
knowledge_base_ids = [
row.linked_knowledge_base_id
for row in rows
if row.linked_knowledge_base_id and not row.linked_import_batch_id
]
related_filters = []
for prompt_id, prompt_version in prompt_keys:
related_filters.append(
(AiQualityImprovementActionModel.linked_prompt_template_id == prompt_id)
& (AiQualityImprovementActionModel.linked_prompt_template_version == prompt_version)
)
if import_batch_ids:
related_filters.append(
AiQualityImprovementActionModel.linked_import_batch_id.in_(import_batch_ids)
)
if knowledge_base_ids:
related_filters.append(
AiQualityImprovementActionModel.linked_knowledge_base_id.in_(knowledge_base_ids)
)
if related_filters:
related_statement = related_statement.where(or_(*related_filters))
related_rows = session.scalars(related_statement).all() if rows else []
related_actions = [_quality_improvement_action_from_model(row) for row in related_rows]
row_actions = [_quality_improvement_action_from_model(row) for row in rows]
related_by_id = {action.id: action for action in related_actions}
related_by_id.update({action.id: action for action in row_actions})
actions = _materialize_improvement_actions(
[_quality_improvement_action_from_model(row) for row in rows],
row_actions,
runs,
all_actions=related_by_id.values(),
)
return _filter_improvement_actions(
actions,
@@ -1682,7 +1755,12 @@ class SqlAlchemyAiPlatformRepository:
)
session.add(row)
session.flush()
return _materialize_improvement_action(_quality_improvement_action_from_model(row), runs)
action_rows = session.scalars(select(AiQualityImprovementActionModel)).all()
return _materialize_improvement_action(
_quality_improvement_action_from_model(row),
runs,
all_actions=[_quality_improvement_action_from_model(action_row) for action_row in action_rows],
)
def update_quality_improvement_action(
self,
@@ -1724,6 +1802,8 @@ class SqlAlchemyAiPlatformRepository:
cutoff = now - timedelta(days=max(row.evaluation_days - 1, 0))
cutoff = cutoff.replace(hour=0, minute=0, second=0, microsecond=0)
if row.resolved_at is not None:
cutoff = min(cutoff, _ensure_utc(row.resolved_at))
audit_rows = session.scalars(
select(AiRunAuditModel)
.where(AiRunAuditModel.created_at >= cutoff)
@@ -1731,7 +1811,12 @@ class SqlAlchemyAiPlatformRepository:
).all()
runs = [_ai_run_audit_from_model(row) for row in audit_rows]
session.flush()
return _materialize_improvement_action(_quality_improvement_action_from_model(row), runs)
action_rows = session.scalars(select(AiQualityImprovementActionModel)).all()
return _materialize_improvement_action(
_quality_improvement_action_from_model(row),
runs,
all_actions=[_quality_improvement_action_from_model(action_row) for action_row in action_rows],
)
def update_ai_run_quality_label(
self,
@@ -2413,20 +2498,38 @@ def _finalize_attribution_groups(groups: dict[str, dict[str, object]]) -> list[A
def _materialize_improvement_actions(
actions: Iterable[AiQualityImprovementAction],
runs: list[AiRunAudit],
*,
all_actions: Iterable[AiQualityImprovementAction] | None = None,
) -> list[AiQualityImprovementAction]:
return [_materialize_improvement_action(action, runs) for action in actions]
action_list = list(actions)
related_actions = list(all_actions) if all_actions is not None else action_list
return [
_materialize_improvement_action(action, runs, all_actions=related_actions)
for action in action_list
]
def _materialize_improvement_action(
action: AiQualityImprovementAction,
runs: list[AiRunAudit],
*,
all_actions: Iterable[AiQualityImprovementAction] | None = None,
) -> AiQualityImprovementAction:
current = _calculate_improvement_action_stats(action, runs)
post_change = (
_calculate_improvement_action_stats(action, runs, since=action.resolved_at)
if action.resolved_at
else {"total_runs": 0, "affected_runs": 0, "pending_labels": 0, "quality_score": 0.0}
)
baseline_rate = _safe_rate(action.baseline_affected_runs, action.baseline_total_runs)
current_rate = _safe_rate(current["affected_runs"], current["total_runs"])
affected_rate_delta = round(current_rate - baseline_rate, 4)
pending_labels_delta = current["pending_labels"] - action.baseline_pending_labels
quality_score_delta = round(current["quality_score"] - action.baseline_quality_score, 2)
post_change_rate = _safe_rate(post_change["affected_runs"], post_change["total_runs"])
post_change_affected_rate_delta = round(post_change_rate - baseline_rate, 4)
post_change_pending_labels_delta = post_change["pending_labels"] - action.baseline_pending_labels
post_change_quality_score_delta = round(post_change["quality_score"] - action.baseline_quality_score, 2)
effect_status, effect_summary = _summarize_improvement_effect(
action,
current_total_runs=current["total_runs"],
@@ -2434,6 +2537,11 @@ def _materialize_improvement_action(
affected_rate_delta=affected_rate_delta,
pending_labels_delta=pending_labels_delta,
quality_score_delta=quality_score_delta,
post_change_total_runs=post_change["total_runs"],
post_change_rate=post_change_rate,
post_change_affected_rate_delta=post_change_affected_rate_delta,
post_change_pending_labels_delta=post_change_pending_labels_delta,
post_change_quality_score_delta=post_change_quality_score_delta,
)
return action.model_copy(
update={
@@ -2441,9 +2549,19 @@ def _materialize_improvement_action(
"current_affected_runs": current["affected_runs"],
"current_pending_labels": current["pending_labels"],
"current_quality_score": current["quality_score"],
"post_change_total_runs": post_change["total_runs"],
"post_change_affected_runs": post_change["affected_runs"],
"post_change_pending_labels": post_change["pending_labels"],
"post_change_quality_score": post_change["quality_score"],
"post_change_affected_rate": post_change_rate,
"post_change_observation_days": _post_change_observation_days(action),
"affected_rate_delta": affected_rate_delta,
"pending_labels_delta": pending_labels_delta,
"quality_score_delta": quality_score_delta,
"post_change_affected_rate_delta": post_change_affected_rate_delta,
"post_change_pending_labels_delta": post_change_pending_labels_delta,
"post_change_quality_score_delta": post_change_quality_score_delta,
"related_change_actions_count": _count_related_change_actions(action, all_actions or ()),
"effect_status": effect_status,
"effect_summary": effect_summary,
}
@@ -2484,9 +2602,12 @@ def _filter_improvement_actions(
def _calculate_improvement_action_stats(
action: AiQualityImprovementActionCreate | AiQualityImprovementAction,
runs: list[AiRunAudit],
*,
since: datetime | None = None,
) -> dict[str, int | float]:
cutoff = datetime.now(UTC) - timedelta(days=max(action.evaluation_days - 1, 0))
cutoff = cutoff.replace(hour=0, minute=0, second=0, microsecond=0)
cutoff = _ensure_utc(since) if since else datetime.now(UTC) - timedelta(days=max(action.evaluation_days - 1, 0))
if not since:
cutoff = cutoff.replace(hour=0, minute=0, second=0, microsecond=0)
matched_runs: list[AiRunAudit] = []
affected_runs: set[str] = set()
pending_labels = 0
@@ -2550,6 +2671,37 @@ def _has_source_issue(run: AiRunAudit) -> bool:
return any(label.code in {"source_insufficient", "source_limited"} for label in run.quality_labels)
def _post_change_observation_days(action: AiQualityImprovementAction) -> int:
if not action.resolved_at:
return 0
elapsed = datetime.now(UTC).date() - _ensure_utc(action.resolved_at).date()
return max(elapsed.days + 1, 1)
def _count_related_change_actions(
action: AiQualityImprovementAction,
actions: Iterable[AiQualityImprovementAction],
) -> int:
change_key = _quality_action_change_key(action)
if change_key is None:
return 0
return sum(1 for candidate in actions if _quality_action_change_key(candidate) == change_key)
def _quality_action_change_key(action: AiQualityImprovementAction) -> tuple[str, str, str | None] | None:
if action.linked_prompt_template_id:
return (
"prompt",
action.linked_prompt_template_id,
action.linked_prompt_template_version,
)
if action.linked_import_batch_id:
return ("knowledge_import", action.linked_import_batch_id, None)
if action.linked_knowledge_base_id:
return ("knowledge_base", action.linked_knowledge_base_id, None)
return None
def _safe_rate(numerator: int | float, denominator: int | float) -> float:
return round(float(numerator) / float(denominator), 4) if denominator else 0.0
@@ -2562,26 +2714,50 @@ def _summarize_improvement_effect(
affected_rate_delta: float,
pending_labels_delta: int | float,
quality_score_delta: float,
post_change_total_runs: int | float,
post_change_rate: float,
post_change_affected_rate_delta: float,
post_change_pending_labels_delta: int | float,
post_change_quality_score_delta: float,
) -> tuple[str, str]:
baseline_rate = _safe_rate(action.baseline_affected_runs, action.baseline_total_runs)
if action.baseline_total_runs <= 0 or current_total_runs <= 0:
resolved = action.status in {"resolved", "false_positive", "closed"}
if action.baseline_total_runs <= 0:
return "pending_data", "Need more AI run data to evaluate this improvement action."
if resolved and post_change_total_runs <= 0:
return (
"pending_data",
"Need post-change AI run data after this action was resolved to evaluate effect.",
)
if not resolved and current_total_runs <= 0:
return "pending_data", "Need more AI run data to evaluate this improvement action."
effective_rate_delta = post_change_affected_rate_delta if resolved else affected_rate_delta
effective_pending_delta = post_change_pending_labels_delta if resolved else pending_labels_delta
effective_quality_delta = post_change_quality_score_delta if resolved else quality_score_delta
improved = (
affected_rate_delta < -0.01
or pending_labels_delta < 0
or quality_score_delta > 1
effective_rate_delta < -0.01
or effective_pending_delta < 0
or effective_quality_delta > 1
)
regressed = (
affected_rate_delta > 0.01
or pending_labels_delta > 0
or quality_score_delta < -1
effective_rate_delta > 0.01
or effective_pending_delta > 0
or effective_quality_delta < -1
)
status = "improved" if improved and not regressed else "regressed" if regressed and not improved else "unchanged"
summary = (
f"Baseline affected rate {baseline_rate:.0%}, current {current_rate:.0%}; "
f"pending labels delta {pending_labels_delta:+.0f}; "
f"quality score delta {quality_score_delta:+.1f}."
)
if resolved:
summary = (
f"Baseline affected rate {baseline_rate:.0%}, post-change {post_change_rate:.0%} "
f"over {post_change_total_runs:.0f} run(s); "
f"pending labels delta {post_change_pending_labels_delta:+.0f}; "
f"quality score delta {post_change_quality_score_delta:+.1f}."
)
else:
summary = (
f"Baseline affected rate {baseline_rate:.0%}, current {current_rate:.0%}; "
f"pending labels delta {pending_labels_delta:+.0f}; "
f"quality score delta {quality_score_delta:+.1f}."
)
return status, summary
@@ -1,3 +1,5 @@
from datetime import UTC, datetime
import pytest
from fastapi.testclient import TestClient
@@ -450,6 +452,63 @@ def test_quality_improvement_action_resolve_api_returns_422_without_real_change(
assert "linked_knowledge_base_id" in resolve_response.json()["detail"]
def test_quality_improvement_action_effect_review_uses_post_resolution_window() -> None:
repository = InMemoryAiPlatformRepository()
repository.seed_defaults()
repository.add_ai_run_audit(
AiRunAudit(
id="run-baseline",
operation="legal_qa",
provider="deepseek",
model="deepseek-chat",
prompt_template_id="legal_qa.default",
retrieved_sources=[],
quality_score=88,
quality_status="needs_review",
quality_labels=[],
created_at=datetime.now(UTC),
)
)
action = repository.create_quality_improvement_action(
AiQualityImprovementActionCreate(
dimension="prompt_template",
key="legal_qa.default",
label="legal_qa.default",
action_type="tune_prompt",
evaluation_days=7,
)
)
assert action.current_total_runs >= 1
resolved = repository.update_quality_improvement_action(
action.id,
AiQualityImprovementActionUpdate(
status="resolved",
linked_prompt_template_id=action.linked_prompt_template_id,
linked_prompt_template_version=action.linked_prompt_template_version,
resolution_note="Prompt improved.",
),
)
repository.add_ai_run_audit(
AiRunAudit(
id="run-post-change",
operation="legal_qa",
provider="deepseek",
model="deepseek-chat",
prompt_template_id="legal_qa.default",
retrieved_sources=[],
quality_score=96,
quality_status="passed",
quality_labels=[],
created_at=resolved.resolved_at or datetime.now(UTC),
)
)
refreshed = repository.list_quality_improvement_actions(limit=5)[0]
assert refreshed.post_change_observation_days >= 1
assert refreshed.post_change_total_runs >= 1
assert refreshed.related_change_actions_count >= 1
assert "post-change" in (refreshed.effect_summary or "")
def test_knowledge_document_can_be_added_and_searched() -> None:
client = make_client()
@@ -375,9 +375,19 @@ class AiQualityImprovementAction(AiQualityImprovementActionCreate):
current_affected_runs: int = 0
current_pending_labels: int = 0
current_quality_score: float = 0.0
post_change_total_runs: int = 0
post_change_affected_runs: int = 0
post_change_pending_labels: int = 0
post_change_quality_score: float = 0.0
post_change_affected_rate: float = 0.0
post_change_observation_days: int = 0
affected_rate_delta: float = 0.0
pending_labels_delta: int = 0
quality_score_delta: float = 0.0
post_change_affected_rate_delta: float = 0.0
post_change_pending_labels_delta: int = 0
post_change_quality_score_delta: float = 0.0
related_change_actions_count: int = 0
effect_status: str = "pending_data"
effect_summary: str | None = None
resolution_note: str | None = None
+15 -1
View File
@@ -128,11 +128,21 @@ def test_ai_platform_client_covers_admin_data_endpoints() -> None:
"current_affected_runs": 1,
"current_pending_labels": 1,
"current_quality_score": 90.0,
"post_change_total_runs": 2,
"post_change_affected_runs": 0,
"post_change_pending_labels": 0,
"post_change_quality_score": 98.0,
"post_change_affected_rate": 0.0,
"post_change_observation_days": 3,
"affected_rate_delta": 0.0,
"pending_labels_delta": 0,
"quality_score_delta": 0.0,
"post_change_affected_rate_delta": -1.0,
"post_change_pending_labels_delta": -1,
"post_change_quality_score_delta": 8.0,
"related_change_actions_count": 2,
"effect_status": "unchanged",
"effect_summary": "Baseline affected rate 100%, current 100%; pending labels delta +0; quality score delta +0.0.",
"effect_summary": "Baseline affected rate 100%, post-change 0% over 2 run(s); pending labels delta -1; quality score delta +8.0.",
"resolution_note": resolution_note,
"created_at": "2026-06-22T00:00:00Z",
"updated_at": "2026-06-22T00:02:00Z",
@@ -817,6 +827,8 @@ def test_ai_platform_client_covers_admin_data_endpoints() -> None:
assert created_action.linked_import_batch_id == "kb-batch-1"
assert created_action.baseline_affected_runs == 1
assert created_action.effect_status == "unchanged"
assert created_action.post_change_total_runs == 2
assert created_action.related_change_actions_count == 2
listed_actions = client.list_quality_improvement_actions(
status="open",
owner="admin",
@@ -837,6 +849,8 @@ def test_ai_platform_client_covers_admin_data_endpoints() -> None:
)
assert updated_action.status == "resolved"
assert updated_action.resolved_at == "2026-06-22T00:05:00Z"
assert updated_action.post_change_total_runs == 2
assert updated_action.effect_summary and "post-change" in updated_action.effect_summary
updated_audit = client.update_ai_run_quality_label(
"run-1",
"citation_present",
@@ -100,9 +100,19 @@ type QualityImprovementAction = {
current_affected_runs: number;
current_pending_labels: number;
current_quality_score: number;
post_change_total_runs: number;
post_change_affected_runs: number;
post_change_pending_labels: number;
post_change_quality_score: number;
post_change_affected_rate: number;
post_change_observation_days: number;
affected_rate_delta: number;
pending_labels_delta: number;
quality_score_delta: number;
post_change_affected_rate_delta: number;
post_change_pending_labels_delta: number;
post_change_quality_score_delta: number;
related_change_actions_count: number;
effect_status?: string;
effect_summary?: string | null;
resolution_note?: string | null;
@@ -774,6 +784,7 @@ function QualityActionItem({
const needsPromptDraft = action.action_type === "tune_prompt";
const needsKnowledgeImport = action.action_type === "improve_knowledge";
const completionBlockReason = getCompletionBlockReason(action);
const hasPostChange = action.post_change_total_runs > 0;
return (
<div className="quality-action-item">
@@ -796,13 +807,29 @@ function QualityActionItem({
<span className="item-meta">{action.description || action.root_cause_hint}</span>
)}
<div className="quality-action-stats">
<span> {formatPercent(baselineRate)} {formatPercent(currentRate)}</span>
<span> {action.baseline_pending_labels} {action.current_pending_labels}</span>
<span> {action.baseline_quality_score} {action.current_quality_score}</span>
<span> {formatPercent(baselineRate)}</span>
<span> {formatPercent(currentRate)}</span>
<span> {hasPostChange ? formatPercent(action.post_change_affected_rate) : "待观察"}</span>
<span> {action.post_change_observation_days || action.evaluation_days} </span>
<span> {action.baseline_pending_labels}</span>
<span> {hasPostChange ? action.post_change_pending_labels : "待观察"}</span>
<span> {action.baseline_quality_score}</span>
<span> {hasPostChange ? action.post_change_quality_score : "待观察"}</span>
<span> {formatSignedPercent(action.affected_rate_delta)}</span>
</div>
{action.resolution_note && <span className="item-meta">{action.resolution_note}</span>}
<LinkedChangeSummary action={action} />
<div className="quality-action-closure">
<span className="item-meta"> {action.related_change_actions_count}</span>
{hasPostChange && (
<span className="item-meta">
{formatSignedPercent(action.post_change_affected_rate_delta)} / {" "}
{action.post_change_pending_labels_delta >= 0
? `+${action.post_change_pending_labels_delta}`
: action.post_change_pending_labels_delta} / {formatSignedPercent(action.post_change_quality_score_delta)}
</span>
)}
</div>
{action.effect_summary && <span className="item-meta">{action.effect_summary}</span>}
{["open", "in_progress"].includes(action.status) && (needsPromptDraft || needsKnowledgeImport) && (
<div className="quality-action-execution">
@@ -812,6 +812,12 @@ select {
background: #f1f5f9;
}
.quality-action-closure {
display: grid;
gap: 6px;
min-width: 0;
}
.quality-action-execution {
display: grid;
gap: 10px;