feat: enhance ai quality effect review
This commit is contained in:
@@ -1759,6 +1759,20 @@ components:
|
||||
current_quality_score:
|
||||
type: number
|
||||
format: float
|
||||
post_change_total_runs:
|
||||
type: integer
|
||||
post_change_affected_runs:
|
||||
type: integer
|
||||
post_change_pending_labels:
|
||||
type: integer
|
||||
post_change_quality_score:
|
||||
type: number
|
||||
format: float
|
||||
post_change_affected_rate:
|
||||
type: number
|
||||
format: float
|
||||
post_change_observation_days:
|
||||
type: integer
|
||||
affected_rate_delta:
|
||||
type: number
|
||||
format: float
|
||||
@@ -1767,6 +1781,16 @@ components:
|
||||
quality_score_delta:
|
||||
type: number
|
||||
format: float
|
||||
post_change_affected_rate_delta:
|
||||
type: number
|
||||
format: float
|
||||
post_change_pending_labels_delta:
|
||||
type: integer
|
||||
post_change_quality_score_delta:
|
||||
type: number
|
||||
format: float
|
||||
related_change_actions_count:
|
||||
type: integer
|
||||
effect_status:
|
||||
type: string
|
||||
enum: [pending_data, improved, unchanged, regressed]
|
||||
|
||||
+197
-21
@@ -9,7 +9,7 @@ from typing import Iterable, Protocol
|
||||
from uuid import uuid4
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import create_engine, delete, select, text
|
||||
from sqlalchemy import create_engine, delete, or_, select, text
|
||||
from sqlalchemy.engine import Engine
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
from sqlalchemy.pool import StaticPool
|
||||
@@ -365,9 +365,19 @@ class AiQualityImprovementAction(AiQualityImprovementActionCreate):
|
||||
current_affected_runs: int = 0
|
||||
current_pending_labels: int = 0
|
||||
current_quality_score: float = 0.0
|
||||
post_change_total_runs: int = 0
|
||||
post_change_affected_runs: int = 0
|
||||
post_change_pending_labels: int = 0
|
||||
post_change_quality_score: float = 0.0
|
||||
post_change_affected_rate: float = 0.0
|
||||
post_change_observation_days: int = 0
|
||||
affected_rate_delta: float = 0.0
|
||||
pending_labels_delta: int = 0
|
||||
quality_score_delta: float = 0.0
|
||||
post_change_affected_rate_delta: float = 0.0
|
||||
post_change_pending_labels_delta: int = 0
|
||||
post_change_quality_score_delta: float = 0.0
|
||||
related_change_actions_count: int = 0
|
||||
effect_status: str = "pending_data"
|
||||
effect_summary: str | None = None
|
||||
resolution_note: str | None = None
|
||||
@@ -896,7 +906,11 @@ class InMemoryAiPlatformRepository:
|
||||
**payload.model_dump(exclude={"title"}),
|
||||
)
|
||||
self.quality_improvement_actions[action.id] = action
|
||||
return _materialize_improvement_action(action, self.ai_run_audits)
|
||||
return _materialize_improvement_action(
|
||||
action,
|
||||
self.ai_run_audits,
|
||||
all_actions=self.quality_improvement_actions.values(),
|
||||
)
|
||||
|
||||
def update_quality_improvement_action(
|
||||
self,
|
||||
@@ -939,7 +953,11 @@ class InMemoryAiPlatformRepository:
|
||||
update["linked_import_batch_id"] = payload.linked_import_batch_id
|
||||
updated = action.model_copy(update=update)
|
||||
self.quality_improvement_actions[action_id] = updated
|
||||
return _materialize_improvement_action(updated, self.ai_run_audits)
|
||||
return _materialize_improvement_action(
|
||||
updated,
|
||||
self.ai_run_audits,
|
||||
all_actions=self.quality_improvement_actions.values(),
|
||||
)
|
||||
|
||||
def update_ai_run_quality_label(
|
||||
self,
|
||||
@@ -1557,9 +1575,21 @@ class SqlAlchemyAiPlatformRepository:
|
||||
.order_by(AiQualityImprovementActionModel.updated_at.desc())
|
||||
.limit(20)
|
||||
).all()
|
||||
all_action_rows = session.scalars(
|
||||
select(AiQualityImprovementActionModel).order_by(
|
||||
AiQualityImprovementActionModel.updated_at.desc()
|
||||
)
|
||||
).all()
|
||||
action_days = [row.evaluation_days for row in action_rows]
|
||||
resolved_since = [
|
||||
_ensure_utc(row.resolved_at)
|
||||
for row in action_rows
|
||||
if row.resolved_at is not None
|
||||
]
|
||||
stats_cutoff = datetime.now(UTC) - timedelta(days=max([days, *action_days], default=days) - 1)
|
||||
stats_cutoff = stats_cutoff.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
if resolved_since:
|
||||
stats_cutoff = min(stats_cutoff, min(resolved_since))
|
||||
rows = session.scalars(
|
||||
select(AiRunAuditModel)
|
||||
.where(AiRunAuditModel.created_at >= stats_cutoff)
|
||||
@@ -1572,6 +1602,7 @@ class SqlAlchemyAiPlatformRepository:
|
||||
actions = _materialize_improvement_actions(
|
||||
[_quality_improvement_action_from_model(row) for row in action_rows],
|
||||
all_runs,
|
||||
all_actions=[_quality_improvement_action_from_model(row) for row in all_action_rows],
|
||||
)
|
||||
return _build_quality_dashboard(
|
||||
dashboard_runs,
|
||||
@@ -1608,15 +1639,57 @@ class SqlAlchemyAiPlatformRepository:
|
||||
max_days = max((row.evaluation_days for row in rows), default=14)
|
||||
cutoff = datetime.now(UTC) - timedelta(days=max(max_days - 1, 0))
|
||||
cutoff = cutoff.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
resolved_since = [
|
||||
_ensure_utc(row.resolved_at)
|
||||
for row in rows
|
||||
if row.resolved_at is not None
|
||||
]
|
||||
if resolved_since:
|
||||
cutoff = min(cutoff, min(resolved_since))
|
||||
audit_rows = session.scalars(
|
||||
select(AiRunAuditModel)
|
||||
.where(AiRunAuditModel.created_at >= cutoff)
|
||||
.order_by(AiRunAuditModel.created_at.desc())
|
||||
).all()
|
||||
runs = [_ai_run_audit_from_model(row) for row in audit_rows]
|
||||
related_statement = select(AiQualityImprovementActionModel)
|
||||
if rows:
|
||||
prompt_keys = [
|
||||
(row.linked_prompt_template_id, row.linked_prompt_template_version)
|
||||
for row in rows
|
||||
if row.linked_prompt_template_id
|
||||
]
|
||||
import_batch_ids = [row.linked_import_batch_id for row in rows if row.linked_import_batch_id]
|
||||
knowledge_base_ids = [
|
||||
row.linked_knowledge_base_id
|
||||
for row in rows
|
||||
if row.linked_knowledge_base_id and not row.linked_import_batch_id
|
||||
]
|
||||
related_filters = []
|
||||
for prompt_id, prompt_version in prompt_keys:
|
||||
related_filters.append(
|
||||
(AiQualityImprovementActionModel.linked_prompt_template_id == prompt_id)
|
||||
& (AiQualityImprovementActionModel.linked_prompt_template_version == prompt_version)
|
||||
)
|
||||
if import_batch_ids:
|
||||
related_filters.append(
|
||||
AiQualityImprovementActionModel.linked_import_batch_id.in_(import_batch_ids)
|
||||
)
|
||||
if knowledge_base_ids:
|
||||
related_filters.append(
|
||||
AiQualityImprovementActionModel.linked_knowledge_base_id.in_(knowledge_base_ids)
|
||||
)
|
||||
if related_filters:
|
||||
related_statement = related_statement.where(or_(*related_filters))
|
||||
related_rows = session.scalars(related_statement).all() if rows else []
|
||||
related_actions = [_quality_improvement_action_from_model(row) for row in related_rows]
|
||||
row_actions = [_quality_improvement_action_from_model(row) for row in rows]
|
||||
related_by_id = {action.id: action for action in related_actions}
|
||||
related_by_id.update({action.id: action for action in row_actions})
|
||||
actions = _materialize_improvement_actions(
|
||||
[_quality_improvement_action_from_model(row) for row in rows],
|
||||
row_actions,
|
||||
runs,
|
||||
all_actions=related_by_id.values(),
|
||||
)
|
||||
return _filter_improvement_actions(
|
||||
actions,
|
||||
@@ -1682,7 +1755,12 @@ class SqlAlchemyAiPlatformRepository:
|
||||
)
|
||||
session.add(row)
|
||||
session.flush()
|
||||
return _materialize_improvement_action(_quality_improvement_action_from_model(row), runs)
|
||||
action_rows = session.scalars(select(AiQualityImprovementActionModel)).all()
|
||||
return _materialize_improvement_action(
|
||||
_quality_improvement_action_from_model(row),
|
||||
runs,
|
||||
all_actions=[_quality_improvement_action_from_model(action_row) for action_row in action_rows],
|
||||
)
|
||||
|
||||
def update_quality_improvement_action(
|
||||
self,
|
||||
@@ -1724,6 +1802,8 @@ class SqlAlchemyAiPlatformRepository:
|
||||
|
||||
cutoff = now - timedelta(days=max(row.evaluation_days - 1, 0))
|
||||
cutoff = cutoff.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
if row.resolved_at is not None:
|
||||
cutoff = min(cutoff, _ensure_utc(row.resolved_at))
|
||||
audit_rows = session.scalars(
|
||||
select(AiRunAuditModel)
|
||||
.where(AiRunAuditModel.created_at >= cutoff)
|
||||
@@ -1731,7 +1811,12 @@ class SqlAlchemyAiPlatformRepository:
|
||||
).all()
|
||||
runs = [_ai_run_audit_from_model(row) for row in audit_rows]
|
||||
session.flush()
|
||||
return _materialize_improvement_action(_quality_improvement_action_from_model(row), runs)
|
||||
action_rows = session.scalars(select(AiQualityImprovementActionModel)).all()
|
||||
return _materialize_improvement_action(
|
||||
_quality_improvement_action_from_model(row),
|
||||
runs,
|
||||
all_actions=[_quality_improvement_action_from_model(action_row) for action_row in action_rows],
|
||||
)
|
||||
|
||||
def update_ai_run_quality_label(
|
||||
self,
|
||||
@@ -2413,20 +2498,38 @@ def _finalize_attribution_groups(groups: dict[str, dict[str, object]]) -> list[A
|
||||
def _materialize_improvement_actions(
|
||||
actions: Iterable[AiQualityImprovementAction],
|
||||
runs: list[AiRunAudit],
|
||||
*,
|
||||
all_actions: Iterable[AiQualityImprovementAction] | None = None,
|
||||
) -> list[AiQualityImprovementAction]:
|
||||
return [_materialize_improvement_action(action, runs) for action in actions]
|
||||
action_list = list(actions)
|
||||
related_actions = list(all_actions) if all_actions is not None else action_list
|
||||
return [
|
||||
_materialize_improvement_action(action, runs, all_actions=related_actions)
|
||||
for action in action_list
|
||||
]
|
||||
|
||||
|
||||
def _materialize_improvement_action(
|
||||
action: AiQualityImprovementAction,
|
||||
runs: list[AiRunAudit],
|
||||
*,
|
||||
all_actions: Iterable[AiQualityImprovementAction] | None = None,
|
||||
) -> AiQualityImprovementAction:
|
||||
current = _calculate_improvement_action_stats(action, runs)
|
||||
post_change = (
|
||||
_calculate_improvement_action_stats(action, runs, since=action.resolved_at)
|
||||
if action.resolved_at
|
||||
else {"total_runs": 0, "affected_runs": 0, "pending_labels": 0, "quality_score": 0.0}
|
||||
)
|
||||
baseline_rate = _safe_rate(action.baseline_affected_runs, action.baseline_total_runs)
|
||||
current_rate = _safe_rate(current["affected_runs"], current["total_runs"])
|
||||
affected_rate_delta = round(current_rate - baseline_rate, 4)
|
||||
pending_labels_delta = current["pending_labels"] - action.baseline_pending_labels
|
||||
quality_score_delta = round(current["quality_score"] - action.baseline_quality_score, 2)
|
||||
post_change_rate = _safe_rate(post_change["affected_runs"], post_change["total_runs"])
|
||||
post_change_affected_rate_delta = round(post_change_rate - baseline_rate, 4)
|
||||
post_change_pending_labels_delta = post_change["pending_labels"] - action.baseline_pending_labels
|
||||
post_change_quality_score_delta = round(post_change["quality_score"] - action.baseline_quality_score, 2)
|
||||
effect_status, effect_summary = _summarize_improvement_effect(
|
||||
action,
|
||||
current_total_runs=current["total_runs"],
|
||||
@@ -2434,6 +2537,11 @@ def _materialize_improvement_action(
|
||||
affected_rate_delta=affected_rate_delta,
|
||||
pending_labels_delta=pending_labels_delta,
|
||||
quality_score_delta=quality_score_delta,
|
||||
post_change_total_runs=post_change["total_runs"],
|
||||
post_change_rate=post_change_rate,
|
||||
post_change_affected_rate_delta=post_change_affected_rate_delta,
|
||||
post_change_pending_labels_delta=post_change_pending_labels_delta,
|
||||
post_change_quality_score_delta=post_change_quality_score_delta,
|
||||
)
|
||||
return action.model_copy(
|
||||
update={
|
||||
@@ -2441,9 +2549,19 @@ def _materialize_improvement_action(
|
||||
"current_affected_runs": current["affected_runs"],
|
||||
"current_pending_labels": current["pending_labels"],
|
||||
"current_quality_score": current["quality_score"],
|
||||
"post_change_total_runs": post_change["total_runs"],
|
||||
"post_change_affected_runs": post_change["affected_runs"],
|
||||
"post_change_pending_labels": post_change["pending_labels"],
|
||||
"post_change_quality_score": post_change["quality_score"],
|
||||
"post_change_affected_rate": post_change_rate,
|
||||
"post_change_observation_days": _post_change_observation_days(action),
|
||||
"affected_rate_delta": affected_rate_delta,
|
||||
"pending_labels_delta": pending_labels_delta,
|
||||
"quality_score_delta": quality_score_delta,
|
||||
"post_change_affected_rate_delta": post_change_affected_rate_delta,
|
||||
"post_change_pending_labels_delta": post_change_pending_labels_delta,
|
||||
"post_change_quality_score_delta": post_change_quality_score_delta,
|
||||
"related_change_actions_count": _count_related_change_actions(action, all_actions or ()),
|
||||
"effect_status": effect_status,
|
||||
"effect_summary": effect_summary,
|
||||
}
|
||||
@@ -2484,9 +2602,12 @@ def _filter_improvement_actions(
|
||||
def _calculate_improvement_action_stats(
|
||||
action: AiQualityImprovementActionCreate | AiQualityImprovementAction,
|
||||
runs: list[AiRunAudit],
|
||||
*,
|
||||
since: datetime | None = None,
|
||||
) -> dict[str, int | float]:
|
||||
cutoff = datetime.now(UTC) - timedelta(days=max(action.evaluation_days - 1, 0))
|
||||
cutoff = cutoff.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
cutoff = _ensure_utc(since) if since else datetime.now(UTC) - timedelta(days=max(action.evaluation_days - 1, 0))
|
||||
if not since:
|
||||
cutoff = cutoff.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
matched_runs: list[AiRunAudit] = []
|
||||
affected_runs: set[str] = set()
|
||||
pending_labels = 0
|
||||
@@ -2550,6 +2671,37 @@ def _has_source_issue(run: AiRunAudit) -> bool:
|
||||
return any(label.code in {"source_insufficient", "source_limited"} for label in run.quality_labels)
|
||||
|
||||
|
||||
def _post_change_observation_days(action: AiQualityImprovementAction) -> int:
|
||||
if not action.resolved_at:
|
||||
return 0
|
||||
elapsed = datetime.now(UTC).date() - _ensure_utc(action.resolved_at).date()
|
||||
return max(elapsed.days + 1, 1)
|
||||
|
||||
|
||||
def _count_related_change_actions(
|
||||
action: AiQualityImprovementAction,
|
||||
actions: Iterable[AiQualityImprovementAction],
|
||||
) -> int:
|
||||
change_key = _quality_action_change_key(action)
|
||||
if change_key is None:
|
||||
return 0
|
||||
return sum(1 for candidate in actions if _quality_action_change_key(candidate) == change_key)
|
||||
|
||||
|
||||
def _quality_action_change_key(action: AiQualityImprovementAction) -> tuple[str, str, str | None] | None:
|
||||
if action.linked_prompt_template_id:
|
||||
return (
|
||||
"prompt",
|
||||
action.linked_prompt_template_id,
|
||||
action.linked_prompt_template_version,
|
||||
)
|
||||
if action.linked_import_batch_id:
|
||||
return ("knowledge_import", action.linked_import_batch_id, None)
|
||||
if action.linked_knowledge_base_id:
|
||||
return ("knowledge_base", action.linked_knowledge_base_id, None)
|
||||
return None
|
||||
|
||||
|
||||
def _safe_rate(numerator: int | float, denominator: int | float) -> float:
|
||||
return round(float(numerator) / float(denominator), 4) if denominator else 0.0
|
||||
|
||||
@@ -2562,26 +2714,50 @@ def _summarize_improvement_effect(
|
||||
affected_rate_delta: float,
|
||||
pending_labels_delta: int | float,
|
||||
quality_score_delta: float,
|
||||
post_change_total_runs: int | float,
|
||||
post_change_rate: float,
|
||||
post_change_affected_rate_delta: float,
|
||||
post_change_pending_labels_delta: int | float,
|
||||
post_change_quality_score_delta: float,
|
||||
) -> tuple[str, str]:
|
||||
baseline_rate = _safe_rate(action.baseline_affected_runs, action.baseline_total_runs)
|
||||
if action.baseline_total_runs <= 0 or current_total_runs <= 0:
|
||||
resolved = action.status in {"resolved", "false_positive", "closed"}
|
||||
if action.baseline_total_runs <= 0:
|
||||
return "pending_data", "Need more AI run data to evaluate this improvement action."
|
||||
if resolved and post_change_total_runs <= 0:
|
||||
return (
|
||||
"pending_data",
|
||||
"Need post-change AI run data after this action was resolved to evaluate effect.",
|
||||
)
|
||||
if not resolved and current_total_runs <= 0:
|
||||
return "pending_data", "Need more AI run data to evaluate this improvement action."
|
||||
effective_rate_delta = post_change_affected_rate_delta if resolved else affected_rate_delta
|
||||
effective_pending_delta = post_change_pending_labels_delta if resolved else pending_labels_delta
|
||||
effective_quality_delta = post_change_quality_score_delta if resolved else quality_score_delta
|
||||
improved = (
|
||||
affected_rate_delta < -0.01
|
||||
or pending_labels_delta < 0
|
||||
or quality_score_delta > 1
|
||||
effective_rate_delta < -0.01
|
||||
or effective_pending_delta < 0
|
||||
or effective_quality_delta > 1
|
||||
)
|
||||
regressed = (
|
||||
affected_rate_delta > 0.01
|
||||
or pending_labels_delta > 0
|
||||
or quality_score_delta < -1
|
||||
effective_rate_delta > 0.01
|
||||
or effective_pending_delta > 0
|
||||
or effective_quality_delta < -1
|
||||
)
|
||||
status = "improved" if improved and not regressed else "regressed" if regressed and not improved else "unchanged"
|
||||
summary = (
|
||||
f"Baseline affected rate {baseline_rate:.0%}, current {current_rate:.0%}; "
|
||||
f"pending labels delta {pending_labels_delta:+.0f}; "
|
||||
f"quality score delta {quality_score_delta:+.1f}."
|
||||
)
|
||||
if resolved:
|
||||
summary = (
|
||||
f"Baseline affected rate {baseline_rate:.0%}, post-change {post_change_rate:.0%} "
|
||||
f"over {post_change_total_runs:.0f} run(s); "
|
||||
f"pending labels delta {post_change_pending_labels_delta:+.0f}; "
|
||||
f"quality score delta {post_change_quality_score_delta:+.1f}."
|
||||
)
|
||||
else:
|
||||
summary = (
|
||||
f"Baseline affected rate {baseline_rate:.0%}, current {current_rate:.0%}; "
|
||||
f"pending labels delta {pending_labels_delta:+.0f}; "
|
||||
f"quality score delta {quality_score_delta:+.1f}."
|
||||
)
|
||||
return status, summary
|
||||
|
||||
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
@@ -450,6 +452,63 @@ def test_quality_improvement_action_resolve_api_returns_422_without_real_change(
|
||||
assert "linked_knowledge_base_id" in resolve_response.json()["detail"]
|
||||
|
||||
|
||||
def test_quality_improvement_action_effect_review_uses_post_resolution_window() -> None:
|
||||
repository = InMemoryAiPlatformRepository()
|
||||
repository.seed_defaults()
|
||||
repository.add_ai_run_audit(
|
||||
AiRunAudit(
|
||||
id="run-baseline",
|
||||
operation="legal_qa",
|
||||
provider="deepseek",
|
||||
model="deepseek-chat",
|
||||
prompt_template_id="legal_qa.default",
|
||||
retrieved_sources=[],
|
||||
quality_score=88,
|
||||
quality_status="needs_review",
|
||||
quality_labels=[],
|
||||
created_at=datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
action = repository.create_quality_improvement_action(
|
||||
AiQualityImprovementActionCreate(
|
||||
dimension="prompt_template",
|
||||
key="legal_qa.default",
|
||||
label="legal_qa.default",
|
||||
action_type="tune_prompt",
|
||||
evaluation_days=7,
|
||||
)
|
||||
)
|
||||
assert action.current_total_runs >= 1
|
||||
resolved = repository.update_quality_improvement_action(
|
||||
action.id,
|
||||
AiQualityImprovementActionUpdate(
|
||||
status="resolved",
|
||||
linked_prompt_template_id=action.linked_prompt_template_id,
|
||||
linked_prompt_template_version=action.linked_prompt_template_version,
|
||||
resolution_note="Prompt improved.",
|
||||
),
|
||||
)
|
||||
repository.add_ai_run_audit(
|
||||
AiRunAudit(
|
||||
id="run-post-change",
|
||||
operation="legal_qa",
|
||||
provider="deepseek",
|
||||
model="deepseek-chat",
|
||||
prompt_template_id="legal_qa.default",
|
||||
retrieved_sources=[],
|
||||
quality_score=96,
|
||||
quality_status="passed",
|
||||
quality_labels=[],
|
||||
created_at=resolved.resolved_at or datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
refreshed = repository.list_quality_improvement_actions(limit=5)[0]
|
||||
assert refreshed.post_change_observation_days >= 1
|
||||
assert refreshed.post_change_total_runs >= 1
|
||||
assert refreshed.related_change_actions_count >= 1
|
||||
assert "post-change" in (refreshed.effect_summary or "")
|
||||
|
||||
|
||||
def test_knowledge_document_can_be_added_and_searched() -> None:
|
||||
client = make_client()
|
||||
|
||||
|
||||
@@ -375,9 +375,19 @@ class AiQualityImprovementAction(AiQualityImprovementActionCreate):
|
||||
current_affected_runs: int = 0
|
||||
current_pending_labels: int = 0
|
||||
current_quality_score: float = 0.0
|
||||
post_change_total_runs: int = 0
|
||||
post_change_affected_runs: int = 0
|
||||
post_change_pending_labels: int = 0
|
||||
post_change_quality_score: float = 0.0
|
||||
post_change_affected_rate: float = 0.0
|
||||
post_change_observation_days: int = 0
|
||||
affected_rate_delta: float = 0.0
|
||||
pending_labels_delta: int = 0
|
||||
quality_score_delta: float = 0.0
|
||||
post_change_affected_rate_delta: float = 0.0
|
||||
post_change_pending_labels_delta: int = 0
|
||||
post_change_quality_score_delta: float = 0.0
|
||||
related_change_actions_count: int = 0
|
||||
effect_status: str = "pending_data"
|
||||
effect_summary: str | None = None
|
||||
resolution_note: str | None = None
|
||||
|
||||
@@ -128,11 +128,21 @@ def test_ai_platform_client_covers_admin_data_endpoints() -> None:
|
||||
"current_affected_runs": 1,
|
||||
"current_pending_labels": 1,
|
||||
"current_quality_score": 90.0,
|
||||
"post_change_total_runs": 2,
|
||||
"post_change_affected_runs": 0,
|
||||
"post_change_pending_labels": 0,
|
||||
"post_change_quality_score": 98.0,
|
||||
"post_change_affected_rate": 0.0,
|
||||
"post_change_observation_days": 3,
|
||||
"affected_rate_delta": 0.0,
|
||||
"pending_labels_delta": 0,
|
||||
"quality_score_delta": 0.0,
|
||||
"post_change_affected_rate_delta": -1.0,
|
||||
"post_change_pending_labels_delta": -1,
|
||||
"post_change_quality_score_delta": 8.0,
|
||||
"related_change_actions_count": 2,
|
||||
"effect_status": "unchanged",
|
||||
"effect_summary": "Baseline affected rate 100%, current 100%; pending labels delta +0; quality score delta +0.0.",
|
||||
"effect_summary": "Baseline affected rate 100%, post-change 0% over 2 run(s); pending labels delta -1; quality score delta +8.0.",
|
||||
"resolution_note": resolution_note,
|
||||
"created_at": "2026-06-22T00:00:00Z",
|
||||
"updated_at": "2026-06-22T00:02:00Z",
|
||||
@@ -817,6 +827,8 @@ def test_ai_platform_client_covers_admin_data_endpoints() -> None:
|
||||
assert created_action.linked_import_batch_id == "kb-batch-1"
|
||||
assert created_action.baseline_affected_runs == 1
|
||||
assert created_action.effect_status == "unchanged"
|
||||
assert created_action.post_change_total_runs == 2
|
||||
assert created_action.related_change_actions_count == 2
|
||||
listed_actions = client.list_quality_improvement_actions(
|
||||
status="open",
|
||||
owner="admin",
|
||||
@@ -837,6 +849,8 @@ def test_ai_platform_client_covers_admin_data_endpoints() -> None:
|
||||
)
|
||||
assert updated_action.status == "resolved"
|
||||
assert updated_action.resolved_at == "2026-06-22T00:05:00Z"
|
||||
assert updated_action.post_change_total_runs == 2
|
||||
assert updated_action.effect_summary and "post-change" in updated_action.effect_summary
|
||||
updated_audit = client.update_ai_run_quality_label(
|
||||
"run-1",
|
||||
"citation_present",
|
||||
|
||||
+30
-3
@@ -100,9 +100,19 @@ type QualityImprovementAction = {
|
||||
current_affected_runs: number;
|
||||
current_pending_labels: number;
|
||||
current_quality_score: number;
|
||||
post_change_total_runs: number;
|
||||
post_change_affected_runs: number;
|
||||
post_change_pending_labels: number;
|
||||
post_change_quality_score: number;
|
||||
post_change_affected_rate: number;
|
||||
post_change_observation_days: number;
|
||||
affected_rate_delta: number;
|
||||
pending_labels_delta: number;
|
||||
quality_score_delta: number;
|
||||
post_change_affected_rate_delta: number;
|
||||
post_change_pending_labels_delta: number;
|
||||
post_change_quality_score_delta: number;
|
||||
related_change_actions_count: number;
|
||||
effect_status?: string;
|
||||
effect_summary?: string | null;
|
||||
resolution_note?: string | null;
|
||||
@@ -774,6 +784,7 @@ function QualityActionItem({
|
||||
const needsPromptDraft = action.action_type === "tune_prompt";
|
||||
const needsKnowledgeImport = action.action_type === "improve_knowledge";
|
||||
const completionBlockReason = getCompletionBlockReason(action);
|
||||
const hasPostChange = action.post_change_total_runs > 0;
|
||||
|
||||
return (
|
||||
<div className="quality-action-item">
|
||||
@@ -796,13 +807,29 @@ function QualityActionItem({
|
||||
<span className="item-meta">{action.description || action.root_cause_hint}</span>
|
||||
)}
|
||||
<div className="quality-action-stats">
|
||||
<span>异常率 {formatPercent(baselineRate)} → {formatPercent(currentRate)}</span>
|
||||
<span>待处理 {action.baseline_pending_labels} → {action.current_pending_labels}</span>
|
||||
<span>质量分 {action.baseline_quality_score} → {action.current_quality_score}</span>
|
||||
<span>改前异常率 {formatPercent(baselineRate)}</span>
|
||||
<span>当前异常率 {formatPercent(currentRate)}</span>
|
||||
<span>完成后异常率 {hasPostChange ? formatPercent(action.post_change_affected_rate) : "待观察"}</span>
|
||||
<span>观察 {action.post_change_observation_days || action.evaluation_days} 天</span>
|
||||
<span>改前待处理 {action.baseline_pending_labels}</span>
|
||||
<span>完成后待处理 {hasPostChange ? action.post_change_pending_labels : "待观察"}</span>
|
||||
<span>改前质量分 {action.baseline_quality_score}</span>
|
||||
<span>完成后质量分 {hasPostChange ? action.post_change_quality_score : "待观察"}</span>
|
||||
<span>变化 {formatSignedPercent(action.affected_rate_delta)}</span>
|
||||
</div>
|
||||
{action.resolution_note && <span className="item-meta">处理说明:{action.resolution_note}</span>}
|
||||
<LinkedChangeSummary action={action} />
|
||||
<div className="quality-action-closure">
|
||||
<span className="item-meta">关联改进项 {action.related_change_actions_count}</span>
|
||||
{hasPostChange && (
|
||||
<span className="item-meta">
|
||||
完成后变化:异常率 {formatSignedPercent(action.post_change_affected_rate_delta)} / 待处理{" "}
|
||||
{action.post_change_pending_labels_delta >= 0
|
||||
? `+${action.post_change_pending_labels_delta}`
|
||||
: action.post_change_pending_labels_delta} / 质量分 {formatSignedPercent(action.post_change_quality_score_delta)}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
{action.effect_summary && <span className="item-meta">效果复盘:{action.effect_summary}</span>}
|
||||
{["open", "in_progress"].includes(action.status) && (needsPromptDraft || needsKnowledgeImport) && (
|
||||
<div className="quality-action-execution">
|
||||
|
||||
@@ -812,6 +812,12 @@ select {
|
||||
background: #f1f5f9;
|
||||
}
|
||||
|
||||
.quality-action-closure {
|
||||
display: grid;
|
||||
gap: 6px;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.quality-action-execution {
|
||||
display: grid;
|
||||
gap: 10px;
|
||||
|
||||
Reference in New Issue
Block a user