diff --git a/SESSION.md b/SESSION.md index 07d4f9e..443546e 100644 --- a/SESSION.md +++ b/SESSION.md @@ -2,7 +2,7 @@ ## Current Goal -Automation execution-layer P21 is closed locally. P21 adds a runtime incident journal with persisted dedupe/aggregation, recommended actions, API/script access, runtime status summary, and workbench visibility. The default next action is P22 `Maintenance And Retention Rollup`. +Automation execution-layer P22 is closed locally. P22 adds a read-only maintenance summary that rolls up visual capture retention, alert delivery/feedback health, runtime watchdog state, and incident journal counts into P23-ready assertions. The default next action is P23 `End-To-End Unattended Harness`. ## Current Branch @@ -14,8 +14,8 @@ Automation execution-layer P21 is closed locally. P21 adds a runtime incident jo ## Automation Status -- state: automation_execution_layer_p21_runtime_incident_journal_closed -- current_phase: P21 Runtime Incident Journal +- state: automation_execution_layer_p22_maintenance_retention_rollup_closed +- current_phase: P22 Maintenance And Retention Rollup - latest_p7_selector_choice: continue_periodic_analysis_decision_monitoring_with_maturity_rollup - latest_p14_slice: webhook_transport_retry - latest_p15_pxx_tasklist_status: ready @@ -52,7 +52,12 @@ Automation execution-layer P21 is closed locally. P21 adds a runtime incident jo - latest_runtime_incident_journal_route_status: ok - latest_runtime_incident_journal_workbench_status: visible - latest_runtime_incident_journal_smoke_status: idle_persisted -- next_execution_layer_phase: P22 Maintenance And Retention Rollup +- latest_p22_slice: maintenance_retention_rollup +- latest_maintenance_summary_status: implemented +- latest_maintenance_summary_route_status: ok +- latest_maintenance_summary_workbench_status: visible +- latest_maintenance_summary_smoke_status: warning_feedback_sample_health +- next_execution_layer_phase: P23 End-To-End Unattended Harness - latest_visual_review_script_status: idle - latest_runtime_visual_watch_task_status: ok - latest_visual_capture_cleanup_task_status: ok @@ -134,6 +139,9 @@ Automation execution-layer P21 is closed locally. P21 adds a runtime incident jo - Added `runtime_incidents` persistence with upsert aggregation by incident key. - Added `RuntimeIncidentJournalService`, `scripts/run_runtime_incident_journal.py`, and `GET /trading/runtime-incidents`. - Extended `runtime_status`, `run_trading_runtime.py`, and the workbench runtime panel with incident summary, severity/source counts, and recommended actions. +- Added `MaintenanceSummaryService` for read-only storage/delivery/feedback/runtime maintenance rollups. +- Added `scripts/run_maintenance_summary.py` and `GET /trading/maintenance-summary`. +- Extended the workbench runtime panel with maintenance status, blockers/warnings, actions, and P23 assertion visibility. ## Current Working State @@ -161,6 +169,8 @@ Automation execution-layer P21 is closed locally. P21 adds a runtime incident jo - P20 visibility is implemented: `scripts/run_operator_digest.py`, `/trading/alerts/operator-digest`, focused tests, and the workbench `Alert Quality` panel expose digest status, counts, warnings, and top queue items. - P21 runtime incident journal is implemented: runtime/watchdog/blocker/noisy incident signals can be normalized into persisted incidents with severity, source, reason, subject, first/last seen timestamps, occurrence count, and recommended action. - P21 visibility is implemented: `/trading/runtime-incidents`, `scripts/run_runtime_incident_journal.py`, `runtime_status.incident_summary`, `run_trading_runtime.py`, and the workbench runtime panel expose incident counts and recommendations. +- P22 maintenance retention rollup is implemented: visual capture retention dry-run stats, alert delivery health, feedback sample health, runtime watchdog, and incident journal health are aggregated into `checks`, `blockers`, `warnings`, `maintenance_actions`, and `p23_assertions`. +- P22 visibility is implemented: `/trading/maintenance-summary`, `scripts/run_maintenance_summary.py`, and the workbench runtime panel expose maintenance state without deleting business data. - Latest `scripts/run_visual_review.py` smoke found no pending reviews: attempted=0, reviewed=0, status=idle. - Latest capture smoke returned `status=ignored` when no watch run was supplied, which is the expected safe no-op. - Latest cleanup smoke deleted one expired temporary artifact through `scripts/run_visual_capture_cleanup.py`. @@ -170,7 +180,7 @@ Automation execution-layer P21 is closed locally. P21 adds a runtime incident jo - Latest `scripts/run_trading_runtime.py` smoke returned blockers=none and includes a healthy `visual_watch` runtime task. - Latest local runtime smoke confirmed `runtime_status.tasks.alert_dispatch.details` now exposes retry and silent-window counters with blockers=none. -## P21 Blockers +## P22 Blockers - none @@ -192,6 +202,7 @@ Automation execution-layer P21 is closed locally. P21 adds a runtime incident jo - P19 channel routing remains notification routing/observability only and must not change execution permission, live readiness, trust bucket, risk governor, or operator confirmation. - P20 operator digest remains read-only review aggregation only and must not change execution permission, live readiness, trust bucket, risk governor, or operator confirmation. - P21 runtime incident journal remains observability/persistence only and must not change execution permission, live readiness, trust bucket, risk governor, or operator confirmation. +- P22 maintenance summary remains read-only and uses visual capture retention dry-run statistics only; it must not delete business data or alter execution permission, live readiness, trust bucket, risk governor, or operator confirmation. - A final rerun of `scripts/run_alert_dispatcher.py` and `scripts/run_trading_runtime.py` hit local PostgreSQL connection timeout because nothing was listening on `127.0.0.1:5432`; the P14 code path is still covered by full unit tests and earlier successful smoke during implementation. - P15 dispatcher smoke succeeded with `suppressed_noise_policy`, `workbench_only`, and `policy_external_allowed` counters present. P15 runtime smoke reached `alert_dispatch` successfully, but the overall runtime returned external environment blockers for market stream disconnected and OKX read transport connection refused. - P16 direct browser smoke succeeded locally with `browser_capture_status=captured`. Script-level capture without a watch run still returns safe no-op `watch_run_missing`. @@ -209,11 +220,13 @@ Automation execution-layer P21 is closed locally. P21 adds a runtime incident jo - P21 local migration applied `043_create_runtime_incidents.sql`. - P21 full validation passed: 691 unit tests, 69 script static checks, `node --check src/web/app.js`, and runtime smoke blockers=none with `Incident journal: idle / 0`. - P21 script/API smoke returned idle persisted incident journal state, which is expected when no current runtime incidents are present. +- P22 full validation passed: 696 unit tests, 70 script static checks, and `node --check src/web/app.js`. +- P22 script/API smoke returned `status=warning` with `warnings=['feedback.sample_health']`, blockers=[], automatic_actions=[], and P23 assertions visible. - Real-live order submission remains intentionally disabled because the OKX order adapter is still not a real-live submitter. ## Exact Next Step -P21 is locally closed. Default next action is P22 `Maintenance And Retention Rollup`: aggregate retention cleanup, alert delivery/feedback, visual capture, incident journal, and watchdog health into a read-only maintenance summary. +P22 is locally closed. Default next action is P23 `End-To-End Unattended Harness`: build a repeatable local harness covering P15-P22 key paths and producing a single JSON report with blockers, warnings, and passed checks. ## Validation Commands Used Recently @@ -239,4 +252,6 @@ D:\ai-exchange\.venv\Scripts\python.exe scripts\run_operator_digest.py D:\ai-exchange\.venv\Scripts\python.exe -m unittest tests.unit.test_runtime_incident_journal_service tests.unit.test_run_runtime_incident_journal tests.unit.test_trading_runtime_service tests.unit.test_api_static_server.ApiStaticServerTests.test_dashboard_uses_canonical_trading_routes D:\ai-exchange\.venv\Scripts\python.exe scripts\apply_migrations.py D:\ai-exchange\.venv\Scripts\python.exe scripts\run_runtime_incident_journal.py +D:\ai-exchange\.venv\Scripts\python.exe -m unittest tests.unit.test_maintenance_summary_service tests.unit.test_run_maintenance_summary tests.unit.test_api_server.ApiServerRoutingTests.test_routes_maintenance_summary tests.unit.test_api_static_server.ApiStaticServerTests.test_dashboard_uses_canonical_trading_routes +D:\ai-exchange\.venv\Scripts\python.exe scripts\run_maintenance_summary.py ``` diff --git a/TASKS.md b/TASKS.md index 9ede5e1..e1d45ba 100644 --- a/TASKS.md +++ b/TASKS.md @@ -13,7 +13,7 @@ ## In Progress -- [x] P21 Runtime Incident Journal local closure complete +- [x] P22 Maintenance And Retention Rollup local closure complete ## Next Up @@ -32,14 +32,15 @@ - [x] P19 local closure complete - [x] P20 local closure complete - [x] P21 local closure complete -- [ ] Start P22 Maintenance And Retention Rollup +- [x] P22 local closure complete +- [ ] Start P23 End-To-End Unattended Harness - [ ] Continue periodic analysis-decision monitoring with maturity rollup in parallel - [ ] Treat browser screenshot capture and additional external media adapters as future enhancements ## Automation Status -- [x] state: automation_execution_layer_p21_runtime_incident_journal_closed -- [x] current_phase: P21 Runtime Incident Journal +- [x] state: automation_execution_layer_p22_maintenance_retention_rollup_closed +- [x] current_phase: P22 Maintenance And Retention Rollup - [x] latest_p7_selector_choice: continue_periodic_analysis_decision_monitoring_with_maturity_rollup - [x] latest_p14_slice: webhook_transport_retry - [x] latest_p15_pxx_tasklist_status: ready @@ -76,7 +77,12 @@ - [x] latest_runtime_incident_journal_route_status: ok - [x] latest_runtime_incident_journal_workbench_status: visible - [x] latest_runtime_incident_journal_smoke_status: idle_persisted -- [x] next_execution_layer_phase: P22 Maintenance And Retention Rollup +- [x] latest_p22_slice: maintenance_retention_rollup +- [x] latest_maintenance_summary_status: implemented +- [x] latest_maintenance_summary_route_status: ok +- [x] latest_maintenance_summary_workbench_status: visible +- [x] latest_maintenance_summary_smoke_status: warning_feedback_sample_health +- [x] next_execution_layer_phase: P23 End-To-End Unattended Harness - [x] latest_visual_review_script_status: idle - [x] latest_runtime_visual_watch_task_status: ok - [x] latest_visual_capture_cleanup_task_status: ok @@ -179,6 +185,11 @@ - [x] Added `runtime_status.incident_summary` - [x] Added incident journal visibility to `run_trading_runtime.py` and the workbench runtime panel - [x] Added focused P21 service, script, API, runtime, and dashboard route tests +- [x] Added `MaintenanceSummaryService` +- [x] Added `scripts/run_maintenance_summary.py` +- [x] Added `GET /trading/maintenance-summary` +- [x] Added maintenance rollup visibility to the workbench runtime panel +- [x] Added focused P22 maintenance service, script, API, and dashboard route tests - [x] Added runtime `visual_watch` task - [x] Added focused service, script, API, and runtime tests - [x] Smoked the visual review script with no pending reviews @@ -206,6 +217,8 @@ - [x] Applied local migration `043_create_runtime_incidents.sql` - [x] Smoked P21 runtime incident journal script/API with idle persisted state - [x] Completed P21 full validation: 691 unit tests, 69 script static checks, dashboard JS syntax, runtime incident script/API smoke, and runtime smoke blockers=none +- [x] Smoked P22 maintenance summary script/API with warning-only feedback sample health and no blockers +- [x] Completed P22 full validation: 696 unit tests, 70 script static checks, dashboard JS syntax, maintenance summary script/API smoke ## Blocked @@ -258,6 +271,7 @@ - [x] `automation_execution_layer_p19_multi_channel_alert_routing` - [x] `automation_execution_layer_p20_operator_digest_review_queue` - [x] `automation_execution_layer_p21_runtime_incident_journal` +- [x] `automation_execution_layer_p22_maintenance_retention_rollup` ## P15-PXX Finite Queue @@ -268,7 +282,7 @@ - [x] `P19` Multi-Channel Alert Routing - [x] `P20` Operator Digest And Review Queue - [x] `P21` Runtime Incident Journal -- [ ] `P22` Maintenance And Retention Rollup +- [x] `P22` Maintenance And Retention Rollup - [ ] `P23` End-To-End Unattended Harness - [ ] `P24` Remote Smoke And Ops Closure - [ ] `PXX` Automation Execution Closure Gate diff --git a/docs/AI_ICT_盯盘告警开发任务清单_v1.md b/docs/AI_ICT_盯盘告警开发任务清单_v1.md index 0df7fc8..7429c14 100644 --- a/docs/AI_ICT_盯盘告警开发任务清单_v1.md +++ b/docs/AI_ICT_盯盘告警开发任务清单_v1.md @@ -411,6 +411,7 @@ candidate -> decision alert -> dispatcher -> 外部消息 -> delivery record - 2026-05-12 P19 已完成 `multi-channel alert routing` 本地闭环：channel router 现在输出 workbench/webhook/WeCom/Email capability 与 health summary，WeCom/Email 在凭据缺失或 adapter 未实现时会以 disabled adapter 形式返回 `credential_missing` / `adapter_not_implemented`，dispatcher、delivery summary、runtime、脚本和 workbench 均可见 per-channel 状态。 - 2026-05-12 P20 已完成 `operator digest / review queue` 本地闭环：operator digest 会把高价值告警、被抑制 delivery、失败/重试 delivery、视觉 mismatch 与待反馈项聚合成 bounded review queue，并通过脚本、API 和 workbench `Alert Quality` 面板暴露。 - 2026-05-12 P21 已完成 `runtime incident journal` 本地闭环：runtime/watchdog/blocker/noisy incident 现在可聚合写入 `runtime_incidents`，每条 incident 带 severity、source、reason、first/last seen 与 recommended action，并通过脚本、API、runtime status 和 workbench runtime panel 暴露。 +- 2026-05-12 P22 已完成 `maintenance / retention rollup` 本地闭环：maintenance summary 以只读方式聚合 visual capture retention dry-run、delivery/feedback、runtime watchdog 与 incident journal，输出 P23 harness 可复用的 checks/blockers/warnings/p23_assertions，不执行未纳入 retention policy 的业务数据删除。 ### 验收 - 非重大错误不打断主循环 diff --git a/docs/AI_ICT_盯盘告警技术设计稿_v1.md b/docs/AI_ICT_盯盘告警技术设计稿_v1.md index e6a7afe..573ca85 100644 --- a/docs/AI_ICT_盯盘告警技术设计稿_v1.md +++ b/docs/AI_ICT_盯盘告警技术设计稿_v1.md @@ -119,6 +119,11 @@ 2026-05-12 P21 已落地：`runtime_incidents` 使用 `incident_key` 做去重聚合，并保存 severity、source、reason、subject、first/last seen、occurrence_count、recommended_action 与 payload。`GET /trading/runtime-incidents`、`scripts/run_runtime_incident_journal.py`、`runtime_status.incident_summary` 与 workbench runtime panel 均可查看当前 incident summary；该层只做可观测性与恢复建议，不改变任何交易权限或 operator confirmation protocol。 +#### `src/services/maintenance/maintenance_summary_service.py` +职责：只读汇总 storage / delivery / feedback / runtime 维护健康状态，为无人值守 harness 提供断言输入。 + +2026-05-12 P22 已落地：maintenance summary 会 dry-run visual capture retention，读取 alert quality summary、runtime watchdog 与 runtime incident journal，输出 `checks`、`blockers`、`warnings`、`maintenance_actions` 与 `p23_assertions`。该 summary 不执行删除；自动清理动作只作为可用 action 暴露，未纳入 retention policy 的业务数据不会被删除。 + #### `src/services/alerts/delivery_policy_service.py` 职责： diff --git a/docs/AI_ICT_自动化执行层_P15-PXX闭环任务清单_v1.md b/docs/AI_ICT_自动化执行层_P15-PXX闭环任务清单_v1.md index dfd3379..74248d6 100644 --- a/docs/AI_ICT_自动化执行层_P15-PXX闭环任务清单_v1.md +++ b/docs/AI_ICT_自动化执行层_P15-PXX闭环任务清单_v1.md @@ -267,6 +267,10 @@ - 维护摘要能作为 P23 harness 的断言输入。 - 不自动删除未纳入 retention policy 的业务数据。 +- 2026-05-12 P22 已完成 `Maintenance And Retention Rollup` 本地闭环：新增 `MaintenanceSummaryService`，聚合 visual capture retention dry-run、alert delivery health、feedback sample health、runtime watchdog 与 incident journal summary，并输出 `checks`、`blockers`、`warnings`、`maintenance_actions` 与 `p23_assertions`。 +- 2026-05-12 P22 已新增 `scripts/run_maintenance_summary.py` 与 `GET /trading/maintenance-summary`，并将 maintenance rollup 接入 workbench runtime panel。 +- 2026-05-12 P22 本地闭环已验证：focused P22 tests 通过，`tests/unit` 全量 696 个测试通过，script static checks 70 个脚本通过，`node --check src/web/app.js` 通过；脚本/API smoke 返回 warning-only `feedback.sample_health`，blockers 为空，且 `p23_assertions.no_unscoped_deletion=true` / `retention_policy_dry_run_only=true`。 + --- ## 13. P23 End-To-End Unattended Harness diff --git a/scripts/run_maintenance_summary.py b/scripts/run_maintenance_summary.py new file mode 100644 index 0000000..1c388c3 --- /dev/null +++ b/scripts/run_maintenance_summary.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +from pathlib import Path +import json +import os +import sys + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from src.api.maintenance import get_maintenance_summary + + +def main(env=None) -> int: + env = os.environ if env is None else env + payload = get_maintenance_summary( + instrument_id=int(env.get("MAINTENANCE_SUMMARY_INSTRUMENT_ID", "1")), + alert_limit=int(env.get("MAINTENANCE_SUMMARY_ALERT_LIMIT", "50")), + visual_limit=int(env.get("MAINTENANCE_SUMMARY_VISUAL_LIMIT", "50")), + incident_limit=int(env.get("MAINTENANCE_SUMMARY_INCIDENT_LIMIT", "50")), + ) + if _get_bool(env, "MAINTENANCE_SUMMARY_OUTPUT_JSON", False): + print(json.dumps(payload, ensure_ascii=False, sort_keys=True, default=str)) + else: + print(f"Maintenance summary status: {payload.get('status', 'unknown')}") + print(f"Blockers: {', '.join(payload.get('blockers') or []) or 'none'}") + print(f"Warnings: {', '.join(payload.get('warnings') or []) or 'none'}") + actions = dict(payload.get("maintenance_actions") or {}) + print(f"Automatic actions: {', '.join(actions.get('automatic_actions') or []) or 'none'}") + print(f"Operator actions: {', '.join(actions.get('operator_actions') or []) or 'none'}") + return 0 if not payload.get("blockers") else 2 + + +def _get_bool(env, key: str, default: bool) -> bool: + value = env.get(key) + if value is None or value == "": + return default + return str(value).strip().lower() in {"1", "true", "yes", "on"} + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/api/__init__.py b/src/api/__init__.py index af612c1..62c2b41 100644 --- a/src/api/__init__.py +++ b/src/api/__init__.py @@ -6,6 +6,7 @@ from src.api.candles import get_recent_candles from src.api.execution_tickets import get_recent_execution_tickets from src.api.health import get_health from src.api.live_operation_audits import get_recent_live_operation_audits +from src.api.maintenance import get_maintenance_summary from src.api.server import route_request, serve_api from src.api.sessions import get_recent_sessions from src.api.signals import get_recent_signals @@ -90,6 +91,7 @@ __all__ = [ "get_recent_sessions", "get_recent_signals", "get_recent_live_operation_audits", + "get_maintenance_summary", "get_recent_trade_plan_candidates", "get_recent_trading_journal", "get_latest_trading_plan", diff --git a/src/api/maintenance.py b/src/api/maintenance.py new file mode 100644 index 0000000..5d98f0f --- /dev/null +++ b/src/api/maintenance.py @@ -0,0 +1,16 @@ +from src.services.maintenance import MaintenanceSummaryService + + +def get_maintenance_summary( + *, + instrument_id: int, + alert_limit: int = 50, + visual_limit: int = 50, + incident_limit: int = 50, +) -> dict: + return MaintenanceSummaryService().build_summary( + instrument_id=instrument_id, + alert_limit=alert_limit, + visual_review_limit=visual_limit, + incident_limit=incident_limit, + ) diff --git a/src/api/server.py b/src/api/server.py index c68a66f..7095782 100644 --- a/src/api/server.py +++ b/src/api/server.py @@ -10,6 +10,7 @@ from src.api.candles import get_recent_candles from src.api.execution_tickets import get_recent_execution_tickets from src.api.health import get_health from src.api.live_operation_audits import get_recent_live_operation_audits +from src.api.maintenance import get_maintenance_summary from src.api.alerts import ( get_alert_deliveries, get_alert_quality_summary, @@ -108,6 +109,7 @@ def build_api_dependencies() -> dict: "candles": get_recent_candles, "execution_tickets": get_recent_execution_tickets, "live_operation_audits": get_recent_live_operation_audits, + "maintenance_summary": get_maintenance_summary, "trade_plan_candidates": get_recent_trade_plan_candidates, "trading_plan_generate": generate_trading_plan, "trading_plan_latest": get_latest_trading_plan, @@ -263,6 +265,13 @@ def route_request( limit=_get_int(query, "limit", 50), refresh=str(refresh).strip().lower() in {"1", "true", "yes", "on"}, ) + if method == "GET" and path == "/trading/maintenance-summary": + return 200, deps["maintenance_summary"]( + instrument_id=_require_int(query, "instrument_id"), + alert_limit=_get_int(query, "alert_limit", 50), + visual_limit=_get_int(query, "visual_limit", 50), + incident_limit=_get_int(query, "incident_limit", 50), + ) if method == "POST" and path == "/trading/runtime/run-once": return 200, deps["trading_runtime_run_once"]() if method == "GET" and path == "/trading/decision-packet": diff --git a/src/services/maintenance/__init__.py b/src/services/maintenance/__init__.py index 1fa5eee..f46624d 100644 --- a/src/services/maintenance/__init__.py +++ b/src/services/maintenance/__init__.py @@ -9,8 +9,10 @@ from src.services.maintenance.historical_compatibility import ( plan_trade_signal_invalidation_repairs, summarize_liquidity_duplicate_groups, ) +from src.services.maintenance.maintenance_summary_service import MaintenanceSummaryService __all__ = [ + "MaintenanceSummaryService", "build_historical_compatibility_summary", "classify_liquidity_duplicate_groups", "derive_historical_fvg_meta", diff --git a/src/services/maintenance/maintenance_summary_service.py b/src/services/maintenance/maintenance_summary_service.py new file mode 100644 index 0000000..68cb566 --- /dev/null +++ b/src/services/maintenance/maintenance_summary_service.py @@ -0,0 +1,232 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Callable, Optional + +from src.config.settings import settings +from src.services.alerts import AlertQualitySummaryService +from src.services.runtime.runtime_incident_journal_service import RuntimeIncidentJournalService +from src.services.visual_watch import VisualCaptureRetentionService + + +class MaintenanceSummaryService: + def __init__( + self, + *, + capture_retention_service: Optional[VisualCaptureRetentionService] = None, + alert_quality_service: Optional[AlertQualitySummaryService] = None, + incident_journal_service: Optional[RuntimeIncidentJournalService] = None, + runtime_status_provider: Optional[Callable[[], dict]] = None, + ) -> None: + self.capture_retention_service = capture_retention_service or VisualCaptureRetentionService() + self.alert_quality_service = alert_quality_service or AlertQualitySummaryService() + self.incident_journal_service = incident_journal_service or RuntimeIncidentJournalService() + self.runtime_status_provider = runtime_status_provider + + def build_summary( + self, + *, + instrument_id: int, + alert_limit: int = 50, + visual_review_limit: int = 50, + incident_limit: int = 50, + ) -> dict: + warnings: list[str] = [] + storage = _safe_section( + lambda: self._build_storage_summary(), + warning_code="storage_summary_unavailable", + warnings=warnings, + fallback={"visual_capture_retention": {"status": "unavailable"}}, + ) + alert_quality = _safe_section( + lambda: self.alert_quality_service.build_summary( + instrument_id=instrument_id, + alert_limit=alert_limit, + visual_review_limit=visual_review_limit, + ), + warning_code="alert_quality_summary_unavailable", + warnings=warnings, + fallback={}, + ) + incident_summary = _safe_section( + lambda: self.incident_journal_service.build_summary(limit=incident_limit), + warning_code="incident_summary_unavailable", + warnings=warnings, + fallback={"status": "unavailable", "incident_count": 0, "severity_counts": {}, "recommended_actions": []}, + ) + runtime_status = _safe_section( + self._load_runtime_status, + warning_code="runtime_status_unavailable", + warnings=warnings, + fallback={"runtime_health": {}, "runtime_watchdog": {}, "current_blockers": []}, + ) + checks = _build_checks( + storage=storage, + alert_quality=alert_quality, + incident_summary=incident_summary, + runtime_status=runtime_status, + ) + blockers = [item["name"] for item in checks if item["status"] == "blocked"] + check_warnings = [item["name"] for item in checks if item["status"] == "warning"] + maintenance_actions = _build_actions( + storage=storage, + alert_quality=alert_quality, + incident_summary=incident_summary, + runtime_status=runtime_status, + ) + return { + "status": "blocked" if blockers else ("warning" if check_warnings or warnings else "healthy"), + "generated_ts": datetime.now(timezone.utc).isoformat(), + "instrument_id": instrument_id, + "window": { + "alert_limit": alert_limit, + "visual_review_limit": visual_review_limit, + "incident_limit": incident_limit, + }, + "storage": storage, + "delivery": dict(alert_quality.get("delivery_summary") or {}), + "feedback": { + "alert": dict(alert_quality.get("alert_feedback_summary") or {}), + "visual": dict(alert_quality.get("visual_feedback_summary") or {}), + "quality_flags": list(alert_quality.get("quality_flags") or []), + }, + "runtime": { + "runtime_health": dict(runtime_status.get("runtime_health") or {}), + "runtime_watchdog": dict(runtime_status.get("runtime_watchdog") or {}), + "incident_summary": dict(incident_summary or {}), + "current_blockers": list(runtime_status.get("current_blockers") or []), + "current_warnings": list(runtime_status.get("current_warnings") or []), + }, + "checks": checks, + "blockers": blockers, + "warnings": warnings + check_warnings, + "maintenance_actions": maintenance_actions, + "p23_assertions": { + "storage_visible": bool(storage), + "delivery_visible": bool(alert_quality.get("delivery_summary") is not None), + "feedback_visible": bool(alert_quality.get("alert_feedback_summary") is not None), + "runtime_visible": bool(runtime_status), + "incident_journal_visible": bool(incident_summary), + "no_unscoped_deletion": True, + "retention_policy_dry_run_only": True, + }, + } + + def _build_storage_summary(self) -> dict: + retention = self.capture_retention_service.cleanup_captures( + retention_hours=settings.visual_capture_retention_hours, + dry_run=True, + max_delete_files=settings.visual_capture_cleanup_max_delete_files, + ) + return { + "visual_capture_retention": retention, + "retention_policy": { + "automatic_cleanup_enabled": bool(settings.visual_capture_cleanup_enabled), + "dry_run_only": True, + "retention_hours": int(settings.visual_capture_retention_hours), + "max_delete_files": int(settings.visual_capture_cleanup_max_delete_files), + "scoped_file_types": [".html", ".json"], + "business_data_deleted": False, + }, + } + + def _load_runtime_status(self) -> dict: + if self.runtime_status_provider is not None: + return dict(self.runtime_status_provider() or {}) + from src.services.runtime import get_trading_runtime_service + + return get_trading_runtime_service().status() + + +def _build_checks(*, storage: dict, alert_quality: dict, incident_summary: dict, runtime_status: dict) -> list[dict]: + retention = dict(storage.get("visual_capture_retention") or {}) + delivery = dict(alert_quality.get("delivery_summary") or {}) + alert_feedback = dict(alert_quality.get("alert_feedback_summary") or {}) + visual_feedback = dict(alert_quality.get("visual_feedback_summary") or {}) + runtime_watchdog = dict(runtime_status.get("runtime_watchdog") or {}) + incident_severity = dict(incident_summary.get("severity_counts") or {}) + checks = [ + { + "name": "storage.visual_capture_retention", + "status": "warning" if int(retention.get("eligible_file_count") or 0) > 0 else "ok", + "detail": f"eligible={int(retention.get('eligible_file_count') or 0)} dry_run=true", + "automation": "automatic_retention_policy_available", + }, + { + "name": "delivery.alert_delivery_health", + "status": "warning" + if int(delivery.get("failed_count") or 0) > 0 + or int(delivery.get("retry_exhausted_skip_count") or 0) > 0 + or int(delivery.get("channel_disabled_count") or 0) > 0 + else "ok", + "detail": ( + f"failed={int(delivery.get('failed_count') or 0)} " + f"retry_exhausted={int(delivery.get('retry_exhausted_skip_count') or 0)} " + f"channel_disabled={int(delivery.get('channel_disabled_count') or 0)}" + ), + "automation": "dispatcher_retry_and_policy_controls", + }, + { + "name": "feedback.sample_health", + "status": "warning" if int(alert_feedback.get("count") or 0) == 0 and int(visual_feedback.get("count") or 0) == 0 else "ok", + "detail": f"alert={int(alert_feedback.get('count') or 0)} visual={int(visual_feedback.get('count') or 0)}", + "automation": "operator_review_required", + }, + { + "name": "runtime.watchdog_health", + "status": "warning" if int(runtime_watchdog.get("stalled_task_count") or 0) > 0 else "ok", + "detail": f"stalled={int(runtime_watchdog.get('stalled_task_count') or 0)}", + "automation": "operator_review_required", + }, + { + "name": "runtime.incident_journal_health", + "status": "blocked" if int(incident_severity.get("blocker") or 0) > 0 else ("warning" if int(incident_summary.get("incident_count") or 0) > 0 else "ok"), + "detail": f"incidents={int(incident_summary.get('incident_count') or 0)} severities={incident_severity}", + "automation": "operator_review_required", + }, + ] + if runtime_status.get("current_blockers"): + checks.append( + { + "name": "runtime.current_blockers", + "status": "blocked", + "detail": ", ".join(runtime_status.get("current_blockers") or []), + "automation": "external_environment_or_runtime_recovery_required", + } + ) + return checks + + +def _build_actions(*, storage: dict, alert_quality: dict, incident_summary: dict, runtime_status: dict) -> dict: + retention = dict(storage.get("visual_capture_retention") or {}) + delivery = dict(alert_quality.get("delivery_summary") or {}) + automatic_actions = [] + operator_actions = [] + if int(retention.get("eligible_file_count") or 0) > 0: + automatic_actions.append("visual_capture_retention_cleanup") + if int(delivery.get("failed_count") or 0) > 0 or int(delivery.get("retry_exhausted_skip_count") or 0) > 0: + operator_actions.append("inspect_alert_delivery_failures") + if int(delivery.get("channel_disabled_count") or 0) > 0: + operator_actions.append("review_disabled_alert_channels") + for action in list(incident_summary.get("recommended_actions") or []): + if action not in operator_actions: + operator_actions.append(action) + if runtime_status.get("current_blockers") and "resolve_runtime_blockers" not in operator_actions: + operator_actions.append("resolve_runtime_blockers") + return { + "automatic_actions": automatic_actions, + "operator_actions": operator_actions, + "policy_notes": [ + "maintenance_summary_is_read_only", + "visual_capture_retention_uses_dry_run_for_summary", + "no_business_data_deletion_outside_retention_policy", + ], + } + + +def _safe_section(fetcher, *, warning_code: str, warnings: list[str], fallback): + try: + return fetcher() + except Exception: + warnings.append(warning_code) + return fallback diff --git a/src/web/app.js b/src/web/app.js index 4584170..fc0c3f8 100644 --- a/src/web/app.js +++ b/src/web/app.js @@ -284,13 +284,14 @@ function renderMarketState(payload, executions) { marketStatePanel.appendChild(panel); } -function renderRuntimeStatus(runtimeStatus, runtimeIncidents) { +function renderRuntimeStatus(runtimeStatus, runtimeIncidents, maintenanceSummary) { clearNode(runtimeStatusPanel); const tasks = runtimeStatus?.tasks || {}; const blockers = runtimeStatus?.current_blockers || []; const incidents = runtimeStatus?.incidents || []; const watchdog = runtimeStatus?.runtime_watchdog || {}; const incidentSummary = runtimeIncidents || runtimeStatus?.incident_summary || {}; + const maintenance = maintenanceSummary || {}; const stalledTasks = watchdog?.stalled_tasks || []; const stalledMap = new Map(stalledTasks.map((item) => [item.name, item])); runtimeStatusPanel.appendChild( @@ -305,6 +306,7 @@ function renderRuntimeStatus(runtimeStatus, runtimeIncidents) { ['watchdog checked', formatTimestamp(watchdog?.checked_ts)], ['incident journal', `${incidentSummary.status || 'idle'} | ${incidentSummary.incident_count ?? 0}`], ['incident severity', formatJsonInline(incidentSummary.severity_counts || {})], + ['maintenance', `${maintenance.status || 'unavailable'} | blockers ${(maintenance.blockers || []).length}`], ['last incident', incidents.length ? `${incidents[incidents.length - 1].code} @ ${formatTimestamp(incidents[incidents.length - 1].observed_ts)}` : 'none'], ]) ); @@ -322,6 +324,19 @@ function renderRuntimeStatus(runtimeStatus, runtimeIncidents) { ] ) ); + grid.appendChild( + buildItemCard( + 'Maintenance Rollup', + maintenance.status || 'unavailable', + [ + ['blockers', formatList(maintenance.blockers || [])], + ['warnings', formatList(maintenance.warnings || [])], + ['automatic actions', formatList((maintenance.maintenance_actions || {}).automatic_actions || [])], + ['operator actions', formatList((maintenance.maintenance_actions || {}).operator_actions || [])], + ['p23 assertions', formatJsonInline(maintenance.p23_assertions || {})], + ] + ) + ); Object.entries(tasks).forEach(([name, task]) => { const stalled = stalledMap.get(name) || null; grid.appendChild( @@ -1326,6 +1341,7 @@ function renderDashboardData({ marketState, runtimeStatus, runtimeIncidents, + maintenanceSummary, decisionPacket, decisionAlerts, visualReviews, @@ -1349,7 +1365,7 @@ function renderDashboardData({ renderHealth(health); renderOverview(weeklyReview, tradeCandidates.items || [], executionTickets.items || [], tradingExecutions.items || []); renderMarketState(marketState, tradingExecutions.items || []); - renderRuntimeStatus(runtimeStatus, runtimeIncidents); + renderRuntimeStatus(runtimeStatus, runtimeIncidents, maintenanceSummary); renderDecisionPacket(decisionPacket, decisionAlerts); renderTrustScore(trustScore); renderLiveReadiness(liveReadiness); @@ -1375,13 +1391,14 @@ async function refreshDashboard() { const limit = limitInput.value || '20'; try { - const [health, candles, tradingPlan, marketState, runtimeStatus, runtimeIncidents, decisionPacket, decisionAlerts, alertQualitySummary, operatorDigest, trustScore, liveReadiness, launchReadiness, sessionView, tradeCandidates, executionTickets, tradingExecutions, journalSummary, operatorNotes, reviewPacket, journalItems, weeklyReview] = await Promise.all([ + const [health, candles, tradingPlan, marketState, runtimeStatus, runtimeIncidents, maintenanceSummary, decisionPacket, decisionAlerts, alertQualitySummary, operatorDigest, trustScore, liveReadiness, launchReadiness, sessionView, tradeCandidates, executionTickets, tradingExecutions, journalSummary, operatorNotes, reviewPacket, journalItems, weeklyReview] = await Promise.all([ fetchJson('/health'), fetchJson(`/candles?${queryString({ instrument_id: instrumentId, timeframe, limit })}`), fetchJson(`/trading/plans/latest?${queryString({ instrument_id: instrumentId })}`).catch(() => null), fetchJson(`/trading/market-state?${queryString({ instrument_id: instrumentId, timeframe })}`).catch(() => ({ market_state: {} })), fetchJson('/trading/runtime-status').catch(() => ({ scheduler_state: 'unavailable', tasks: {}, current_blockers: ['runtime_status_unavailable'] })), fetchJson(`/trading/runtime-incidents?${queryString({ limit: 20, refresh: 0 })}`).catch(() => ({ status: 'unavailable', persisted: false, incident_count: 0, severity_counts: {}, source_counts: {}, recommended_actions: [], items: [] })), + fetchJson(`/trading/maintenance-summary?${queryString({ instrument_id: instrumentId, alert_limit: 50, visual_limit: 50, incident_limit: 50 })}`).catch(() => ({ status: 'unavailable', blockers: ['maintenance_summary_unavailable'], warnings: [], maintenance_actions: {}, p23_assertions: {} })), fetchJson(`/trading/decision-packet?${queryString({ instrument_id: instrumentId })}`).catch(() => ({ status: 'unavailable', action_recommendation: 'monitor' })), fetchJson(`/trading/alerts?${queryString({ instrument_id: instrumentId, limit: 10 })}`).catch(() => ({ items: [] })), fetchJson(`/trading/alerts/quality-summary?${queryString({ instrument_id: instrumentId, limit: 50, visual_limit: 50 })}`).catch(() => ({ recent_alert_count: 0, recent_visual_review_count: 0, delivery_summary: {}, alert_feedback_summary: {}, visual_feedback_summary: {}, visual_review_summary: {}, quality_flags: [], operator_summary: 'alert_quality_unavailable' })), @@ -1411,6 +1428,7 @@ async function refreshDashboard() { marketState, runtimeStatus, runtimeIncidents, + maintenanceSummary, decisionPacket, decisionAlerts, alertQualitySummary, diff --git a/tests/unit/test_api_server.py b/tests/unit/test_api_server.py index 41f57c7..b12437f 100644 --- a/tests/unit/test_api_server.py +++ b/tests/unit/test_api_server.py @@ -108,6 +108,27 @@ class ApiServerRoutingTests(unittest.TestCase): self.assertEqual(payload["top_limit"], 3) self.assertEqual(payload["review_queue"][0]["queue_type"], "failed_retry") + def test_routes_maintenance_summary(self) -> None: + status, payload = route_request( + "/trading/maintenance-summary", + {"instrument_id": ["1"], "alert_limit": ["25"], "visual_limit": ["15"], "incident_limit": ["5"]}, + dependencies={ + "maintenance_summary": lambda **kwargs: { + "instrument_id": kwargs["instrument_id"], + "alert_limit": kwargs["alert_limit"], + "visual_limit": kwargs["visual_limit"], + "incident_limit": kwargs["incident_limit"], + "status": "healthy", + } + }, + ) + + self.assertEqual(status, 200) + self.assertEqual(payload["instrument_id"], 1) + self.assertEqual(payload["alert_limit"], 25) + self.assertEqual(payload["visual_limit"], 15) + self.assertEqual(payload["incident_limit"], 5) + def test_routes_trading_alert_silent_window_status(self) -> None: status, payload = route_request( "/trading/alerts/silent-window-status", diff --git a/tests/unit/test_api_static_server.py b/tests/unit/test_api_static_server.py index 40240e7..48413d9 100644 --- a/tests/unit/test_api_static_server.py +++ b/tests/unit/test_api_static_server.py @@ -103,6 +103,7 @@ class ApiStaticServerTests(unittest.TestCase): "/trading/alerts/quality-summary", "/trading/alerts/operator-digest", "/trading/runtime-incidents", + "/trading/maintenance-summary", "/trading/visual-watch/cleanup-captures", "/trading/tickets", "/trading/executions", diff --git a/tests/unit/test_maintenance_summary_service.py b/tests/unit/test_maintenance_summary_service.py new file mode 100644 index 0000000..f750869 --- /dev/null +++ b/tests/unit/test_maintenance_summary_service.py @@ -0,0 +1,113 @@ +import unittest + +from src.services.maintenance import MaintenanceSummaryService + + +class _CaptureRetentionService: + def __init__(self, payload=None) -> None: + self.payload = payload or { + "status": "completed", + "eligible_file_count": 2, + "deleted_file_count": 2, + "deleted_bytes": 100, + } + self.calls = [] + + def cleanup_captures(self, **kwargs): + self.calls.append(kwargs) + return dict(self.payload) + + +class _AlertQualityService: + def __init__(self, payload=None) -> None: + self.payload = payload or { + "delivery_summary": { + "failed_count": 1, + "retry_exhausted_skip_count": 0, + "channel_disabled_count": 1, + }, + "alert_feedback_summary": {"count": 0}, + "visual_feedback_summary": {"count": 0}, + "quality_flags": ["feedback_sample_missing"], + } + + def build_summary(self, **kwargs): + return dict(self.payload) + + +class _IncidentJournalService: + def __init__(self, payload=None) -> None: + self.payload = payload or { + "status": "completed", + "incident_count": 1, + "severity_counts": {"blocker": 1}, + "recommended_actions": ["check_market_truth_stream_proxy_and_backfill"], + } + + def build_summary(self, *, limit: int = 50): + return dict(self.payload) + + +class MaintenanceSummaryServiceTests(unittest.TestCase): + def test_build_summary_rolls_up_storage_delivery_feedback_and_runtime(self) -> None: + capture_service = _CaptureRetentionService() + service = MaintenanceSummaryService( + capture_retention_service=capture_service, + alert_quality_service=_AlertQualityService(), + incident_journal_service=_IncidentJournalService(), + runtime_status_provider=lambda: { + "current_blockers": ["market_stream_degraded:BTC-USDT:market_stream_disconnected"], + "current_warnings": [], + "runtime_watchdog": {"stalled_task_count": 1}, + "runtime_health": {"level": "blocked"}, + }, + ) + + summary = service.build_summary(instrument_id=1) + + self.assertEqual(summary["status"], "blocked") + self.assertTrue(capture_service.calls[0]["dry_run"]) + self.assertIn("storage", summary) + self.assertIn("delivery", summary) + self.assertIn("feedback", summary) + self.assertIn("runtime", summary) + self.assertIn("visual_capture_retention_cleanup", summary["maintenance_actions"]["automatic_actions"]) + self.assertIn("inspect_alert_delivery_failures", summary["maintenance_actions"]["operator_actions"]) + self.assertIn("check_market_truth_stream_proxy_and_backfill", summary["maintenance_actions"]["operator_actions"]) + self.assertTrue(summary["p23_assertions"]["storage_visible"]) + self.assertTrue(summary["p23_assertions"]["no_unscoped_deletion"]) + self.assertTrue(summary["p23_assertions"]["retention_policy_dry_run_only"]) + + def test_build_summary_is_healthy_when_no_maintenance_findings_exist(self) -> None: + service = MaintenanceSummaryService( + capture_retention_service=_CaptureRetentionService( + {"status": "completed", "eligible_file_count": 0, "deleted_file_count": 0} + ), + alert_quality_service=_AlertQualityService( + { + "delivery_summary": {"failed_count": 0, "retry_exhausted_skip_count": 0, "channel_disabled_count": 0}, + "alert_feedback_summary": {"count": 1}, + "visual_feedback_summary": {"count": 1}, + "quality_flags": [], + } + ), + incident_journal_service=_IncidentJournalService( + {"status": "idle", "incident_count": 0, "severity_counts": {}, "recommended_actions": []} + ), + runtime_status_provider=lambda: { + "current_blockers": [], + "current_warnings": [], + "runtime_watchdog": {"stalled_task_count": 0}, + "runtime_health": {"level": "ok"}, + }, + ) + + summary = service.build_summary(instrument_id=1) + + self.assertEqual(summary["status"], "healthy") + self.assertEqual(summary["blockers"], []) + self.assertEqual(summary["maintenance_actions"]["automatic_actions"], []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_run_maintenance_summary.py b/tests/unit/test_run_maintenance_summary.py new file mode 100644 index 0000000..f693409 --- /dev/null +++ b/tests/unit/test_run_maintenance_summary.py @@ -0,0 +1,52 @@ +import contextlib +import io +import json +import unittest +from unittest.mock import patch + +from scripts import run_maintenance_summary + + +class RunMaintenanceSummaryTests(unittest.TestCase): + def test_main_outputs_json_summary(self) -> None: + payload = { + "status": "healthy", + "blockers": [], + "warnings": [], + "maintenance_actions": {"automatic_actions": [], "operator_actions": []}, + } + output = io.StringIO() + with patch("scripts.run_maintenance_summary.get_maintenance_summary", return_value=payload) as summary: + with contextlib.redirect_stdout(output): + exit_code = run_maintenance_summary.main( + env={ + "MAINTENANCE_SUMMARY_OUTPUT_JSON": "1", + "MAINTENANCE_SUMMARY_INSTRUMENT_ID": "2", + "MAINTENANCE_SUMMARY_ALERT_LIMIT": "3", + "MAINTENANCE_SUMMARY_VISUAL_LIMIT": "4", + "MAINTENANCE_SUMMARY_INCIDENT_LIMIT": "5", + } + ) + + self.assertEqual(exit_code, 0) + summary.assert_called_once_with(instrument_id=2, alert_limit=3, visual_limit=4, incident_limit=5) + self.assertEqual(json.loads(output.getvalue())["status"], "healthy") + + def test_main_returns_nonzero_when_blocked(self) -> None: + payload = { + "status": "blocked", + "blockers": ["runtime.current_blockers"], + "warnings": [], + "maintenance_actions": {"automatic_actions": [], "operator_actions": ["resolve_runtime_blockers"]}, + } + output = io.StringIO() + with patch("scripts.run_maintenance_summary.get_maintenance_summary", return_value=payload): + with contextlib.redirect_stdout(output): + exit_code = run_maintenance_summary.main(env={}) + + self.assertEqual(exit_code, 2) + self.assertIn("runtime.current_blockers", output.getvalue()) + + +if __name__ == "__main__": + unittest.main()