feat(alpha): complete closed-alpha readiness stage

Co-authored-by: multica-agent <github@multica.ai>
2026-09-03 20:40:40 +05:00
parent b5a9e2d61f
commit e67167648b
8 changed files with 384 additions and 40 deletions
--- a/apps/web/server/utils/alpha-observability.test.ts
+++ b/apps/web/server/utils/alpha-observability.test.ts
@@ -0,0 +1,82 @@
+import { describe, expect, it } from 'vitest'
+import { summarizeAlphaUsage, summarizeRoundObservability, type AlphaRoundRow, type AlphaUsageRow } from './alpha-observability'
+
+const usage = (patch: Partial<AlphaUsageRow>): AlphaUsageRow => ({
+  campaign_id: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa',
+  job_id: 'bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb',
+  request_kind: 'round_plan',
+  input_tokens: 1_200,
+  output_tokens: 300,
+  cost_usd: 0.01,
+  latency_ms: 1_000,
+  created_at: '2026-09-01T10:00:00.000Z',
+  ...patch,
+})
+
+const round = (patch: Partial<AlphaRoundRow>): AlphaRoundRow => ({
+  id: 'cccccccc-cccc-4ccc-8ccc-cccccccccccc',
+  campaign_id: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa',
+  number: 4,
+  status: 'resolved',
+  queued_at: '2026-09-01T10:00:00.000Z',
+  resolved_at: '2026-09-01T10:00:08.000Z',
+  error: null,
+  created_at: '2026-09-01T10:00:00.000Z',
+  ...patch,
+})
+
+describe('closed-alpha observability', () => {
+  it('summarizes provider usage without treating absent latency as zero-latency work', () => {
+    expect(summarizeAlphaUsage([
+      usage({ latency_ms: 1_000, cost_usd: '0.010' }),
+      usage({ latency_ms: null, input_tokens: 800, output_tokens: 200, cost_usd: '0.005' }),
+    ])).toEqual({
+      requests: 2,
+      inputTokens: 2_000,
+      outputTokens: 500,
+      costUsd: 0.015,
+      averageLatencyMs: 1_000,
+    })
+  })
+
+  it('reports true per-round cost, context size, latency and failures', () => {
+    const otherJob = 'dddddddd-dddd-4ddd-8ddd-dddddddddddd'
+    const result = summarizeRoundObservability([
+      usage({ request_kind: 'round_plan', input_tokens: 1_200, cost_usd: 0.01 }),
+      usage({ request_kind: 'round_plan', input_tokens: 1_200, cost_usd: 0.01 }), // paid retry
+      usage({ request_kind: 'round_resolution', input_tokens: 1_700, cost_usd: 0.02 }),
+      usage({ job_id: otherJob, request_kind: 'round_plan', input_tokens: 800, cost_usd: 0.01 }),
+      usage({ job_id: null, request_kind: 'character-draft', input_tokens: 9_999, cost_usd: 9 }),
+    ], [
+      round({}),
+      round({
+        id: 'eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee',
+        number: 5,
+        status: 'resolved',
+        queued_at: '2026-09-02T10:00:00.000Z',
+        resolved_at: '2026-09-02T10:00:12.000Z',
+        created_at: '2026-09-02T10:00:00.000Z',
+      }),
+      round({
+        id: 'ffffffff-ffff-4fff-8fff-ffffffffffff',
+        number: 6,
+        status: 'failed',
+        queued_at: '2026-09-03T10:00:00.000Z',
+        resolved_at: null,
+        error: 'Provider timed out',
+        created_at: '2026-09-03T10:00:00.000Z',
+      }),
+    ], new Map([['aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa', 'Signal Below']]))
+
+    expect(result).toMatchObject({
+      completed: 2,
+      failed: 1,
+      failureRate: 1 / 3,
+      averageCostUsd: 0.025,
+      averageLatencyMs: 10_000,
+      p95LatencyMs: 12_000,
+      averageContextTokens: 1_000,
+      latestFailures: [{ campaign: 'Signal Below', round: 6, message: 'Provider timed out' }],
+    })
+  })
+})