feat(alpha): complete closed-alpha readiness stage

Co-authored-by: multica-agent <github@multica.ai>

apps/web/server/utils/alpha-observability.test.ts

@@ -0,0 +1,82 @@
+import { describe, expect, it } from 'vitest'
+import { summarizeAlphaUsage, summarizeRoundObservability, type AlphaRoundRow, type AlphaUsageRow } from './alpha-observability'
+
+const usage = (patch: Partial<AlphaUsageRow>): AlphaUsageRow => ({
+  campaign_id: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa',
+  job_id: 'bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb',
+  request_kind: 'round_plan',
+  input_tokens: 1_200,
+  output_tokens: 300,
+  cost_usd: 0.01,
+  latency_ms: 1_000,
+  created_at: '2026-09-01T10:00:00.000Z',
+  ...patch,
+})
+
+const round = (patch: Partial<AlphaRoundRow>): AlphaRoundRow => ({
+  id: 'cccccccc-cccc-4ccc-8ccc-cccccccccccc',
+  campaign_id: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa',
+  number: 4,
+  status: 'resolved',
+  queued_at: '2026-09-01T10:00:00.000Z',
+  resolved_at: '2026-09-01T10:00:08.000Z',
+  error: null,
+  created_at: '2026-09-01T10:00:00.000Z',
+  ...patch,
+})
+
+describe('closed-alpha observability', () => {
+  it('summarizes provider usage without treating absent latency as zero-latency work', () => {
+    expect(summarizeAlphaUsage([
+      usage({ latency_ms: 1_000, cost_usd: '0.010' }),
+      usage({ latency_ms: null, input_tokens: 800, output_tokens: 200, cost_usd: '0.005' }),
+    ])).toEqual({
+      requests: 2,
+      inputTokens: 2_000,
+      outputTokens: 500,
+      costUsd: 0.015,
+      averageLatencyMs: 1_000,
+    })
+  })
+
+  it('reports true per-round cost, context size, latency and failures', () => {
+    const otherJob = 'dddddddd-dddd-4ddd-8ddd-dddddddddddd'
+    const result = summarizeRoundObservability([
+      usage({ request_kind: 'round_plan', input_tokens: 1_200, cost_usd: 0.01 }),
+      usage({ request_kind: 'round_plan', input_tokens: 1_200, cost_usd: 0.01 }), // paid retry
+      usage({ request_kind: 'round_resolution', input_tokens: 1_700, cost_usd: 0.02 }),
+      usage({ job_id: otherJob, request_kind: 'round_plan', input_tokens: 800, cost_usd: 0.01 }),
+      usage({ job_id: null, request_kind: 'character-draft', input_tokens: 9_999, cost_usd: 9 }),
+    ], [
+      round({}),
+      round({
+        id: 'eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee',
+        number: 5,
+        status: 'resolved',
+        queued_at: '2026-09-02T10:00:00.000Z',
+        resolved_at: '2026-09-02T10:00:12.000Z',
+        created_at: '2026-09-02T10:00:00.000Z',
+      }),
+      round({
+        id: 'ffffffff-ffff-4fff-8fff-ffffffffffff',
+        number: 6,
+        status: 'failed',
+        queued_at: '2026-09-03T10:00:00.000Z',
+        resolved_at: null,
+        error: 'Provider timed out',
+        created_at: '2026-09-03T10:00:00.000Z',
+      }),
+    ], new Map([['aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa', 'Signal Below']]))
+
+    expect(result).toMatchObject({
+      completed: 2,
+      failed: 1,
+      failureRate: 1 / 3,
+      averageCostUsd: 0.025,
+      averageLatencyMs: 10_000,
+      p95LatencyMs: 12_000,
+      averageContextTokens: 1_000,
+      latestFailures: [{ campaign: 'Signal Below', round: 6, message: 'Provider timed out' }],
+    })
+  })
+})

apps/web/server/utils/alpha-observability.ts

@@ -0,0 +1,97 @@
+export interface AlphaUsageRow {
+  campaign_id: string | null
+  job_id: string | null
+  request_kind: string
+  input_tokens: number
+  output_tokens: number
+  cost_usd: number | string
+  latency_ms: number | null
+  created_at: string
+}
+
+export interface AlphaRoundRow {
+  id: string
+  campaign_id: string
+  number: number
+  status: string
+  queued_at: string | null
+  resolved_at: string | null
+  error: string | null
+  created_at: string
+}
+
+const roundRequestKinds = new Set(['round_plan', 'round_resolution', 'story_summary'])
+
+function average(values: number[]): number {
+  return values.length ? values.reduce((sum, value) => sum + value, 0) / values.length : 0
+}
+
+function percentile(values: number[], percentileValue: number): number {
+  if (!values.length) return 0
+  const sorted = [...values].sort((left, right) => left - right)
+  return sorted[Math.max(0, Math.ceil(sorted.length * percentileValue) - 1)] ?? 0
+}
+
+export function summarizeAlphaUsage(rows: AlphaUsageRow[]) {
+  const latencies = rows.flatMap(row => row.latency_ms === null ? [] : [Number(row.latency_ms)])
+  return {
+    requests: rows.length,
+    inputTokens: rows.reduce((sum, row) => sum + Number(row.input_tokens), 0),
+    outputTokens: rows.reduce((sum, row) => sum + Number(row.output_tokens), 0),
+    costUsd: rows.reduce((sum, row) => sum + Number(row.cost_usd), 0),
+    averageLatencyMs: Math.round(average(latencies)),
+  }
+}
+
+export function summarizeRoundObservability(
+  usageRows: AlphaUsageRow[],
+  roundRows: AlphaRoundRow[],
+  campaignNames: ReadonlyMap<string, string>,
+) {
+  const jobs = new Map<string, { costUsd: number; contextTokens: number }>()
+  for (const row of usageRows) {
+    if (!row.job_id || !roundRequestKinds.has(row.request_kind)) continue
+    const current = jobs.get(row.job_id) ?? { costUsd: 0, contextTokens: 0 }
+    current.costUsd += Number(row.cost_usd)
+    // The planning request is the direct serialization of RoundContext. Keep
+    // the largest attempt so retries increase true cost without inflating the
+    // reported size of the context itself.
+    if (row.request_kind === 'round_plan') {
+      current.contextTokens = Math.max(current.contextTokens, Number(row.input_tokens))
+    }
+    jobs.set(row.job_id, current)
+  }
+
+  const completed = roundRows.filter(round => round.status === 'resolved')
+  const failed = roundRows.filter(round => round.status === 'failed')
+  const endToEndLatencies = completed.flatMap(round => {
+    if (!round.queued_at || !round.resolved_at) return []
+    const elapsed = Date.parse(round.resolved_at) - Date.parse(round.queued_at)
+    return Number.isFinite(elapsed) && elapsed >= 0 ? [elapsed] : []
+  })
+  const costs = [...jobs.values()].map(job => job.costUsd)
+  const contextSizes = [...jobs.values()].flatMap(job => job.contextTokens > 0 ? [job.contextTokens] : [])
+  const observedOutcomes = completed.length + failed.length
+
+  return {
+    completed: completed.length,
+    failed: failed.length,
+    failureRate: observedOutcomes ? failed.length / observedOutcomes : 0,
+    averageCostUsd: average(costs),
+    averageLatencyMs: Math.round(average(endToEndLatencies)),
+    p95LatencyMs: Math.round(percentile(endToEndLatencies, 0.95)),
+    averageContextTokens: Math.round(average(contextSizes)),
+    latestFailures: failed
+      .slice()
+      .sort((left, right) => right.created_at.localeCompare(left.created_at))
+      .slice(0, 3)
+      .map(round => ({
+        id: round.id,
+        campaignId: round.campaign_id,
+        campaign: campaignNames.get(round.campaign_id) ?? 'Private campaign',
+        round: Number(round.number),
+        message: round.error?.trim() || 'The round stopped before narration was committed.',
+        occurredAt: round.created_at,
+      })),
+  }
+}