pavel444-byte
e67167648b
Some checks failed
CI / validate (push) Has been cancelled
Co-authored-by: multica-agent <github@multica.ai>
83 lines
2.9 KiB
TypeScript
83 lines
2.9 KiB
TypeScript
import { describe, expect, it } from 'vitest'
|
|
import { summarizeAlphaUsage, summarizeRoundObservability, type AlphaRoundRow, type AlphaUsageRow } from './alpha-observability'
|
|
|
|
const usage = (patch: Partial<AlphaUsageRow>): AlphaUsageRow => ({
|
|
campaign_id: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa',
|
|
job_id: 'bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb',
|
|
request_kind: 'round_plan',
|
|
input_tokens: 1_200,
|
|
output_tokens: 300,
|
|
cost_usd: 0.01,
|
|
latency_ms: 1_000,
|
|
created_at: '2026-09-01T10:00:00.000Z',
|
|
...patch,
|
|
})
|
|
|
|
const round = (patch: Partial<AlphaRoundRow>): AlphaRoundRow => ({
|
|
id: 'cccccccc-cccc-4ccc-8ccc-cccccccccccc',
|
|
campaign_id: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa',
|
|
number: 4,
|
|
status: 'resolved',
|
|
queued_at: '2026-09-01T10:00:00.000Z',
|
|
resolved_at: '2026-09-01T10:00:08.000Z',
|
|
error: null,
|
|
created_at: '2026-09-01T10:00:00.000Z',
|
|
...patch,
|
|
})
|
|
|
|
describe('closed-alpha observability', () => {
|
|
it('summarizes provider usage without treating absent latency as zero-latency work', () => {
|
|
expect(summarizeAlphaUsage([
|
|
usage({ latency_ms: 1_000, cost_usd: '0.010' }),
|
|
usage({ latency_ms: null, input_tokens: 800, output_tokens: 200, cost_usd: '0.005' }),
|
|
])).toEqual({
|
|
requests: 2,
|
|
inputTokens: 2_000,
|
|
outputTokens: 500,
|
|
costUsd: 0.015,
|
|
averageLatencyMs: 1_000,
|
|
})
|
|
})
|
|
|
|
it('reports true per-round cost, context size, latency and failures', () => {
|
|
const otherJob = 'dddddddd-dddd-4ddd-8ddd-dddddddddddd'
|
|
const result = summarizeRoundObservability([
|
|
usage({ request_kind: 'round_plan', input_tokens: 1_200, cost_usd: 0.01 }),
|
|
usage({ request_kind: 'round_plan', input_tokens: 1_200, cost_usd: 0.01 }), // paid retry
|
|
usage({ request_kind: 'round_resolution', input_tokens: 1_700, cost_usd: 0.02 }),
|
|
usage({ job_id: otherJob, request_kind: 'round_plan', input_tokens: 800, cost_usd: 0.01 }),
|
|
usage({ job_id: null, request_kind: 'character-draft', input_tokens: 9_999, cost_usd: 9 }),
|
|
], [
|
|
round({}),
|
|
round({
|
|
id: 'eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee',
|
|
number: 5,
|
|
status: 'resolved',
|
|
queued_at: '2026-09-02T10:00:00.000Z',
|
|
resolved_at: '2026-09-02T10:00:12.000Z',
|
|
created_at: '2026-09-02T10:00:00.000Z',
|
|
}),
|
|
round({
|
|
id: 'ffffffff-ffff-4fff-8fff-ffffffffffff',
|
|
number: 6,
|
|
status: 'failed',
|
|
queued_at: '2026-09-03T10:00:00.000Z',
|
|
resolved_at: null,
|
|
error: 'Provider timed out',
|
|
created_at: '2026-09-03T10:00:00.000Z',
|
|
}),
|
|
], new Map([['aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa', 'Signal Below']]))
|
|
|
|
expect(result).toMatchObject({
|
|
completed: 2,
|
|
failed: 1,
|
|
failureRate: 1 / 3,
|
|
averageCostUsd: 0.025,
|
|
averageLatencyMs: 10_000,
|
|
p95LatencyMs: 12_000,
|
|
averageContextTokens: 1_000,
|
|
latestFailures: [{ campaign: 'Signal Below', round: 6, message: 'Provider timed out' }],
|
|
})
|
|
})
|
|
})
|