pavel444-byte
e67167648b
Some checks failed
CI / validate (push) Has been cancelled
Co-authored-by: multica-agent <github@multica.ai>
98 lines
3.7 KiB
TypeScript
98 lines
3.7 KiB
TypeScript
export interface AlphaUsageRow {
|
|
campaign_id: string | null
|
|
job_id: string | null
|
|
request_kind: string
|
|
input_tokens: number
|
|
output_tokens: number
|
|
cost_usd: number | string
|
|
latency_ms: number | null
|
|
created_at: string
|
|
}
|
|
|
|
export interface AlphaRoundRow {
|
|
id: string
|
|
campaign_id: string
|
|
number: number
|
|
status: string
|
|
queued_at: string | null
|
|
resolved_at: string | null
|
|
error: string | null
|
|
created_at: string
|
|
}
|
|
|
|
const roundRequestKinds = new Set(['round_plan', 'round_resolution', 'story_summary'])
|
|
|
|
function average(values: number[]): number {
|
|
return values.length ? values.reduce((sum, value) => sum + value, 0) / values.length : 0
|
|
}
|
|
|
|
function percentile(values: number[], percentileValue: number): number {
|
|
if (!values.length) return 0
|
|
const sorted = [...values].sort((left, right) => left - right)
|
|
return sorted[Math.max(0, Math.ceil(sorted.length * percentileValue) - 1)] ?? 0
|
|
}
|
|
|
|
export function summarizeAlphaUsage(rows: AlphaUsageRow[]) {
|
|
const latencies = rows.flatMap(row => row.latency_ms === null ? [] : [Number(row.latency_ms)])
|
|
return {
|
|
requests: rows.length,
|
|
inputTokens: rows.reduce((sum, row) => sum + Number(row.input_tokens), 0),
|
|
outputTokens: rows.reduce((sum, row) => sum + Number(row.output_tokens), 0),
|
|
costUsd: rows.reduce((sum, row) => sum + Number(row.cost_usd), 0),
|
|
averageLatencyMs: Math.round(average(latencies)),
|
|
}
|
|
}
|
|
|
|
export function summarizeRoundObservability(
|
|
usageRows: AlphaUsageRow[],
|
|
roundRows: AlphaRoundRow[],
|
|
campaignNames: ReadonlyMap<string, string>,
|
|
) {
|
|
const jobs = new Map<string, { costUsd: number; contextTokens: number }>()
|
|
for (const row of usageRows) {
|
|
if (!row.job_id || !roundRequestKinds.has(row.request_kind)) continue
|
|
const current = jobs.get(row.job_id) ?? { costUsd: 0, contextTokens: 0 }
|
|
current.costUsd += Number(row.cost_usd)
|
|
// The planning request is the direct serialization of RoundContext. Keep
|
|
// the largest attempt so retries increase true cost without inflating the
|
|
// reported size of the context itself.
|
|
if (row.request_kind === 'round_plan') {
|
|
current.contextTokens = Math.max(current.contextTokens, Number(row.input_tokens))
|
|
}
|
|
jobs.set(row.job_id, current)
|
|
}
|
|
|
|
const completed = roundRows.filter(round => round.status === 'resolved')
|
|
const failed = roundRows.filter(round => round.status === 'failed')
|
|
const endToEndLatencies = completed.flatMap(round => {
|
|
if (!round.queued_at || !round.resolved_at) return []
|
|
const elapsed = Date.parse(round.resolved_at) - Date.parse(round.queued_at)
|
|
return Number.isFinite(elapsed) && elapsed >= 0 ? [elapsed] : []
|
|
})
|
|
const costs = [...jobs.values()].map(job => job.costUsd)
|
|
const contextSizes = [...jobs.values()].flatMap(job => job.contextTokens > 0 ? [job.contextTokens] : [])
|
|
const observedOutcomes = completed.length + failed.length
|
|
|
|
return {
|
|
completed: completed.length,
|
|
failed: failed.length,
|
|
failureRate: observedOutcomes ? failed.length / observedOutcomes : 0,
|
|
averageCostUsd: average(costs),
|
|
averageLatencyMs: Math.round(average(endToEndLatencies)),
|
|
p95LatencyMs: Math.round(percentile(endToEndLatencies, 0.95)),
|
|
averageContextTokens: Math.round(average(contextSizes)),
|
|
latestFailures: failed
|
|
.slice()
|
|
.sort((left, right) => right.created_at.localeCompare(left.created_at))
|
|
.slice(0, 3)
|
|
.map(round => ({
|
|
id: round.id,
|
|
campaignId: round.campaign_id,
|
|
campaign: campaignNames.get(round.campaign_id) ?? 'Private campaign',
|
|
round: Number(round.number),
|
|
message: round.error?.trim() || 'The round stopped before narration was committed.',
|
|
occurredAt: round.created_at,
|
|
})),
|
|
}
|
|
}
|