feat(alpha): complete closed-alpha readiness stage
Some checks failed
CI / validate (push) Has been cancelled

Co-authored-by: multica-agent <github@multica.ai>
This commit is contained in:
2026-09-03 20:40:40 +05:00
parent b5a9e2d61f
commit e67167648b
8 changed files with 384 additions and 40 deletions

View File

@@ -74,3 +74,11 @@ If the worker reports `POST /rest/v1/rpc/claim_ai_job 404` and `/rest/v1/profile
3. The campaign screen exposes the current summary, important memories, and active goals. The dashboard shows 30-day tokens, cost, latency, and the current daily request allowance.
4. Campaign owners may correct HP, inventory, or statuses through the audited character-state API; the reason and before/after values are retained in `audit_entries`.
5. `pnpm test` includes twenty long-memory regression scenarios. `pnpm e2e:full-party` verifies the hosted two-player/AI-companion flow end to end.
## Weeks 910 closed-alpha flow
1. A dismissible field guide on the dashboard takes a new tester from world creation through confirmation, invitations, and their first ready action. Its progress follows real drafts and campaigns and is stored per account on the device.
2. Round controls remain available in a compact panel on tablet and phone layouts. Owners can create and copy invitations, configure AI takeover, continue without a missing player, and retry a failed round without switching to desktop.
3. The dashboard reports 30-day round failures, mean and p95 queue-to-resolution latency, true cost per AI job (including paid retries), and the mean planning-context size. Recent failures link directly back to their private campaign; only campaign owners receive round operations data.
4. Before inviting testers, run `pnpm alpha:verify`. It executes all unit and regression tests, strict workspace typechecking, and the production build. On a configured alpha environment, also run `pnpm e2e:full-party` while `pnpm dev:all` is active to verify the real two-account invitation and round path.
5. Team and invited-user checks must cover narrow phone, tablet, and desktop widths; a normal all-ready round; owner continuation with an absent player; failed-round retry; and a return after at least three resolved rounds to confirm durable memory. Treat inaccessible round controls, private-data exposure, duplicate narration, partial state after failure, or an incorrect server-owned roll as release-blocking.

View File

@@ -0,0 +1,78 @@
<script setup lang="ts">
interface OnboardingCampaign {
id: string
}
interface OnboardingDraft {
id: string
}
const props = defineProps<{
userId: string
campaigns: OnboardingCampaign[]
drafts: OnboardingDraft[]
}>()
const visible = ref(false)
const storageKey = computed(() => `dng-alpha-onboarding:${props.userId}`)
const completed = computed(() => props.campaigns.length ? 2 : props.drafts.length ? 1 : 0)
const nextRoute = computed(() => {
if (props.campaigns[0]) return `/campaign/${props.campaigns[0].id}`
if (props.drafts[0]) return `/worlds/new?session=${props.drafts[0].id}`
return '/worlds/new'
})
const nextLabel = computed(() => props.campaigns.length
? 'OPEN YOUR CAMPAIGN'
: props.drafts.length ? 'CONTINUE YOUR WORLD' : 'CREATE YOUR FIRST WORLD')
function restoreVisibility() {
try {
visible.value = localStorage.getItem(storageKey.value) !== 'dismissed'
} catch {
visible.value = true
}
}
function dismiss() {
try {
localStorage.setItem(storageKey.value, 'dismissed')
} catch {
// Storage may be disabled in a private browsing context. The guide can
// still be dismissed for the current page lifetime.
}
visible.value = false
}
onMounted(restoreVisibility)
watch(() => props.userId, restoreVisibility)
</script>
<template>
<section v-if="visible" class="onboarding" aria-labelledby="onboarding-title">
<header>
<div>
<small>ALPHA FIELD GUIDE · {{ completed + 1 }} / 3</small>
<h2 id="onboarding-title">START YOUR FIRST STORY.</h2>
<p>Three short steps take you from a single idea to a shared, persistent campaign.</p>
</div>
<button type="button" aria-label="Dismiss getting started guide" @click="dismiss">×</button>
</header>
<div class="progress" aria-hidden="true"><i :style="{ width: `${Math.max(8, completed / 3 * 100)}%` }" /></div>
<ol>
<li :class="{ done: completed >= 1, active: completed === 0 }">
<b>01</b><span><strong>DESCRIBE A UNIVERSE</strong><small>Answer the coauthors focused questions.</small></span>
</li>
<li :class="{ done: completed >= 2, active: completed === 1 }">
<b>02</b><span><strong>CONFIRM THE STARTER</strong><small>Review the world, companions, and opening scene.</small></span>
</li>
<li :class="{ active: completed >= 2 }">
<b>03</b><span><strong>ASSEMBLE &amp; ACT</strong><small>Invite friends, write an action, then mark it ready.</small></span>
</li>
</ol>
<NuxtLink :to="nextRoute">{{ nextLabel }} <span></span></NuxtLink>
</section>
</template>
<style scoped>
.onboarding{margin-top:34px;border:1px solid #414139;background:linear-gradient(120deg,rgba(217,247,95,.08),transparent 56%),#0d0d0c}.onboarding>header{display:flex;justify-content:space-between;gap:24px;padding:25px 26px 20px}.onboarding header small{font:600 7px var(--mono);letter-spacing:.17em;color:var(--acid)}.onboarding h2{margin:10px 0 7px;font:600 clamp(20px,2.8vw,34px)/1.05 var(--display);letter-spacing:-.045em}.onboarding header p{margin:0;max-width:620px;color:var(--muted);font-size:11px;line-height:1.6}.onboarding header button{align-self:start;width:34px;height:34px;border:1px solid var(--line);background:#0b0b0a;color:var(--muted);font-size:20px}.progress{height:2px;background:#242422}.progress i{display:block;height:100%;min-width:8px;background:var(--acid);transition:width .3s ease}.onboarding ol{list-style:none;display:grid;grid-template-columns:repeat(3,minmax(0,1fr));margin:0;padding:0}.onboarding li{display:grid;grid-template-columns:28px minmax(0,1fr);gap:10px;padding:20px 24px;border-right:1px solid var(--line);opacity:.46}.onboarding li:last-child{border-right:0}.onboarding li.active,.onboarding li.done{opacity:1}.onboarding li>b{font:600 8px var(--mono);color:var(--muted)}.onboarding li.done>b,.onboarding li.active>b{color:var(--acid)}.onboarding li.done>b::after{content:'✓';display:block;margin-top:6px}.onboarding li span{display:grid;gap:7px}.onboarding li strong{font:600 8px var(--mono);letter-spacing:.09em}.onboarding li small{color:var(--muted);font-size:9px;line-height:1.5}.onboarding>a{display:flex;align-items:center;justify-content:space-between;padding:17px 25px;border-top:1px solid var(--line);background:var(--acid);color:#090909;text-decoration:none;font:700 8px var(--mono);letter-spacing:.12em}.onboarding>a span{font-size:15px}@media(max-width:760px){.onboarding ol{grid-template-columns:1fr}.onboarding li{border-right:0;border-bottom:1px solid var(--line)}.onboarding li:last-child{border-bottom:0}}@media(max-width:480px){.onboarding>header{padding:22px 20px}.onboarding li{padding:17px 20px}.onboarding>a{padding:17px 20px}}
</style>

File diff suppressed because one or more lines are too long

View File

@@ -16,6 +16,16 @@ interface DraftSession {
interface UsageSummary {
quota: { usedToday: number; dailyRequestLimit: number; dailyTokenLimit: number }
totals: { requests: number; inputTokens: number; outputTokens: number; costUsd: number; averageLatencyMs: number }
rounds: {
completed: number
failed: number
failureRate: number
averageCostUsd: number
averageLatencyMs: number
p95LatencyMs: number
averageContextTokens: number
latestFailures: Array<{ id: string; campaignId: string; campaign: string; round: number; message: string; occurredAt: string }>
}
}
const { api } = useDngApi()
@@ -26,6 +36,7 @@ const drafts = ref<DraftSession[]>([])
const usage = ref<UsageSummary | null>(null)
const loading = ref(true)
const error = ref('')
const actionError = ref('')
const renamingCampaignId = ref<string | null>(null)
const newName = ref('')
@@ -37,6 +48,7 @@ const displayName = computed(() => {
const value = auth.session.value?.user.user_metadata?.display_name
return typeof value === 'string' && value.trim() ? value.trim() : 'ADVENTURER'
})
const userId = computed(() => auth.session.value?.user.id ?? '')
function draftTitle(draft: DraftSession) {
return draft.generated_world?.title || 'UNFINISHED UNIVERSE'
@@ -52,7 +64,18 @@ function compactNumber(value: number) {
return new Intl.NumberFormat('en', { notation: 'compact', maximumFractionDigits: 1 }).format(value)
}
function duration(value: number) {
if (value < 1_000) return `${Math.round(value)}ms`
return `${(value / 1_000).toFixed(value < 10_000 ? 1 : 0)}s`
}
function messageFrom(cause: unknown) {
const value = cause as { data?: { statusMessage?: string; message?: string }; message?: string }
return value.data?.statusMessage ?? value.data?.message ?? value.message ?? 'The request could not be completed.'
}
function openRename(campaign: CampaignRow) {
actionError.value = ''
renamingCampaignId.value = campaign.id
newName.value = campaign.title
}
@@ -66,18 +89,26 @@ function handleCampaignAction(payload: { action: 'rename' | 'delete'; campaignId
async function confirmRename() {
if (!renamingCampaignId.value || !newName.value.trim()) return
actionError.value = ''
try {
await api(`/api/v1/campaigns/${renamingCampaignId.value}`, { method: 'PATCH', body: { title: newName.value } })
campaigns.value = campaigns.value.map(c => c.id === renamingCampaignId.value ? { ...c, title: newName.value.trim(), updated_at: new Date().toISOString() } : c)
} catch { /* handled silently */ } finally { renamingCampaignId.value = null; newName.value = '' }
renamingCampaignId.value = null
newName.value = ''
} catch (cause) {
actionError.value = messageFrom(cause)
}
}
async function requestDelete(campaign: CampaignRow) {
if (!confirm(`Delete "${campaign.title}"? This cannot be undone.`)) return
actionError.value = ''
try {
await api(`/api/v1/campaigns/${campaign.id}`, { method: 'DELETE' })
campaigns.value = campaigns.value.filter(c => c.id !== campaign.id)
} catch { /* handled silently — refresh on next mount */ }
} catch (cause) {
actionError.value = messageFrom(cause)
}
}
onMounted(async () => {
@@ -108,13 +139,28 @@ onMounted(async () => {
<NuxtLink to="/worlds/new" class="create-button"><span></span> CREATE A UNIVERSE</NuxtLink>
</section>
<AlphaOnboarding v-if="!loading && !error && userId" :user-id="userId" :campaigns="campaigns" :drafts="drafts" />
<p v-if="actionError" class="action-error" role="alert"><span>{{ actionError }}</span><button type="button" @click="actionError = ''">DISMISS</button></p>
<section v-if="usage" class="usage-deck" aria-label="AI usage dashboard">
<div><small>AI REQUESTS / TODAY</small><b>{{ usage.quota.usedToday }}<em>/ {{ usage.quota.dailyRequestLimit }}</em></b></div>
<div><small>TOKENS / 30 DAYS</small><b>{{ compactNumber(usage.totals.inputTokens + usage.totals.outputTokens) }}</b></div>
<div><small>EST. COST / 30 DAYS</small><b>${{ usage.totals.costUsd.toFixed(3) }}</b></div>
<div><small>AVG. AI LATENCY</small><b>{{ (usage.totals.averageLatencyMs / 1000).toFixed(1) }}s</b></div>
<div><small>AVG. COST / ROUND</small><b>{{ usage.rounds.completed ? `$${usage.rounds.averageCostUsd.toFixed(3)}` : '—' }}</b></div>
<div><small>AVG. ROUND LATENCY</small><b>{{ usage.rounds.completed ? duration(usage.rounds.averageLatencyMs) : '—' }}<em v-if="usage.rounds.p95LatencyMs">P95 {{ duration(usage.rounds.p95LatencyMs) }}</em></b></div>
<div><small>AVG. ROUND CONTEXT</small><b>{{ usage.rounds.averageContextTokens ? compactNumber(usage.rounds.averageContextTokens) : '—' }}<em v-if="usage.rounds.averageContextTokens">TOKENS</em></b></div>
<div :class="{ warning: usage.rounds.failed }"><small>ROUND FAILURES / 30 DAYS</small><b>{{ usage.rounds.failed }}<em>/ {{ usage.rounds.completed + usage.rounds.failed }}</em></b></div>
</section>
<details v-if="usage?.rounds.latestFailures.length" class="failure-log">
<summary>RECENT ROUND FAILURES <span>{{ usage.rounds.failed }} IN 30 DAYS</span></summary>
<article v-for="failure in usage.rounds.latestFailures" :key="failure.id">
<div><b>{{ failure.campaign }}</b><small>ROUND {{ failure.round }} · {{ new Date(failure.occurredAt).toLocaleString() }}</small></div>
<p>{{ failure.message }}</p>
<NuxtLink :to="`/campaign/${failure.campaignId}`">OPEN CAMPAIGN </NuxtLink>
</article>
</details>
<div class="filter-row">
<button v-for="item in ['all','active','drafts']" :key="item" :class="{active:filter===item}" @click="filter=item as typeof filter">{{ item }}</button>
<span>{{ campaigns.length }} CAMPAIGNS · {{ drafts.length }} DRAFTS</span>
@@ -151,8 +197,9 @@ onMounted(async () => {
<form @submit.prevent="confirmRename" class="rename-panel">
<small>RENAME UNIVERSE</small>
<input v-model="newName" required maxlength="200" autofocus placeholder="Universe name" aria-label="New universe name" />
<p v-if="actionError" class="dialog-error" role="alert">{{ actionError }}</p>
<div class="rename-actions">
<button type="button" class="ghost-button" @click="renamingCampaignId = null; newName = ''">CANCEL</button>
<button type="button" class="ghost-button" @click="renamingCampaignId = null; newName = ''; actionError = ''">CANCEL</button>
<button class="acid-button" type="submit">SAVE</button>
</div>
</form>
@@ -170,7 +217,9 @@ onMounted(async () => {
.dash-head p{color:var(--muted)}
.create-button{display:flex;align-items:center;gap:18px;padding:18px 22px;background:var(--acid);color:#090909;text-decoration:none;font:600 10px var(--mono);letter-spacing:.12em}
.create-button span{font-size:20px}
.usage-deck{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));margin-top:34px;border:1px solid var(--line);background:#0d0d0c}.usage-deck>div{display:grid;gap:10px;padding:18px 20px;border-right:1px solid var(--line)}.usage-deck>div:last-child{border-right:0}.usage-deck small{font:500 7px var(--mono);letter-spacing:.14em;color:var(--muted)}.usage-deck b{font:600 20px var(--mono);color:var(--acid)}.usage-deck em{margin-left:4px;color:var(--muted);font:500 9px var(--mono);font-style:normal}
.action-error{display:flex;align-items:center;justify-content:space-between;gap:20px;margin:22px 0 0;padding:13px 15px;border:1px solid #6a3a31;background:#251613;color:#ffb29f;font:500 9px/1.5 var(--mono)}.action-error button{border:0;background:none;color:inherit;font:600 7px var(--mono);letter-spacing:.1em}
.usage-deck{display:grid;grid-template-columns:repeat(6,minmax(0,1fr));margin-top:34px;border:1px solid var(--line);background:#0d0d0c}.usage-deck>div{display:grid;align-content:start;gap:10px;min-width:0;padding:18px 16px;border-right:1px solid var(--line)}.usage-deck>div:last-child{border-right:0}.usage-deck small{font:500 7px/1.4 var(--mono);letter-spacing:.12em;color:var(--muted)}.usage-deck b{display:flex;flex-wrap:wrap;align-items:baseline;gap:5px;font:600 clamp(15px,1.6vw,20px) var(--mono);color:var(--acid)}.usage-deck em{color:var(--muted);font:500 7px var(--mono);font-style:normal}.usage-deck .warning b{color:#ff9d85}
.failure-log{margin-top:12px;border:1px solid #56362f;background:#140f0e}.failure-log summary{display:flex;justify-content:space-between;gap:20px;padding:15px 18px;color:#ffab98;font:600 8px var(--mono);letter-spacing:.12em;cursor:pointer}.failure-log summary span{color:var(--muted);font-weight:500}.failure-log article{display:grid;grid-template-columns:minmax(150px,.4fr) minmax(0,1fr) auto;gap:18px;align-items:center;padding:14px 18px;border-top:1px solid #382521}.failure-log article>div{display:grid;gap:5px}.failure-log article b{font:600 9px var(--display)}.failure-log article small{font:500 7px var(--mono);color:var(--muted)}.failure-log article p{margin:0;color:#d7a89e;font-size:10px;line-height:1.5;overflow-wrap:anywhere}.failure-log article a{color:var(--acid);text-decoration:none;font:600 7px var(--mono);letter-spacing:.09em;white-space:nowrap}
.filter-row{display:flex;align-items:center;gap:10px;margin:58px 0 24px;border-bottom:1px solid var(--line)}
.filter-row button{padding:0 4px 16px;margin-right:18px;background:none;border:0;color:var(--muted);font:500 9px var(--mono);text-transform:uppercase;letter-spacing:.14em}
.filter-row button.active{color:var(--ink);border-bottom:2px solid var(--acid)}
@@ -205,8 +254,10 @@ onMounted(async () => {
.rename-panel small{font:600 8px var(--mono);letter-spacing:.18em;color:var(--acid)}
.rename-panel input{box-sizing:border-box;width:100%;min-height:48px;padding:0 14px;border:1px solid var(--line);background:#0e0e0d;color:var(--ink);font:12px var(--body);outline:none}
.rename-panel input:focus{border-color:var(--acid)}
.dialog-error{margin:0;padding:11px 13px;border:1px solid #6a3a31;background:#251613;color:#ffb29f;font:500 8px/1.5 var(--mono)}
.rename-actions{display:flex;justify-content:flex-end;gap:10px}
@media(max-width:1050px){.world-grid{grid-template-columns:1fr}}
@media(max-width:850px){.dash-head{align-items:start;flex-direction:column}.usage-deck{grid-template-columns:repeat(2,1fr)}.usage-deck>div:nth-child(2){border-right:0}.usage-deck>div:nth-child(-n+2){border-bottom:1px solid var(--line)}.active-world{grid-template-columns:1fr}.world-art{min-height:280px}.system-strip{gap:16px;flex-wrap:wrap}}
@media(max-width:520px){.dash-head h1{font-size:clamp(34px,12vw,52px)}.create-button{width:100%;justify-content:center}.filter-row>span{display:none}.world-info,.new-card{padding:28px}.active-world{min-height:580px}.world-info div{align-items:flex-start;flex-direction:column}.system-strip{flex-direction:column}}
@media(max-width:1100px){.usage-deck{grid-template-columns:repeat(3,1fr)}.usage-deck>div{border-bottom:1px solid var(--line)}.usage-deck>div:nth-child(3n){border-right:0}.usage-deck>div:nth-last-child(-n+3){border-bottom:0}}
@media(max-width:850px){.dash-head{align-items:start;flex-direction:column}.active-world{grid-template-columns:1fr}.world-art{min-height:280px}.system-strip{gap:16px;flex-wrap:wrap}.failure-log article{grid-template-columns:1fr}.failure-log article a{justify-self:start}}
@media(max-width:520px){.dash-head h1{font-size:clamp(34px,12vw,52px)}.create-button{width:100%;justify-content:center}.usage-deck{grid-template-columns:repeat(2,1fr)}.usage-deck>div:nth-child(3n){border-right:1px solid var(--line)}.usage-deck>div:nth-child(2n){border-right:0}.usage-deck>div:nth-last-child(-n+3){border-bottom:1px solid var(--line)}.usage-deck>div:nth-last-child(-n+2){border-bottom:0}.filter-row>span{display:none}.world-info,.new-card{padding:28px}.active-world{min-height:580px}.world-info div{align-items:flex-start;flex-direction:column}.system-strip{flex-direction:column}.failure-log summary{align-items:flex-start;flex-direction:column}}
</style>

View File

@@ -1,14 +1,10 @@
import { requireStageTwoUser, stageTwoApiError, stageTwoDatabase } from '~/server/utils/stage-two-supabase'
interface UsageRow {
campaign_id: string | null
request_kind: string
input_tokens: number
output_tokens: number
cost_usd: number | string
latency_ms: number | null
created_at: string
}
import {
summarizeAlphaUsage,
summarizeRoundObservability,
type AlphaRoundRow,
type AlphaUsageRow,
} from '~/server/utils/alpha-observability'
export default defineEventHandler(async (event) => {
try {
@@ -17,30 +13,32 @@ export default defineEventHandler(async (event) => {
since.setUTCDate(since.getUTCDate() - 30)
const today = new Date()
today.setUTCHours(0, 0, 0, 0)
const [usage, quotaEvents] = await Promise.all([
stageTwoDatabase<UsageRow[]>(
`ai_usage?select=campaign_id,request_kind,input_tokens,output_tokens,cost_usd,latency_ms,created_at&user_id=eq.${user.id}&created_at=gte.${encodeURIComponent(since.toISOString())}&order=created_at.desc&limit=1000`,
const [usage, quotaEvents, ownedCampaigns] = await Promise.all([
stageTwoDatabase<AlphaUsageRow[]>(
`ai_usage?select=campaign_id,job_id,request_kind,input_tokens,output_tokens,cost_usd,latency_ms,created_at&user_id=eq.${user.id}&created_at=gte.${encodeURIComponent(since.toISOString())}&order=created_at.desc&limit=1000`,
),
stageTwoDatabase<Array<{ created_at: string }>>(
`ai_quota_events?select=created_at&user_id=eq.${user.id}&created_at=gte.${encodeURIComponent(today.toISOString())}`,
),
stageTwoDatabase<Array<{ id: string; title: string }>>(
`campaigns?select=id,title&owner_id=eq.${user.id}`,
),
])
const campaignIds = [...new Set(usage.flatMap(row => row.campaign_id ? [row.campaign_id] : []))]
const campaigns = campaignIds.length
? await stageTwoDatabase<Array<{ id: string; title: string }>>(`campaigns?select=id,title&id=in.(${campaignIds.join(',')})`)
const knownCampaignIds = new Set(ownedCampaigns.map(campaign => campaign.id))
const additionalCampaignIds = campaignIds.filter(id => !knownCampaignIds.has(id))
const additionalCampaigns = additionalCampaignIds.length
? await stageTwoDatabase<Array<{ id: string; title: string }>>(`campaigns?select=id,title&id=in.(${additionalCampaignIds.join(',')})`)
: []
const campaigns = [...ownedCampaigns, ...additionalCampaigns]
const campaignNames = new Map(campaigns.map(campaign => [campaign.id, campaign.title]))
const summarize = (rows: UsageRow[]) => {
const latencies = rows.flatMap(row => row.latency_ms === null ? [] : [Number(row.latency_ms)])
return {
requests: rows.length,
inputTokens: rows.reduce((sum, row) => sum + Number(row.input_tokens), 0),
outputTokens: rows.reduce((sum, row) => sum + Number(row.output_tokens), 0),
costUsd: rows.reduce((sum, row) => sum + Number(row.cost_usd), 0),
averageLatencyMs: latencies.length ? Math.round(latencies.reduce((sum, value) => sum + value, 0) / latencies.length) : 0,
}
}
const grouped = new Map<string, UsageRow[]>()
const ownedCampaignIds = ownedCampaigns.map(campaign => campaign.id)
const rounds = ownedCampaignIds.length
? await stageTwoDatabase<AlphaRoundRow[]>(
`rounds?select=id,campaign_id,number,status,queued_at,resolved_at,error,created_at&campaign_id=in.(${ownedCampaignIds.join(',')})&status=in.(resolved,failed)&created_at=gte.${encodeURIComponent(since.toISOString())}&order=created_at.desc&limit=1000`,
)
: []
const grouped = new Map<string, AlphaUsageRow[]>()
for (const row of usage) {
const key = row.campaign_id ?? 'coauthor'
grouped.set(key, [...(grouped.get(key) ?? []), row])
@@ -48,11 +46,12 @@ export default defineEventHandler(async (event) => {
return {
periodDays: 30,
quota: { usedToday: quotaEvents.length, dailyRequestLimit: 60, dailyTokenLimit: 250000 },
totals: summarize(usage),
totals: summarizeAlphaUsage(usage),
rounds: summarizeRoundObservability(usage, rounds, campaignNames),
groups: [...grouped].map(([id, rows]) => ({
id,
label: id === 'coauthor' ? 'World coauthor' : campaignNames.get(id) ?? 'Private campaign',
...summarize(rows),
...summarizeAlphaUsage(rows),
})).sort((left, right) => right.costUsd - left.costUsd),
}
} catch (error) {

View File

@@ -0,0 +1,82 @@
import { describe, expect, it } from 'vitest'
import { summarizeAlphaUsage, summarizeRoundObservability, type AlphaRoundRow, type AlphaUsageRow } from './alpha-observability'
const usage = (patch: Partial<AlphaUsageRow>): AlphaUsageRow => ({
campaign_id: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa',
job_id: 'bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb',
request_kind: 'round_plan',
input_tokens: 1_200,
output_tokens: 300,
cost_usd: 0.01,
latency_ms: 1_000,
created_at: '2026-09-01T10:00:00.000Z',
...patch,
})
const round = (patch: Partial<AlphaRoundRow>): AlphaRoundRow => ({
id: 'cccccccc-cccc-4ccc-8ccc-cccccccccccc',
campaign_id: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa',
number: 4,
status: 'resolved',
queued_at: '2026-09-01T10:00:00.000Z',
resolved_at: '2026-09-01T10:00:08.000Z',
error: null,
created_at: '2026-09-01T10:00:00.000Z',
...patch,
})
describe('closed-alpha observability', () => {
it('summarizes provider usage without treating absent latency as zero-latency work', () => {
expect(summarizeAlphaUsage([
usage({ latency_ms: 1_000, cost_usd: '0.010' }),
usage({ latency_ms: null, input_tokens: 800, output_tokens: 200, cost_usd: '0.005' }),
])).toEqual({
requests: 2,
inputTokens: 2_000,
outputTokens: 500,
costUsd: 0.015,
averageLatencyMs: 1_000,
})
})
it('reports true per-round cost, context size, latency and failures', () => {
const otherJob = 'dddddddd-dddd-4ddd-8ddd-dddddddddddd'
const result = summarizeRoundObservability([
usage({ request_kind: 'round_plan', input_tokens: 1_200, cost_usd: 0.01 }),
usage({ request_kind: 'round_plan', input_tokens: 1_200, cost_usd: 0.01 }), // paid retry
usage({ request_kind: 'round_resolution', input_tokens: 1_700, cost_usd: 0.02 }),
usage({ job_id: otherJob, request_kind: 'round_plan', input_tokens: 800, cost_usd: 0.01 }),
usage({ job_id: null, request_kind: 'character-draft', input_tokens: 9_999, cost_usd: 9 }),
], [
round({}),
round({
id: 'eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee',
number: 5,
status: 'resolved',
queued_at: '2026-09-02T10:00:00.000Z',
resolved_at: '2026-09-02T10:00:12.000Z',
created_at: '2026-09-02T10:00:00.000Z',
}),
round({
id: 'ffffffff-ffff-4fff-8fff-ffffffffffff',
number: 6,
status: 'failed',
queued_at: '2026-09-03T10:00:00.000Z',
resolved_at: null,
error: 'Provider timed out',
created_at: '2026-09-03T10:00:00.000Z',
}),
], new Map([['aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa', 'Signal Below']]))
expect(result).toMatchObject({
completed: 2,
failed: 1,
failureRate: 1 / 3,
averageCostUsd: 0.025,
averageLatencyMs: 10_000,
p95LatencyMs: 12_000,
averageContextTokens: 1_000,
latestFailures: [{ campaign: 'Signal Below', round: 6, message: 'Provider timed out' }],
})
})
})

View File

@@ -0,0 +1,97 @@
export interface AlphaUsageRow {
campaign_id: string | null
job_id: string | null
request_kind: string
input_tokens: number
output_tokens: number
cost_usd: number | string
latency_ms: number | null
created_at: string
}
export interface AlphaRoundRow {
id: string
campaign_id: string
number: number
status: string
queued_at: string | null
resolved_at: string | null
error: string | null
created_at: string
}
const roundRequestKinds = new Set(['round_plan', 'round_resolution', 'story_summary'])
function average(values: number[]): number {
return values.length ? values.reduce((sum, value) => sum + value, 0) / values.length : 0
}
function percentile(values: number[], percentileValue: number): number {
if (!values.length) return 0
const sorted = [...values].sort((left, right) => left - right)
return sorted[Math.max(0, Math.ceil(sorted.length * percentileValue) - 1)] ?? 0
}
export function summarizeAlphaUsage(rows: AlphaUsageRow[]) {
const latencies = rows.flatMap(row => row.latency_ms === null ? [] : [Number(row.latency_ms)])
return {
requests: rows.length,
inputTokens: rows.reduce((sum, row) => sum + Number(row.input_tokens), 0),
outputTokens: rows.reduce((sum, row) => sum + Number(row.output_tokens), 0),
costUsd: rows.reduce((sum, row) => sum + Number(row.cost_usd), 0),
averageLatencyMs: Math.round(average(latencies)),
}
}
export function summarizeRoundObservability(
usageRows: AlphaUsageRow[],
roundRows: AlphaRoundRow[],
campaignNames: ReadonlyMap<string, string>,
) {
const jobs = new Map<string, { costUsd: number; contextTokens: number }>()
for (const row of usageRows) {
if (!row.job_id || !roundRequestKinds.has(row.request_kind)) continue
const current = jobs.get(row.job_id) ?? { costUsd: 0, contextTokens: 0 }
current.costUsd += Number(row.cost_usd)
// The planning request is the direct serialization of RoundContext. Keep
// the largest attempt so retries increase true cost without inflating the
// reported size of the context itself.
if (row.request_kind === 'round_plan') {
current.contextTokens = Math.max(current.contextTokens, Number(row.input_tokens))
}
jobs.set(row.job_id, current)
}
const completed = roundRows.filter(round => round.status === 'resolved')
const failed = roundRows.filter(round => round.status === 'failed')
const endToEndLatencies = completed.flatMap(round => {
if (!round.queued_at || !round.resolved_at) return []
const elapsed = Date.parse(round.resolved_at) - Date.parse(round.queued_at)
return Number.isFinite(elapsed) && elapsed >= 0 ? [elapsed] : []
})
const costs = [...jobs.values()].map(job => job.costUsd)
const contextSizes = [...jobs.values()].flatMap(job => job.contextTokens > 0 ? [job.contextTokens] : [])
const observedOutcomes = completed.length + failed.length
return {
completed: completed.length,
failed: failed.length,
failureRate: observedOutcomes ? failed.length / observedOutcomes : 0,
averageCostUsd: average(costs),
averageLatencyMs: Math.round(average(endToEndLatencies)),
p95LatencyMs: Math.round(percentile(endToEndLatencies, 0.95)),
averageContextTokens: Math.round(average(contextSizes)),
latestFailures: failed
.slice()
.sort((left, right) => right.created_at.localeCompare(left.created_at))
.slice(0, 3)
.map(round => ({
id: round.id,
campaignId: round.campaign_id,
campaign: campaignNames.get(round.campaign_id) ?? 'Private campaign',
round: Number(round.number),
message: round.error?.trim() || 'The round stopped before narration was committed.',
occurredAt: round.created_at,
})),
}
}

View File

@@ -20,6 +20,7 @@
"build": "pnpm -r --if-present build",
"prepare:web": "pnpm --filter @dng/web exec nuxt prepare",
"test": "pnpm prepare:web && pnpm test:unit",
"alpha:verify": "pnpm test && pnpm typecheck && pnpm build",
"test:unit": "vitest run",
"test:watch": "vitest",
"typecheck": "pnpm -r --if-present typecheck",