feat: Use wizard agent context pack

README.md

@@ -22,7 +22,7 @@ Seam integration. Run inside a project, it will:
    for JavaScript, or pip, poetry, or uv for Python.
 4. **Install the Seam plugin** —
    [`seamapi/seam-plugin`](https://github.com/seamapi/seam-plugin),
-   which provides the Seam integration skills and the `seam-docs` MCP
+   which provides the Seam integration skills and the `seam` MCP
    for the developer's AI coding assistant.
    Claude Code is detected via `.claude` or `CLAUDE.md`,
    in which case the wizard prints the slash commands to run,

eval/README.md

@@ -31,8 +31,9 @@ SEAM_API_KEY=seam_… SEAM_WIZARD_HARNESS=pi npm run eval
 ```
 
 `SEAM_API_KEY` is a dev key; it is exchanged for a short-lived inference token
-(the Anthropic key never leaves the Seam proxy). Nothing here runs in normal CI —
-it is a manual/opt-in harness.
+(the Anthropic key never leaves the Seam proxy). The eval prints its temporary
+log-file path before starting and mirrors all progress there. Nothing here runs
+in normal CI — it is a manual/opt-in harness.
 
 ## Fixtures
 

eval/real-runner.test.ts

@@ -0,0 +1,48 @@
+import { expect, test } from 'vitest'
+
+import { formatIntegrateEvent } from './real-runner.js'
+
+test('formatIntegrateEvent: prints live step and agent activity', () => {
+  expect(
+    [
+      formatIntegrateEvent({
+        kind: 'step_start',
+        id: 'inspect',
+        label: 'Inspect the project',
+        index: 0,
+        total: 2,
+      }),
+      formatIntegrateEvent({ kind: 'thinking', text: 'Read files\nThen edit' }),
+      formatIntegrateEvent({ kind: 'tool', name: 'Read', detail: 'app.ts' }),
+      formatIntegrateEvent({
+        kind: 'tool_done',
+        name: 'Read',
+        detail: 'app.ts',
+        elapsed_ms: 230,
+      }),
+      formatIntegrateEvent({ kind: 'text', text: 'Implemented it.' }),
+      formatIntegrateEvent({
+        kind: 'turn_done',
+        index: 1,
+        elapsed_ms: 8_400,
+      }),
+      formatIntegrateEvent(
+        {
+          kind: 'step_done',
+          id: 'inspect',
+          index: 0,
+          total: 2,
+          cost_usd: 1.25,
+        },
+        12_500,
+      ),
+    ].join('\n'),
+  ).toBe(`  step 1/2: Inspect the project
+    thinking: Read files
+      Then edit
+    tool: Read app.ts
+    tool: Read app.ts · done · 0.2s
+    agent: Implemented it.
+    turn 1: done · 8.4s
+  step 1/2: done · 12.5s · $1.25`)
+})

eval/real-runner.ts

@@ -1,14 +1,17 @@
 import { getInferenceBaseUrl } from 'lib/api.js'
 import { buildIntegrationSteps } from 'lib/steps/build-plan.js'
-import { runIntegration } from 'lib/steps/integrate.js'
+import { type IntegrateEvent, runIntegration } from 'lib/steps/integrate.js'
 
 import type { CaseRunner } from './run-case.js'
 
 // The real runner: writes the integration into the fixture with the same
 // runIntegration the wizard uses. The harness is selected out-of-band by
 // SEAM_WIZARD_HARNESS (the switchboard reads it), so a case just sets that env
 // before running. Cost/ok come from the terminal `done` event.
-export function createRealRunner(token: string): CaseRunner {
+export function createRealRunner(
+  token: string,
+  write: (message: string) => void,
+): CaseRunner {
   return async (workDir, spec, context) => {
     const steps = buildIntegrationSteps({
       mode: spec.mode,
@@ -18,6 +21,7 @@ export function createRealRunner(token: string): CaseRunner {
 
     let ok = false
     let costUsd: number | null = null
+    let stepStartedAt: number | null = null
     // The first step-failure reason — runIntegration catches per-step errors and
     // reports them as events rather than throwing, so capture it here to surface
     // why a case failed instead of leaving it a silent `no`.
@@ -32,6 +36,13 @@ export function createRealRunner(token: string): CaseRunner {
       mode: spec.mode,
       signal: context.signal,
       onEvent: (event) => {
+        if (event.kind === 'step_start') stepStartedAt = Date.now()
+        const elapsedMs =
+          (event.kind === 'step_done' || event.kind === 'step_failed') &&
+          stepStartedAt != null
+            ? Date.now() - stepStartedAt
+            : null
+        write(formatIntegrateEvent(event, elapsedMs))
         if (event.kind === 'step_failed' && error == null) {
           error = event.reason
         }
@@ -44,3 +55,42 @@ export function createRealRunner(token: string): CaseRunner {
     return { ok, costUsd, ...(error != null ? { error } : {}) }
   }
 }
+
+export function formatIntegrateEvent(
+  event: IntegrateEvent,
+  elapsedMs: number | null = null,
+): string {
+  if (event.kind === 'step_start') {
+    return `  step ${event.index + 1}/${event.total}: ${event.label}`
+  }
+  if (event.kind === 'step_done') {
+    return `  step ${event.index + 1}/${event.total}: done${formatDuration(elapsedMs)}${formatCost(event.cost_usd)}`
+  }
+  if (event.kind === 'step_failed') {
+    return `  step ${event.index + 1}/${event.total}: failed${formatDuration(elapsedMs)} — ${event.reason}`
+  }
+  if (event.kind === 'thinking') return formatBlock('thinking', event.text)
+  if (event.kind === 'text') return formatBlock('agent', event.text)
+  if (event.kind === 'tool') {
+    return `    tool: ${event.name}${event.detail.length > 0 ? ` ${event.detail}` : ''}`
+  }
+  if (event.kind === 'tool_done') {
+    return `    tool: ${event.name}${event.detail.length > 0 ? ` ${event.detail}` : ''} · done${formatDuration(event.elapsed_ms)}`
+  }
+  if (event.kind === 'turn_done') {
+    return `    turn ${event.index}: done${formatDuration(event.elapsed_ms)}`
+  }
+  return `  agent run ${event.ok ? 'complete' : 'failed'}${formatCost(event.cost_usd)}`
+}
+
+function formatBlock(label: string, text: string): string {
+  return `    ${label}: ${text.replaceAll('\n', '\n      ')}`
+}
+
+function formatCost(costUsd: number | null): string {
+  return costUsd == null ? '' : ` · $${costUsd.toFixed(2)}`
+}
+
+export function formatDuration(elapsedMs: number | null): string {
+  return elapsedMs == null ? '' : ` · ${(elapsedMs / 1000).toFixed(1)}s`
+}

eval/run-case.ts

@@ -55,7 +55,6 @@ export async function runCase(args: {
     error = caught instanceof Error ? caught.message : String(caught)
   }
 
-  const elapsedSec = Math.round((now() - startedAt) / 1000)
   const { diff, changedFiles } = captureDiff(workDir)
 
   let score
@@ -68,6 +67,7 @@ export async function runCase(args: {
     score = await scorer({ goal, diff, mode: spec.mode }).catch(() => undefined)
   }
 
+  const elapsedSec = Math.round((now() - startedAt) / 1000)
   return {
     fixture: spec.fixture,
     mode: spec.mode,

eval/run.ts

@@ -1,12 +1,20 @@
-import { existsSync, readdirSync, readFileSync } from 'node:fs'
+import { randomUUID } from 'node:crypto'
+import {
+  appendFileSync,
+  existsSync,
+  readdirSync,
+  readFileSync,
+  writeFileSync,
+} from 'node:fs'
+import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 
 import parseArgs from 'minimist'
 
 import { exchangeWizardInferenceToken, getInferenceBaseUrl } from 'lib/api.js'
 import type { BuildMode } from 'lib/steps/build-plan.js'
 
-import { createRealRunner } from './real-runner.js'
+import { createRealRunner, formatDuration } from './real-runner.js'
 import { formatReport } from './report.js'
 import { runCase } from './run-case.js'
 import { createLlmJudge } from './score.js'
@@ -15,6 +23,7 @@ import type { CaseResult, EvalCase, FixtureConfig } from './types.js'
 // Where the fixture apps live (top-level, so tsc doesn't compile them).
 const FIXTURES_DIR = join(process.cwd(), 'eval', 'fixtures')
 const MODES: BuildMode[] = ['full_api', 'customer_portal']
+let logFile: string | null = null
 
 // Flatten a repeated/comma-separated CLI flag into a list of values.
 function toList(value: unknown): string[] {
@@ -31,6 +40,15 @@ function toList(value: unknown): string[] {
 // model calls, so it costs money and takes minutes. The harness is selected by
 // SEAM_WIZARD_HARNESS (default anthropic).
 async function main(): Promise<void> {
+  // `--fixture a,b` / `--mode full_api` narrow the run to specific cases (each
+  // real case costs money + minutes), else run every fixture × mode.
+  const args = parseArgs(process.argv.slice(2), {
+    string: ['fixture', 'mode'],
+  })
+  logFile = join(tmpdir(), `seam-wizard-eval-${randomUUID()}.log`)
+  writeFileSync(logFile, '', { flag: 'wx', mode: 0o600 })
+  write(`Logging to ${logFile}`)
+
   const apiKey = process.env['SEAM_API_KEY']
   if (apiKey == null || apiKey.length === 0) {
     write('Set SEAM_API_KEY (a dev key) to run the eval.')
@@ -40,9 +58,6 @@ async function main(): Promise<void> {
 
   const harness = process.env['SEAM_WIZARD_HARNESS'] ?? 'anthropic'
 
-  // `--fixture a,b` / `--mode full_api` narrow the run to specific cases (each
-  // real case costs money + minutes), else run every fixture × mode.
-  const args = parseArgs(process.argv.slice(2), { string: ['fixture', 'mode'] })
   const fixtureFilter = toList(args['fixture'])
   const modeFilter = toList(args['mode'])
 
@@ -75,8 +90,11 @@ async function main(): Promise<void> {
     return
   }
 
+  write(`Starting ${harness} eval…`)
+  const tokenStartedAt = Date.now()
   const session = await exchangeWizardInferenceToken(apiKey)
-  const runner = createRealRunner(session.token)
+  write(`Inference token ready${formatDuration(Date.now() - tokenStartedAt)}`)
+  const runner = createRealRunner(session.token, write)
   const scorer = createLlmJudge({
     base_url: getInferenceBaseUrl(),
     token: session.token,
@@ -95,10 +113,23 @@ async function main(): Promise<void> {
         config,
         signal: controller.signal,
         runner,
-        scorer,
+        scorer: async (input) => {
+          write('  scoring diff…')
+          const scoringStartedAt = Date.now()
+          try {
+            return await scorer(input)
+          } finally {
+            write(
+              `  scoring finished${formatDuration(Date.now() - scoringStartedAt)}`,
+            )
+          }
+        },
         now: () => Date.now(),
       })
       results.push(result)
+      write(
+        `  case ${result.ok ? 'complete' : 'failed'} · ${result.elapsedSec}s`,
+      )
     }
   }
 
@@ -127,11 +158,19 @@ function readFixtureConfig(fixture: string): FixtureConfig {
 }
 
 function write(message: string): void {
-  process.stdout.write(`${message}\n`)
+  const line = `${message}\n`
+  process.stdout.write(line)
+  if (logFile != null) appendFileSync(logFile, line)
 }
 
 await main().catch((error: unknown) => {
   const { message } = error instanceof Error ? error : new Error(String(error))
-  process.stderr.write(`Eval failed: ${message}\n`)
+  const line = `Eval failed: ${message}\n`
+  process.stderr.write(line)
+  if (logFile != null) {
+    try {
+      appendFileSync(logFile, line)
+    } catch {}
+  }
   process.exitCode = 1
 })