Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -555,11 +555,13 @@ program
'--requirements <revision>',
'Run exactly the scenarios a spec revision requires, frozen at its release (e.g. 2026-07-28). Replaces --suite and --spec-version'
)
.option('--timeout <ms>', 'Per-scenario timeout in milliseconds', '30000')
.option('--verbose', 'Show verbose output (JSON instead of pretty print)')
.action(async (options, cmd) => {
try {
// Validate options with Zod
const validated = ServerOptionsSchema.parse(options);
const timeout = parseInt(options.timeout, 10);

const verbose = options.verbose ?? false;
const outputDir = options.outputDir;
Expand Down Expand Up @@ -588,7 +590,8 @@ program
validated.scenario,
outputDir,
specVersionFilter,
options.force ?? false
options.force ?? false,
timeout
);

// Inapplicable scenario/spec-version combination (already logged by
Expand Down Expand Up @@ -675,7 +678,8 @@ program
specVersionFilter,
// a requirement set decides membership, so its choice outranks a
// scenario's own applicability window at the pinned revision
Boolean(requirements)
Boolean(requirements),
timeout
)
);
allResults.push({ scenario: scenarioName, checks: result.checks });
Expand Down
46 changes: 46 additions & 0 deletions src/runner/server.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,52 @@ describe('runServerConformanceTest spec-version applicability', () => {
}, 60000);
});

describe('runServerConformanceTest per-scenario timeout', () => {
// A server that completes the TCP handshake and then never writes a byte.
// Before the runner bounded `scenario.run`, this hung the whole suite: the
// scenario had no timeout of its own, so nothing downstream ever ran.
let server: http.Server;
let url: string;

beforeEach(async () => {
server = http.createServer(() => {
// Deliberately never respond, and never destroy the socket.
});
await new Promise<void>((resolve) =>
server.listen(0, '127.0.0.1', resolve)
);
const port = (server.address() as AddressInfo).port;
url = `http://127.0.0.1:${port}/mcp`;
});

afterEach(async () => {
server.closeAllConnections();
await new Promise<void>((resolve) => server.close(() => resolve()));
});

test('fails the scenario instead of hanging when the server never responds', async () => {
const start = Date.now();
const result = await runServerConformanceTest(
url,
'server-initialize',
undefined,
undefined,
false,
1000
);
const elapsed = Date.now() - start;

const timeoutCheck = result.checks.find((c) => c.id === 'scenario-timeout');
expect(timeoutCheck).toBeDefined();
expect(timeoutCheck?.status).toBe('FAILURE');
expect(timeoutCheck?.errorMessage).toContain('1000ms');

// The bound is what makes this a failure rather than a stall; without it
// the call never returns and this assertion is never reached.
expect(elapsed).toBeLessThan(15000);
}, 30000);
});

describe('runServerConformanceTest wire selection for draft-only scenarios', () => {
// Regression: the CLI used to silently emit the legacy initialize+session
// wire when running a draft-only scenario, producing requests with no
Expand Down
55 changes: 53 additions & 2 deletions src/runner/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,59 @@ function formatMarkdown(text: string): string {
);
}

/**
* Bound `scenario.run` so a server that accepts connections but never answers
* fails one scenario instead of stalling the whole suite.
*
* The losing promise is left pending on purpose: a scenario blocked on a socket
* read has no cancellation channel, so there is nothing to await. Its rejection
* is swallowed to keep a late failure from surfacing as an unhandled rejection
* against whichever scenario happens to be running by then.
*/
async function runScenarioBounded(
run: Promise<ConformanceCheck[]>,
scenarioName: string,
timeout: number
): Promise<ConformanceCheck[]> {
const timedOut = Symbol('timed-out');
let timeoutHandle: NodeJS.Timeout | undefined;

run.catch(() => {});

const result = await Promise.race([
run,
new Promise<typeof timedOut>((resolve) => {
timeoutHandle = setTimeout(() => resolve(timedOut), timeout);
})
]);
clearTimeout(timeoutHandle);

if (result !== timedOut) {
return result;
}

console.log(`\nScenario timed out after ${timeout}ms`);
return [
{
id: 'scenario-timeout',
name: 'Scenario completes within the timeout',
description:
'The scenario must finish within the configured timeout. A server that ' +
'accepts the connection but never responds leaves it running forever.',
status: 'FAILURE',
timestamp: new Date().toISOString(),
errorMessage: `Scenario '${scenarioName}' did not complete within ${timeout}ms. The server under test accepted the connection but did not finish the exchange.`
}
];
}

export async function runServerConformanceTest(
serverUrl: string,
scenarioName: string,
outputDir?: string,
specVersion?: SpecVersion,
force = false
force = false,
timeout: number = 30000
): Promise<{
checks: ConformanceCheck[];
resultDir?: string;
Expand Down Expand Up @@ -98,7 +145,11 @@ export async function runServerConformanceTest(
connect: (opts) => connectFor(resolvedSpecVersion)(serverUrl, opts)
};
resetWireValidation();
const checks = await scenario.run(ctx);
const checks = await runScenarioBounded(
scenario.run(ctx),
scenarioName,
timeout
);
checks.push(...wireSchemaChecks(resolvedSpecVersion));

if (resultDir) {
Expand Down