diff --git a/packages/runner/src/run.ts b/packages/runner/src/run.ts index 4adf8f671b7c..4d05d8efd8ff 100644 --- a/packages/runner/src/run.ts +++ b/packages/runner/src/run.ts @@ -1,4 +1,4 @@ -import type { Awaitable } from '@vitest/utils' +import type { Awaitable, TestError } from '@vitest/utils' import type { DiffOptions } from '@vitest/utils/diff' import type { FileSpecification, VitestRunner } from './types/runner' import type { @@ -34,6 +34,46 @@ const now = globalThis.performance ? globalThis.performance.now.bind(globalThis. const unixNow = Date.now const { clearTimeout, setTimeout } = getSafeTimers() +/** + * Normalizes retry configuration to extract individual values. + * Handles both number and object forms. + */ +function getRetryCount(retry: number | { count?: number } | undefined): number { + if (retry === undefined) + return 0 + if (typeof retry === 'number') + return retry + return retry.count ?? 0 +} + +function getRetryDelay(retry: number | { delay?: number } | undefined): number { + if (retry === undefined) + return 0 + if (typeof retry === 'number') + return 0 + return retry.delay ?? 0 +} + +function getRetryCondition( + retry: number | { condition?: string | ((error: Error) => boolean) } | undefined, +): string | ((error: Error) => boolean) | undefined { + if (retry === undefined) + return undefined + if (typeof retry === 'number') + return undefined + return retry.condition +} + +function getRetryStrategy( + retry: number | { strategy?: 'immediate' | 'test-file' | 'deferred' } | undefined, +): 'immediate' | 'test-file' | 'deferred' { + if (retry === undefined) + return 'immediate' + if (typeof retry === 'number') + return 'immediate' + return retry.strategy ?? 'immediate' +} + function updateSuiteHookState( task: Task, name: keyof SuiteHooks, @@ -266,6 +306,51 @@ async function callCleanupHooks(runner: VitestRunner, cleanups: unknown[]) { } } +/** + * Determines if a test should be retried based on its retryCondition configuration + */ +function shouldRetryTest(test: Test, errors: TestError[] | undefined): boolean { + const condition = getRetryCondition(test.retry) + + // No errors means test passed, shouldn't get here but handle it + if (!errors || errors.length === 0) { + return false + } + + // No condition means always retry + if (!condition) { + return true + } + + // Check only the most recent error (last in array) against the condition + const error = errors[errors.length - 1] + + if (typeof condition === 'string') { + // String condition is treated as regex pattern + const regex = new RegExp(condition, 'i') + return regex.test(error.message || '') + } + else if (typeof condition === 'function') { + // Function condition is called with error object + try { + // Convert TestError back to Error-like object for the condition function + const errorObj = Object.assign(new Error(error.message), { + name: error.name, + stack: error.stack, + cause: error.cause, + }) + return condition(errorObj) + } + catch (e) { + // If condition function throws, treat as no match + console.error('retryCondition function threw error:', e) + return false + } + } + + return false +} + export async function runTest(test: Test, runner: VitestRunner): Promise { await runner.onBeforeRunTask?.(test) @@ -299,7 +384,7 @@ export async function runTest(test: Test, runner: VitestRunner): Promise { const repeats = test.repeats ?? 0 for (let repeatCount = 0; repeatCount <= repeats; repeatCount++) { - const retry = test.retry ?? 0 + const retry = getRetryCount(test.retry) for (let retryCount = 0; retryCount <= retry; retryCount++) { let beforeEachCleanups: unknown[] = [] try { @@ -407,9 +492,34 @@ export async function runTest(test: Test, runner: VitestRunner): Promise { } if (retryCount < retry) { + // Check if we should retry based on the error condition + const shouldRetry = shouldRetryTest(test, test.result.errors) + + if (!shouldRetry) { + // Error doesn't match retry condition, stop retrying + break + } + + // Check retry strategy + const strategy = getRetryStrategy(test.retry) + + if (strategy === 'test-file' || strategy === 'deferred') { + // For test-file and deferred strategies, exit the retry loop + // test-file: let the suite handle it + // deferred: let the global runner handle it + break + } + + // For immediate strategy (default), retry immediately // reset state when retry test test.result.state = 'run' test.result.retryCount = (test.result.retryCount ?? 0) + 1 + + // Apply retry delay if configured + const delay = getRetryDelay(test.retry) + if (delay > 0) { + await new Promise(resolve => setTimeout(resolve, delay)) + } } // update retry info @@ -468,6 +578,40 @@ function markTasksAsSkipped(suite: Suite, runner: VitestRunner) { }) } +/** + * Collects tests that failed and have test-file retry strategy + */ +function collectDeferredRetryTests(suite: Suite, strategy: 'test-file' | 'deferred'): Test[] { + const tests: Test[] = [] + + function collectFromTask(task: Task) { + if (task.type === 'test') { + const retry = getRetryCount(task.retry) + const retryCount = task.result?.retryCount ?? 0 + const testStrategy = getRetryStrategy(task.retry) + + if ( + testStrategy === strategy + && task.result?.state === 'fail' + && retryCount < retry + ) { + tests.push(task) + } + } + else if (task.type === 'suite') { + for (const child of task.tasks) { + collectFromTask(child) + } + } + } + + for (const task of suite.tasks) { + collectFromTask(task) + } + + return tests +} + export async function runSuite(suite: Suite, runner: VitestRunner): Promise { await runner.onBeforeRunSuite?.(suite) @@ -544,6 +688,48 @@ export async function runSuite(suite: Suite, runner: VitestRunner): Promise 0) { + await new Promise(resolve => setTimeout(resolve, delay)) + } + + // Run the test again + await runTest(test, runner) + + // If test passed, stop retrying (use any to bypass TypeScript flow analysis) + if (test.result && (test.result as any).state === 'pass') { + break + } + + // Check retry condition for next attempt + if (i + 1 < retry && test.result) { + const shouldRetry = shouldRetryTest(test, test.result.errors) + if (!shouldRetry) { + break + } + } + } + } + } } catch (e) { failTask(suite.result, e, runner.config.diffOptions) @@ -615,6 +801,52 @@ export async function runFiles(files: File[], runner: VitestRunner): Promise 0) { + await new Promise(resolve => setTimeout(resolve, delay)) + } + + // Run the test again + await runTest(test, runner) + + // If test passed, stop retrying + if (test.result && (test.result as any).state === 'pass') { + break + } + + // Check retry condition for next attempt + if (i + 1 < retry && test.result) { + const shouldRetry = shouldRetryTest(test, test.result.errors) + if (!shouldRetry) { + break + } + } + } + } } const workerRunners = new WeakSet() diff --git a/packages/runner/src/types/runner.ts b/packages/runner/src/types/runner.ts index 895253e274ac..4712fd5e6709 100644 --- a/packages/runner/src/types/runner.ts +++ b/packages/runner/src/types/runner.ts @@ -35,7 +35,12 @@ export interface VitestRunnerConfig { maxConcurrency: number testTimeout: number hookTimeout: number - retry: number + retry: number | { + count?: number + delay?: number + condition?: string | ((error: Error) => boolean) + strategy?: 'immediate' | 'test-file' | 'deferred' + } includeTaskLocation?: boolean diffOptions?: DiffOptions } diff --git a/packages/runner/src/types/tasks.ts b/packages/runner/src/types/tasks.ts index 8847538c60e3..4082ead5b60f 100644 --- a/packages/runner/src/types/tasks.ts +++ b/packages/runner/src/types/tasks.ts @@ -53,10 +53,42 @@ export interface TaskBase { */ result?: TaskResult /** - * The amount of times the task should be retried if it fails. + * Retry configuration for the task. + * - If a number, specifies how many times to retry + * - If an object, allows fine-grained retry control * @default 0 */ - retry?: number + retry?: number | { + /** + * The number of times to retry the test if it fails. + * @default 0 + */ + count?: number + /** + * Delay in milliseconds between retry attempts. + * @default 0 + */ + delay?: number + /** + * Condition to determine if a test should be retried based on the error. + * - If a string, treated as a regular expression to match against error message + * - If a function, called with the error object; return true to retry + * + * NOTE: Functions can only be used in test files, not in vitest.config.ts, + * because the configuration is serialized when passed to worker threads. + * + * @default undefined (retry on all errors) + */ + condition?: string | ((error: Error) => boolean) + /** + * Strategy for when to retry failed tests. + * - 'immediate': Retry immediately after failure (default) + * - 'test-file': Defer retries until after all tests in the file complete + * - 'deferred': Defer retries until after all test files complete + * @default 'immediate' + */ + strategy?: 'immediate' | 'test-file' | 'deferred' + } /** * The amount of times the task should be repeated after the successful run. * If the task fails, it will not be retried unless `retry` is specified. @@ -433,12 +465,42 @@ export interface TestOptions { */ timeout?: number /** - * Times to retry the test if fails. Useful for making flaky tests more stable. - * When retries is up, the last test error will be thrown. - * + * Retry configuration for the test. + * - If a number, specifies how many times to retry + * - If an object, allows fine-grained retry control * @default 0 */ - retry?: number + retry?: number | { + /** + * The number of times to retry the test if it fails. + * @default 0 + */ + count?: number + /** + * Delay in milliseconds between retry attempts. + * @default 0 + */ + delay?: number + /** + * Condition to determine if a test should be retried based on the error. + * - If a string, treated as a regular expression to match against error message + * - If a function, called with the error object; return true to retry + * + * NOTE: Functions can only be used in test files, not in vitest.config.ts, + * because the configuration is serialized when passed to worker threads. + * + * @default undefined (retry on all errors) + */ + condition?: string | ((error: Error) => boolean) + /** + * Strategy for when to retry failed tests. + * - 'immediate': Retry immediately after failure (default) + * - 'test-file': Defer retries until after all tests in the file complete + * - 'deferred': Defer retries until after all test files complete + * @default 'immediate' + */ + strategy?: 'immediate' | 'test-file' | 'deferred' + } /** * How many times the test will run again. * Only inner tests will repeat if set on `describe()`, nested `describe()` will inherit parent's repeat by default. diff --git a/packages/vitest/src/node/cli/cli-config.ts b/packages/vitest/src/node/cli/cli-config.ts index d72ff4508976..285e1f10ceb2 100644 --- a/packages/vitest/src/node/cli/cli-config.ts +++ b/packages/vitest/src/node/cli/cli-config.ts @@ -541,6 +541,7 @@ export const cliOptionsConfig: VitestCLIOptions = { description: 'Retry the test specific number of times if it fails (default: `0`)', argument: '', + subcommands: null, }, diff: { description: diff --git a/packages/vitest/src/node/config/serializeConfig.ts b/packages/vitest/src/node/config/serializeConfig.ts index 879a8a039985..5cc81eabce90 100644 --- a/packages/vitest/src/node/config/serializeConfig.ts +++ b/packages/vitest/src/node/config/serializeConfig.ts @@ -6,6 +6,20 @@ export function serializeConfig(project: TestProject): SerializedConfig { const viteConfig = project._vite?.config const optimizer = config.deps?.optimizer || {} + // Handle retry configuration serialization + let retry = config.retry + if (retry && typeof retry === 'object' && typeof retry.condition === 'function') { + console.warn( + 'Warning: retry.condition function cannot be used in vitest.config.ts. ' + + 'Use a string pattern instead, or define the function in your test file.', + ) + // Remove the function from serialized config + retry = { + ...retry, + condition: undefined, + } + } + return { // TODO: remove functions from environmentOptions environmentOptions: config.environmentOptions, @@ -35,7 +49,7 @@ export function serializeConfig(project: TestProject): SerializedConfig { snapshotSerializers: config.snapshotSerializers, // TODO: non serializable function? diff: config.diff, - retry: config.retry, + retry, disableConsoleIntercept: config.disableConsoleIntercept, root: config.root, name: config.name, diff --git a/packages/vitest/src/node/reporters/reported-tasks.ts b/packages/vitest/src/node/reporters/reported-tasks.ts index f1df17ff4981..a8f759f2673e 100644 --- a/packages/vitest/src/node/reporters/reported-tasks.ts +++ b/packages/vitest/src/node/reporters/reported-tasks.ts @@ -500,7 +500,12 @@ export interface TaskOptions { readonly fails: boolean | undefined readonly concurrent: boolean | undefined readonly shuffle: boolean | undefined - readonly retry: number | undefined + readonly retry: number | { + count?: number + delay?: number + condition?: string | ((error: Error) => boolean) + strategy?: 'immediate' | 'test-file' | 'deferred' + } | undefined readonly repeats: number | undefined readonly mode: 'run' | 'only' | 'skip' | 'todo' } diff --git a/packages/vitest/src/node/types/config.ts b/packages/vitest/src/node/types/config.ts index c355a03cf16b..986899215964 100644 --- a/packages/vitest/src/node/types/config.ts +++ b/packages/vitest/src/node/types/config.ts @@ -782,11 +782,44 @@ export interface InlineConfig { bail?: number /** - * Retry the test specific number of times if it fails. + * Retry configuration for tests. + * - If a number, specifies how many times to retry failed tests + * - If an object, allows fine-grained retry control * * @default 0 */ - retry?: number + retry?: number | { + /** + * The number of times to retry the test if it fails. + * @default 0 + */ + count?: number + /** + * Delay in milliseconds between retry attempts. + * Useful for tests that interact with rate-limited APIs or need time to recover. + * @default 0 + */ + delay?: number + /** + * Condition to determine if a test should be retried based on the error. + * - If a string, treated as a regular expression to match against error message + * + * ⚠️ WARNING: Function form is NOT supported in vitest.config.ts + * because configurations are serialized when passed to worker threads. + * Use the function form only in test files directly. + * + * @default undefined (retry on all errors) + */ + condition?: string + /** + * Strategy for when to retry failed tests. + * - 'immediate': Retry immediately after failure (default) + * - 'test-file': Defer retries until after all tests in the file complete + * - 'deferred': Defer retries until after all test files complete + * @default 'immediate' + */ + strategy?: 'immediate' | 'test-file' | 'deferred' + } /** * Show full diff when snapshot fails instead of a patch. diff --git a/packages/vitest/src/runtime/config.ts b/packages/vitest/src/runtime/config.ts index b62b052e0a7b..7321b9722d47 100644 --- a/packages/vitest/src/runtime/config.ts +++ b/packages/vitest/src/runtime/config.ts @@ -78,7 +78,12 @@ export interface SerializedConfig { truncateThreshold?: number } | undefined diff: string | SerializedDiffOptions | undefined - retry: number + retry: number | { + count?: number + delay?: number + condition?: string | ((error: Error) => boolean) + strategy?: 'immediate' | 'test-file' | 'deferred' + } includeTaskLocation: boolean | undefined inspect: boolean | string | undefined inspectBrk: boolean | string | undefined diff --git a/packages/vitest/src/runtime/runners/index.ts b/packages/vitest/src/runtime/runners/index.ts index a0d53634b3b2..58e781d4b895 100644 --- a/packages/vitest/src/runtime/runners/index.ts +++ b/packages/vitest/src/runtime/runners/index.ts @@ -89,6 +89,20 @@ export async function resolveTestRunner( state.durations.prepare = 0 state.durations.environment = 0 }) + + // Strip function conditions from retry config before sending via RPC + // Functions cannot be cloned by structured clone algorithm + const sanitizeRetryConditions = (task: any) => { + if (task.retry && typeof task.retry === 'object' && typeof task.retry.condition === 'function') { + // Remove function condition - it can't be serialized + task.retry = { ...task.retry, condition: undefined } + } + if (task.tasks) { + task.tasks.forEach(sanitizeRetryConditions) + } + } + files.forEach(sanitizeRetryConditions) + rpc().onCollected(files) await originalOnCollected?.call(testRunner, files) } diff --git a/test/core/test/retry-condition.test.ts b/test/core/test/retry-condition.test.ts new file mode 100644 index 000000000000..ef0093977a76 --- /dev/null +++ b/test/core/test/retry-condition.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from 'vitest' + +// Test with string regex condition that eventually passes +let matchingCount = 0 +it('retry with matching condition', { + retry: { + count: 5, + condition: 'retry', + }, +}, () => { + matchingCount += 1 + if (matchingCount < 3) { + throw new Error('Please retry this test') + } + // Third attempt succeeds +}) + +it('verify matching condition retried', () => { + expect(matchingCount).toBe(3) +}) + +// Test with no condition (should retry all errors) +let noConditionCount = 0 +it('retry without condition', { retry: 2 }, () => { + noConditionCount += 1 + expect(noConditionCount).toBe(3) +}) + +it('verify no condition retried all attempts', () => { + expect(noConditionCount).toBe(3) +}) + +// Test with function condition +let functionCount = 0 +const condition = (error: Error) => error.name === 'TimeoutError' + +it('retry with function condition', { + retry: { + count: 5, + condition, + }, +}, () => { + functionCount += 1 + const err: any = new Error('Test failed') + err.name = 'TimeoutError' + if (functionCount < 3) { + throw err + } + // Third attempt succeeds +}) + +it('verify function condition worked', () => { + expect(functionCount).toBe(3) +}) + +describe('retry condition with describe', { + retry: { + count: 2, + condition: 'flaky', + }, +}, () => { + let describeCount = 0 + it('test should inherit retryCondition from describe block', () => { + describeCount += 1 + if (describeCount === 1) { + throw new Error('Flaky test error') + } + // Second attempt succeeds + }) +}) diff --git a/test/core/test/retry-delay.test.ts b/test/core/test/retry-delay.test.ts new file mode 100644 index 000000000000..103d224ce8fa --- /dev/null +++ b/test/core/test/retry-delay.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from 'vitest' + +let delayCount = 0 +let delayStart = 0 + +it('retry with delay', { + retry: { + count: 2, + delay: 100, + }, +}, () => { + if (delayCount === 0) { + delayStart = Date.now() + } + delayCount += 1 + expect(delayCount).toBe(3) +}) + +it('verify delay was applied', () => { + const duration = Date.now() - delayStart + expect(delayCount).toBe(3) + // With 2 retries and 100ms delay each, should take at least 200ms + expect(duration).toBeGreaterThanOrEqual(200) +}) + +let zeroDelayCount = 0 + +it('retry with zero delay', { + retry: { + count: 2, + delay: 0, + }, +}, () => { + zeroDelayCount += 1 + expect(zeroDelayCount).toBe(3) +}) + +it('verify zero delay test passed', () => { + expect(zeroDelayCount).toBe(3) +}) + +describe('retry delay with describe', { + retry: { + count: 2, + delay: 50, + }, +}, () => { + let describeCount = 0 + it('test should inherit retryDelay from describe block', () => { + describeCount += 1 + expect(describeCount).toBe(2) + }) +}) diff --git a/test/core/test/retry-strategy.test.ts b/test/core/test/retry-strategy.test.ts new file mode 100644 index 000000000000..d777ba8e36e6 --- /dev/null +++ b/test/core/test/retry-strategy.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from 'vitest' + +// Test with immediate strategy (default) +let immediateCount = 0 +it('retry with immediate strategy', { retry: 2 }, () => { + immediateCount += 1 + expect(immediateCount).toBe(3) +}) + +it('verify immediate strategy retried', () => { + expect(immediateCount).toBe(3) +}) + +// Test with test-file strategy +// Note: With test-file strategy, the test retries after all tests in the file complete. +// The test itself validates that it ran 3 times (1 initial + 2 retries) by eventually passing. +let testFileCount = 0 +it('retry with test-file strategy', { + retry: { + count: 2, + strategy: 'test-file', + }, +}, () => { + testFileCount += 1 + // This will fail on attempts 1 and 2, pass on attempt 3 + expect(testFileCount).toBe(3) +}) + +// Test with deferred strategy +// Note: With deferred strategy, the test retries after all test files complete. +// The test itself validates that it ran 3 times (1 initial + 2 retries) by eventually passing. +let deferredCount = 0 +it('retry with deferred strategy', { + retry: { + count: 2, + strategy: 'deferred', + }, +}, () => { + deferredCount += 1 + // This will fail on attempts 1 and 2, pass on attempt 3 + expect(deferredCount).toBe(3) +}) + +describe('retry strategy with describe', { + retry: { + count: 2, + strategy: 'immediate', + }, +}, () => { + let describeCount = 0 + it('test should inherit retryStrategy from describe block', () => { + describeCount += 1 + expect(describeCount).toBe(2) + }) +})