feat: add sequence.groupOrder option (#7852)

Author: sheremet-va

docs/config/index.md

@@ -2034,7 +2034,7 @@ export default defineConfig({
 
 ### sequence
 
-- **Type**: `{ sequencer?, shuffle?, seed?, hooks?, setupFiles? }`
+- **Type**: `{ sequencer?, shuffle?, seed?, hooks?, setupFiles?, groupOrder }`
 
 Options for how tests should be sorted.
 
@@ -2053,6 +2053,71 @@ A custom class that defines methods for sharding and sorting. You can extend `Ba
 
 Sharding is happening before sorting, and only if `--shard` option is provided.
 
+If [`sequencer.groupOrder`](#groupOrder) is specified, the sequencer will be called once for each group and pool.
+
+#### groupOrder <Version>3.2.0</Version> {#groupOrder}
+
+- **Type:** `number`
+- **Default:** `0`
+
+Controls the order in which this project runs its tests when using multiple [projects](/guide/projects).
+
+- Projects with the same group order number will run together, and groups are run from lowest to highest.
+- If you don’t set this option, all projects run in parallel.
+- If several projects use the same group order, they will run at the same time.
+
+This setting only affects the order in which projects run, not the order of tests within a project.
+To control test isolation or the order of tests inside a project, use the [`isolate`](#isolate) and [`sequence.sequencer`](#sequence-sequencer) options.
+
+::: details Example
+Consider this example:
+
+```ts
+import { defineConfig } from 'vitest/config'
+
+export default defineConfig({
+  test: {
+    projects: [
+      {
+        test: {
+          name: 'slow',
+          sequence: {
+            groupOrder: 0,
+          },
+        },
+      },
+      {
+        test: {
+          name: 'fast',
+          sequence: {
+            groupOrder: 0,
+          },
+        },
+      },
+      {
+        test: {
+          name: 'flaky',
+          sequence: {
+            groupOrder: 1,
+          },
+        },
+      },
+    ],
+  },
+})
+```
+
+Tests in these projects will run in this order:
+
+```
+ 0. slow  |
+          |> running together
+ 0. fast  |
+
+ 1. flaky |> runs after slow and fast alone
+```
+:::
+
 #### sequence.shuffle
 
 - **Type**: `boolean | { files?, tests? }`

docs/guide/cli-generated.md

@@ -89,9 +89,7 @@ Hide logs for skipped tests
 - **CLI:** `--reporter <name>`
 - **Config:** [reporters](/config/#reporters)
 
-Specify which reporter(s) to use. Available values:
-
-`default`, `basic`, `blob`, `verbose`, `dot`, `json`, `tap`, `tap-flat`, `junit`, `hanging-process`, `github-actions`.
+Specify reporters (default, basic, blob, verbose, dot, json, tap, tap-flat, junit, hanging-process, github-actions)
 
 ### outputFile
 

packages/vitest/src/node/cli/cli-config.ts

@@ -549,6 +549,7 @@ export const cliOptionsConfig: VitestCLIOptions = {
           'Changes the order in which setup files are executed. Accepted values are: "list" and "parallel". If set to "list", will run setup files in the order they are defined. If set to "parallel", will run setup files in parallel (default: `"parallel"`)',
         argument: '<order>',
       },
+      groupOrder: null,
     },
   },
   inspect: {

packages/vitest/src/node/config/resolveConfig.ts

@@ -772,6 +772,7 @@ export function resolveConfig(
       ? RandomSequencer
       : BaseSequencer
   }
+  resolved.sequence.groupOrder ??= 0
   resolved.sequence.hooks ??= 'stack'
   if (resolved.sequence.sequencer === RandomSequencer) {
     resolved.sequence.seed ??= Date.now()

packages/vitest/src/node/pool.ts

@@ -148,6 +148,7 @@ export function createPool(ctx: Vitest): ProcessPool {
       }
     }
 
+    const poolConcurrentPromises = new Map<string, Promise<ProcessPool>>()
     const customPools = new Map<string, ProcessPool>()
     async function resolveCustomPool(filepath: string) {
       if (customPools.has(filepath)) {
@@ -178,26 +179,44 @@ export function createPool(ctx: Vitest): ProcessPool {
       return poolInstance as ProcessPool
     }
 
-    const filesByPool: Record<LocalPool, TestSpecification[]> = {
-      forks: [],
-      threads: [],
-      vmThreads: [],
-      vmForks: [],
-      typescript: [],
+    function getConcurrentPool(pool: string, fn: () => Promise<ProcessPool>) {
+      if (poolConcurrentPromises.has(pool)) {
+        return poolConcurrentPromises.get(pool)!
+      }
+      const promise = fn().finally(() => {
+        poolConcurrentPromises.delete(pool)
+      })
+      poolConcurrentPromises.set(pool, promise)
+      return promise
+    }
+
+    function getCustomPool(pool: string) {
+      return getConcurrentPool(pool, () => resolveCustomPool(pool))
     }
 
+    function getBrowserPool() {
+      return getConcurrentPool('browser', async () => {
+        const { createBrowserPool } = await import('@vitest/browser')
+        return createBrowserPool(ctx)
+      })
+    }
+
+    const groupedSpecifications: Record<string, TestSpecification[]> = {}
+    const groups = new Set<number>()
+
     const factories: Record<LocalPool, () => ProcessPool> = {
       vmThreads: () => createVmThreadsPool(ctx, options),
+      vmForks: () => createVmForksPool(ctx, options),
       threads: () => createThreadsPool(ctx, options),
       forks: () => createForksPool(ctx, options),
-      vmForks: () => createVmForksPool(ctx, options),
       typescript: () => createTypecheckPool(ctx),
     }
 
     for (const spec of files) {
-      const { pool } = spec[2]
-      filesByPool[pool] ??= []
-      filesByPool[pool].push(spec)
+      const group = spec[0].config.sequence.groupOrder ?? 0
+      groups.add(group)
+      groupedSpecifications[group] ??= []
+      groupedSpecifications[group].push(spec)
     }
 
     const Sequencer = ctx.config.sequence.sequencer
@@ -210,35 +229,55 @@ export function createPool(ctx: Vitest): ProcessPool {
       return sequencer.sort(specs)
     }
 
-    await Promise.all(
-      Object.entries(filesByPool).map(async (entry) => {
-        const [pool, files] = entry as [Pool, TestSpecification[]]
-
-        if (!files.length) {
-          return null
-        }
-
-        const specs = await sortSpecs(files)
-
-        if (pool in factories) {
-          const factory = factories[pool]
-          pools[pool] ??= factory()
-          return pools[pool]![method](specs, invalidate)
-        }
-
-        if (pool === 'browser') {
-          pools[pool] ??= await (async () => {
-            const { createBrowserPool } = await import('@vitest/browser')
-            return createBrowserPool(ctx)
-          })()
-          return pools[pool]![method](specs, invalidate)
-        }
-
-        const poolHandler = await resolveCustomPool(pool)
-        pools[poolHandler.name] ??= poolHandler
-        return poolHandler[method](specs, invalidate)
-      }),
-    )
+    const sortedGroups = Array.from(groups).sort()
+    for (const group of sortedGroups) {
+      const specifications = groupedSpecifications[group]
+
+      if (!specifications?.length) {
+        continue
+      }
+
+      const filesByPool: Record<LocalPool, TestSpecification[]> = {
+        forks: [],
+        threads: [],
+        vmThreads: [],
+        vmForks: [],
+        typescript: [],
+      }
+
+      specifications.forEach((specification) => {
+        const pool = specification[2].pool
+        filesByPool[pool] ??= []
+        filesByPool[pool].push(specification)
+      })
+
+      await Promise.all(
+        Object.entries(filesByPool).map(async (entry) => {
+          const [pool, files] = entry as [Pool, TestSpecification[]]
+
+          if (!files.length) {
+            return null
+          }
+
+          const specs = await sortSpecs(files)
+
+          if (pool in factories) {
+            const factory = factories[pool]
+            pools[pool] ??= factory()
+            return pools[pool]![method](specs, invalidate)
+          }
+
+          if (pool === 'browser') {
+            pools.browser ??= await getBrowserPool()
+            return pools.browser[method](specs, invalidate)
+          }
+
+          const poolHandler = await getCustomPool(pool)
+          pools[poolHandler.name] ??= poolHandler
+          return poolHandler[method](specs, invalidate)
+        }),
+      )
+    }
   }
 
   return {

packages/vitest/src/node/types/config.ts

@@ -64,6 +64,15 @@ interface SequenceOptions {
    * @default BaseSequencer
    */
   sequencer?: TestSequencerConstructor
+  /**
+   * Controls the order in which this project runs its tests when using multiple [projects](/guide/projects).
+   *
+   * - Projects with the same group order number will run together, and groups are run from lowest to highest.
+   * - If you don’t set this option, all projects run in parallel.
+   * - If several projects use the same group order, they will run at the same time.
+   * @default 0
+   */
+  groupOrder?: number
   /**
    * Should files and tests run in random order.
    * @default false
@@ -1066,6 +1075,7 @@ export interface ResolvedConfig
     shuffle?: boolean
     concurrent?: boolean
     seed: number
+    groupOrder: number
   }
 
   typecheck: Omit<TypecheckConfig, 'enabled'> & {

test/cli/test/group-order.test.ts

@@ -0,0 +1,53 @@
+import { expect, test } from 'vitest'
+import { runInlineTests } from '../../test-utils'
+
+test('tests run according to the group order', async () => {
+  const { stdout, stderr } = await runInlineTests({
+    'example.1.test.ts': `test('1')`,
+    'example.2.test.ts': `test('2')`,
+    'example.2-2.test.ts': `test('2-2')`,
+    'example.3.test.ts': `test('3')`,
+  }, {
+    globals: true,
+    // run projects in the opposite order!
+    projects: [
+      {
+        test: {
+          name: '3',
+          include: ['./example.3.test.ts'],
+          sequence: {
+            groupOrder: 1,
+          },
+        },
+      },
+      {
+        test: {
+          include: ['./example.2.test.ts', './example.2-2.test.ts'],
+          name: '2',
+          sequence: {
+            groupOrder: 2,
+          },
+        },
+      },
+      {
+        test: {
+          name: '1',
+          include: ['./example.1.test.ts'],
+          sequence: {
+            groupOrder: 3,
+          },
+        },
+      },
+    ],
+  })
+  expect(stderr).toBe('')
+
+  const tests = stdout.split('\n').filter(c => c.startsWith(' ✓')).join('\n').replace(/\d+ms/g, '<time>')
+
+  expect(tests).toMatchInlineSnapshot(`
+    " ✓ |3| example.3.test.ts > 3 <time>
+     ✓ |2| example.2-2.test.ts > 2-2 <time>
+     ✓ |2| example.2.test.ts > 2 <time>
+     ✓ |1| example.1.test.ts > 1 <time>"
+  `)
+})