Skip to content

Commit 69ee466

Browse files
potetopoteto
committed
[ci] Parallelize yarn build
ghstack-source-id: c93d81e Pull Request resolved: #30071
1 parent 361b752 commit 69ee466

File tree

5 files changed

+978
-44
lines changed

5 files changed

+978
-44
lines changed

‎.circleci/config.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ jobs:
9797
steps:
9898
- checkout
9999
- setup_node_modules
100-
- run: yarn build
100+
- run: yarn build --ci=circleci
101101
- persist_to_workspace:
102102
root: .
103103
paths:

‎.github/workflows/runtime_build.yml‎

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,11 @@ jobs:
1111
build_and_lint:
1212
name: yarn build and lint
1313
runs-on: ubuntu-latest
14+
strategy:
15+
matrix:
16+
# yml is dumb. update the --total arg to yarn build if you change the number of workers
17+
worker_id: [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]
18+
release_channel: [stable, experimental]
1419
steps:
1520
- uses: actions/checkout@v4
1621
- uses: actions/setup-node@v4
@@ -25,11 +30,18 @@ jobs:
2530
path: "**/node_modules"
2631
key: ${{ runner.arch }}-${{ runner.os }}-modules-${{ hashFiles('yarn.lock') }}
2732
- run: yarn install --frozen-lockfile
28-
- run: yarn build
29-
- run: yarn lint-build
30-
- name: Cache build
31-
uses: actions/cache@v4
32-
id: build_cache
33+
- run: yarn build --index=${{ matrix.worker_id }} --total=20 --r=${{ matrix.release_channel }} --ci=github
34+
env:
35+
CI: github
36+
RELEASE_CHANNEL: ${{ matrix.release_channel }}
37+
NODE_INDEX: ${{ matrix.worker_id }}
38+
- name: Lint build
39+
run: yarn lint-build
40+
- name: Display structure of build
41+
run: ls -R build
42+
- name: Archive build
43+
uses: actions/upload-artifact@v4
3344
with:
34-
path: "build/**"
35-
key: yarn-build-${{ runner.arch }}-${{ runner.os }}-modules-${{ hashFiles('yarn.lock') }}
45+
name: build_${{ matrix.worker_id }}_${{ matrix.release_channel }}
46+
path: |
47+
build

‎scripts/rollup/build-all-release-channels.js‎

Lines changed: 84 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ const {
1515
canaryChannelLabel,
1616
rcNumber,
1717
} = require('../../ReactVersions');
18+
const yargs = require('yargs');
19+
const {buildEverything} = require('./build-ghaction');
1820

1921
// Runs the build script for both stable and experimental release channels,
2022
// by configuring an environment variable.
@@ -51,44 +53,88 @@ fs.writeFileSync(
5153
`export default '${PLACEHOLDER_REACT_VERSION}';\n`
5254
);
5355

54-
if (process.env.CIRCLE_NODE_TOTAL) {
55-
// In CI, we use multiple concurrent processes. Allocate half the processes to
56-
// build the stable channel, and the other half for experimental. Override
57-
// the environment variables to "trick" the underlying build script.
58-
const total = parseInt(process.env.CIRCLE_NODE_TOTAL, 10);
59-
const halfTotal = Math.floor(total / 2);
60-
const index = parseInt(process.env.CIRCLE_NODE_INDEX, 10);
61-
if (index < halfTotal) {
62-
const nodeTotal = halfTotal;
63-
const nodeIndex = index;
64-
buildForChannel('stable', nodeTotal, nodeIndex);
65-
processStable('./build');
56+
const argv = yargs.wrap(yargs.terminalWidth()).options({
57+
releaseChannel: {
58+
alias: 'r',
59+
describe: 'Build the given release channel.',
60+
requiresArg: true,
61+
type: 'string',
62+
choices: ['experimental', 'stable'],
63+
},
64+
index: {
65+
alias: 'i',
66+
describe: 'Worker id.',
67+
requiresArg: true,
68+
type: 'number',
69+
},
70+
total: {
71+
alias: 't',
72+
describe: 'Total number of workers.',
73+
requiresArg: true,
74+
type: 'number',
75+
},
76+
ci: {
77+
describe: 'Run tests in CI',
78+
requiresArg: false,
79+
type: 'choices',
80+
choices: ['circleci', 'github'],
81+
},
82+
}).argv;
83+
84+
async function main() {
85+
if (argv.ci === 'github') {
86+
await buildEverything(argv.index, argv.total);
87+
switch (argv.releaseChannel) {
88+
case 'stable': {
89+
processStable('./build');
90+
break;
91+
}
92+
case 'experimental': {
93+
processExperimental('./build');
94+
break;
95+
}
96+
default:
97+
throw new Error(`Unknown release channel ${argv.releaseChannel}`);
98+
}
99+
} else if (argv.ci === 'circleci') {
100+
// In CI, we use multiple concurrent processes. Allocate half the processes to
101+
// build the stable channel, and the other half for experimental. Override
102+
// the environment variables to "trick" the underlying build script.
103+
const total = parseInt(process.env.CIRCLE_NODE_TOTAL, 10);
104+
const halfTotal = Math.floor(total / 2);
105+
const index = parseInt(process.env.CIRCLE_NODE_INDEX, 10);
106+
if (index < halfTotal) {
107+
const nodeTotal = halfTotal;
108+
const nodeIndex = index;
109+
buildForChannel('stable', nodeTotal, nodeIndex);
110+
processStable('./build');
111+
} else {
112+
const nodeTotal = total - halfTotal;
113+
const nodeIndex = index - halfTotal;
114+
buildForChannel('experimental', nodeTotal, nodeIndex);
115+
processExperimental('./build');
116+
}
66117
} else {
67-
const nodeTotal = total - halfTotal;
68-
const nodeIndex = index - halfTotal;
69-
buildForChannel('experimental', nodeTotal, nodeIndex);
70-
processExperimental('./build');
118+
// Running locally, no concurrency. Move each channel's build artifacts into
119+
// a temporary directory so that they don't conflict.
120+
buildForChannel('stable', '', '');
121+
const stableDir = tmp.dirSync().name;
122+
crossDeviceRenameSync('./build', stableDir);
123+
processStable(stableDir);
124+
buildForChannel('experimental', '', '');
125+
const experimentalDir = tmp.dirSync().name;
126+
crossDeviceRenameSync('./build', experimentalDir);
127+
processExperimental(experimentalDir);
128+
129+
// Then merge the experimental folder into the stable one. processExperimental
130+
// will have already removed conflicting files.
131+
//
132+
// In CI, merging is handled automatically by CircleCI's workspace feature.
133+
mergeDirsSync(experimentalDir + '/', stableDir + '/');
134+
135+
// Now restore the combined directory back to its original name
136+
crossDeviceRenameSync(stableDir, './build');
71137
}
72-
} else {
73-
// Running locally, no concurrency. Move each channel's build artifacts into
74-
// a temporary directory so that they don't conflict.
75-
buildForChannel('stable', '', '');
76-
const stableDir = tmp.dirSync().name;
77-
crossDeviceRenameSync('./build', stableDir);
78-
processStable(stableDir);
79-
buildForChannel('experimental', '', '');
80-
const experimentalDir = tmp.dirSync().name;
81-
crossDeviceRenameSync('./build', experimentalDir);
82-
processExperimental(experimentalDir);
83-
84-
// Then merge the experimental folder into the stable one. processExperimental
85-
// will have already removed conflicting files.
86-
//
87-
// In CI, merging is handled automatically by CircleCI's workspace feature.
88-
mergeDirsSync(experimentalDir + '/', stableDir + '/');
89-
90-
// Now restore the combined directory back to its original name
91-
crossDeviceRenameSync(stableDir, './build');
92138
}
93139

94140
function buildForChannel(channel, nodeTotal, nodeIndex) {
@@ -455,3 +501,5 @@ function mergeDirsSync(source, destination) {
455501
}
456502
}
457503
}
504+
505+
main();

0 commit comments

Comments
 (0)