Skip to content

Commit 9cc216e

Browse files
vlsivlsi
committed
ci: add randomized matrix for better test coverage
See https://github.com/vlsi/github-actions-random-matrix
1 parent 5f1c8db commit 9cc216e

File tree

6 files changed

+464
-26
lines changed

6 files changed

+464
-26
lines changed

‎.github/workflows/main.yml‎

Lines changed: 43 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -18,40 +18,58 @@ concurrency:
1818
cancel-in-progress: true
1919

2020
jobs:
21-
windows:
22-
name: 'Windows (JDK 17)'
23-
runs-on: windows-latest
21+
matrix_prep:
22+
name: Matrix Preparation
23+
runs-on: ubuntu-latest
24+
outputs:
25+
matrix: ${{ steps.set-matrix.outputs.matrix }}
26+
env:
27+
# Number of jobs to generate in matrix.js
28+
MATRIX_JOBS: 4
2429
steps:
25-
- uses: actions/checkout@v3
26-
- name: 'Set up JDK 17'
27-
uses: actions/setup-java@v3
28-
with:
29-
java-version: 17
30-
distribution: 'zulu'
31-
- uses: burrunan/gradle-cache-action@v1
32-
name: Test
33-
with:
34-
job-id: jdk17
35-
multi-cache-enabled: false
36-
# An explicit skip for Sha512 tasks is required due to https://github.com/gradle/gradle/issues/16789
37-
arguments: --scan --no-parallel build -x distTar -x distTarSource -x distTarSha512 -x distTarSourceSha512
30+
- uses: actions/checkout@v3
31+
- id: set-matrix
32+
run: |
33+
node .github/workflows/matrix.js
3834
39-
mac:
40-
name: 'macOS (JDK 17)'
41-
runs-on: macos-latest
35+
test:
36+
needs: matrix_prep
37+
name: '${{ matrix.name }}'
38+
runs-on: ${{ matrix.os }}
39+
env:
40+
TZ: ${{ matrix.tz }}
41+
strategy:
42+
matrix: ${{fromJson(needs.matrix_prep.outputs.matrix)}}
43+
fail-fast: false
44+
# max-parallel: 4
4245
steps:
43-
- uses: actions/checkout@v3
44-
- name: 'Set up JDK 17'
46+
- uses: actions/checkout@v2
47+
with:
48+
fetch-depth: 50
49+
- name: Set up Java ${{ matrix.java_version }}, ${{ matrix.java_distribution }}
4550
uses: actions/setup-java@v3
4651
with:
47-
java-version: 17
48-
distribution: 'zulu'
52+
java-version: ${{ matrix.java_version }}
53+
distribution: ${{ matrix.java_distribution }}
54+
architecture: x64
55+
- name: Steps to reproduce
56+
uses: actions/github-script@v6
57+
with:
58+
script: |
59+
console.log('The following command might help reproducing CI results, use Java ${{ matrix.java_version }}')
60+
console.log('TZ="${{ matrix.tz }}" _JAVA_OPTIONS="${{ matrix.extraJvmArgs }}" ./gradlew build -x distTar -x distTarSource -x distTarSha512 -x distTarSourceSha512 ${{ matrix.extraGradleArgs }} -PtestExtraJvmArgs="${{ matrix.testExtraJvmArgs }}" -PtestDisableCaching="${{ matrix.testDisableCaching }}"')
4961
- uses: burrunan/gradle-cache-action@v1
5062
name: Test
5163
with:
52-
job-id: jdk14
64+
job-id: jdk${{ matrix.java_version }}
5365
multi-cache-enabled: false
54-
arguments: --scan --no-parallel build -x distTar -x distTarSource -x distTarSha512 -x distTarSourceSha512 -Dskip.test_TestDNSCacheManager.testWithCustomResolverAnd1Server=true
66+
# An explicit skip for Sha512 tasks is required due to https://github.com/gradle/gradle/issues/16789
67+
arguments: --scan --no-parallel build -x distTar -x distTarSource -x distTarSha512 -x distTarSourceSha512 ${{ matrix.extraGradleArgs }}
68+
properties: |
69+
testExtraJvmArgs=${{ matrix.testExtraJvmArgs }}
70+
testDisableCaching=${{ matrix.testDisableCaching }}
71+
env:
72+
_JAVA_OPTIONS: ${{ matrix.extraJvmArgs }}
5573

5674
errorprone:
5775
name: 'Error Prone (JDK 11)'

‎.github/workflows/matrix.js‎

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
// The script generates a random subset of valid jdk, os, timezone, and other axes.
2+
// You can preview the results by running "node matrix.js"
3+
// See https://github.com/vlsi/github-actions-random-matrix
4+
let fs = require('fs');
5+
let os = require('os');
6+
let {MatrixBuilder} = require('./matrix_builder');
7+
const matrix = new MatrixBuilder();
8+
matrix.addAxis({
9+
name: 'java_distribution',
10+
values: [
11+
{value: 'corretto', weight: 1},
12+
{value: 'liberica', weight: 1},
13+
{value: 'microsoft', weight: 1},
14+
{value: 'oracle', weight: 1},
15+
{value: 'semeru', weight: 4},
16+
{value: 'temurin', weight: 1},
17+
{value: 'zulu', weight: 1},
18+
]
19+
});
20+
21+
matrix.addAxis({
22+
name: 'java_version',
23+
// Strings allow versions like 18-ea
24+
values: [
25+
'8',
26+
'11',
27+
'17',
28+
]
29+
});
30+
31+
matrix.addAxis({
32+
name: 'tz',
33+
values: [
34+
'America/New_York',
35+
'Pacific/Chatham',
36+
'UTC'
37+
]
38+
});
39+
40+
matrix.addAxis({
41+
name: 'os',
42+
title: x => x.replace('-latest', ''),
43+
values: [
44+
// TODO: X11 is not available. Un-comment when https://github.com/burrunan/gradle-cache-action/issues/48 is resolved
45+
// 'ubuntu-latest',
46+
'windows-latest',
47+
'macos-latest'
48+
]
49+
});
50+
51+
// Test cases when Object#hashCode produces the same results
52+
// It allows capturing cases when the code uses hashCode as a unique identifier
53+
matrix.addAxis({
54+
name: 'hash',
55+
values: [
56+
{value: 'regular', title: '', weight: 42},
57+
{value: 'same', title: 'same hashcode', weight: 1}
58+
]
59+
});
60+
matrix.addAxis({
61+
name: 'locale',
62+
title: x => x.language + '_' + x.country,
63+
values: [
64+
{language: 'de', country: 'DE'},
65+
{language: 'fr', country: 'FR'},
66+
// TODO: fix :src:dist-check:batchBUG_62847
67+
// Fails with "ERROR o.a.j.u.JMeterUtils: Could not find resources for 'ru_EN'"
68+
// {language: 'ru', country: 'RU'},
69+
{language: 'tr', country: 'TR'},
70+
]
71+
});
72+
73+
matrix.setNamePattern(['java_version', 'java_distribution', 'hash', 'os', 'tz', 'locale']);
74+
75+
// Semeru uses OpenJ9 jit which has no option for making hash codes the same
76+
matrix.exclude({java_distribution: {value: 'semeru'}, hash: {value: 'same'}});
77+
// Microsoft Java has no distribution for 8
78+
matrix.exclude({java_distribution: {value: 'microsoft'}, java_version: '8'});
79+
// Oracle JDK is only supported for JDK 17 and later
80+
matrix.exclude({java_distribution: {value: 'oracle'}, java_version: ['8', '11']});
81+
// Ensure at least one job with "same" hashcode exists
82+
matrix.generateRow({hash: {value: 'same'}});
83+
// Ensure at least one Windows and at least one Linux job is present (macOS is almost the same as Linux)
84+
matrix.generateRow({os: 'windows-latest'});
85+
// TODO: un-comment when xvfb will be possible
86+
// matrix.generateRow({os: 'ubuntu-latest'});
87+
// Ensure there will be at least one job with Java 8
88+
matrix.generateRow({java_version: "8"});
89+
// Ensure there will be at least one job with Java 11
90+
matrix.generateRow({java_version: "11"});
91+
// Ensure there will be at least one job with Java 17
92+
matrix.generateRow({java_version: "17"});
93+
const include = matrix.generateRows(process.env.MATRIX_JOBS || 5);
94+
if (include.length === 0) {
95+
throw new Error('Matrix list is empty');
96+
}
97+
include.sort((a, b) => a.name.localeCompare(b.name, undefined, {numeric: true}));
98+
include.forEach(v => {
99+
// Pass locale via Gradle arguments in case it won't be inherited from _JAVA_OPTIONS
100+
// In fact, _JAVA_OPTIONS is non-standard and might be ignored by some JVMs
101+
let gradleArgs = [
102+
`-Duser.country=${v.locale.country}`,
103+
`-Duser.language=${v.locale.language}`,
104+
];
105+
if (v.hash.value === 'same' && v.java_version <= 8) {
106+
// processSiteXslt fails with VerifyError when running with Java 8 and the same hashcode
107+
// Skip the task in that case
108+
//java.lang.VerifyError: (class: website_style, method: issue_separator signature: (Lcom/sun/org/apache/xala...)
109+
// Illegal target of jump or branch
110+
gradleArgs.push('-x :src:dist:processSiteXslt');
111+
// javadoc tool seems take too much CPU when there are many hash collisions
112+
gradleArgs.push('-x :src:dist:javadocAggregate');
113+
}
114+
v.extraGradleArgs = gradleArgs.join(' ');
115+
});
116+
include.forEach(v => {
117+
let jvmArgs = [];
118+
// Extra JVM arguments passed to test execution
119+
let testJvmArgs = [];
120+
if (v.hash.value === 'same') {
121+
testJvmArgs.push('-XX:+UnlockExperimentalVMOptions', '-XX:hashCode=2');
122+
}
123+
// Gradle does not work in tr_TR locale, so pass locale to test only: https://github.com/gradle/gradle/issues/17361
124+
jvmArgs.push(`-Duser.country=${v.locale.country}`);
125+
jvmArgs.push(`-Duser.language=${v.locale.language}`);
126+
v.java_distribution = v.java_distribution.value;
127+
if (v.java_distribution !== 'semeru' && Math.random() > 0.5) {
128+
// The following options randomize instruction selection in JIT compiler
129+
// so it might reveal missing synchronization in TestNG code
130+
v.name += ', stress JIT';
131+
v.testDisableCaching = 'JIT randomization should not be cached';
132+
jvmArgs.push('-XX:+UnlockDiagnosticVMOptions');
133+
if (v.java_version >= 8) {
134+
// Randomize instruction scheduling in GCM
135+
// share/opto/c2_globals.hpp
136+
jvmArgs.push('-XX:+StressGCM');
137+
// Randomize instruction scheduling in LCM
138+
// share/opto/c2_globals.hpp
139+
jvmArgs.push('-XX:+StressLCM');
140+
}
141+
if (v.java_version >= 16) {
142+
// Randomize worklist traversal in IGVN
143+
// share/opto/c2_globals.hpp
144+
jvmArgs.push('-XX:+StressIGVN');
145+
}
146+
if (v.java_version >= 17) {
147+
// Randomize worklist traversal in CCP
148+
// share/opto/c2_globals.hpp
149+
jvmArgs.push('-XX:+StressCCP');
150+
}
151+
}
152+
v.extraJvmArgs = jvmArgs.join(' ');
153+
v.testExtraJvmArgs = testJvmArgs.join(' ::: ');
154+
delete v.hash;
155+
});
156+
157+
console.log(include);
158+
159+
let filePath = process.env['GITHUB_OUTPUT'] || '';
160+
if (filePath) {
161+
fs.appendFileSync(filePath, `matrix<<MATRIX_BODY${os.EOL}${JSON.stringify({include})}${os.EOL}MATRIX_BODY${os.EOL}`, {
162+
encoding: 'utf8'
163+
});
164+
}

0 commit comments

Comments
 (0)