Skip to content

Commit ac53fda

Browse files
theturtle32theturtle32
authored
Merge pull request #497 from theturtle32/restructure-benchmarks
Restructure benchmarks and use Vitest's native comparison
2 parents c3c9272 + 490af8f commit ac53fda

File tree

6 files changed

+376
-178
lines changed

6 files changed

+376
-178
lines changed

‎package.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -56,9 +56,9 @@
5656
"test:browser:ui": "playwright test --ui",
5757
"test:autobahn": "cd test/autobahn && ./run-wstest.js",
5858
"bench": "vitest bench --run --config vitest.bench.config.mjs",
59-
"bench:baseline": "node test/benchmark/track-performance.mjs save",
60-
"bench:compare": "node test/benchmark/track-performance.mjs compare",
61-
"bench:check": "node test/benchmark/track-performance.mjs check",
59+
"bench:baseline": "pnpm run bench -- --outputJson test/benchmark/baseline.json",
60+
"bench:compare": "pnpm run bench -- --compare test/benchmark/baseline.json",
61+
"bench:check": "pnpm run bench -- --compare test/benchmark/baseline.json",
6262
"lint": "eslint lib/**/*.js test/**/*.js",
6363
"lint:fix": "eslint lib/**/*.js test/**/*.js --fix"
6464
},

‎test/benchmark/README.md‎

Lines changed: 37 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,25 @@
11
# WebSocket-Node Performance Benchmarks
22

3-
This directory contains performance benchmarks for critical WebSocket operations.
3+
This directory contains performance benchmarks for critical WebSocket operations using Vitest's built-in benchmarking functionality.
44

55
## Running Benchmarks
66

77
```bash
88
# Run all benchmarks
99
pnpm run bench
1010

11-
# Compare with previous results
11+
# Save current results as baseline
12+
pnpm run bench:baseline
13+
14+
# Compare with baseline (shows ⇑/⇓ indicators)
1215
pnpm run bench:compare
16+
17+
# Check for regressions (exits with error on performance drops)
18+
pnpm run bench:check
1319
```
1420

21+
Note: `bench:check` is the same as `bench:compare` but is intended for CI environments where you want the build to fail on performance regressions.
22+
1523
## Benchmark Suites
1624

1725
### Frame Operations (`frame-operations.bench.mjs`)
@@ -46,15 +54,35 @@ Benchmarks output operations per second (hz) and timing statistics:
4654

4755
## Performance Baselines
4856

49-
These benchmarks establish baseline performance for regression detection:
50-
1. Frame serialization should maintain 4M+ ops/sec for small frames
51-
2. Connection operations should maintain 25K+ ops/sec
52-
3. Large message handling (64KB) should not degrade significantly
57+
Baseline results are stored in `baseline.json` using Vitest's JSON format. When running `bench:compare` or `bench:check`, Vitest automatically compares current results against the baseline and shows:
58+
- `[1.05x] ⇑` for improvements (faster)
59+
- `[0.95x] ⇓` for regressions (slower)
60+
- Baseline values for reference
61+
62+
Expected performance ranges:
63+
1. Frame serialization: 3-4.5M ops/sec
64+
2. Message sending: 100K-900K ops/sec (varies by size)
65+
3. Ping/Pong: 1.5-2M ops/sec
66+
4. Connection creation: 30K ops/sec
67+
68+
## Benchmark Structure
69+
70+
Each operation is in its own `describe` block to prevent Vitest from treating them as alternative implementations for comparison. This structure ensures each operation is measured independently:
71+
72+
```javascript
73+
describe('Send Ping Frame', () => {
74+
bench('send ping frame', () => {
75+
sharedConnection.ping();
76+
});
77+
});
78+
```
5379

5480
## Adding New Benchmarks
5581

5682
When adding benchmarks:
5783
1. Pre-allocate buffers and data outside the benchmark loop
58-
2. Use descriptive test names with size information
59-
3. Focus on operations that directly impact production performance
60-
4. Avoid testing implementation details
84+
2. Create shared connections at module scope (not inside benchmark functions)
85+
3. Use descriptive test names with size information
86+
4. Put each unique operation in its own `describe` block
87+
5. Focus on operations that directly impact production performance
88+
6. Avoid testing implementation details

‎test/benchmark/baseline.json‎

Lines changed: 290 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,294 @@
11
{
2-
"timestamp": "2025-10-06T17:27:59.641Z",
3-
"results": {
4-
"WebSocketConnection Performance 7359ms": {
5-
"create connection instance": 28847.63,
6-
"send small UTF-8 message": 914160.4,
7-
"send medium UTF-8 message (1KB)": 108086.21,
8-
"send binary message (1KB)": 222918.81,
9-
"send ping frame": 2375454.67,
10-
"send pong frame": 1935975.19
2+
"files": [
3+
{
4+
"filepath": "/home/ubuntu/code/websocket-node/test/benchmark/connection-operations.bench.mjs",
5+
"groups": [
6+
{
7+
"fullName": "test/benchmark/connection-operations.bench.mjs > Connection Creation",
8+
"benchmarks": [
9+
{
10+
"id": "347648886_0_0",
11+
"sampleCount": 14950,
12+
"name": "create connection instance",
13+
"rank": 1,
14+
"rme": 5.721365271168026,
15+
"totalTime": 500.017038999993,
16+
"min": 0.02091599999994287,
17+
"max": 5.860308999999916,
18+
"hz": 29898.981102522404,
19+
"period": 0.03344595578595271,
20+
"mean": 0.03344595578595271,
21+
"variance": 0.014250024909513178,
22+
"sd": 0.1193734681975571,
23+
"sem": 0.0009763088259937304,
24+
"df": 14949,
25+
"critical": 1.96,
26+
"moe": 0.0019135652989477115,
27+
"p75": 0.035286000000041895,
28+
"p99": 0.07499600000005557,
29+
"p995": 0.0863120000001345,
30+
"p999": 0.40689399999996567
31+
}
32+
]
33+
},
34+
{
35+
"fullName": "test/benchmark/connection-operations.bench.mjs > Send Small UTF-8 Message",
36+
"benchmarks": [
37+
{
38+
"id": "347648886_1_0",
39+
"sampleCount": 433879,
40+
"name": "send small UTF-8 message",
41+
"rank": 1,
42+
"rme": 11.43139905071693,
43+
"totalTime": 506.6114810000274,
44+
"min": 0.00028599999996004044,
45+
"max": 9.61037800000031,
46+
"hz": 856433.4135174811,
47+
"period": 0.0011676330981679856,
48+
"mean": 0.0011676330981679856,
49+
"variance": 0.0020121856762223642,
50+
"sd": 0.04485739265965382,
51+
"sem": 0.000068100407601955,
52+
"df": 433878,
53+
"critical": 1.96,
54+
"moe": 0.00013347679889983178,
55+
"p75": 0.0007550000000264845,
56+
"p99": 0.0038449999997283157,
57+
"p995": 0.01220100000000457,
58+
"p999": 0.02408300000001873
59+
}
60+
]
61+
},
62+
{
63+
"fullName": "test/benchmark/connection-operations.bench.mjs > Send Medium UTF-8 Message (1KB)",
64+
"benchmarks": [
65+
{
66+
"id": "347648886_2_0",
67+
"sampleCount": 48440,
68+
"name": "send medium UTF-8 message (1KB)",
69+
"rank": 1,
70+
"rme": 4.570568475755207,
71+
"totalTime": 500.00057599997626,
72+
"min": 0.0008000000002539309,
73+
"max": 5.454282000000148,
74+
"hz": 96879.88839437316,
75+
"period": 0.010322059785300914,
76+
"mean": 0.010322059785300914,
77+
"variance": 0.0028065008097464196,
78+
"sd": 0.0529764174869009,
79+
"sem": 0.0002407024543854945,
80+
"df": 48439,
81+
"critical": 1.96,
82+
"moe": 0.0004717768105955692,
83+
"p75": 0.015486000000237254,
84+
"p99": 0.039346000000023196,
85+
"p995": 0.04509699999971417,
86+
"p999": 0.1321459999999206
87+
}
88+
]
89+
},
90+
{
91+
"fullName": "test/benchmark/connection-operations.bench.mjs > Send Binary Message (1KB)",
92+
"benchmarks": [
93+
{
94+
"id": "347648886_3_0",
95+
"sampleCount": 108833,
96+
"name": "send binary message (1KB)",
97+
"rank": 1,
98+
"rme": 6.408247345850968,
99+
"totalTime": 500.0128869999694,
100+
"min": 0.0002959999997074192,
101+
"max": 8.217849999999999,
102+
"hz": 217660.39002112093,
103+
"period": 0.004594313186257563,
104+
"mean": 0.004594313186257563,
105+
"variance": 0.0024556597086718116,
106+
"sd": 0.04955461339443394,
107+
"sem": 0.00015021171062164863,
108+
"df": 108832,
109+
"critical": 1.96,
110+
"moe": 0.0002944149528184313,
111+
"p75": 0.004922000000078697,
112+
"p99": 0.027522000000317348,
113+
"p995": 0.03348600000026636,
114+
"p999": 0.11101899999994203
115+
}
116+
]
117+
},
118+
{
119+
"fullName": "test/benchmark/connection-operations.bench.mjs > Send Ping Frame",
120+
"benchmarks": [
121+
{
122+
"id": "347648886_4_0",
123+
"sampleCount": 1062702,
124+
"name": "send ping frame",
125+
"rank": 1,
126+
"rme": 20.14485654032809,
127+
"totalTime": 508.32634000000144,
128+
"min": 0.00018199999976786785,
129+
"max": 32.92630800000006,
130+
"hz": 2090590.0725112867,
131+
"period": 0.0004783338508819984,
132+
"mean": 0.0004783338508819984,
133+
"variance": 0.002568561363662073,
134+
"sd": 0.05068097634874523,
135+
"sem": 0.00004916309594081911,
136+
"df": 1062701,
137+
"critical": 1.96,
138+
"moe": 0.00009635966804400545,
139+
"p75": 0.00024899999971239595,
140+
"p99": 0.0006320000002233428,
141+
"p995": 0.0008950000001277658,
142+
"p999": 0.011792999999670428
143+
}
144+
]
145+
},
146+
{
147+
"fullName": "test/benchmark/connection-operations.bench.mjs > Send Pong Frame",
148+
"benchmarks": [
149+
{
150+
"id": "347648886_5_0",
151+
"sampleCount": 962038,
152+
"name": "send pong frame",
153+
"rank": 1,
154+
"rme": 18.913749130520372,
155+
"totalTime": 500.0001680007763,
156+
"min": 0.00019799999972747173,
157+
"max": 31.614644000000226,
158+
"hz": 1924075.353507694,
159+
"period": 0.0005197301645057433,
160+
"mean": 0.0005197301645057433,
161+
"variance": 0.0024198652313353448,
162+
"sd": 0.049192125704581466,
163+
"sem": 0.00005015329564809036,
164+
"df": 962037,
165+
"critical": 1.96,
166+
"moe": 0.0000983004594702571,
167+
"p75": 0.0003349999997226405,
168+
"p99": 0.0009099999997488339,
169+
"p995": 0.0018499999996492988,
170+
"p999": 0.011822000000393018
171+
}
172+
]
173+
}
174+
]
11175
},
12-
"WebSocketFrame Performance 9745ms": {
13-
"serialize small text frame (17 bytes, unmasked)": 4240401.67,
14-
"serialize small text frame (17 bytes, masked)": 3070532.44,
15-
"serialize medium binary frame (1KB)": 4418217.61,
16-
"serialize large binary frame (64KB)": 3918678.38
176+
{
177+
"filepath": "/home/ubuntu/code/websocket-node/test/benchmark/frame-operations.bench.mjs",
178+
"groups": [
179+
{
180+
"fullName": "test/benchmark/frame-operations.bench.mjs > Serialize Small Text Frame (Unmasked)",
181+
"benchmarks": [
182+
{
183+
"id": "2141590085_0_0",
184+
"sampleCount": 2221574,
185+
"name": "serialize small text frame (17 bytes, unmasked)",
186+
"rank": 1,
187+
"rme": 0.9893597558817244,
188+
"totalTime": 500.0001229996669,
189+
"min": 0.0001449999999749707,
190+
"max": 0.5502369999994698,
191+
"hz": 4443146.90698882,
192+
"period": 0.00022506570701658686,
193+
"mean": 0.00022506570701658686,
194+
"variance": 0.00000286731744387617,
195+
"sd": 0.0016933155181111906,
196+
"sem": 0.0000011360762905677454,
197+
"df": 2221573,
198+
"critical": 1.96,
199+
"moe": 0.000002226709529512781,
200+
"p75": 0.00019599999905040022,
201+
"p99": 0.0005499999988387572,
202+
"p995": 0.0006460000004153699,
203+
"p999": 0.006724000000758679
204+
}
205+
]
206+
},
207+
{
208+
"fullName": "test/benchmark/frame-operations.bench.mjs > Serialize Small Text Frame (Masked)",
209+
"benchmarks": [
210+
{
211+
"id": "2141590085_1_0",
212+
"sampleCount": 1534250,
213+
"name": "serialize small text frame (17 bytes, masked)",
214+
"rank": 1,
215+
"rme": 3.3082056682221554,
216+
"totalTime": 500.0079150010697,
217+
"min": 0.0002190000013797544,
218+
"max": 6.977795999999216,
219+
"hz": 3068451.4264073553,
220+
"period": 0.0003258972885781781,
221+
"mean": 0.0003258972885781781,
222+
"variance": 0.00004642270968057881,
223+
"sd": 0.006813421290407544,
224+
"sem": 0.00000550069008843143,
225+
"df": 1534249,
226+
"critical": 1.96,
227+
"moe": 0.000010781352573325602,
228+
"p75": 0.0002859999985957984,
229+
"p99": 0.0007029999997030245,
230+
"p995": 0.0009759999993548263,
231+
"p999": 0.008757999999943422
232+
}
233+
]
234+
},
235+
{
236+
"fullName": "test/benchmark/frame-operations.bench.mjs > Serialize Medium Binary Frame (1KB)",
237+
"benchmarks": [
238+
{
239+
"id": "2141590085_2_0",
240+
"sampleCount": 2164023,
241+
"name": "serialize medium binary frame (1KB)",
242+
"rank": 1,
243+
"rme": 1.6649018583076653,
244+
"totalTime": 500.0305290022534,
245+
"min": 0.00015799999891896732,
246+
"max": 2.1783230000000913,
247+
"hz": 4327781.754282142,
248+
"period": 0.00023106525623907575,
249+
"mean": 0.00023106525623907575,
250+
"variance": 0.000008336740867673836,
251+
"sd": 0.0028873414878870557,
252+
"sem": 0.0000019627600739937454,
253+
"df": 2164022,
254+
"critical": 1.96,
255+
"moe": 0.000003847009745027741,
256+
"p75": 0.00020399999993969686,
257+
"p99": 0.0005010000004403992,
258+
"p995": 0.0005930000006628688,
259+
"p999": 0.006003999998938525
260+
}
261+
]
262+
},
263+
{
264+
"fullName": "test/benchmark/frame-operations.bench.mjs > Serialize Large Binary Frame (64KB)",
265+
"benchmarks": [
266+
{
267+
"id": "2141590085_3_0",
268+
"sampleCount": 2251276,
269+
"name": "serialize large binary frame (64KB)",
270+
"rank": 1,
271+
"rme": 1.1766836796382618,
272+
"totalTime": 500.00007300055404,
273+
"min": 0.00015700000039942097,
274+
"max": 1.1734879999985424,
275+
"hz": 4502551.342622515,
276+
"period": 0.00022209630138665985,
277+
"mean": 0.00022209630138665985,
278+
"variance": 0.000004002383606964811,
279+
"sd": 0.002000595812992922,
280+
"sem": 0.0000013333525160699149,
281+
"df": 2251275,
282+
"critical": 1.96,
283+
"moe": 0.000002613370931497033,
284+
"p75": 0.0001940000001923181,
285+
"p99": 0.0005029999992984813,
286+
"p995": 0.0006020000000717118,
287+
"p999": 0.005696000000170898
288+
}
289+
]
290+
}
291+
]
17292
}
18-
}
293+
]
19294
}

0 commit comments

Comments
 (0)