99 "bytes"
1010 "context"
1111 "fmt"
12- "math/rand"
1312 "reflect"
1413 "strconv"
1514 "strings"
@@ -123,15 +122,6 @@ func TestFileSSTSinkExtendOneFile(t *testing.T) {
123122 require .Equal (t , 1 , len (progDetails .Files ))
124123}
125124
126- func randomValue (n int64 ) []byte {
127- // Create random data so that it does not compress well.
128- b := make ([]byte , n )
129- for i := range b {
130- b [i ] = byte (rand .Int ())
131- }
132- return b
133- }
134-
135125// TestFileSSTSinkWrite tests the contents of flushed files and the internal
136126// unflushed files of the FileSSTSink under different write scenarios. Each test
137127// writes a sequence of exportedSpans into a FileSSTSink. The test then verifies
@@ -143,12 +133,6 @@ func TestFileSSTSinkWrite(t *testing.T) {
143133 defer log .Scope (t ).Close (t )
144134
145135 ctx := context .Background ()
146- testTargetFileSize := int64 (10 << 10 )
147-
148- // Override the fileSpanByteLimit so we can test going over the limit without
149- // needing large buffers that may oom the test node.
150- defer func (oldLimit int64 ) { fileSpanByteLimit = oldLimit }(fileSpanByteLimit )
151- fileSpanByteLimit = testTargetFileSize / 2
152136
153137 type testCase struct {
154138 name string
@@ -161,7 +145,8 @@ func TestFileSSTSinkWrite(t *testing.T) {
161145 //
162146 // TODO (msbutler): we currently don't test expected error handling. If this
163147 // is non-empty, we just skip the test.
164- errorExplanation string
148+ errorExplanation string
149+ noSSTSizeOverride bool
165150 }
166151
167152 for _ , tt := range []testCase {{name : "out-of-order-key-boundary" ,
@@ -293,7 +278,7 @@ func TestFileSSTSinkWrite(t *testing.T) {
293278 {
294279 name : "size-flush" ,
295280 exportSpans : []ExportedSpan {
296- newExportedSpanBuilder ("a" , "c" ).withKVs ([]kvAndTS {{key : "a" , timestamp : 10 , value : randomValue ( testTargetFileSize )}, {key : "b" , timestamp : 10 }}).build (),
281+ newExportedSpanBuilder ("a" , "c" ).withKVs ([]kvAndTS {{key : "a" , timestamp : 10 , value : make ([] byte , 20 << 20 )}, {key : "b" , timestamp : 10 }}).build (),
297282 newExportedSpanBuilder ("d" , "f" ).withKVs ([]kvAndTS {{key : "d" , timestamp : 10 }, {key : "e" , timestamp : 10 }}).build (),
298283 },
299284 flushedSpans : []roachpb.Spans {
@@ -307,7 +292,7 @@ func TestFileSSTSinkWrite(t *testing.T) {
307292 // No flush can occur between two versions of the same key. Further, we must combine flushes which split a row.
308293 name : "no-size-flush-if-mid-mvcc" ,
309294 exportSpans : []ExportedSpan {
310- newRawExportedSpanBuilder (s2k0 ("a" ), s2k0 ("c" ), s2k0 ("c" )).withKVs ([]kvAndTS {{key : "a" , timestamp : 10 , value : randomValue ( testTargetFileSize )}, {key : "c" , timestamp : 10 }}).build (),
295+ newRawExportedSpanBuilder (s2k0 ("a" ), s2k0 ("c" ), s2k0 ("c" )).withKVs ([]kvAndTS {{key : "a" , timestamp : 10 , value : make ([] byte , 20 << 20 )}, {key : "c" , timestamp : 10 }}).build (),
311296 newRawExportedSpanBuilder (s2k0 ("c" ), s2k0 ("f" ), s2k0 ("f" )).withKVs ([]kvAndTS {{key : "c" , timestamp : 8 }, {key : "f" , timestamp : 10 }}).build (),
312297 },
313298 flushedSpans : []roachpb.Spans {},
@@ -320,9 +305,9 @@ func TestFileSSTSinkWrite(t *testing.T) {
320305 name : "no-size-flush-mid-col-family" ,
321306 exportSpans : []ExportedSpan {
322307 newRawExportedSpanBuilder (s2kWithColFamily ("c" , 0 ), s2kWithColFamily ("c" , 1 ), s2kWithColFamily ("c" , 1 )).withKVs ([]kvAndTS {
323- {key : "c" , timestamp : 10 , value : randomValue ( testTargetFileSize )}}).build (),
308+ {key : "c" , timestamp : 10 , value : make ([] byte , 20 << 20 )}}).build (),
324309 newRawExportedSpanBuilder (s2kWithColFamily ("c" , 1 ), s2kWithColFamily ("c" , 2 ), s2kWithColFamily ("c" , 2 )).withKVs ([]kvAndTS {
325- {key : "c" , timestamp : 10 , value : randomValue ( testTargetFileSize )}}).buildWithEncoding (func (stingedKey string ) roachpb.Key { return s2kWithColFamily (stingedKey , 1 ) }),
310+ {key : "c" , timestamp : 10 , value : make ([] byte , 20 << 20 )}}).buildWithEncoding (func (stingedKey string ) roachpb.Key { return s2kWithColFamily (stingedKey , 1 ) }),
326311 },
327312 flushedSpans : []roachpb.Spans {},
328313 unflushedSpans : []roachpb.Spans {
@@ -333,7 +318,7 @@ func TestFileSSTSinkWrite(t *testing.T) {
333318 // It's safe to flush at the range boundary.
334319 name : "size-flush-at-range-boundary" ,
335320 exportSpans : []ExportedSpan {
336- newRawExportedSpanBuilder (s2k ("a" ), s2k ("d" ), s2k ("d" )).withKVs ([]kvAndTS {{key : "a" , timestamp : 10 , value : randomValue ( testTargetFileSize )}, {key : "c" , timestamp : 10 }}).build (),
321+ newRawExportedSpanBuilder (s2k ("a" ), s2k ("d" ), s2k ("d" )).withKVs ([]kvAndTS {{key : "a" , timestamp : 10 , value : make ([] byte , 20 << 20 )}, {key : "c" , timestamp : 10 }}).build (),
337322 },
338323 flushedSpans : []roachpb.Spans {
339324 {{Key : s2k ("a" ), EndKey : s2k ("d" )}},
@@ -347,7 +332,7 @@ func TestFileSSTSinkWrite(t *testing.T) {
347332 // row between two column families.
348333 name : "trim-resume-key" ,
349334 exportSpans : []ExportedSpan {
350- newRawExportedSpanBuilder (s2k0 ("a" ), s2k0 ("c" ), s2k ("c" )).withKVs ([]kvAndTS {{key : "a" , timestamp : 10 , value : randomValue ( testTargetFileSize )}}).build (),
335+ newRawExportedSpanBuilder (s2k0 ("a" ), s2k0 ("c" ), s2k ("c" )).withKVs ([]kvAndTS {{key : "a" , timestamp : 10 , value : make ([] byte , 20 << 20 )}}).build (),
351336 },
352337 flushedSpans : []roachpb.Spans {
353338 {{Key : s2k0 ("a" ), EndKey : s2k ("c" )}},
@@ -359,23 +344,24 @@ func TestFileSSTSinkWrite(t *testing.T) {
359344 // even if the next span's start key matches the file's end key.
360345 name : "file-size-cut" ,
361346 exportSpans : []ExportedSpan {
362- newExportedSpanBuilder ("a" , "c" ).withKVs ([]kvAndTS {{key : "a" , timestamp : 10 , value : randomValue ( fileSpanByteLimit )}, {key : "b" , timestamp : 10 }}).build (),
347+ newExportedSpanBuilder ("a" , "c" ).withKVs ([]kvAndTS {{key : "a" , timestamp : 10 , value : make ([] byte , 64 << 20 )}, {key : "b" , timestamp : 10 }}).build (),
363348 newExportedSpanBuilder ("c" , "f" ).withKVs ([]kvAndTS {{key : "c" , timestamp : 10 }, {key : "e" , timestamp : 10 }}).build (),
364349 },
365350 flushedSpans : []roachpb.Spans {},
366351 unflushedSpans : []roachpb.Spans {
367352 {{Key : s2k0 ("a" ), EndKey : s2k0 ("c" )}, {Key : s2k0 ("c" ), EndKey : s2k0 ("f" )}},
368353 },
354+ noSSTSizeOverride : true ,
369355 },
370356 {
371357 // No file cut can occur between the two column families of the same row,
372358 // even if the file is sufficiently large to get cut.
373359 name : "no-file-cut-mid-col-family" ,
374360 exportSpans : []ExportedSpan {
375361 newRawExportedSpanBuilder (s2kWithColFamily ("c" , 0 ), s2kWithColFamily ("c" , 1 ), s2kWithColFamily ("c" , 1 )).withKVs ([]kvAndTS {
376- {key : "c" , timestamp : 10 , value : randomValue ( testTargetFileSize )}}).build (),
362+ {key : "c" , timestamp : 10 , value : make ([] byte , 65 << 20 )}}).build (),
377363 newRawExportedSpanBuilder (s2kWithColFamily ("c" , 1 ), s2kWithColFamily ("c" , 2 ), s2kWithColFamily ("c" , 2 )).withKVs ([]kvAndTS {
378- {key : "c" , timestamp : 10 , value : randomValue ( testTargetFileSize / 2 )}}).buildWithEncoding (func (stingedKey string ) roachpb.Key { return s2kWithColFamily (stingedKey , 1 ) }),
364+ {key : "c" , timestamp : 10 , value : make ([] byte , 20 << 20 )}}).buildWithEncoding (func (stingedKey string ) roachpb.Key { return s2kWithColFamily (stingedKey , 1 ) }),
379365 },
380366 flushedSpans : []roachpb.Spans {},
381367 unflushedSpans : []roachpb.Spans {
@@ -391,7 +377,9 @@ func TestFileSSTSinkWrite(t *testing.T) {
391377 return
392378 }
393379 st := cluster .MakeTestingClusterSettings ()
394- targetFileSize .Override (ctx , & st .SV , testTargetFileSize )
380+ if ! tt .noSSTSizeOverride {
381+ targetFileSize .Override (ctx , & st .SV , 10 << 10 )
382+ }
395383
396384 sink , store := fileSSTSinkTestSetup (t , st , elide )
397385 defer func () {
@@ -546,7 +534,7 @@ func TestFileSSTSinkStats(t *testing.T) {
546534 sinkStats {hlc.Timestamp {WallTime : 10 }, 3 , 3 , 0 , 0 , 0 , 1 }},
547535 {
548536 // Write an exported span that comes after all spans so far. This span has enough data for a size flush.
549- newExportedSpanBuilder ("g" , "h" ).withKVs ([]kvAndTS {{key : "g" , timestamp : 10 , value : randomValue ( 10 << 10 )}}).build (),
537+ newExportedSpanBuilder ("g" , "h" ).withKVs ([]kvAndTS {{key : "g" , timestamp : 10 , value : make ([] byte , 20 << 20 )}}).build (),
550538 sinkStats {hlc.Timestamp {WallTime : 0 }, 0 , 4 , 1 , 0 , 1 , 1 }},
551539 {
552540 // Write the first exported span after the flush.
0 commit comments