Skip to content

Commit 5728d8a

Browse files
SachumanSachuman
committed
metamorphic: add lazy load SeekPrefixGE opt
This commit enables new lazy load optimization on metamorphic test with probability of 20% Part of #2002 Previous #5256
1 parent 9d1f319 commit 5728d8a

File tree

11 files changed

+80
-15
lines changed

11 files changed

+80
-15
lines changed

‎metamorphic/cockroachkvs.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,16 @@ var CockroachKeyFormat = KeyFormat{
6464
}
6565
return cockroachkvs.NewMVCCTimeIntervalFilter(minWallTime, maxWallTime)
6666
},
67+
ParseMaximumSuffixProperty: func(s string) pebble.MaximumSuffixProperty {
68+
return cockroachkvs.MaxMVCCTimestampProperty{}
69+
},
70+
FormatMaximumSuffixProperty: func(prop pebble.MaximumSuffixProperty) string {
71+
if prop == nil {
72+
return ""
73+
}
74+
return "maxsuffixprop"
75+
},
76+
MaximumSuffixProperty: cockroachkvs.MaxMVCCTimestampProperty{},
6777
}
6878

6979
type cockroachKeyGenerator struct {

‎metamorphic/config.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -374,6 +374,9 @@ type KeyFormat struct {
374374
NewGenerator func(*keyManager, *rand.Rand, OpConfig) KeyGenerator
375375
NewSuffixFilterMask func() pebble.BlockPropertyFilterMask
376376
NewSuffixBlockPropertyFilter func(min []byte, max []byte) sstable.BlockPropertyFilter
377+
ParseMaximumSuffixProperty func(string) pebble.MaximumSuffixProperty
378+
FormatMaximumSuffixProperty func(pebble.MaximumSuffixProperty) string
379+
MaximumSuffixProperty pebble.MaximumSuffixProperty
377380
}
378381

379382
func (kf KeyFormat) apply(ro *runAndCompareOptions) { ro.keyFormat = kf }

‎metamorphic/diagram.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ func TryToGenerateDiagram(keyFormat KeyFormat, opsData []byte) (string, error) {
1515
ops, err := parse(opsData, parserOpts{
1616
parseFormattedUserKey: keyFormat.ParseFormattedKey,
1717
parseFormattedUserKeySuffix: keyFormat.ParseFormattedKeySuffix,
18+
parseMaximumSuffixProperty: keyFormat.ParseMaximumSuffixProperty,
1819
})
1920
if err != nil {
2021
return "", err

‎metamorphic/generator.go‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,12 +48,16 @@ type iterOpts struct {
4848
// see IterOptions.UseL6Filters.
4949
useL6Filters bool
5050

51+
// MaximumSuffixProperty is the maximum suffix property used during the lazy
52+
// position of SeekPrefixGE optimization.
53+
maximumSuffixProperty pebble.MaximumSuffixProperty
54+
5155
// NB: If adding or removing fields, ensure IsZero is in sync.
5256
}
5357

5458
func (o iterOpts) IsZero() bool {
5559
return o.lower == nil && o.upper == nil && o.keyTypes == 0 &&
56-
o.maskSuffix == nil && o.filterMin == nil && o.filterMax == nil && !o.useL6Filters
60+
o.maskSuffix == nil && o.filterMin == nil && o.filterMax == nil && !o.useL6Filters && o.maximumSuffixProperty == nil
5761
}
5862

5963
// GenerateOps generates n random operations, drawing randomness from the
@@ -566,6 +570,11 @@ func (g *generator) newIter() {
566570
opts.useL6Filters = true
567571
}
568572

573+
// With 20% probability, enable the lazy positioning SeekPrefixGE optimization.
574+
if g.rng.Float64() <= 0.2 {
575+
opts.maximumSuffixProperty = g.keyManager.kf.MaximumSuffixProperty
576+
}
577+
569578
g.itersLastOpts[iterID] = opts
570579
g.iterVisibleKeys[iterID] = g.keyManager.getSetOfVisibleKeys(readerID)
571580
g.iterReaderID[iterID] = readerID
@@ -1562,6 +1571,10 @@ func (g *generator) maybeMutateOptions(readerID objID, opts *iterOpts) {
15621571
if g.rng.Float64() <= 0.1 {
15631572
opts.useL6Filters = !opts.useL6Filters
15641573
}
1574+
// With 20% probability, clear existing maximum suffix property.
1575+
if opts.maximumSuffixProperty != nil && g.rng.IntN(5) == 0 {
1576+
opts.maximumSuffixProperty = nil
1577+
}
15651578
}
15661579
}
15671580

‎metamorphic/meta.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -509,6 +509,7 @@ func RunOnce(t TestingT, runDir string, seed uint64, historyPath string, rOpts .
509509
ops, err := parse(opsData, parserOpts{
510510
parseFormattedUserKey: testOpts.KeyFormat.ParseFormattedKey,
511511
parseFormattedUserKeySuffix: testOpts.KeyFormat.ParseFormattedKeySuffix,
512+
parseMaximumSuffixProperty: testOpts.KeyFormat.ParseMaximumSuffixProperty,
512513
})
513514
require.NoError(t, err)
514515

‎metamorphic/ops.go‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1206,10 +1206,10 @@ func (o *newIterOp) run(t *Test, h historyRecorder) {
12061206
}
12071207

12081208
func (o *newIterOp) formattedString(kf KeyFormat) string {
1209-
return fmt.Sprintf("%s = %s.NewIter(%q, %q, %d /* key types */, %q, %q, %t /* use L6 filters */, %q /* masking suffix */)",
1209+
return fmt.Sprintf("%s = %s.NewIter(%q, %q, %d /* key types */, %q, %q, %t /* use L6 filters */, %q /* masking suffix */, %q /* maximum suffix property */)",
12101210
o.iterID, o.readerID, kf.FormatKey(o.lower), kf.FormatKey(o.upper),
12111211
o.keyTypes, kf.FormatKeySuffix(o.filterMax), kf.FormatKeySuffix(o.filterMin),
1212-
o.useL6Filters, kf.FormatKeySuffix(o.maskSuffix))
1212+
o.useL6Filters, kf.FormatKeySuffix(o.maskSuffix), kf.FormatMaximumSuffixProperty(o.maximumSuffixProperty))
12131213
}
12141214

12151215
func (o *newIterOp) receiver() objID { return o.readerID }
@@ -1273,10 +1273,10 @@ func (o *newIterUsingCloneOp) run(t *Test, h historyRecorder) {
12731273
}
12741274

12751275
func (o *newIterUsingCloneOp) formattedString(kf KeyFormat) string {
1276-
return fmt.Sprintf("%s = %s.Clone(%t, %q, %q, %d /* key types */, %q, %q, %t /* use L6 filters */, %q /* masking suffix */)",
1276+
return fmt.Sprintf("%s = %s.Clone(%t, %q, %q, %d /* key types */, %q, %q, %t /* use L6 filters */, %q /* masking suffix */, %q /* maximum suffix property */)",
12771277
o.iterID, o.existingIterID, o.refreshBatch, kf.FormatKey(o.lower),
12781278
kf.FormatKey(o.upper), o.keyTypes, kf.FormatKeySuffix(o.filterMax),
1279-
kf.FormatKeySuffix(o.filterMin), o.useL6Filters, kf.FormatKeySuffix(o.maskSuffix))
1279+
kf.FormatKeySuffix(o.filterMin), o.useL6Filters, kf.FormatKeySuffix(o.maskSuffix), kf.FormatMaximumSuffixProperty(o.maximumSuffixProperty))
12801280
}
12811281

12821282
func (o *newIterUsingCloneOp) receiver() objID { return o.existingIterID }
@@ -1370,10 +1370,10 @@ func (o *iterSetOptionsOp) run(t *Test, h historyRecorder) {
13701370
}
13711371

13721372
func (o *iterSetOptionsOp) formattedString(kf KeyFormat) string {
1373-
return fmt.Sprintf("%s.SetOptions(%q, %q, %d /* key types */, %q, %q, %t /* use L6 filters */, %q /* masking suffix */)",
1373+
return fmt.Sprintf("%s.SetOptions(%q, %q, %d /* key types */, %q, %q, %t /* use L6 filters */, %q /* masking suffix */, %q /* maximum suffix property */)",
13741374
o.iterID, kf.FormatKey(o.lower), kf.FormatKey(o.upper),
13751375
o.keyTypes, kf.FormatKeySuffix(o.filterMax), kf.FormatKeySuffix(o.filterMin),
1376-
o.useL6Filters, kf.FormatKeySuffix(o.maskSuffix))
1376+
o.useL6Filters, kf.FormatKeySuffix(o.maskSuffix), kf.FormatMaximumSuffixProperty(o.maximumSuffixProperty))
13771377
}
13781378

13791379
func iterOptions(kf KeyFormat, o iterOpts) *pebble.IterOptions {
@@ -1394,8 +1394,9 @@ func iterOptions(kf KeyFormat, o iterOpts) *pebble.IterOptions {
13941394
RangeKeyMasking: pebble.RangeKeyMasking{
13951395
Suffix: o.maskSuffix,
13961396
},
1397-
UseL6Filters: o.useL6Filters,
1398-
DebugRangeKeyStack: debugIterators,
1397+
UseL6Filters: o.useL6Filters,
1398+
DebugRangeKeyStack: debugIterators,
1399+
MaximumSuffixProperty: o.maximumSuffixProperty,
13991400
}
14001401
if opts.RangeKeyMasking.Suffix != nil {
14011402
opts.RangeKeyMasking.Filter = kf.NewSuffixFilterMask

‎metamorphic/parser.go‎

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -93,9 +93,9 @@ func opArgs(op op) (receiverID *objID, targetID *objID, args []interface{}) {
9393
case *newIndexedBatchOp:
9494
return &t.dbID, &t.batchID, nil
9595
case *newIterOp:
96-
return &t.readerID, &t.iterID, []interface{}{&t.lower, &t.upper, &t.keyTypes, &t.filterMax, &t.filterMin, &t.useL6Filters, &t.maskSuffix}
96+
return &t.readerID, &t.iterID, []interface{}{&t.lower, &t.upper, &t.keyTypes, &t.filterMax, &t.filterMin, &t.useL6Filters, &t.maskSuffix, &t.maximumSuffixProperty}
9797
case *newIterUsingCloneOp:
98-
return &t.existingIterID, &t.iterID, []interface{}{&t.refreshBatch, &t.lower, &t.upper, &t.keyTypes, &t.filterMax, &t.filterMin, &t.useL6Filters, &t.maskSuffix}
98+
return &t.existingIterID, &t.iterID, []interface{}{&t.refreshBatch, &t.lower, &t.upper, &t.keyTypes, &t.filterMax, &t.filterMin, &t.useL6Filters, &t.maskSuffix, &t.maximumSuffixProperty}
9999
case *newSnapshotOp:
100100
return &t.dbID, &t.snapID, []interface{}{&t.bounds}
101101
case *newExternalObjOp:
@@ -119,7 +119,7 @@ func opArgs(op op) (receiverID *objID, targetID *objID, args []interface{}) {
119119
case *iterSetBoundsOp:
120120
return &t.iterID, nil, []interface{}{&t.lower, &t.upper}
121121
case *iterSetOptionsOp:
122-
return &t.iterID, nil, []interface{}{&t.lower, &t.upper, &t.keyTypes, &t.filterMax, &t.filterMin, &t.useL6Filters, &t.maskSuffix}
122+
return &t.iterID, nil, []interface{}{&t.lower, &t.upper, &t.keyTypes, &t.filterMax, &t.filterMin, &t.useL6Filters, &t.maskSuffix, &t.maximumSuffixProperty}
123123
case *singleDeleteOp:
124124
return &t.writerID, nil, []interface{}{&t.key, &t.maybeReplaceDelete}
125125
case *rangeKeyDeleteOp:
@@ -197,6 +197,7 @@ type parserOpts struct {
197197
allowUndefinedObjs bool
198198
parseFormattedUserKey func(string) UserKey
199199
parseFormattedUserKeySuffix func(string) UserKeySuffix
200+
parseMaximumSuffixProperty func(string) pebble.MaximumSuffixProperty
200201
}
201202

202203
func parse(src []byte, opts parserOpts) (_ []op, err error) {
@@ -319,6 +320,22 @@ func (p *parser) parseUserKeySuffix(pos token.Pos, lit string) UserKeySuffix {
319320
return p.opts.parseFormattedUserKeySuffix(s)
320321
}
321322

323+
func (p *parser) parseMaximumSuffixProperty(
324+
pos token.Pos, lit string,
325+
) pebble.MaximumSuffixProperty {
326+
s, err := strconv.Unquote(lit)
327+
if err != nil {
328+
panic(p.errorf(pos, "%s", err))
329+
}
330+
if len(s) == 0 {
331+
return nil
332+
}
333+
if p.opts.parseMaximumSuffixProperty == nil {
334+
return nil
335+
}
336+
return p.opts.parseMaximumSuffixProperty(s)
337+
}
338+
322339
func unquoteBytes(lit string) []byte {
323340
s, err := strconv.Unquote(lit)
324341
if err != nil {
@@ -408,6 +425,10 @@ func (p *parser) parseArgs(op op, methodName string, args []interface{}) {
408425
}
409426
*t = pebble.FormatMajorVersion(val)
410427

428+
case *pebble.MaximumSuffixProperty:
429+
elem.expectToken(p, token.STRING)
430+
*t = p.parseMaximumSuffixProperty(elem.pos, elem.lit)
431+
411432
default:
412433
panic(p.errorf(pos, "%s: unsupported arg[%d] type: %T", methodName, i, args[i]))
413434
}

‎metamorphic/parser_test.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ func TestParser(t *testing.T) {
2121
ops, err := parse([]byte(d.Input), parserOpts{
2222
parseFormattedUserKey: kf.ParseFormattedKey,
2323
parseFormattedUserKeySuffix: kf.ParseFormattedKeySuffix,
24+
parseMaximumSuffixProperty: kf.ParseMaximumSuffixProperty,
2425
})
2526
if err != nil {
2627
return err.Error()
@@ -46,7 +47,9 @@ func TestParserRandom(t *testing.T) {
4647
ops := g.generate(10000)
4748
src := formatOps(km.kf, ops)
4849

49-
parsedOps, err := parse([]byte(src), parserOpts{})
50+
parsedOps, err := parse([]byte(src), parserOpts{
51+
parseMaximumSuffixProperty: km.kf.ParseMaximumSuffixProperty,
52+
})
5053
if err != nil {
5154
t.Fatalf("%s\n%s", err, src)
5255
}
@@ -67,6 +70,7 @@ func TestParserNilBounds(t *testing.T) {
6770
parsedOps, err := parse([]byte(formatted), parserOpts{
6871
parseFormattedUserKey: kf.ParseFormattedKey,
6972
parseFormattedUserKeySuffix: kf.ParseFormattedKeySuffix,
73+
parseMaximumSuffixProperty: kf.ParseMaximumSuffixProperty,
7074
})
7175
require.NoError(t, err)
7276
require.Equal(t, 1, len(parsedOps))

‎metamorphic/simplify.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ func TryToSimplifyKeys(keyFormat KeyFormat, opsData []byte, retainSuffixes bool)
2121
ops, err := parse(opsData, parserOpts{
2222
parseFormattedUserKey: keyFormat.ParseFormattedKey,
2323
parseFormattedUserKeySuffix: keyFormat.ParseFormattedKeySuffix,
24+
parseMaximumSuffixProperty: keyFormat.ParseMaximumSuffixProperty,
2425
})
2526
if err != nil {
2627
panic(err)

‎metamorphic/testdata/diagram‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ snap9 = db2.NewSnapshot("a", "z")
55
db2.RangeKeyDelete("d", "f")
66
db2.Compact("g", "i", true /* parallelize */)
77
db2.Replicate(db1, "b", "f")
8-
iter25 = db1.NewIter("", "", 2 /* key types */, "", "", false /* use L6 filters */, "@7" /* masking suffix */)
8+
iter25 = db1.NewIter("", "", 2 /* key types */, "", "", false /* use L6 filters */, "@7" /* masking suffix */, "" /* maximum suffix property */)
99
iter25.SeekGE("e", "")
1010
----
1111
Init(2 /* dbs */, 49 /* batches */, 63 /* iters */, 45 /* snapshots */, 0 /* externalObjs */)
@@ -14,7 +14,7 @@ iter25.SeekGE("e", "")
1414
|-----------| db2.RangeKeyDelete("d", "f")
1515
|-----------| db2.Compact("g", "i", true /* parallelize */)
1616
|-----------------------| db2.Replicate(db1, "b", "f")
17-
iter25 = db1.NewIter("", "", 2 /* key types */, "", "", false /* use L6 filters */, "@7" /* masking suffix */)
17+
iter25 = db1.NewIter("", "", 2 /* key types */, "", "", false /* use L6 filters */, "@7" /* masking suffix */, "" /* maximum suffix property */)
1818
| iter25.SeekGE("e", "")
1919
|-----|-----|-----|-----|-----|-----|-----|-----|-----|
2020
a b c d e f g h i z

0 commit comments

Comments
 (0)