diff --git a/processor/error/benchmark_test.go b/processor/error/benchmark_test.go deleted file mode 100644 index 575fbf0488a..00000000000 --- a/processor/error/benchmark_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package error - -import ( - "testing" - - "github.com/elastic/apm-server/config" - "github.com/elastic/apm-server/tests/loader" -) - -func BenchmarkEventWithFileLoading(b *testing.B) { - processor := NewProcessor() - for i := 0; i < b.N; i++ { - data, _ := loader.LoadValidData("error") - err := processor.Validate(data) - if err != nil { - panic(err) - } - - payload, err := processor.Decode(data) - if err != nil { - b.Fatalf("Error: %v", err) - } - payload.Transform(config.Config{}) - } -} - -func BenchmarkEventFileLoadingOnce(b *testing.B) { - processor := NewProcessor() - data, _ := loader.LoadValidData("error") - for i := 0; i < b.N; i++ { - err := processor.Validate(data) - if err != nil { - panic(err) - } - - payload, err := processor.Decode(data) - if err != nil { - b.Fatalf("Error: %v", err) - } - payload.Transform(config.Config{}) - } -} diff --git a/processor/error/package_tests/processor_test.go b/processor/error/package_tests/processor_test.go index 57c66d69d1c..78aadf909ce 100644 --- a/processor/error/package_tests/processor_test.go +++ b/processor/error/package_tests/processor_test.go @@ -15,52 +15,77 @@ import ( "github.com/elastic/apm-server/tests" ) -// ensure all valid documents pass through the whole validation and transformation process -func TestProcessorBackendOK(t *testing.T) { - requestInfo := []tests.RequestInfo{ +var ( + backendRequestInfo = []tests.RequestInfo{ {Name: "TestProcessErrorMinimalService", Path: "data/valid/error/minimal_service.json"}, {Name: "TestProcessErrorMinimalProcess", Path: "data/valid/error/minimal_process.json"}, {Name: "TestProcessErrorFull", Path: "data/valid/error/payload.json"}, {Name: "TestProcessErrorNullValues", Path: "data/valid/error/null_values.json"}, {Name: "TestProcessErrorAugmentedIP", Path: "data/valid/error/augmented_payload_backend.json"}, } - conf := config.Config{ExcludeFromGrouping: nil} - tests.TestProcessRequests(t, er.NewProcessor(), conf, requestInfo, map[string]string{}) -} -func TestProcessorMinimalPayloadOK(t *testing.T) { - requestInfo := []tests.RequestInfo{ + backendRequestInfoIgnoreTimestamp = []tests.RequestInfo{ {Name: "TestProcessErrorMinimalPayloadException", Path: "data/valid/error/minimal_payload_exception.json"}, {Name: "TestProcessErrorMinimalPayloadLog", Path: "data/valid/error/minimal_payload_log.json"}, } - conf := config.Config{ExcludeFromGrouping: nil} - tests.TestProcessRequests(t, er.NewProcessor(), conf, requestInfo, map[string]string{"@timestamp": "-"}) -} -func TestProcessorFrontendOK(t *testing.T) { - requestInfo := []tests.RequestInfo{ + frontendRequestInfo = []tests.RequestInfo{ {Name: "TestProcessErrorFrontend", Path: "data/valid/error/frontend.json"}, {Name: "TestProcessErrorFrontendNoSmap", Path: "data/valid/error/frontend_app.e2e-bundle.json"}, {Name: "TestProcessErrorFrontendMinifiedSmap", Path: "data/valid/error/frontend_app.e2e-bundle.min.json"}, {Name: "TestProcessErrorAugmentedUserAgentAndIP", Path: "data/valid/error/augmented_payload_frontend.json"}, } +) + +// ensure all valid documents pass through the whole validation and transformation process +func TestProcessorBackendOK(t *testing.T) { + conf := config.Config{ExcludeFromGrouping: nil} + tests.TestProcessRequests(t, er.NewProcessor(), conf, backendRequestInfo, map[string]string{}) +} + +func TestProcessorMinimalPayloadOK(t *testing.T) { + conf := config.Config{ExcludeFromGrouping: nil} + tests.TestProcessRequests(t, er.NewProcessor(), conf, backendRequestInfoIgnoreTimestamp, map[string]string{"@timestamp": "-"}) +} + +func TestProcessorFrontendOK(t *testing.T) { mapper := sourcemap.SmapMapper{Accessor: &fakeAcc{}} conf := config.Config{ SmapMapper: &mapper, LibraryPattern: regexp.MustCompile("^test/e2e|~"), ExcludeFromGrouping: regexp.MustCompile("^\\s*$|^/webpack|^[/][^/]*$"), } - tests.TestProcessRequests(t, er.NewProcessor(), conf, requestInfo, map[string]string{}) + tests.TestProcessRequests(t, er.NewProcessor(), conf, frontendRequestInfo, map[string]string{}) } -type fakeAcc struct{} +type fakeAcc struct { + *testing.B + benchmarkingConsumer *s.Consumer +} + +var testSourcemapInfo = struct { + name, file string +}{ + name: "http://localhost:8000/test/e2e/general-usecase/app.e2e-bundle.min.js", + file: "app.e2e-bundle.min.js.map", +} + +func (ac *fakeAcc) PreFetch() error { + var err error + ac.benchmarkingConsumer, err = ac.Fetch(sourcemap.Id{ServiceName: testSourcemapInfo.name}) + return err +} func (ac *fakeAcc) Fetch(smapId sourcemap.Id) (*s.Consumer, error) { file := "bundle.js.map" - if smapId.Path == "http://localhost:8000/test/e2e/general-usecase/app.e2e-bundle.min.js" { - file = "app.e2e-bundle.min.js.map" + if smapId.Path == testSourcemapInfo.name { + // only not nil if PreFetch called + // PreFetch only called from benchmarks, optionally + if ac.B != nil && ac.benchmarkingConsumer != nil { + return ac.benchmarkingConsumer, nil + } + file = testSourcemapInfo.file } - current, _ := os.Getwd() path := filepath.Join(current, "../../../tests/data/valid/sourcemap/", file) fileBytes, err := ioutil.ReadFile(path) @@ -69,4 +94,23 @@ func (ac *fakeAcc) Fetch(smapId sourcemap.Id) (*s.Consumer, error) { } return s.Parse("", fileBytes) } -func (a *fakeAcc) Remove(smapId sourcemap.Id) {} +func (ac *fakeAcc) Remove(smapId sourcemap.Id) {} + +func BenchmarkBackendProcessor(b *testing.B) { + tests.BenchmarkProcessRequests(b, er.NewProcessor(), config.Config{ExcludeFromGrouping: nil}, backendRequestInfo) + tests.BenchmarkProcessRequests(b, er.NewProcessor(), config.Config{ExcludeFromGrouping: nil}, backendRequestInfoIgnoreTimestamp) +} + +func BenchmarkFrontendProcessor(b *testing.B) { + accessor := &fakeAcc{B: b} + if err := accessor.PreFetch(); err != nil { + b.Fatal(err) + } + mapper := sourcemap.SmapMapper{Accessor: accessor} + conf := config.Config{ + SmapMapper: &mapper, + LibraryPattern: regexp.MustCompile("^test/e2e|~"), + ExcludeFromGrouping: regexp.MustCompile("^\\s*$|^/webpack|^[/][^/]*$"), + } + tests.BenchmarkProcessRequests(b, er.NewProcessor(), conf, frontendRequestInfo) +} diff --git a/processor/transaction/benchmark_test.go b/processor/transaction/benchmark_test.go deleted file mode 100644 index 3c61191ad00..00000000000 --- a/processor/transaction/benchmark_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package transaction - -import ( - "testing" - - "github.com/elastic/apm-server/config" - "github.com/elastic/apm-server/tests/loader" -) - -func BenchmarkWithFileLoading(b *testing.B) { - processor := NewProcessor() - for i := 0; i < b.N; i++ { - data, _ := loader.LoadValidData("transaction") - err := processor.Validate(data) - if err != nil { - b.Fatalf("Error: %v", err) - } - payload, err := processor.Decode(data) - if err != nil { - b.Fatalf("Error: %v", err) - } - payload.Transform(config.Config{}) - } -} - -func BenchmarkTransactionFileLoadingOnce(b *testing.B) { - processor := NewProcessor() - data, _ := loader.LoadValidData("transaction") - for i := 0; i < b.N; i++ { - err := processor.Validate(data) - if err != nil { - payload, err := processor.Decode(data) - if err != nil { - b.Fatalf("Error: %v", err) - } - payload.Transform(config.Config{}) - b.Fatalf("Error: %v", err) - } - } -} diff --git a/processor/transaction/package_tests/processor_test.go b/processor/transaction/package_tests/processor_test.go index 7ce042a7a27..e073aaa017f 100644 --- a/processor/transaction/package_tests/processor_test.go +++ b/processor/transaction/package_tests/processor_test.go @@ -9,9 +9,8 @@ import ( "github.com/elastic/apm-server/tests" ) -// ensure all valid documents pass through the whole validation and transformation process -func TestTransactionProcessorOK(t *testing.T) { - requestInfo := []tests.RequestInfo{ +var ( + backendRequestInfo = []tests.RequestInfo{ {Name: "TestProcessTransactionFull", Path: "data/valid/transaction/payload.json"}, {Name: "TestProcessTransactionNullValues", Path: "data/valid/transaction/null_values.json"}, {Name: "TestProcessSystemNull", Path: "data/valid/transaction/system_null.json"}, @@ -22,25 +21,44 @@ func TestTransactionProcessorOK(t *testing.T) { {Name: "TestProcessTransactionEmpty", Path: "data/valid/transaction/transaction_empty_values.json"}, {Name: "TestProcessTransactionAugmentedIP", Path: "data/valid/transaction/augmented_payload_backend.json"}, } - tests.TestProcessRequests(t, transaction.NewProcessor(), config.Config{}, requestInfo, map[string]string{}) -} -func TestMinimalTransactionProcessorOK(t *testing.T) { - requestInfo := []tests.RequestInfo{ + backendRequestInfoIgnoreTimestamp = []tests.RequestInfo{ {Name: "TestProcessTransactionMinimalPayload", Path: "data/valid/transaction/minimal_payload.json"}, } - tests.TestProcessRequests(t, transaction.NewProcessor(), config.Config{}, requestInfo, map[string]string{"@timestamp": "-"}) -} -func TestProcessorFrontendOK(t *testing.T) { - requestInfo := []tests.RequestInfo{ + frontendRequestInfo = []tests.RequestInfo{ {Name: "TestProcessTransactionFrontend", Path: "data/valid/transaction/frontend.json"}, {Name: "TestProcessTransactionAugmentedMerge", Path: "data/valid/transaction/augmented_payload_frontend.json"}, {Name: "TestProcessTransactionAugmented", Path: "data/valid/transaction/augmented_payload_frontend_no_context.json"}, } +) + +// ensure all valid documents pass through the whole validation and transformation process +func TestTransactionProcessorOK(t *testing.T) { + tests.TestProcessRequests(t, transaction.NewProcessor(), config.Config{}, backendRequestInfo, map[string]string{}) +} + +func TestMinimalTransactionProcessorOK(t *testing.T) { + tests.TestProcessRequests(t, transaction.NewProcessor(), config.Config{}, backendRequestInfoIgnoreTimestamp, map[string]string{"@timestamp": "-"}) +} + +func TestProcessorFrontendOK(t *testing.T) { + conf := config.Config{ + LibraryPattern: regexp.MustCompile("/test/e2e|~"), + ExcludeFromGrouping: regexp.MustCompile("^~/test"), + } + tests.TestProcessRequests(t, transaction.NewProcessor(), conf, frontendRequestInfo, map[string]string{"@timestamp": "-"}) +} + +func BenchmarkBackendProcessor(b *testing.B) { + tests.BenchmarkProcessRequests(b, transaction.NewProcessor(), config.Config{}, backendRequestInfo) + tests.BenchmarkProcessRequests(b, transaction.NewProcessor(), config.Config{}, backendRequestInfoIgnoreTimestamp) +} + +func BenchmarkFrontendProcessor(b *testing.B) { conf := config.Config{ LibraryPattern: regexp.MustCompile("/test/e2e|~"), ExcludeFromGrouping: regexp.MustCompile("^~/test"), } - tests.TestProcessRequests(t, transaction.NewProcessor(), conf, requestInfo, map[string]string{"@timestamp": "-"}) + tests.BenchmarkProcessRequests(b, transaction.NewProcessor(), conf, frontendRequestInfo) } diff --git a/tests/approvals.go b/tests/approvals.go index 99479bc3b13..75e3da48b11 100644 --- a/tests/approvals.go +++ b/tests/approvals.go @@ -122,7 +122,6 @@ func TestProcessRequests(t *testing.T, p processor.Processor, config config.Conf verifyErr := ApproveJson(receivedJson, info.Name, ignored) if verifyErr != nil { assert.Fail(fmt.Sprintf("Test %s failed with error: %s", info.Name, verifyErr.Error())) - } } } diff --git a/tests/common_bench.go b/tests/common_bench.go new file mode 100644 index 00000000000..befb36c5022 --- /dev/null +++ b/tests/common_bench.go @@ -0,0 +1,97 @@ +package tests + +import ( + "testing" + + "github.com/elastic/apm-server/config" + "github.com/elastic/apm-server/processor" + "github.com/elastic/apm-server/tests/loader" +) + +func benchmarkValidate(b *testing.B, p processor.Processor, requestInfo RequestInfo) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + b.StopTimer() + data, err := loader.LoadData(requestInfo.Path) + if err != nil { + b.Error(err) + } + b.StartTimer() + if err := p.Validate(data); err != nil { + b.Error(err) + } + } +} + +func benchmarkDecode(b *testing.B, p processor.Processor, requestInfo RequestInfo) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + b.StopTimer() + data, err := loader.LoadData(requestInfo.Path) + if err != nil { + b.Error(err) + } + b.StartTimer() + if _, err := p.Decode(data); err != nil { + b.Error(err) + } + } +} + +func benchmarkTransform(b *testing.B, p processor.Processor, config config.Config, requestInfo RequestInfo) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + b.StopTimer() + data, err := loader.LoadData(requestInfo.Path) + if err != nil { + b.Error(err) + } + if payload, err := p.Decode(data); err != nil { + b.Error(err) + } else { + b.StartTimer() + payload.Transform(config) + } + } +} + +func benchmarkProcessRequest(b *testing.B, p processor.Processor, config config.Config, requestInfo RequestInfo) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + b.StopTimer() + data, err := loader.LoadData(requestInfo.Path) + if err != nil { + b.Error(err) + } + b.StartTimer() + if err := p.Validate(data); err != nil { + b.Error(err) + } + if payload, err := p.Decode(data); err != nil { + b.Error(err) + } else { + payload.Transform(config) + } + } +} + +func BenchmarkProcessRequests(b *testing.B, p processor.Processor, config config.Config, requestInfo []RequestInfo) { + for _, info := range requestInfo { + validate := func(b *testing.B) { + benchmarkValidate(b, p, info) + } + decode := func(b *testing.B) { + benchmarkDecode(b, p, info) + } + transform := func(b *testing.B) { + benchmarkTransform(b, p, config, info) + } + processRequest := func(b *testing.B) { + benchmarkProcessRequest(b, p, config, info) + } + b.Run(info.Name+"Validate", validate) + b.Run(info.Name+"Decode", decode) + b.Run(info.Name+"Transform", transform) + b.Run(info.Name+"ProcessRequest", processRequest) + } +} diff --git a/tests/loader/loader.go b/tests/loader/loader.go index e705e04d532..071f62e6f5d 100644 --- a/tests/loader/loader.go +++ b/tests/loader/loader.go @@ -64,5 +64,6 @@ func unmarshalData(filePath string, err error) (map[string]interface{}, error) { if err != nil { return data, err } + defer r.Close() return decoder.DecodeJSONData(r) }