benchmark each phase of request processing

processor/error/package_tests/processor_test.go

@@ -15,52 +15,77 @@ import (
 	"github.com/elastic/apm-server/tests"
 )
 
-// ensure all valid documents pass through the whole validation and transformation process
-func TestProcessorBackendOK(t *testing.T) {
-	requestInfo := []tests.RequestInfo{
+var (
+	backendRequestInfo = []tests.RequestInfo{
 		{Name: "TestProcessErrorMinimalService", Path: "data/valid/error/minimal_service.json"},
 		{Name: "TestProcessErrorMinimalProcess", Path: "data/valid/error/minimal_process.json"},
 		{Name: "TestProcessErrorFull", Path: "data/valid/error/payload.json"},
 		{Name: "TestProcessErrorNullValues", Path: "data/valid/error/null_values.json"},
 		{Name: "TestProcessErrorAugmentedIP", Path: "data/valid/error/augmented_payload_backend.json"},
 	}
-	conf := config.Config{ExcludeFromGrouping: nil}
-	tests.TestProcessRequests(t, er.NewProcessor(), conf, requestInfo, map[string]string{})
-}
 
-func TestProcessorMinimalPayloadOK(t *testing.T) {
-	requestInfo := []tests.RequestInfo{
+	backendRequestInfoIgnoreTimestamp = []tests.RequestInfo{
 		{Name: "TestProcessErrorMinimalPayloadException", Path: "data/valid/error/minimal_payload_exception.json"},
 		{Name: "TestProcessErrorMinimalPayloadLog", Path: "data/valid/error/minimal_payload_log.json"},
 	}
-	conf := config.Config{ExcludeFromGrouping: nil}
-	tests.TestProcessRequests(t, er.NewProcessor(), conf, requestInfo, map[string]string{"@timestamp": "-"})
-}
 
-func TestProcessorFrontendOK(t *testing.T) {
-	requestInfo := []tests.RequestInfo{
+	frontendRequestInfo = []tests.RequestInfo{
 		{Name: "TestProcessErrorFrontend", Path: "data/valid/error/frontend.json"},
 		{Name: "TestProcessErrorFrontendNoSmap", Path: "data/valid/error/frontend_app.e2e-bundle.json"},
 		{Name: "TestProcessErrorFrontendMinifiedSmap", Path: "data/valid/error/frontend_app.e2e-bundle.min.json"},
 		{Name: "TestProcessErrorAugmentedUserAgentAndIP", Path: "data/valid/error/augmented_payload_frontend.json"},
 	}
+)
+
+// ensure all valid documents pass through the whole validation and transformation process
+func TestProcessorBackendOK(t *testing.T) {
+	conf := config.Config{ExcludeFromGrouping: nil}
+	tests.TestProcessRequests(t, er.NewProcessor(), conf, backendRequestInfo, map[string]string{})
+}
+
+func TestProcessorMinimalPayloadOK(t *testing.T) {
+	conf := config.Config{ExcludeFromGrouping: nil}
+	tests.TestProcessRequests(t, er.NewProcessor(), conf, backendRequestInfoIgnoreTimestamp, map[string]string{"@timestamp": "-"})
+}
+
+func TestProcessorFrontendOK(t *testing.T) {
 	mapper := sourcemap.SmapMapper{Accessor: &fakeAcc{}}
 	conf := config.Config{
 		SmapMapper:          &mapper,
 		LibraryPattern:      regexp.MustCompile("^test/e2e|~"),
 		ExcludeFromGrouping: regexp.MustCompile("^\\s*$|^/webpack|^[/][^/]*$"),
 	}
-	tests.TestProcessRequests(t, er.NewProcessor(), conf, requestInfo, map[string]string{})
+	tests.TestProcessRequests(t, er.NewProcessor(), conf, frontendRequestInfo, map[string]string{})
 }
 
-type fakeAcc struct{}
+type fakeAcc struct {
+	*testing.B
+	benchmarkingConsumer *s.Consumer
+}
+
+var testSourcemapInfo = struct {
+	name, file string
+}{
+	name: "http://localhost:8000/test/e2e/general-usecase/app.e2e-bundle.min.js",
+	file: "app.e2e-bundle.min.js.map",
+}
+
+func (ac *fakeAcc) PreFetch() error {
+	var err error
+	ac.benchmarkingConsumer, err = ac.Fetch(sourcemap.Id{ServiceName: testSourcemapInfo.name})
+	return err
+}
 
 func (ac *fakeAcc) Fetch(smapId sourcemap.Id) (*s.Consumer, error) {
 	file := "bundle.js.map"
-	if smapId.Path == "http://localhost:8000/test/e2e/general-usecase/app.e2e-bundle.min.js" {
-		file = "app.e2e-bundle.min.js.map"
+	if smapId.Path == testSourcemapInfo.name {
+		// only not nil if PreFetch called
+		// PreFetch only called from benchmarks, optionally
+		if ac.B != nil && ac.benchmarkingConsumer != nil {
+			return ac.benchmarkingConsumer, nil
+		}
+		file = testSourcemapInfo.file
 	}
-
 	current, _ := os.Getwd()
 	path := filepath.Join(current, "../../../tests/data/valid/sourcemap/", file)
 	fileBytes, err := ioutil.ReadFile(path)
@@ -69,4 +94,23 @@ func (ac *fakeAcc) Fetch(smapId sourcemap.Id) (*s.Consumer, error) {
 	}
 	return s.Parse("", fileBytes)
 }
-func (a *fakeAcc) Remove(smapId sourcemap.Id) {}
+func (ac *fakeAcc) Remove(smapId sourcemap.Id) {}
+
+func BenchmarkBackendProcessor(b *testing.B) {
+	tests.BenchmarkProcessRequests(b, er.NewProcessor(), config.Config{ExcludeFromGrouping: nil}, backendRequestInfo)
+	tests.BenchmarkProcessRequests(b, er.NewProcessor(), config.Config{ExcludeFromGrouping: nil}, backendRequestInfoIgnoreTimestamp)
+}
+
+func BenchmarkFrontendProcessor(b *testing.B) {
+	accessor := &fakeAcc{B: b}
+	if err := accessor.PreFetch(); err != nil {
+		b.Fatal(err)
+	}
+	mapper := sourcemap.SmapMapper{Accessor: accessor}
+	conf := config.Config{
+		SmapMapper:          &mapper,
+		LibraryPattern:      regexp.MustCompile("^test/e2e|~"),
+		ExcludeFromGrouping: regexp.MustCompile("^\\s*$|^/webpack|^[/][^/]*$"),
+	}
+	tests.BenchmarkProcessRequests(b, er.NewProcessor(), conf, frontendRequestInfo)
+}

processor/transaction/package_tests/processor_test.go

@@ -9,9 +9,8 @@ import (
 	"github.com/elastic/apm-server/tests"
 )
 
-// ensure all valid documents pass through the whole validation and transformation process
-func TestTransactionProcessorOK(t *testing.T) {
-	requestInfo := []tests.RequestInfo{
+var (
+	backendRequestInfo = []tests.RequestInfo{
 		{Name: "TestProcessTransactionFull", Path: "data/valid/transaction/payload.json"},
 		{Name: "TestProcessTransactionNullValues", Path: "data/valid/transaction/null_values.json"},
 		{Name: "TestProcessSystemNull", Path: "data/valid/transaction/system_null.json"},
@@ -22,25 +21,44 @@ func TestTransactionProcessorOK(t *testing.T) {
 		{Name: "TestProcessTransactionEmpty", Path: "data/valid/transaction/transaction_empty_values.json"},
 		{Name: "TestProcessTransactionAugmentedIP", Path: "data/valid/transaction/augmented_payload_backend.json"},
 	}
-	tests.TestProcessRequests(t, transaction.NewProcessor(), config.Config{}, requestInfo, map[string]string{})
-}
 
-func TestMinimalTransactionProcessorOK(t *testing.T) {
-	requestInfo := []tests.RequestInfo{
+	backendRequestInfoIgnoreTimestamp = []tests.RequestInfo{
 		{Name: "TestProcessTransactionMinimalPayload", Path: "data/valid/transaction/minimal_payload.json"},
 	}
-	tests.TestProcessRequests(t, transaction.NewProcessor(), config.Config{}, requestInfo, map[string]string{"@timestamp": "-"})
-}
 
-func TestProcessorFrontendOK(t *testing.T) {
-	requestInfo := []tests.RequestInfo{
+	frontendRequestInfo = []tests.RequestInfo{
 		{Name: "TestProcessTransactionFrontend", Path: "data/valid/transaction/frontend.json"},
 		{Name: "TestProcessTransactionAugmentedMerge", Path: "data/valid/transaction/augmented_payload_frontend.json"},
 		{Name: "TestProcessTransactionAugmented", Path: "data/valid/transaction/augmented_payload_frontend_no_context.json"},
 	}
+)
+
+// ensure all valid documents pass through the whole validation and transformation process
+func TestTransactionProcessorOK(t *testing.T) {
+	tests.TestProcessRequests(t, transaction.NewProcessor(), config.Config{}, backendRequestInfo, map[string]string{})
+}
+
+func TestMinimalTransactionProcessorOK(t *testing.T) {
+	tests.TestProcessRequests(t, transaction.NewProcessor(), config.Config{}, backendRequestInfoIgnoreTimestamp, map[string]string{"@timestamp": "-"})
+}
+
+func TestProcessorFrontendOK(t *testing.T) {
+	conf := config.Config{
+		LibraryPattern:      regexp.MustCompile("/test/e2e|~"),
+		ExcludeFromGrouping: regexp.MustCompile("^~/test"),
+	}
+	tests.TestProcessRequests(t, transaction.NewProcessor(), conf, frontendRequestInfo, map[string]string{"@timestamp": "-"})
+}
+
+func BenchmarkBackendProcessor(b *testing.B) {
+	tests.BenchmarkProcessRequests(b, transaction.NewProcessor(), config.Config{}, backendRequestInfo)
+	tests.BenchmarkProcessRequests(b, transaction.NewProcessor(), config.Config{}, backendRequestInfoIgnoreTimestamp)
+}
+
+func BenchmarkFrontendProcessor(b *testing.B) {
 	conf := config.Config{
 		LibraryPattern:      regexp.MustCompile("/test/e2e|~"),
 		ExcludeFromGrouping: regexp.MustCompile("^~/test"),
 	}
-	tests.TestProcessRequests(t, transaction.NewProcessor(), conf, requestInfo, map[string]string{"@timestamp": "-"})
+	tests.BenchmarkProcessRequests(b, transaction.NewProcessor(), conf, frontendRequestInfo)
 }

tests/approvals.go

@@ -122,7 +122,6 @@ func TestProcessRequests(t *testing.T, p processor.Processor, config config.Conf
 		verifyErr := ApproveJson(receivedJson, info.Name, ignored)
 		if verifyErr != nil {
 			assert.Fail(fmt.Sprintf("Test %s failed with error: %s", info.Name, verifyErr.Error()))
-
 		}
 	}
 }

tests/common_bench.go

@@ -0,0 +1,97 @@
+package tests
+
+import (
+	"testing"
+
+	"github.com/elastic/apm-server/config"
+	"github.com/elastic/apm-server/processor"
+	"github.com/elastic/apm-server/tests/loader"
+)
+
+func benchmarkValidate(b *testing.B, p processor.Processor, requestInfo RequestInfo) {
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		b.StopTimer()
+		data, err := loader.LoadData(requestInfo.Path)
+		if err != nil {
+			b.Error(err)
+		}
+		b.StartTimer()
+		if err := p.Validate(data); err != nil {
+			b.Error(err)
+		}
+	}
+}
+
+func benchmarkDecode(b *testing.B, p processor.Processor, requestInfo RequestInfo) {
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		b.StopTimer()
+		data, err := loader.LoadData(requestInfo.Path)
+		if err != nil {
+			b.Error(err)
+		}
+		b.StartTimer()
+		if _, err := p.Decode(data); err != nil {
+			b.Error(err)
+		}
+	}
+}
+
+func benchmarkTransform(b *testing.B, p processor.Processor, config config.Config, requestInfo RequestInfo) {
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		b.StopTimer()
+		data, err := loader.LoadData(requestInfo.Path)
+		if err != nil {
+			b.Error(err)
+		}
+		if payload, err := p.Decode(data); err != nil {
+			b.Error(err)
+		} else {
+			b.StartTimer()
+			payload.Transform(config)
+		}
+	}
+}
+
+func benchmarkProcessRequest(b *testing.B, p processor.Processor, config config.Config, requestInfo RequestInfo) {
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		b.StopTimer()
+		data, err := loader.LoadData(requestInfo.Path)
+		if err != nil {
+			b.Error(err)
+		}
+		b.StartTimer()
+		if err := p.Validate(data); err != nil {
+			b.Error(err)
+		}
+		if payload, err := p.Decode(data); err != nil {
+			b.Error(err)
+		} else {
+			payload.Transform(config)
+		}
+	}
+}
+
+func BenchmarkProcessRequests(b *testing.B, p processor.Processor, config config.Config, requestInfo []RequestInfo) {
+	for _, info := range requestInfo {
+		validate := func(b *testing.B) {
+			benchmarkValidate(b, p, info)
+		}
+		decode := func(b *testing.B) {
+			benchmarkDecode(b, p, info)
+		}
+		transform := func(b *testing.B) {
+			benchmarkTransform(b, p, config, info)
+		}
+		processRequest := func(b *testing.B) {
+			benchmarkProcessRequest(b, p, config, info)
+		}
+		b.Run(info.Name+"Validate", validate)
+		b.Run(info.Name+"Decode", decode)
+		b.Run(info.Name+"Transform", transform)
+		b.Run(info.Name+"ProcessRequest", processRequest)
+	}
+}

tests/loader/loader.go

@@ -64,5 +64,6 @@ func unmarshalData(filePath string, err error) (map[string]interface{}, error) {
 	if err != nil {
 		return data, err
 	}
+	defer r.Close()
 	return decoder.DecodeJSONData(r)
 }