Add query-frontend, query-scheduler and querier worker. (grafana/phlare#496)

* Introducing frontend and scheduler

* revert prometheus change

* Hook up the frontend with query-scheduler and querier

* Ensure missing tenant id is propagated correctly.

* Refactoring querier transport/handler.

* Update helm

* Update the documentation to reflect new changes

* Removes todos

* make go/mod

* make docs/validate

* make lint

* make lint

* make helm/check

* fixes race

* fixes timeout issues with scheduler-querier

Author: cyriltovena

Makefile

@@ -48,13 +48,16 @@ test: go/test ## Run unit tests
 .PHONY: generate
 generate: $(BIN)/buf $(BIN)/protoc-gen-go $(BIN)/protoc-gen-go-vtproto $(BIN)/protoc-gen-openapiv2 $(BIN)/protoc-gen-grpc-gateway $(BIN)/protoc-gen-connect-go $(BIN)/protoc-gen-connect-go-mux $(BIN)/gomodifytags ## Regenerate protobuf
 	rm -Rf api/openapiv2/gen/ api/gen
-	PATH=$(BIN) $(BIN)/buf generate
+	find pkg/ \( -name \*.pb.go -o -name \*.connect\*.go \) -delete
+	cd api/ && PATH=$(BIN) $(BIN)/buf generate
+	cd pkg && PATH=$(BIN) $(BIN)/buf generate
 	PATH=$(BIN):$(PATH) ./tools/add-parquet-tags.sh
 	go run ./tools/doc-generator/ ./docs/sources/operators-guide/configure/reference-configuration-parameters/index.template > docs/sources/operators-guide/configure/reference-configuration-parameters/index.md
 
 .PHONY: buf/lint
 buf/lint: $(BIN)/buf
-	$(BIN)/buf lint || true # TODO: Fix linting problems and remove the always true
+	cd api/ && $(BIN)/buf lint || true # TODO: Fix linting problems and remove the always true
+	cd pkg && $(BIN)/buf lint || true # TODO: Fix linting problems and remove the always true
 
 .PHONY: go/test
 go/test: $(BIN)/gotestsum
@@ -120,7 +123,8 @@ fmt: $(BIN)/golangci-lint $(BIN)/buf $(BIN)/tk ## Automatically fix some lint er
 	git ls-files '*.go' | grep -v 'vendor/' | xargs gofmt -s -w
 	# TODO: Reenable once golangci-lint support go 1.18 properly
 	# $(BIN)/golangci-lint run --fix
-	$(BIN)/buf format -w .
+	cd api/ && $(BIN)/buf format -w .
+	cd pkg && $(BIN)/buf format -w .
 	$(BIN)/tk fmt ./operations/phlare/jsonnet/ tools/monitoring/
 
 .PHONY: check/unstaged-changes
@@ -227,7 +231,7 @@ $(BIN)/kind: Makefile go.mod
 
 $(BIN)/tk: Makefile go.mod $(BIN)/jb
 	@mkdir -p $(@D)
-	GOBIN=$(abspath $(@D)) $(GO) install github.com/grafana/tanka/cmd/tk@v0.22.1
+	GOBIN=$(abspath $(@D)) $(GO) install github.com/grafana/tanka/cmd/tk@v0.24.0
 
 $(BIN)/jb: Makefile go.mod
 	@mkdir -p $(@D)

buf.gen.yaml renamed to api/buf.gen.yaml

@@ -8,31 +8,31 @@ managed:
 
 plugins:
   - name: go
-    out: api/gen/proto/go
+    out: gen/proto/go
     opt: paths=source_relative
 
   - name: go-vtproto
-    out: api/gen/proto/go
+    out: gen/proto/go
     opt:
       - paths=source_relative
       - features=marshal+unmarshal+size+pool+grpc+pool
       - pool=github.com/grafana/phlare/api/gen/proto/go/google/v1.Profile
       - pool=github.com/grafana/phlare/api/gen/proto/go/google/v1.Sample
 
   - name: connect-go
-    out: api/gen/proto/go
+    out: gen/proto/go
     opt: paths=source_relative
 
   - name: connect-go-mux
-    out: api/gen/proto/go
+    out: gen/proto/go
     opt: paths=source_relative
 
   - name: grpc-gateway
-    out: api/gen/proto/go
+    out: gen/proto/go
     opt:
       - paths=source_relative
 
   - name: openapiv2
-    out: api/openapiv2/gen/
+    out: openapiv2/gen/
     strategy: all
     opt: allow_merge=true,merge_file_name=phlare

buf.work.yaml renamed to api/buf.work.yaml

@@ -1,3 +1,3 @@
 version: v1
 directories:
-  - api/
+  - .

api/go.mod

@@ -20,9 +20,9 @@ require (
 	github.com/prometheus/client_golang v1.14.0 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/procfs v0.8.0 // indirect
-	golang.org/x/net v0.4.0 // indirect
-	golang.org/x/sys v0.3.0 // indirect
-	golang.org/x/text v0.5.0 // indirect
+	golang.org/x/net v0.5.0 // indirect
+	golang.org/x/sys v0.4.0 // indirect
+	golang.org/x/text v0.6.0 // indirect
 )
 
 replace google.golang.org/grpc => google.golang.org/grpc v1.44.0

api/go.sum

@@ -68,20 +68,20 @@ golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU=
-golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
+golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw=
+golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
-golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18=
+golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM=
-golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k=
+golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

docs/sources/operators-guide/architecture/about-grafana-phlare-architecture/index.md

@@ -22,6 +22,7 @@ Most components are stateless and do not require any data persisted between proc
 ### The write path
 
 [//]: # "To edit open with https://mermaid.live/edit#pako{...}"
+
 <p align="center">
   <img alt="Architecture of Grafana Phlare's write path" width="200px" src="https://mermaid.ink/svg/pako:eNqNUc9PwyAU_lcavGzJtrqi3cbBg9GzB008rDtQeLQoLQ08nMvS_11onHr09vH9gvc4E2ElEEaUsUfRcofZy33VZ1nw4Gb7V6cR_GGeLZd3mdQena4DWpccf46TrPsGPMKkXfAkWJ8o62f7p_oNBGY-RuAwT6zHk4HpskxpY9iV2qlF7LXvwK4opd94edQSW1YMn78h6_8dIQvSgeu4lnHSc6qoCLbQQUVYhBIUDwYrUvVjtIZBcoRHqeMzCVPceFgQHud8PvWCMHQBLqYHzRvHux-XsVxCDJ0Jnoa01iYuKVYK2yvdJD44E-kWcfAsz5O8ajS2oV4J2-Vey_QH7ceuzMui3PKCQrmh_JZSKer1bquKm7WSm-t1wck4jl9KVZdq" />
   </a>
@@ -43,12 +44,17 @@ By default, each profile series is replicated to three ingesters, and each inges
 ### The read path
 
 [//]: # "To edit open with https://mermaid.live/edit#pako{...}"
+
 <p align="center">
-  <img alt="Architecture of Grafana Phlare's read path" width="400px" src="https://mermaid.ink/svg/pako:eNqNkTFPwzAQhf-K5S6t1DY0gbTxwIBgRgK2poNrnxODEwf7TKmq_HfsqoDExPb03vfO9vlEhZVAGVXGHkTLHZKXu7onJHhw0-0TcOl3M7JY3JL3AE6DS-FFnm3dN-Dxr0-E7WJCvLX9GbM-AdZPt4_7VxBIPFoHu1lyPR4NnE8kShvDJqpSc4_OvgGbFEVx0YuDltiyfPj8LVn_7wqd0w5cx7WMzz2lETXFFjqoKYtSguLBYE3rfoxoGCRHeJA6XpMyxY2HOeUB7fOxF5ShC_AN3WveON79UMZyCbF0ongc0m4b7TGOFLZXukl-cCbaLeLgWZaleNlobMN-GdeWeS3TR7QfVZmVebnheQHluuA3RSHFflVtVH69UnJ9tco5HcfxC3-dl_E" />
+  <img alt="Architecture of Grafana Phlare's read path" width="400px" src="https://mermaid.ink/svg/pako:eNqNkU1PwzAMQP9KlF02aV1ZC92WAwcEZyTgtu6QNU4bSJOSOIxq6n8nnfg-7eY8P1uOfaSVFUAZldoeqoY7JE83pSEkeHDT7QNw4XczkiTX5DWA6xPprEEwYnT-kl-SrxoQQYP7shS4c9LK1ODxPyeVbWOGeGvNSbN-FKyfbu_3z1Ah8Wgd7GYj9dhrOE1PpNKaTeRGzj06-wJskuf5Z5wclMCGZd37T5H1Z5fQOW3BtVyJuLrj2KKk2EALJWUxFCB50FjS0gxR5QHtY28qytAFmNPQCY5wq3jteEuZ5Np_0zuh4me-obZcQHweKfbdeKdaeYwtK2ukqkcenI64Qew8S9MxvagVNmG_iGtLvRLjUZu3TZEWWbHmWQ7FKudXeS6q_XKzltnlUorVxTLjdBiGD-Nas38" />
   </a>
 </p>
 
-Queries coming into Grafana Phlare arrive at the [querier]({{< relref "../components/querier" >}}). The queriers connect to the ingesters to fetch all the data needed to execute a query. For more information about how the query is executed, refer to [querier]({{< relref "../components/querier.md" >}}).
+Queries coming into Grafana Phlare arrive at [query-frontend]({{< relref "../components/query-frontend" >}}) component which is responsible for accelerating queries and dispatching them to the [query-scheduler]({{< relref "../components/query-scheduler" >}}).
+
+The [query-scheduler]({{< relref "../components/query-scheduler" >}}) maintains a queue of queries and ensures that each tenant's queries are fairly executed.
+
+The [querier]({{< relref "../components/querier" >}}) act as workers, pulling queries from the queue in the query-scheduler. The queriers connect to the ingesters to fetch all the data needed to execute a query. For more information about how the query is executed, refer to [querier]({{< relref "../components/querier.md" >}}).
 
 ## Long-term storage
 

docs/sources/operators-guide/architecture/components/query-frontend/index.md

@@ -0,0 +1,23 @@
+---
+title: "Grafana Phlare query-frontend"
+menuTitle: "Query-frontend"
+description: "The query-frontend accelerates queries."
+weight: 60
+---
+
+# Grafana Phlare query-frontend
+
+The query-frontend is a stateless component that provides the same API as the [querier]({{< relref "../querier.md" >}}) and can be used to accelerate the read path and ensure fair scheduling between tenants using the [query-scheduler]({{< relref "../query-scheduler/index.md" >}}).
+
+In this situation, queriers act as workers that pull jobs from the queue, execute them, and return the results to the query-frontend for aggregation.
+
+We recommend that you run at least two query-frontend replicas for high-availability reasons.
+
+> Because the [query-scheduler]({{< relref "../query-scheduler" >}}) is a mandatory component when using the query-frontend, you must run at least one query-scheduler replica.
+
+The following steps describe how a query moves through the query-frontend.
+
+1. A query-frontend receives a query.
+1. The query-frontend places the query in an queue by communicating with the query-scheduler, where it waits to be picked up by a querier.
+1. A querier picks up the query from the queue and executes it.
+1. A querier or queriers return the result to query-frontend, which then aggregates and forwards the results to the client.

docs/sources/operators-guide/architecture/components/query-scheduler/index.md

@@ -0,0 +1,37 @@
+---
+title: "Grafana Phlare query-scheduler"
+menuTitle: "Query-scheduler"
+description: "The query-scheduler distributes work to queriers."
+weight: 120
+---
+
+# Grafana Phlare query-scheduler
+
+The query-scheduler is a stateless component that retains a queue of queries to execute, and distributes the workload to available [queriers]({{< relref "../querier.md" >}}).
+
+The query-scheduler is a required component when using the [query-frontend]({{< relref "../query-frontend/index.md" >}}).
+
+![Query-scheduler architecture](query-scheduler-architecture.png)
+
+[//]: # "Diagram source at https://docs.google.com/presentation/d/1bHp8_zcoWCYoNU2AhO2lSagQyuIrghkCncViSqn14cU/edit"
+
+The following flow describes how a query moves through a Grafana Phlare cluster:
+
+1. The [query-frontend]({{< relref "../query-frontend/index.md" >}}) receives queries, and then either splits and shards them, or serves them from the cache.
+1. The query-frontend enqueues the queries into a query-scheduler.
+1. The query-scheduler stores the queries in an in-memory queue where they wait for a querier to pick them up.
+1. Queriers pick up the queries, and executes them.
+1. The querier sends results back to query-frontend, which then forwards the results to the client.
+
+## Benefits of using the query-scheduler
+
+Query-scheduler enables the scaling of query-frontends. To learn more, see Mimir [Query Frontend](/docs/mimir/latest/operators-guide/architecture/components/query-frontend/#why-query-frontend-scalability-is-limited) documentation.
+
+## Configuration
+
+To use the query-scheduler, query-frontends and queriers need to discover the addresses of query-scheduler instances.
+To advertise itself, the query-scheduler uses Ring-based service discovery which is configured via the [memberlist configuration]({{< relref "../../../configure/configuring-memberlist.md" >}}).
+
+## Operational considerations
+
+For high-availability, run two query-scheduler replicas.