Add block opened metrics (grafana/phlare#785)

* Add block opened metrics

* Add fake metrics to tests to fix it

* Update pkg/phlaredb/metrics.go

Co-authored-by: Christian Simon <[email protected]>

---------

Co-authored-by: Christian Simon <[email protected]>

Author: cyriltovena

pkg/phlaredb/block_querier.go

@@ -373,7 +373,10 @@ func NewSingleBlockQuerierFromMeta(phlarectx context.Context, bucketReader phlar
 
 func (b *singleBlockQuerier) Close() error {
 	b.openLock.Lock()
-	defer b.openLock.Unlock()
+	defer func() {
+		b.openLock.Unlock()
+		b.metrics.blockOpened.Dec()
+	}()
 	errs := multierror.New()
 	if b.index != nil {
 		err := b.index.Close()
@@ -1076,6 +1079,7 @@ func (q *singleBlockQuerier) Open(ctx context.Context) error {
 	if err := q.openFiles(ctx); err != nil {
 		return err
 	}
+	q.metrics.blockOpened.Inc()
 	q.opened = true
 	return nil
 }

pkg/phlaredb/block_querier_test.go

@@ -101,7 +101,10 @@ func TestQuerierBlockEviction(t *testing.T) {
 	for _, tc := range testCases {
 		q := BlockQuerier{queriers: make([]*singleBlockQuerier, len(tc.blocks))}
 		for i, b := range tc.blocks {
-			q.queriers[i] = &singleBlockQuerier{meta: &block.Meta{ULID: ulid.MustParse(b)}}
+			q.queriers[i] = &singleBlockQuerier{
+				meta:    &block.Meta{ULID: ulid.MustParse(b)},
+				metrics: newBlocksMetrics(nil),
+			}
 		}
 
 		evicted, err := q.evict(ulid.MustParse(blockToEvict))

pkg/phlaredb/metrics.go

@@ -29,6 +29,7 @@ type headMetrics struct {
 
 	sampleValuesIngested *prometheus.CounterVec
 	sampleValuesReceived *prometheus.CounterVec
+	samples              prometheus.Gauge
 
 	flushedFileSizeBytes        *prometheus.HistogramVec
 	flushedBlockSizeBytes       prometheus.Histogram
@@ -145,6 +146,10 @@ func newHeadMetrics(reg prometheus.Registerer) *headMetrics {
 			//  [512KB, 1MB, 2MB, 4MB, 8MB, 16MB, 32MB, 64MB, 128MB, 256MB, 512MB]
 			Buckets: prometheus.ExponentialBuckets(512*1024, 2, 11),
 		}),
+		samples: prometheus.NewGauge(prometheus.GaugeOpts{
+			Name: "pyroscope_head_samples",
+			Help: "Number of samples in the head.",
+		}),
 	}
 
 	m.register(reg)
@@ -192,6 +197,7 @@ type blocksMetrics struct {
 	query *query.Metrics
 
 	blockOpeningLatency prometheus.Histogram
+	blockOpened         prometheus.Gauge
 }
 
 func newBlocksMetrics(reg prometheus.Registerer) *blocksMetrics {
@@ -201,8 +207,13 @@ func newBlocksMetrics(reg prometheus.Registerer) *blocksMetrics {
 			Name: "pyroscopedb_block_opening_duration",
 			Help: "Latency of opening a block in seconds",
 		}),
+		blockOpened: prometheus.NewGauge(prometheus.GaugeOpts{
+			Name: "pyroscopedb_blocks_currently_open",
+			Help: "Number of blocks opened",
+		}),
 	}
 	m.blockOpeningLatency = util.RegisterOrGet(reg, m.blockOpeningLatency)
+	m.blockOpened = util.RegisterOrGet(reg, m.blockOpened)
 	return m
 }
 

pkg/phlaredb/profile_store.go

@@ -279,6 +279,7 @@ func (s *profileStore) cutRowGroup(count int) (err error) {
 	defer s.profilesLock.Unlock()
 	for i := range s.slice[:count] {
 		// don't retain profiles and samples in memory as re-slice.
+		s.metrics.samples.Sub(float64(len(s.slice[i].Samples)))
 		s.slice[i] = nil
 	}
 	// reset slice and metrics
@@ -420,6 +421,7 @@ func (s *profileStore) ingest(_ context.Context, profiles []*schemav1.Profile, l
 
 		// add to slice
 		s.slice = append(s.slice, p)
+		s.metrics.samples.Add(float64(len(p.Samples)))
 
 	}