chore: Add v2 support for micro services integration test (#4194)

* chore: Add v2 support for microservices integration test

This adds support for v2 storage layer in the microservices integration
test.

* Skip inconsistent query results with v2

* Add @aleks-p's suggestions

* Fix data races

* Reduce segment-writer heartbeats

* Move the lock up

* Check if all service names are available before starting the test

Author: simonswine

pkg/experiment/metastore/compaction/scheduler/metrics.go

@@ -77,9 +77,6 @@ func (c *statsCollector) Collect(ch chan<- prometheus.Metric) {
 }
 
 func (c *statsCollector) collectStats(fn func(level int, stats queueStats)) {
-	c.s.mu.Lock()
-	defer c.s.mu.Unlock()
-
 	for i, q := range c.s.queue.levels {
 		// Note that some levels may be empty.
 		if q == nil || q.jobs == nil {
@@ -118,7 +115,10 @@ func (c *statsCollector) collectStats(fn func(level int, stats queueStats)) {
 }
 
 func (c *statsCollector) collectMetrics() []prometheus.Metric {
-	metrics := make([]prometheus.Metric, 0, 8*len(c.s.queue.levels))
+	c.s.mu.Lock()
+	defer c.s.mu.Unlock()
+
+	var metrics = make([]prometheus.Metric, 0, 8*len(c.s.queue.levels))
 	c.collectStats(func(l int, stats queueStats) {
 		level := strconv.Itoa(l)
 		metrics = append(metrics,