improves analytics

Author: petethepig

go.mod

@@ -4,6 +4,7 @@ go 1.14
 
 require (
 	github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59
+	github.com/clarkduvall/hyperloglog v0.0.0-20171127014514-a0107a5d8004 // indirect
 	github.com/cosmtrek/air v1.21.2
 	github.com/creack/pty v1.1.11 // indirect
 	github.com/davecgh/go-spew v1.1.1

go.sum

@@ -17,6 +17,8 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5O
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/clarkduvall/hyperloglog v0.0.0-20171127014514-a0107a5d8004 h1:mK6JroY6bLiPS3s6QCYOSjRyErFc2iHNkhhmRfF0nHo=
+github.com/clarkduvall/hyperloglog v0.0.0-20171127014514-a0107a5d8004/go.mod h1:drodPoQNro6QBO6TJ/MpMZbz8Bn2eSDtRN6jpG4VGw8=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=

pkg/analytics/analytics.go

@@ -11,6 +11,7 @@ import (
 	"github.com/google/uuid"
 	"github.com/pyroscope-io/pyroscope/pkg/build"
 	"github.com/pyroscope-io/pyroscope/pkg/config"
+	"github.com/pyroscope-io/pyroscope/pkg/server"
 	"github.com/pyroscope-io/pyroscope/pkg/storage"
 	"github.com/sirupsen/logrus"
 )
@@ -19,10 +20,11 @@ const url = "https://analytics.pyroscope.io/api/events"
 const gracePeriod = 5 * time.Minute
 const uploadFrequency = 24 * time.Hour
 
-func NewService(cfg *config.Config, s *storage.Storage) *Service {
+func NewService(cfg *config.Config, s *storage.Storage, c *server.Controller) *Service {
 	return &Service{
 		cfg: cfg,
 		s:   s,
+		c:   c,
 		httpClient: &http.Client{
 			Transport: &http.Transport{
 				MaxConnsPerHost: 1,
@@ -36,6 +38,7 @@ func NewService(cfg *config.Config, s *storage.Storage) *Service {
 type Service struct {
 	cfg        *config.Config
 	s          *storage.Storage
+	c          *server.Controller
 	httpClient *http.Client
 	uploads    int
 	stopCh     chan struct{}
@@ -58,6 +61,13 @@ type metrics struct {
 	BadgerDicts      int       `json:"badger_dicts"`
 	BadgerDimensions int       `json:"badger_dimensions"`
 	BadgerSegments   int       `json:"badger_segments"`
+	ControllerIndex  int       `json:"controller_index"`
+	ControllerIngest int       `json:"controller_ingest"`
+	ControllerRender int       `json:"controller_render"`
+	SpyRbspy         int       `json:"spy_rbspy"`
+	SpyPyspy         int       `json:"spy_pyspy"`
+	SpyGospy         int       `json:"spy_gospy"`
+	AppsCount        int       `json:"apps_count"`
 }
 
 func (s *Service) Start() {
@@ -89,6 +99,8 @@ func (s *Service) sendReport() {
 	runtime.ReadMemStats(&ms)
 	du := s.s.DiskUsage()
 
+	controllerStats := s.c.Stats()
+
 	m := metrics{
 		InstallID:        s.s.InstallID(),
 		RunID:            uuid.New().String(),
@@ -106,6 +118,13 @@ func (s *Service) sendReport() {
 		BadgerDicts:      int(du["dicts"]),
 		BadgerDimensions: int(du["dimensions"]),
 		BadgerSegments:   int(du["segments"]),
+		ControllerIndex:  controllerStats["index"],
+		ControllerIngest: controllerStats["ingest"],
+		ControllerRender: controllerStats["render"],
+		SpyRbspy:         controllerStats["ingest:rbspy"],
+		SpyPyspy:         controllerStats["ingest:pyspy"],
+		SpyGospy:         controllerStats["ingest:gospy"],
+		AppsCount:        s.c.AppsCount(),
 	}
 
 	buf, err := json.Marshal(m)

pkg/cli/cli.go

@@ -267,12 +267,12 @@ func startServer(cfg *config.Config) {
 	go agent.SelfProfile(cfg, u, "pyroscope.server.cpu{}")
 	go printRAMUsage()
 	go printDiskUsage(cfg)
+	c := server.New(cfg, s)
 	if !cfg.Server.AnalyticsOptOut {
-		analyticsService := analytics.NewService(cfg, s)
+		analyticsService := analytics.NewService(cfg, s, c)
 		go analyticsService.Start()
 		atexit.Register(func() { analyticsService.Stop() })
 	}
-	c := server.New(cfg, s)
 	// if you ever change this line, make sure to update this homebrew test:
 	//   https://github.com/pyroscope-io/homebrew-brew/blob/main/Formula/pyroscope.rb#L94
 	log.Info("starting HTTP server")

pkg/server/controller.go

@@ -3,10 +3,12 @@ package server
 import (
 	golog "log"
 	"net/http"
+	"sync"
 	"time"
 
 	_ "net/http/pprof"
 
+	"github.com/clarkduvall/hyperloglog"
 	"github.com/markbates/pkger"
 	"github.com/pyroscope-io/pyroscope/pkg/build"
 	"github.com/pyroscope-io/pyroscope/pkg/config"
@@ -22,12 +24,20 @@ func init() {
 type Controller struct {
 	cfg *config.Config
 	s   *storage.Storage
+
+	statsMutex sync.Mutex
+	stats      map[string]int
+
+	appStats *hyperloglog.HyperLogLogPlus
 }
 
 func New(cfg *config.Config, s *storage.Storage) *Controller {
+	appStats, _ := hyperloglog.NewPlus(uint8(18))
 	return &Controller{
-		cfg: cfg,
-		s:   s,
+		cfg:      cfg,
+		s:        s,
+		stats:    make(map[string]int),
+		appStats: appStats,
 	}
 }
 
@@ -44,7 +54,10 @@ func (ctrl *Controller) Start() {
 	} else {
 		fs = http.FileServer(http.Dir("./webapp/public"))
 	}
-	mux.HandleFunc("/", fs.ServeHTTP)
+	mux.HandleFunc("/", func(rw http.ResponseWriter, r *http.Request) {
+		ctrl.statsInc("index")
+		fs.ServeHTTP(rw, r)
+	})
 
 	logger := log.New()
 	w := logger.Writer()

pkg/server/ingest.go

@@ -96,5 +96,9 @@ func (ctrl *Controller) ingestHandler(w http.ResponseWriter, r *http.Request) {
 	if err != nil {
 		log.Fatal(err)
 	}
+	ctrl.statsInc("ingest")
+	ctrl.statsInc("ingest:" + ip.spyName)
+	k := *ip.storageKey
+	ctrl.appStats.Add(hashString(k.AppName()))
 	w.WriteHeader(200)
 }

pkg/server/render.go

@@ -27,6 +27,7 @@ func (ctrl *Controller) renderHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	resultTree, tl, spyName, sampleRate, err := ctrl.s.Get(startTime, endTime, storageKey)
+	ctrl.statsInc("render")
 	if err != nil {
 		panic(err) // TODO: handle
 	}

pkg/server/stats.go

@@ -0,0 +1,26 @@
+package server
+
+import "github.com/spaolacci/murmur3"
+
+const seed = 6231912
+
+type hashString string
+
+func (hs hashString) Sum64() uint64 {
+	return murmur3.Sum64WithSeed([]byte(hs), seed)
+}
+
+func (ctrl *Controller) statsInc(name string) {
+	ctrl.statsMutex.Lock()
+	defer ctrl.statsMutex.Unlock()
+
+	ctrl.stats[name]++
+}
+
+func (ctrl *Controller) Stats() map[string]int {
+	return ctrl.stats
+}
+
+func (ctrl *Controller) AppsCount() int {
+	return int(ctrl.appStats.Count())
+}

pkg/storage/key.go

@@ -130,3 +130,7 @@ func (k *Key) Hashed() []byte {
 	binary.LittleEndian.PutUint64(b[8:16], u2)
 	return b
 }
+
+func (k *Key) AppName() string {
+	return k.labels["__name__"]
+}