Merge #155848

155848: sql: automatically cleanup automatic stats jobs r=michae2 a=dt

Automatic stats jobs are frequent -- sometimes almost continious -- background process in many clusters, constantly being created and running as part of its normal operation. In some clusters these completed jobs have been observed to make up more than 90% of the retained content of the jobs system, but in most user-visible surfaces, they are usually filtered out or excluded as they have typically been found to be of low relevance or utility to users looking for more notable, discrete cluster events tied to less continious jobs.

This change introduces automatic, immediate cleanup of automatic stats jobs so that they are eagerly removed rather than being retained for the full job retention period. This should dramatically reduce the number of these in the jobs system to just those are executing and could be relevant to the cluster's operation, and those which were manually run (and thus likely of concern to a user), as well as those which failed which could also be of interest.

Release note (ops change): successfully completed automatic SQL stats collecton jobs now automatically purged rather than being retained for the full default job retention period.
Epic: CRDB-55121.

Co-authored-by: David Taylor <[email protected]>

Author: craig[bot]

pkg/sql/create_stats.go

@@ -91,6 +91,12 @@ var errorOnConcurrentCreateStats = settings.RegisterBoolSetting(
 	false,
 	settings.WithPublic)
 
+var automaticStatsJobAutoCleanup = settings.RegisterBoolSetting(
+	settings.ApplicationLevel,
+	"sql.stats.automatic_stats_job_auto_cleanup.enabled",
+	"set to true to enable automatic cleanup of completed AUTO CREATE STATISTICS jobs",
+	true)
+
 const nonIndexColHistogramBuckets = 2
 
 // createStatsNode is a planNode implemented in terms of a function. The
@@ -203,6 +209,12 @@ func (n *createStatsNode) runJob(ctx context.Context) error {
 				return nil
 			}
 		}
+	} else if automaticStatsJobAutoCleanup.Get(n.p.ExecCfg().SV()) {
+		if name := job.Details().(jobspb.CreateStatsDetails).Name; name == jobspb.AutoStatsName || name == jobspb.AutoPartialStatsName {
+			if err := n.p.ExecCfg().JobRegistry.DeleteTerminalJobByID(ctx, job.ID()); err != nil {
+				log.Dev.Warningf(ctx, "failed to auto-delete terminal automatic stats job: %v", err)
+			}
+		}
 	}
 	return err
 }

pkg/sql/create_stats_test.go

@@ -303,6 +303,7 @@ func TestAutoPartialStatsJobDescription(t *testing.T) {
 	// Disable automatic statistics collection.
 	sqlRunner.Exec(t, `SET CLUSTER SETTING sql.stats.automatic_collection.enabled = false;`)
 	sqlRunner.Exec(t, `SET CLUSTER SETTING sql.stats.automatic_partial_collection.enabled = false;`)
+	sqlRunner.Exec(t, `SET CLUSTER SETTING sql.stats.automatic_stats_job_auto_cleanup.enabled = false;`)
 
 	// Create a test table.
 	sqlRunner.Exec(t, `CREATE TABLE test (id INT PRIMARY KEY, value INT);`)