add `inject-per-series-metadata'

See the discussion here #2551
This frees users from having to join on the various label series, which
is useful when infrastructure has agents that can't do recording rules.

Author: jacobstr

internal/store/builder.go

@@ -75,6 +75,7 @@ type Builder struct {
 	buildCustomResourceStoresFunc ksmtypes.BuildCustomResourceStoresFunc
 	allowAnnotationsList          map[string][]string
 	allowLabelsList               map[string][]string
+	injectPerSeriesMetadata       bool
 	utilOptions                   *options.Options
 	// namespaceFilter is inside fieldSelectorFilter
 	fieldSelectorFilter string
@@ -242,6 +243,13 @@ func (b *Builder) allowList(list map[string][]string) (map[string][]string, erro
 	return m, nil
 }
 
+// WithAllowAnnotations configures which annotations can be returned for metrics
+func (b *Builder) WithInjectPerSeriesMetadata(inject bool) error {
+	var err error
+	b.injectPerSeriesMetadata = inject
+	return err
+}
+
 // WithAllowAnnotations configures which annotations can be returned for metrics
 func (b *Builder) WithAllowAnnotations(annotations map[string][]string) error {
 	var err error
@@ -460,7 +468,7 @@ func (b *Builder) buildStorageClassStores() []cache.Store {
 }
 
 func (b *Builder) buildPodStores() []cache.Store {
-	return b.buildStoresFunc(podMetricFamilies(b.allowAnnotationsList["pods"], b.allowLabelsList["pods"]), &v1.Pod{}, createPodListWatch, b.useAPIServerCache)
+	return b.buildStoresFunc(podMetricFamilies(b.injectPerSeriesMetadata, b.allowAnnotationsList["pods"], b.allowLabelsList["pods"]), &v1.Pod{}, createPodListWatch, b.useAPIServerCache)
 }
 
 func (b *Builder) buildCsrStores() []cache.Store {

internal/store/pod.go

@@ -40,7 +40,12 @@ var (
 	podStatusReasons           = []string{"Evicted", "NodeAffinity", "NodeLost", "Shutdown", "UnexpectedAdmissionError"}
 )
 
-func podMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator {
+func podMetricFamilies(injectPerSeriesMetadata bool, allowAnnotationsList []string, allowLabelsList []string) []generator.FamilyGenerator {
+	mc := &MetricConfig{
+		InjectPerSeriesMetadata: injectPerSeriesMetadata,
+		AllowAnnotations:        allowAnnotationsList,
+		AllowLabels:             allowLabelsList,
+	}
 	return []generator.FamilyGenerator{
 		createPodCompletionTimeFamilyGenerator(),
 		createPodContainerInfoFamilyGenerator(),
@@ -82,7 +87,7 @@ func podMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generat
 		createPodSpecVolumesPersistentVolumeClaimsInfoFamilyGenerator(),
 		createPodSpecVolumesPersistentVolumeClaimsReadonlyFamilyGenerator(),
 		createPodStartTimeFamilyGenerator(),
-		createPodStatusPhaseFamilyGenerator(),
+		createPodStatusPhaseFamilyGenerator(mc),
 		createPodStatusQosClassFamilyGenerator(),
 		createPodStatusReadyFamilyGenerator(),
 		createPodStatusReadyTimeFamilyGenerator(),
@@ -1333,7 +1338,7 @@ func createPodStartTimeFamilyGenerator() generator.FamilyGenerator {
 	)
 }
 
-func createPodStatusPhaseFamilyGenerator() generator.FamilyGenerator {
+func createPodStatusPhaseFamilyGenerator(mc *MetricConfig) generator.FamilyGenerator {
 	return *generator.NewFamilyGeneratorWithStability(
 		"kube_pod_status_phase",
 		"The pods current phase.",
@@ -1361,13 +1366,12 @@ func createPodStatusPhaseFamilyGenerator() generator.FamilyGenerator {
 
 			ms := make([]*metric.Metric, len(phases))
 
-			for i, p := range phases {
-				ms[i] = &metric.Metric{
-
+			for i, ph := range phases {
+				ms[i] = injectLabelsAndAnnos(&metric.Metric{
 					LabelKeys:   []string{"phase"},
-					LabelValues: []string{p.n},
-					Value:       boolFloat64(p.v),
-				}
+					LabelValues: []string{ph.n},
+					Value:       boolFloat64(ph.v),
+				}, mc, &p.ObjectMeta)
 			}
 
 			return &metric.Family{

internal/store/pod_test.go

@@ -2175,8 +2175,8 @@ func TestPodStore(t *testing.T) {
 	}
 
 	for i, c := range cases {
-		c.Func = generator.ComposeMetricGenFuncs(podMetricFamilies(c.AllowAnnotationsList, c.AllowLabelsList))
-		c.Headers = generator.ExtractMetricFamilyHeaders(podMetricFamilies(c.AllowAnnotationsList, c.AllowLabelsList))
+		c.Func = generator.ComposeMetricGenFuncs(podMetricFamilies(false, c.AllowAnnotationsList, c.AllowLabelsList))
+		c.Headers = generator.ExtractMetricFamilyHeaders(podMetricFamilies(false, c.AllowAnnotationsList, c.AllowLabelsList))
 		if err := c.run(); err != nil {
 			t.Errorf("unexpected collecting result in %vth run:\n%s", i, err)
 		}
@@ -2186,7 +2186,7 @@ func TestPodStore(t *testing.T) {
 func BenchmarkPodStore(b *testing.B) {
 	b.ReportAllocs()
 
-	f := generator.ComposeMetricGenFuncs(podMetricFamilies(nil, nil))
+	f := generator.ComposeMetricGenFuncs(podMetricFamilies(false, nil, nil))
 
 	pod := &v1.Pod{
 		ObjectMeta: metav1.ObjectMeta{

internal/store/utils.go

@@ -26,6 +26,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 
 	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/validation"
 
 	"k8s.io/kube-state-metrics/v2/pkg/metric"
@@ -38,6 +39,12 @@ var (
 	conditionStatuses  = []v1.ConditionStatus{v1.ConditionTrue, v1.ConditionFalse, v1.ConditionUnknown}
 )
 
+type MetricConfig struct {
+	InjectPerSeriesMetadata bool
+	AllowAnnotations        []string
+	AllowLabels             []string
+}
+
 func resourceVersionMetric(rv string) []*metric.Metric {
 	v, err := strconv.ParseFloat(rv, 64)
 	if err != nil {
@@ -175,6 +182,17 @@ func isPrefixedNativeResource(name v1.ResourceName) bool {
 	return strings.Contains(string(name), v1.ResourceDefaultNamespacePrefix)
 }
 
+// convenience wrapper to inject allow-listed labels and annotations to a metric if per-series injection is enabled.
+func injectLabelsAndAnnos(m *metric.Metric, metricConfig *MetricConfig, obj *metav1.ObjectMeta) *metric.Metric {
+	if !metricConfig.InjectPerSeriesMetadata {
+		return m
+	}
+	labelKeys, labelValues := createPrometheusLabelKeysValues("label", obj.Labels, metricConfig.AllowLabels)
+	annotationKeys, annotationValues := createPrometheusLabelKeysValues("annotation", obj.Annotations, metricConfig.AllowAnnotations)
+	m.LabelKeys, m.LabelValues = mergeKeyValues(m.LabelKeys, m.LabelValues, annotationKeys, annotationValues, labelKeys, labelValues)
+	return m
+}
+
 // createPrometheusLabelKeysValues takes in passed kubernetes annotations/labels
 // and associated allowed list in kubernetes label format.
 // It returns only those allowed annotations/labels that exist in the list and converts them to Prometheus labels.

pkg/app/server.go

@@ -264,6 +264,9 @@ func RunKubeStateMetrics(ctx context.Context, opts *options.Options) error {
 	if err := storeBuilder.WithAllowLabels(opts.LabelsAllowList); err != nil {
 		return fmt.Errorf("failed to set up labels allowlist: %v", err)
 	}
+	if err := storeBuilder.WithInjectPerSeriesMetadata(opts.InjectPerSeriesMetadata); err != nil {
+		return fmt.Errorf("failed to configure per series metadata injection: %v", err)
+	}
 
 	ksmMetricsRegistry.MustRegister(
 		collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),

pkg/options/options.go

@@ -47,6 +47,8 @@ type Options struct {
 	MetricOptInList      MetricSet       `yaml:"metric_opt_in_list"`
 	Resources            ResourceSet     `yaml:"resources"`
 
+	InjectPerSeriesMetadata bool `yaml:"inject_per_series_metadata"`
+
 	cmd                      *cobra.Command
 	Apiserver                string   `yaml:"apiserver"`
 	CustomResourceConfig     string   `yaml:"custom_resource_config"`
@@ -164,6 +166,7 @@ func (o *Options) AddFlags(cmd *cobra.Command) {
 	o.cmd.Flags().Var(&o.LabelsAllowList, "metric-labels-allowlist", "Comma-separated list of additional Kubernetes label keys that will be used in the resource' labels metric. By default the labels metrics are not exposed. To include them, provide a list of resource names in their plural form and Kubernetes label keys you would like to allow for them (Example: '=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)'. A single '*' can be provided per resource instead to allow any labels, but that has severe performance implications (Example: '=pods=[*]'). Additionally, an asterisk (*) can be provided as a key, which will resolve to all resources, i.e., assuming '--resources=deployments,pods', '=*=[*]' will resolve to '=deployments=[*],pods=[*]'.")
 	o.cmd.Flags().Var(&o.MetricAllowlist, "metric-allowlist", "Comma-separated list of metrics to be exposed. This list comprises of exact metric names and/or regex patterns. The allowlist and denylist are mutually exclusive.")
 	o.cmd.Flags().Var(&o.MetricDenylist, "metric-denylist", "Comma-separated list of metrics not to be enabled. This list comprises of exact metric names and/or regex patterns. The allowlist and denylist are mutually exclusive.")
+	o.cmd.Flags().BoolVar(&o.InjectPerSeriesMetadata, "inject-per-series-metadata", false, "Propagate labels and annotations from object metadata to all respective metrics rather than simply populating kube_<resource>_labels and kube_<resource>_annotations. Honors metric-annotations-allowlist and metric-labels-allowlist.")
 	o.cmd.Flags().Var(&o.MetricOptInList, "metric-opt-in-list", "Comma-separated list of metrics which are opt-in and not enabled by default. This is in addition to the metric allow- and denylists")
 	o.cmd.Flags().Var(&o.Namespaces, "namespaces", fmt.Sprintf("Comma-separated list of namespaces to be enabled. Defaults to %q", &DefaultNamespaces))
 	o.cmd.Flags().Var(&o.NamespacesDenylist, "namespaces-denylist", "Comma-separated list of namespaces not to be enabled. If namespaces and namespaces-denylist are both set, only namespaces that are excluded in namespaces-denylist will be used.")