OCPBUGS-59768: metrics endpoints security - add client cert auth to kube-rbac-proxy

anik120 · anik120 · commit ab50f2985b08 · 2025-09-12T12:24:28.000-04:00
**Problem:**
OLM metrics endpoints were exposed without proper authentication, allowing unauthorized access to sensitive operational data.

**Root Cause:**
kube-rbac-proxy configuration was missing critical authentication argument `--client-ca-file`

**Solution:**
- Add missing authentication argument to kube-rbac-proxy in package-server-manager
- Mount client CA certificate from new `metrics-client-ca` ConfigMap
- Update ServiceMonitors to use `scrapeClass: tls-client-certificate-auth`
- Create ConfigMap with service CA injection for automatic CA bundle sync

**Security Impact:**
- **Before:** Any HTTPS client could access metrics endpoints
- **After:** Only authorized Prometheus service account (`system:serviceaccount:openshift-monitoring:prometheus-k8s`) can access metrics
diff --git a/manifests/0000_50_olm_02-metrics-client-ca.configmap.yaml b/manifests/0000_50_olm_02-metrics-client-ca.configmap.yaml
@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: metrics-client-ca
+  namespace: openshift-operator-lifecycle-manager
+  annotations:
+    include.release.openshift.io/self-managed-high-availability: "true"
+    service.beta.openshift.io/inject-cabundle: "true"
+    include.release.openshift.io/ibm-cloud-managed: "true"
+    capability.openshift.io/name: "OperatorLifecycleManager"
+    include.release.openshift.io/hypershift: "true"
+data: {}
diff --git a/manifests/0000_50_olm_06-psm-operator.deployment.ibm-cloud-managed.yaml b/manifests/0000_50_olm_06-psm-operator.deployment.ibm-cloud-managed.yaml
@@ -35,6 +35,7 @@ spec:
             - --upstream=http://127.0.0.1:9090/
             - --tls-cert-file=/etc/tls/private/tls.crt
             - --tls-private-key-file=/etc/tls/private/tls.key
+            - --client-ca-file=/etc/tls/client/client-ca-file
             - --logtostderr=true
           image: quay.io/openshift/origin-kube-rbac-proxy:latest
           imagePullPolicy: IfNotPresent
@@ -57,6 +58,9 @@ spec:
           volumeMounts:
             - mountPath: /etc/tls/private
               name: package-server-manager-serving-cert
+            - mountPath: /etc/tls/client
+              name: metrics-client-ca
+              readOnly: true
         - name: package-server-manager
           securityContext:
             allowPrivilegeEscalation: false
@@ -120,3 +124,6 @@ spec:
         - name: package-server-manager-serving-cert
           secret:
             secretName: package-server-manager-serving-cert
+        - name: metrics-client-ca
+          configMap:
+            name: metrics-client-ca
diff --git a/manifests/0000_50_olm_06-psm-operator.deployment.yaml b/manifests/0000_50_olm_06-psm-operator.deployment.yaml
@@ -35,6 +35,7 @@ spec:
             - --upstream=http://127.0.0.1:9090/
             - --tls-cert-file=/etc/tls/private/tls.crt
             - --tls-private-key-file=/etc/tls/private/tls.key
+            - --client-ca-file=/etc/tls/client/client-ca-file
             - --logtostderr=true
           image: quay.io/openshift/origin-kube-rbac-proxy:latest
           imagePullPolicy: IfNotPresent
@@ -57,6 +58,9 @@ spec:
           volumeMounts:
             - mountPath: /etc/tls/private
               name: package-server-manager-serving-cert
+            - mountPath: /etc/tls/client
+              name: metrics-client-ca
+              readOnly: true
         - name: package-server-manager
           securityContext:
             allowPrivilegeEscalation: false
@@ -121,3 +125,6 @@ spec:
         - name: package-server-manager-serving-cert
           secret:
             secretName: package-server-manager-serving-cert
+        - name: metrics-client-ca
+          configMap:
+            name: metrics-client-ca
diff --git a/manifests/0000_50_olm_06-psm-operator.servicemonitor.yaml b/manifests/0000_50_olm_06-psm-operator.servicemonitor.yaml
@@ -9,13 +9,12 @@ metadata:
     capability.openshift.io/name: "OperatorLifecycleManager"
 spec:
   endpoints:
-    - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
-      interval: 30s
+    - interval: 30s
       port: metrics
       scheme: https
       tlsConfig:
-        caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt
         serverName: package-server-manager-metrics.openshift-operator-lifecycle-manager.svc
+  scrapeClass: tls-client-certificate-auth
   namespaceSelector:
     matchNames:
       - openshift-operator-lifecycle-manager
diff --git a/manifests/0000_90_olm_00-service-monitor.yaml b/manifests/0000_90_olm_00-service-monitor.yaml
@@ -53,7 +53,6 @@ metadata:
     include.release.openshift.io/hypershift: "true"
 spec:
   endpoints:
-    - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
       interval: 30s
       metricRelabelings:
         - action: drop
@@ -63,8 +62,8 @@ spec:
       port: https-metrics
       scheme: https
       tlsConfig:
-        caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt
         serverName: olm-operator-metrics.openshift-operator-lifecycle-manager.svc
+  scrapeClass: tls-client-certificate-auth
   jobLabel: component
   namespaceSelector:
     matchNames:
@@ -87,7 +86,6 @@ metadata:
     include.release.openshift.io/hypershift: "true"
 spec:
   endpoints:
-    - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
       interval: 30s
       metricRelabelings:
         - action: drop
@@ -97,8 +95,8 @@ spec:
       port: https-metrics
       scheme: https
       tlsConfig:
-        caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt
         serverName: catalog-operator-metrics.openshift-operator-lifecycle-manager.svc
+  scrapeClass: tls-client-certificate-auth
   jobLabel: component
   namespaceSelector:
     matchNames:
diff --git a/microshift-manifests/0000_50_olm_02-metrics-client-ca.configmap.yaml b/microshift-manifests/0000_50_olm_02-metrics-client-ca.configmap.yaml
@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: metrics-client-ca
+  namespace: openshift-operator-lifecycle-manager
+  annotations:
+    include.release.openshift.io/self-managed-high-availability: "true"
+    service.beta.openshift.io/inject-cabundle: "true"
+    include.release.openshift.io/ibm-cloud-managed: "true"
+    capability.openshift.io/name: "OperatorLifecycleManager"
+    include.release.openshift.io/hypershift: "true"
+data: {}
diff --git a/microshift-manifests/0000_50_olm_06-psm-operator.deployment.ibm-cloud-managed.yaml b/microshift-manifests/0000_50_olm_06-psm-operator.deployment.ibm-cloud-managed.yaml
@@ -35,6 +35,7 @@ spec:
             - --upstream=http://127.0.0.1:9090/
             - --tls-cert-file=/etc/tls/private/tls.crt
             - --tls-private-key-file=/etc/tls/private/tls.key
+            - --client-ca-file=/etc/tls/client/client-ca-file
             - --logtostderr=true
           image: quay.io/openshift/origin-kube-rbac-proxy:latest
           imagePullPolicy: IfNotPresent
@@ -57,6 +58,9 @@ spec:
           volumeMounts:
             - mountPath: /etc/tls/private
               name: package-server-manager-serving-cert
+            - mountPath: /etc/tls/client
+              name: metrics-client-ca
+              readOnly: true
         - name: package-server-manager
           securityContext:
             allowPrivilegeEscalation: false
@@ -120,3 +124,6 @@ spec:
         - name: package-server-manager-serving-cert
           secret:
             secretName: package-server-manager-serving-cert
+        - name: metrics-client-ca
+          configMap:
+            name: metrics-client-ca
diff --git a/microshift-manifests/0000_50_olm_06-psm-operator.deployment.yaml b/microshift-manifests/0000_50_olm_06-psm-operator.deployment.yaml
@@ -35,6 +35,7 @@ spec:
             - --upstream=http://127.0.0.1:9090/
             - --tls-cert-file=/etc/tls/private/tls.crt
             - --tls-private-key-file=/etc/tls/private/tls.key
+            - --client-ca-file=/etc/tls/client/client-ca-file
             - --logtostderr=true
           image: quay.io/openshift/origin-kube-rbac-proxy:latest
           imagePullPolicy: IfNotPresent
@@ -57,6 +58,9 @@ spec:
           volumeMounts:
             - mountPath: /etc/tls/private
               name: package-server-manager-serving-cert
+            - mountPath: /etc/tls/client
+              name: metrics-client-ca
+              readOnly: true
         - name: package-server-manager
           securityContext:
             allowPrivilegeEscalation: false
@@ -121,3 +125,6 @@ spec:
         - name: package-server-manager-serving-cert
           secret:
             secretName: package-server-manager-serving-cert
+        - name: metrics-client-ca
+          configMap:
+            name: metrics-client-ca
diff --git a/microshift-manifests/0000_50_olm_06-psm-operator.servicemonitor.yaml b/microshift-manifests/0000_50_olm_06-psm-operator.servicemonitor.yaml
@@ -9,13 +9,12 @@ metadata:
     capability.openshift.io/name: "OperatorLifecycleManager"
 spec:
   endpoints:
-    - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
-      interval: 30s
+    - interval: 30s
       port: metrics
       scheme: https
       tlsConfig:
-        caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt
         serverName: package-server-manager-metrics.openshift-operator-lifecycle-manager.svc
+  scrapeClass: tls-client-certificate-auth
   namespaceSelector:
     matchNames:
       - openshift-operator-lifecycle-manager
diff --git a/microshift-manifests/kustomization.yaml b/microshift-manifests/kustomization.yaml
@@ -15,6 +15,7 @@ resources:
   - 0000_50_olm_00-pprof-secret.yaml
   - 0000_50_olm_00-subscriptions.crd.yaml
   - 0000_50_olm_01-networkpolicies.yaml
+  - 0000_50_olm_02-metrics-client-ca.configmap.yaml
   - 0000_50_olm_02-olm-operator.serviceaccount.yaml
   - 0000_50_olm_03-olmconfig.yaml
   - 0000_50_olm_03-services.yaml
diff --git a/scripts/generate_crds_manifests.sh b/scripts/generate_crds_manifests.sh
@@ -123,6 +123,18 @@ spec:
       name: quay.io/openshift/origin-kube-rbac-proxy:latest
 EOF
 
+cat << EOF > manifests/0000_50_olm_02-metrics-client-ca.configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: metrics-client-ca
+  namespace: openshift-operator-lifecycle-manager
+  annotations:
+    include.release.openshift.io/self-managed-high-availability: "true"
+    service.beta.openshift.io/inject-cabundle: "true"
+data: {}
+EOF
+
 cat << EOF > manifests/0000_50_olm_06-psm-operator.networkpolicy.yaml
 apiVersion: networking.k8s.io/v1
 kind: NetworkPolicy
@@ -196,6 +208,7 @@ spec:
           - --upstream=http://127.0.0.1:9090/
           - --tls-cert-file=/etc/tls/private/tls.crt
           - --tls-private-key-file=/etc/tls/private/tls.key
+          - --client-ca-file=/etc/tls/client/client-ca-file
           - --logtostderr=true
           image: quay.io/openshift/origin-kube-rbac-proxy:latest
           imagePullPolicy: IfNotPresent
@@ -218,6 +231,9 @@ spec:
           volumeMounts:
           - mountPath: /etc/tls/private
             name: package-server-manager-serving-cert
+          - mountPath: /etc/tls/client
+            name: metrics-client-ca
+            readOnly: true
         - name: package-server-manager
           securityContext:
             allowPrivilegeEscalation: false
@@ -282,6 +298,9 @@ spec:
       - name: package-server-manager-serving-cert
         secret:
           secretName: package-server-manager-serving-cert
+      - name: metrics-client-ca
+        configMap:
+          name: metrics-client-ca
 EOF
 
 cat << EOF > manifests/0000_50_olm_06-psm-operator.service.yaml
@@ -315,13 +334,12 @@ metadata:
     include.release.openshift.io/self-managed-high-availability: "true"
 spec:
   endpoints:
-  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
-    interval: 30s
+  - interval: 30s
     port: metrics
     scheme: https
     tlsConfig:
-      caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt
       serverName: package-server-manager-metrics.openshift-operator-lifecycle-manager.svc
+  scrapeClass: tls-client-certificate-auth
   namespaceSelector:
     matchNames:
     - openshift-operator-lifecycle-manager
@@ -616,6 +634,11 @@ for file in ${microshift_manifests_files}; do
   echo "  - $(realpath --relative-to "${ROOT_DIR}/microshift-manifests" "${file}")" >> "${ROOT_DIR}/microshift-manifests/kustomization.yaml"
 done
 
+# Update ServiceMonitors to use client certificate authentication instead of bearer tokens
+${SED} -i '/bearerTokenFile:/d' manifests/0000_90_olm_00-service-monitor.yaml
+${SED} -i '/caFile:/d' manifests/0000_90_olm_00-service-monitor.yaml
+${SED} -i 's/serverName: \(.*\)$/serverName: \1\n  scrapeClass: tls-client-certificate-auth/' manifests/0000_90_olm_00-service-monitor.yaml
+
 # Now we need to get rid of these args from the olm-operator deployment:
 #
 # - --writeStatusName
