Emit metrics on failed credential fetching and allow 0.1% pod restart (#537)

Author: xdu31

tests/assets/eks-pod-identity/pod-default.yaml

@@ -43,8 +43,8 @@ spec:
         start_epoch=$(date +%s%3N)
         # fetch credentials
         status_code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 2 -H "Authorization: $AUTH_TOKEN" http://169.254.170.23/v1/credentials)
+        end_epoch=$(date +%s%3N)
         if [ "$status_code" -eq 200 ]; then
-          end_epoch=$(date +%s%3N)
           printf "Endpoint is reachable at try %d\n" "$i"
 
           latency_ms=$((end_epoch - start_epoch))

tests/tekton-resources/tasks/generators/clusterloader/load-pod-identity.yaml

@@ -290,7 +290,13 @@ spec:
       int_p999=$(echo "$p999" | awk '{printf "%d", $1}')
       echo "p99.9 is $p999"
 
-      if [ "$int_p999" -lt 1 ] && [ "$total_samples" -eq "$expected_pods" ]; then
+      # Calculate 0.1% tolerance (expected_pods * 0.001)
+      tolerance=$(echo "$expected_pods" | awk '{printf "%.0f", $1 * 0.001}')
+      max_allowed=$((expected_pods + tolerance))
+
+      echo "Tolerance: $tolerance, max allowed samples: $max_allowed"
+
+      if [ "$int_p999" -lt 1 ] && [ "$total_samples" -ge "$expected_pods" ] && [ "$total_samples" -le "$max_allowed" ]; then
         echo "1" | tee $(results.datapoint.path)
       else
         echo "0" | tee $(results.datapoint.path)