Merge pull request #357 from opsani/fred/eng-549-opsani-devprometheus-success_rate-and

Fix success/error rate metrics

Author: linkous8

servo/connectors/opsani_dev.py

@@ -172,24 +172,24 @@ def generate_prometheus_config(
                 servo.connectors.prometheus.PrometheusMetric(
                     "main_success_rate",
                     servo.types.Unit.requests_per_second,
-                    query='avg(rate(envoy_cluster_upstream_rq_xx{opsani_role!="tuning", envoy_response_code_class=~"2|3"}[1m]))',
+                    query='avg(sum by(kubernetes_pod_name)(rate(envoy_cluster_upstream_rq_xx{opsani_role!="tuning", envoy_response_code_class=~"2|3"}[1m])))',
                 ),
                 servo.connectors.prometheus.PrometheusMetric(
                     "tuning_success_rate",
                     servo.types.Unit.requests_per_second,
-                    query='avg(rate(envoy_cluster_upstream_rq_xx{opsani_role="tuning", envoy_response_code_class=~"2|3"}[1m]))',
+                    query='avg(sum by(kubernetes_pod_name)(rate(envoy_cluster_upstream_rq_xx{opsani_role="tuning", envoy_response_code_class=~"2|3"}[1m])))',
                     absent=servo.connectors.prometheus.AbsentMetricPolicy.zero
                 ),
                 servo.connectors.prometheus.PrometheusMetric(
                     "main_error_rate",
                     servo.types.Unit.requests_per_second,
-                    query='avg(rate(envoy_cluster_upstream_rq_xx{opsani_role!="tuning", envoy_response_code_class=~"4|5"}[1m]))',
+                    query='avg(sum by(kubernetes_pod_name)(rate(envoy_cluster_upstream_rq_xx{opsani_role!="tuning", envoy_response_code_class=~"4|5"}[1m])))',
                     absent=servo.connectors.prometheus.AbsentMetricPolicy.zero
                 ),
                 servo.connectors.prometheus.PrometheusMetric(
                     "tuning_error_rate",
                     servo.types.Unit.requests_per_second,
-                    query='avg(rate(envoy_cluster_upstream_rq_xx{opsani_role="tuning", envoy_response_code_class=~"4|5"}[1m]))',
+                    query='avg(sum by(kubernetes_pod_name)(rate(envoy_cluster_upstream_rq_xx{opsani_role="tuning", envoy_response_code_class=~"4|5"}[1m])))',
                     absent=servo.connectors.prometheus.AbsentMetricPolicy.zero
                 ),
                 servo.connectors.prometheus.PrometheusMetric(