Skip to content

Commit 45fbf68

Browse files
nikhilsinhaparseablenikhilsinhaparseable
committed
improve object store related metrics
1 parent e9e013d commit 45fbf68

File tree

6 files changed

+2237
-493
lines changed

6 files changed

+2237
-493
lines changed

‎src/metrics/storage.rs‎

Lines changed: 45 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -16,150 +16,109 @@
1616
*
1717
*/
1818

19+
use crate::metrics::METRICS_NAMESPACE;
1920
use actix_web_prometheus::PrometheusMetrics;
21+
use once_cell::sync::Lazy;
22+
use prometheus::{CounterVec, HistogramOpts, HistogramVec, Opts};
23+
24+
// Global storage metric used by all storage providers
25+
pub static STORAGE_REQUEST_RESPONSE_TIME: Lazy<HistogramVec> = Lazy::new(|| {
26+
HistogramVec::new(
27+
HistogramOpts::new("storage_request_response_time", "Storage Request Latency")
28+
.namespace(METRICS_NAMESPACE),
29+
&["provider", "method", "status"],
30+
)
31+
.expect("metric can be created")
32+
});
33+
34+
// Global storage metric for tracking number of files scanned
35+
pub static STORAGE_FILES_SCANNED: Lazy<CounterVec> = Lazy::new(|| {
36+
CounterVec::new(
37+
Opts::new(
38+
"storage_files_scanned_total",
39+
"Total number of files scanned in storage operations",
40+
)
41+
.namespace(METRICS_NAMESPACE),
42+
&["provider", "operation"],
43+
)
44+
.expect("metric can be created")
45+
});
2046

2147
pub trait StorageMetrics {
2248
fn register_metrics(&self, handler: &PrometheusMetrics);
2349
}
2450

2551
pub mod localfs {
26-
use crate::{metrics::METRICS_NAMESPACE, storage::FSConfig};
27-
use once_cell::sync::Lazy;
28-
use prometheus::{HistogramOpts, HistogramVec};
29-
30-
use super::StorageMetrics;
52+
use crate::storage::FSConfig;
3153

32-
pub static REQUEST_RESPONSE_TIME: Lazy<HistogramVec> = Lazy::new(|| {
33-
HistogramVec::new(
34-
HistogramOpts::new("local_fs_response_time", "FileSystem Request Latency")
35-
.namespace(METRICS_NAMESPACE),
36-
&["method", "status"],
37-
)
38-
.expect("metric can be created")
39-
});
54+
use super::{STORAGE_FILES_SCANNED, STORAGE_REQUEST_RESPONSE_TIME, StorageMetrics};
4055

4156
impl StorageMetrics for FSConfig {
4257
fn register_metrics(&self, handler: &actix_web_prometheus::PrometheusMetrics) {
4358
handler
4459
.registry
45-
.register(Box::new(REQUEST_RESPONSE_TIME.clone()))
60+
.register(Box::new(STORAGE_REQUEST_RESPONSE_TIME.clone()))
61+
.expect("metric can be registered");
62+
handler
63+
.registry
64+
.register(Box::new(STORAGE_FILES_SCANNED.clone()))
4665
.expect("metric can be registered");
4766
}
4867
}
4968
}
5069

5170
pub mod s3 {
52-
use crate::{metrics::METRICS_NAMESPACE, storage::S3Config};
53-
use once_cell::sync::Lazy;
54-
use prometheus::{HistogramOpts, HistogramVec};
71+
use crate::storage::S3Config;
5572

56-
use super::StorageMetrics;
57-
58-
pub static REQUEST_RESPONSE_TIME: Lazy<HistogramVec> = Lazy::new(|| {
59-
HistogramVec::new(
60-
HistogramOpts::new("s3_response_time", "S3 Request Latency")
61-
.namespace(METRICS_NAMESPACE),
62-
&["method", "status"],
63-
)
64-
.expect("metric can be created")
65-
});
66-
67-
pub static QUERY_LAYER_STORAGE_REQUEST_RESPONSE_TIME: Lazy<HistogramVec> = Lazy::new(|| {
68-
HistogramVec::new(
69-
HistogramOpts::new("query_s3_response_time", "S3 Request Latency")
70-
.namespace(METRICS_NAMESPACE),
71-
&["method", "status"],
72-
)
73-
.expect("metric can be created")
74-
});
73+
use super::{STORAGE_FILES_SCANNED, STORAGE_REQUEST_RESPONSE_TIME, StorageMetrics};
7574

7675
impl StorageMetrics for S3Config {
7776
fn register_metrics(&self, handler: &actix_web_prometheus::PrometheusMetrics) {
7877
handler
7978
.registry
80-
.register(Box::new(REQUEST_RESPONSE_TIME.clone()))
79+
.register(Box::new(STORAGE_REQUEST_RESPONSE_TIME.clone()))
8180
.expect("metric can be registered");
8281
handler
8382
.registry
84-
.register(Box::new(QUERY_LAYER_STORAGE_REQUEST_RESPONSE_TIME.clone()))
83+
.register(Box::new(STORAGE_FILES_SCANNED.clone()))
8584
.expect("metric can be registered");
8685
}
8786
}
8887
}
8988

9089
pub mod azureblob {
91-
use crate::{metrics::METRICS_NAMESPACE, storage::AzureBlobConfig};
92-
use once_cell::sync::Lazy;
93-
use prometheus::{HistogramOpts, HistogramVec};
94-
95-
use super::StorageMetrics;
90+
use crate::storage::AzureBlobConfig;
9691

97-
pub static REQUEST_RESPONSE_TIME: Lazy<HistogramVec> = Lazy::new(|| {
98-
HistogramVec::new(
99-
HistogramOpts::new("azr_blob_response_time", "AzureBlob Request Latency")
100-
.namespace(METRICS_NAMESPACE),
101-
&["method", "status"],
102-
)
103-
.expect("metric can be created")
104-
});
105-
106-
pub static QUERY_LAYER_STORAGE_REQUEST_RESPONSE_TIME: Lazy<HistogramVec> = Lazy::new(|| {
107-
HistogramVec::new(
108-
HistogramOpts::new("query_azr_blob_response_time", "AzureBlob Request Latency")
109-
.namespace(METRICS_NAMESPACE),
110-
&["method", "status"],
111-
)
112-
.expect("metric can be created")
113-
});
92+
use super::{STORAGE_FILES_SCANNED, STORAGE_REQUEST_RESPONSE_TIME, StorageMetrics};
11493

11594
impl StorageMetrics for AzureBlobConfig {
11695
fn register_metrics(&self, handler: &actix_web_prometheus::PrometheusMetrics) {
11796
handler
11897
.registry
119-
.register(Box::new(REQUEST_RESPONSE_TIME.clone()))
98+
.register(Box::new(STORAGE_REQUEST_RESPONSE_TIME.clone()))
12099
.expect("metric can be registered");
121100
handler
122101
.registry
123-
.register(Box::new(QUERY_LAYER_STORAGE_REQUEST_RESPONSE_TIME.clone()))
102+
.register(Box::new(STORAGE_FILES_SCANNED.clone()))
124103
.expect("metric can be registered");
125104
}
126105
}
127106
}
128107

129108
pub mod gcs {
130-
use crate::{metrics::METRICS_NAMESPACE, storage::GcsConfig};
131-
use once_cell::sync::Lazy;
132-
use prometheus::{HistogramOpts, HistogramVec};
109+
use crate::storage::GcsConfig;
133110

134-
use super::StorageMetrics;
135-
136-
pub static REQUEST_RESPONSE_TIME: Lazy<HistogramVec> = Lazy::new(|| {
137-
HistogramVec::new(
138-
HistogramOpts::new("gcs_response_time", "GCS Request Latency")
139-
.namespace(METRICS_NAMESPACE),
140-
&["method", "status"],
141-
)
142-
.expect("metric can be created")
143-
});
144-
145-
pub static QUERY_LAYER_STORAGE_REQUEST_RESPONSE_TIME: Lazy<HistogramVec> = Lazy::new(|| {
146-
HistogramVec::new(
147-
HistogramOpts::new("query_gcs_response_time", "GCS Request Latency")
148-
.namespace(METRICS_NAMESPACE),
149-
&["method", "status"],
150-
)
151-
.expect("metric can be created")
152-
});
111+
use super::{STORAGE_FILES_SCANNED, STORAGE_REQUEST_RESPONSE_TIME, StorageMetrics};
153112

154113
impl StorageMetrics for GcsConfig {
155114
fn register_metrics(&self, handler: &actix_web_prometheus::PrometheusMetrics) {
156115
handler
157116
.registry
158-
.register(Box::new(REQUEST_RESPONSE_TIME.clone()))
117+
.register(Box::new(STORAGE_REQUEST_RESPONSE_TIME.clone()))
159118
.expect("metric can be registered");
160119
handler
161120
.registry
162-
.register(Box::new(QUERY_LAYER_STORAGE_REQUEST_RESPONSE_TIME.clone()))
121+
.register(Box::new(STORAGE_FILES_SCANNED.clone()))
163122
.expect("metric can be registered");
164123
}
165124
}

0 commit comments

Comments
 (0)