refactor: construct once

Author: Devdutt Shenoi

src/storage/azure_blob.rs

@@ -163,7 +163,7 @@ impl ObjectStorageProvider for AzureBlobConfig {
         RuntimeConfig::new().with_object_store_registry(Arc::new(object_store_registry))
     }
 
-    fn get_object_store(&self) -> Arc<dyn super::ObjectStorage> {
+    fn construct_client(&self) -> Arc<dyn super::ObjectStorage> {
         let azure = self.get_default_builder().build().unwrap();
         // limit objectstore to a concurrent request limit
         let azure = LimitStore::new(azure, super::MAX_OBJECT_STORE_REQUESTS);

src/storage/localfs.rs

@@ -67,7 +67,7 @@ impl ObjectStorageProvider for FSConfig {
         RuntimeConfig::new()
     }
 
-    fn get_object_store(&self) -> Arc<dyn ObjectStorage> {
+    fn construct_client(&self) -> Arc<dyn ObjectStorage> {
         Arc::new(LocalFS::new(self.root.clone()))
     }
 

src/storage/object_storage.rs

@@ -46,6 +46,7 @@ use bytes::Bytes;
 use chrono::Local;
 use datafusion::{datasource::listing::ListingTableUrl, execution::runtime_env::RuntimeConfig};
 use itertools::Itertools;
+use once_cell::sync::OnceCell;
 use relative_path::RelativePath;
 use relative_path::RelativePathBuf;
 
@@ -60,7 +61,12 @@ use std::{
 
 pub trait ObjectStorageProvider: StorageMetrics + std::fmt::Debug + Send + Sync {
     fn get_datafusion_runtime(&self) -> RuntimeConfig;
-    fn get_object_store(&self) -> Arc<dyn ObjectStorage>;
+    fn construct_client(&self) -> Arc<dyn ObjectStorage>;
+    fn get_object_store(&self) -> Arc<dyn ObjectStorage> {
+        static STORE: OnceCell<Arc<dyn ObjectStorage>> = OnceCell::new();
+
+        STORE.get_or_init(|| self.construct_client()).clone()
+    }
     fn get_endpoint(&self) -> String;
     fn register_store_metrics(&self, handler: &PrometheusMetrics);
 }

src/storage/s3.rs

@@ -298,7 +298,7 @@ impl ObjectStorageProvider for S3Config {
         RuntimeConfig::new().with_object_store_registry(Arc::new(object_store_registry))
     }
 
-    fn get_object_store(&self) -> Arc<dyn ObjectStorage> {
+    fn construct_client(&self) -> Arc<dyn ObjectStorage> {
         let s3 = self.get_default_builder().build().unwrap();
 
         // limit objectstore to a concurrent request limit