diff --git a/crates/catalog/loader/Cargo.toml b/crates/catalog/loader/Cargo.toml new file mode 100644 index 0000000000..d29edad051 --- /dev/null +++ b/crates/catalog/loader/Cargo.toml @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +edition = { workspace = true } +homepage = { workspace = true } +name = "iceberg-catalog-loader" +rust-version = { workspace = true } +version = { workspace = true } + +categories = ["database"] +description = "Apache Iceberg Catalog Loader API" +keywords = ["iceberg", "catalog"] +license = { workspace = true } +repository = { workspace = true } + +[dependencies] +iceberg = { workspace = true } +iceberg-catalog-rest = {workspace = true} +tokio = { workspace = true } +async-trait = {workspace = true} diff --git a/crates/catalog/loader/DEPENDENCIES.rust.tsv b/crates/catalog/loader/DEPENDENCIES.rust.tsv new file mode 100644 index 0000000000..4dac7186cd --- /dev/null +++ b/crates/catalog/loader/DEPENDENCIES.rust.tsv @@ -0,0 +1,338 @@ +crate 0BSD Apache-2.0 Apache-2.0 WITH LLVM-exception BSD-2-Clause BSD-3-Clause BSL-1.0 CC0-1.0 ISC MIT MPL-2.0 Unicode-3.0 Unlicense Zlib +addr2line@0.24.2 X X +adler2@2.0.0 X X X +adler32@1.2.0 X +ahash@0.8.11 X X +aho-corasick@1.1.3 X X +alloc-no-stdlib@2.0.4 X +alloc-stdlib@0.2.2 X +allocator-api2@0.2.21 X X +android-tzdata@0.1.1 X X +android_system_properties@0.1.5 X X +anyhow@1.0.95 X X +apache-avro@0.17.0 X +array-init@2.1.0 X X +arrayvec@0.7.6 X X +arrow-arith@55.2.0 X +arrow-array@55.2.0 X +arrow-buffer@55.2.0 X +arrow-cast@55.2.0 X +arrow-data@55.2.0 X +arrow-ipc@55.2.0 X +arrow-ord@55.2.0 X +arrow-schema@55.2.0 X +arrow-select@55.2.0 X +arrow-string@55.2.0 X +as-any@0.3.2 X X +async-lock@3.4.0 X X +async-trait@0.1.88 X X +atoi@2.0.0 X +atomic-waker@1.1.2 X X +autocfg@1.4.0 X X +backon@1.5.1 X +backtrace@0.3.74 X X +base64@0.22.1 X X +bigdecimal@0.4.8 X X +bimap@0.6.3 X X +bitflags@2.8.0 X X +block-buffer@0.10.4 X X +brotli@8.0.1 X X +brotli-decompressor@5.0.0 X X +bumpalo@3.17.0 X X +bytemuck@1.21.0 X X X +byteorder@1.5.0 X X +bytes@1.10.0 X +cc@1.2.14 X X +cfg-if@1.0.0 X X +chrono@0.4.41 X X +concurrent-queue@2.5.0 X X +const-oid@0.9.6 X X +const-random@0.1.18 X X +const-random-macro@0.1.16 X X +core-foundation-sys@0.8.7 X X +core2@0.4.0 X X +cpufeatures@0.2.17 X X +crc32c@0.6.8 X X +crc32fast@1.4.2 X X +crossbeam-channel@0.5.15 X X +crossbeam-epoch@0.9.18 X X +crossbeam-utils@0.8.21 X X +crunchy@0.2.3 X +crypto-common@0.1.6 X X +darling@0.20.10 X +darling_core@0.20.10 X +darling_macro@0.20.10 X +dary_heap@0.3.7 X X +derive_builder@0.20.2 X X +derive_builder_core@0.20.2 X X +derive_builder_macro@0.20.2 X X +digest@0.10.7 X X +displaydoc@0.2.5 X X +dissimilar@1.0.9 X +either@1.15.0 X X +equivalent@1.0.2 X X +event-listener@5.4.0 X X +event-listener-strategy@0.5.3 X X +expect-test@1.5.1 X X +fastrand@2.3.0 X X +flatbuffers@25.2.10 X +flate2@1.1.1 X X +fnv@1.0.7 X X +form_urlencoded@1.2.1 X X +futures@0.3.31 X X +futures-channel@0.3.31 X X +futures-core@0.3.31 X X +futures-executor@0.3.31 X X +futures-io@0.3.31 X X +futures-macro@0.3.31 X X +futures-sink@0.3.31 X X +futures-task@0.3.31 X X +futures-util@0.3.31 X X +generator@0.8.4 X X +generic-array@0.14.7 X +getrandom@0.2.15 X X +getrandom@0.3.1 X X +gimli@0.31.1 X X +gloo-timers@0.3.0 X X +h2@0.4.7 X +half@2.6.0 X X +hashbrown@0.14.5 X X +hashbrown@0.15.2 X X +heck@0.5.0 X X +hermit-abi@0.3.9 X X +hex@0.4.3 X X +hmac@0.12.1 X X +home@0.5.11 X X +http@1.3.1 X X +http-body@1.0.1 X +http-body-util@0.1.2 X +httparse@1.10.0 X X +httpdate@1.0.3 X X +hyper@1.6.0 X +hyper-rustls@0.27.5 X X X +hyper-util@0.1.10 X +iana-time-zone@0.1.61 X X +iana-time-zone-haiku@0.1.2 X X +iceberg@0.6.0 X +iceberg-catalog-loader@0.6.0 X +iceberg-catalog-rest@0.6.0 X +iceberg_test_utils@0.6.0 X +icu_collections@1.5.0 X +icu_locid@1.5.0 X +icu_locid_transform@1.5.0 X +icu_locid_transform_data@1.5.0 X +icu_normalizer@1.5.0 X +icu_normalizer_data@1.5.0 X +icu_properties@1.5.1 X +icu_properties_data@1.5.0 X +icu_provider@1.5.0 X +icu_provider_macros@1.5.0 X +ident_case@1.0.1 X X +idna@1.0.3 X X +idna_adapter@1.2.0 X X +indexmap@2.9.0 X X +integer-encoding@3.0.4 X +io-uring@0.7.8 X X +ipnet@2.11.0 X X +itertools@0.13.0 X X +itoa@1.0.14 X X +jobserver@0.1.32 X X +js-sys@0.3.77 X X +lazy_static@1.5.0 X X +lexical-core@1.0.5 X X +lexical-parse-float@1.0.5 X X +lexical-parse-integer@1.0.5 X X +lexical-util@1.0.6 X X +lexical-write-float@1.0.5 X X +lexical-write-integer@1.0.5 X X +libc@0.2.174 X X +libflate@2.1.0 X +libflate_lz77@2.1.0 X +libm@0.2.11 X X +libz-rs-sys@0.5.0 X +litemap@0.7.4 X +lock_api@0.4.12 X X +log@0.4.25 X X +loom@0.7.2 X +lz4_flex@0.11.3 X +matchers@0.1.0 X +md-5@0.10.6 X X +memchr@2.7.4 X X +mime@0.3.17 X X +miniz_oxide@0.8.8 X X X +mio@1.0.3 X +moka@0.12.10 X X +murmur3@0.5.2 X X +nu-ansi-term@0.46.0 X +num@0.4.3 X X +num-bigint@0.4.6 X X +num-complex@0.4.6 X X +num-integer@0.1.46 X X +num-iter@0.1.45 X X +num-rational@0.4.2 X X +num-traits@0.2.19 X X +num_cpus@1.16.0 X X +object@0.36.7 X X +once_cell@1.21.1 X X +opendal@0.54.0 X +ordered-float@2.10.1 X +ordered-float@4.6.0 X +overload@0.1.1 X +parking@2.2.1 X X +parking_lot@0.12.3 X X +parking_lot_core@0.9.10 X X +parquet@55.2.0 X +paste@1.0.15 X X +percent-encoding@2.3.1 X X +pin-project-lite@0.2.16 X X +pin-utils@0.1.0 X X +pkg-config@0.3.31 X X +portable-atomic@1.10.0 X X +ppv-lite86@0.2.20 X X +proc-macro2@1.0.93 X X +quad-rand@0.2.3 X +quick-xml@0.37.4 X +quote@1.0.40 X X +rand@0.8.5 X X +rand_chacha@0.3.1 X X +rand_core@0.6.4 X X +redox_syscall@0.5.8 X +regex@1.11.1 X X +regex-automata@0.1.10 X X +regex-automata@0.4.9 X X +regex-lite@0.1.6 X X +regex-syntax@0.6.29 X X +regex-syntax@0.8.5 X X +reqsign@0.16.3 X +reqwest@0.12.12 X X +ring@0.17.13 X X +rle-decode-fast@1.0.3 X X +roaring@0.11.2 X X +rust_decimal@1.37.1 X +rustc-demangle@0.1.24 X X +rustc_version@0.4.1 X X +rustls@0.23.23 X X X +rustls-pemfile@2.2.0 X X X +rustls-pki-types@1.11.0 X X +rustls-webpki@0.102.8 X +rustversion@1.0.19 X X +ryu@1.0.19 X X +scoped-tls@1.0.1 X X +scopeguard@1.2.0 X X +semver@1.0.25 X X +seq-macro@0.3.5 X X +serde@1.0.217 X X +serde_bytes@0.11.15 X X +serde_derive@1.0.217 X X +serde_json@1.0.138 X X +serde_repr@0.1.19 X X +serde_urlencoded@0.7.1 X X +serde_with@3.14.0 X X +serde_with_macros@3.14.0 X X +sha1@0.10.6 X X +sha2@0.10.8 X X +sharded-slab@0.1.7 X +shlex@1.3.0 X X +simdutf8@0.1.5 X X +slab@0.4.9 X +smallvec@1.14.0 X X +snap@1.1.1 X +socket2@0.5.8 X X +socket2@0.6.0 X X +stable_deref_trait@1.2.0 X X +static_assertions@1.1.0 X X +strsim@0.11.1 X +strum@0.26.3 X +strum@0.27.1 X +strum_macros@0.26.4 X +strum_macros@0.27.1 X +subtle@2.6.1 X +syn@2.0.101 X X +sync_wrapper@1.0.2 X +synstructure@0.13.1 X +tagptr@0.2.0 X X +thiserror@1.0.69 X X +thiserror-impl@1.0.69 X X +thread_local@1.1.8 X X +threadpool@1.8.1 X X +thrift@0.17.0 X +tiny-keccak@2.0.2 X +tinystr@0.7.6 X +tokio@1.47.0 X +tokio-macros@2.5.0 X +tokio-rustls@0.26.1 X X +tokio-util@0.7.15 X +tower@0.5.2 X +tower-layer@0.3.3 X +tower-service@0.3.3 X +tracing@0.1.41 X +tracing-attributes@0.1.28 X +tracing-core@0.1.33 X +tracing-log@0.2.0 X +tracing-subscriber@0.3.19 X +try-lock@0.2.5 X +twox-hash@1.6.3 X +twox-hash@2.1.0 X +typed-builder@0.19.1 X X +typed-builder@0.20.1 X X +typed-builder-macro@0.19.1 X X +typed-builder-macro@0.20.1 X X +typenum@1.17.0 X X +unicode-ident@1.0.16 X X X +untrusted@0.9.0 X +url@2.5.4 X X +utf16_iter@1.0.5 X X +utf8_iter@1.0.4 X X +uuid@1.17.0 X X +version_check@0.9.5 X X +want@0.3.1 X +wasi@0.11.0+wasi-snapshot-preview1 X X X +wasi@0.13.3+wasi-0.2.2 X X X +wasm-bindgen@0.2.100 X X +wasm-bindgen-backend@0.2.100 X X +wasm-bindgen-futures@0.4.50 X X +wasm-bindgen-macro@0.2.100 X X +wasm-bindgen-macro-support@0.2.100 X X +wasm-bindgen-shared@0.2.100 X X +wasm-streams@0.4.2 X X +web-sys@0.3.77 X X +webpki-roots@0.26.8 X +winapi@0.3.9 X X +winapi-i686-pc-windows-gnu@0.4.0 X X +winapi-x86_64-pc-windows-gnu@0.4.0 X X +windows@0.58.0 X X +windows-core@0.52.0 X X +windows-core@0.58.0 X X +windows-implement@0.58.0 X X +windows-interface@0.58.0 X X +windows-link@0.1.1 X X +windows-registry@0.2.0 X X +windows-result@0.2.0 X X +windows-strings@0.1.0 X X +windows-sys@0.52.0 X X +windows-sys@0.59.0 X X +windows-targets@0.52.6 X X +windows_aarch64_gnullvm@0.52.6 X X +windows_aarch64_msvc@0.52.6 X X +windows_i686_gnu@0.52.6 X X +windows_i686_gnullvm@0.52.6 X X +windows_i686_msvc@0.52.6 X X +windows_x86_64_gnu@0.52.6 X X +windows_x86_64_gnullvm@0.52.6 X X +windows_x86_64_msvc@0.52.6 X X +wit-bindgen-rt@0.33.0 X X X +write16@1.0.0 X X +writeable@0.5.5 X +yoke@0.7.5 X +yoke-derive@0.7.5 X +zerocopy@0.7.35 X X X +zerocopy-derive@0.7.35 X X X +zerofrom@0.1.5 X +zerofrom-derive@0.1.5 X +zeroize@1.8.1 X X +zerovec@0.10.4 X +zerovec-derive@0.10.3 X +zlib-rs@0.5.0 X +zstd@0.13.2 X +zstd-safe@7.2.1 X X +zstd-sys@2.0.13+zstd.1.5.6 X X diff --git a/crates/catalog/loader/src/lib.rs b/crates/catalog/loader/src/lib.rs new file mode 100644 index 0000000000..e5fce46822 --- /dev/null +++ b/crates/catalog/loader/src/lib.rs @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashMap; +use std::sync::Arc; + +use async_trait::async_trait; +use iceberg::{Catalog, CatalogBuilder, Error, ErrorKind, Result}; +use iceberg_catalog_rest::RestCatalogBuilder; + +#[async_trait] +pub trait BoxedCatalogBuilder { + async fn load( + self: Box, + name: String, + props: HashMap, + ) -> Result>; +} + +#[async_trait] +impl BoxedCatalogBuilder for T { + async fn load( + self: Box, + name: String, + props: HashMap, + ) -> Result> { + let builder = *self; + Ok(Arc::new(builder.load(name, props).await?) as Arc) + } +} + +pub fn load(r#type: &str) -> Result> { + match r#type { + "rest" => Ok(Box::new(RestCatalogBuilder::default()) as Box), + _ => Err(Error::new( + ErrorKind::FeatureUnsupported, + format!("Unsupported catalog type: {}", r#type), + )), + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use iceberg_catalog_rest::REST_CATALOG_PROP_URI; + + use crate::load; + + #[tokio::test] + async fn test_load_rest_catalog() { + let catalog_loader = load("rest").unwrap(); + let catalog = catalog_loader + .load( + "rest".to_string(), + HashMap::from([ + ( + REST_CATALOG_PROP_URI.to_string(), + "http://localhost:8080".to_string(), + ), + ("key".to_string(), "value".to_string()), + ]), + ) + .await; + + assert!(catalog.is_ok()); + } +} diff --git a/crates/catalog/rest/src/catalog.rs b/crates/catalog/rest/src/catalog.rs index 7d81982f55..0021ec8bf2 100644 --- a/crates/catalog/rest/src/catalog.rs +++ b/crates/catalog/rest/src/catalog.rs @@ -19,14 +19,15 @@ use std::any::Any; use std::collections::HashMap; +use std::future::Future; use std::str::FromStr; use async_trait::async_trait; use iceberg::io::{self, FileIO}; use iceberg::table::Table; use iceberg::{ - Catalog, Error, ErrorKind, Namespace, NamespaceIdent, Result, TableCommit, TableCreation, - TableIdent, + Catalog, CatalogBuilder, Error, ErrorKind, Namespace, NamespaceIdent, Result, TableCommit, + TableCreation, TableIdent, }; use itertools::Itertools; use reqwest::header::{ @@ -45,13 +46,92 @@ use crate::types::{ RegisterTableRequest, RenameTableRequest, }; +/// REST catalog URI +pub const REST_CATALOG_PROP_URI: &str = "uri"; +/// REST catalog warehouse location +pub const REST_CATALOG_PROP_WAREHOUSE: &str = "warehouse"; + const ICEBERG_REST_SPEC_VERSION: &str = "0.14.1"; const CARGO_PKG_VERSION: &str = env!("CARGO_PKG_VERSION"); const PATH_V1: &str = "v1"; +/// Builder for [`RestCatalog`]. +#[derive(Debug)] +pub struct RestCatalogBuilder(RestCatalogConfig); + +impl Default for RestCatalogBuilder { + fn default() -> Self { + Self(RestCatalogConfig { + name: None, + uri: "".to_string(), + warehouse: None, + props: HashMap::new(), + client: None, + }) + } +} + +impl CatalogBuilder for RestCatalogBuilder { + type C = RestCatalog; + + fn load( + mut self, + name: impl Into, + props: HashMap, + ) -> impl Future> + Send { + self.0.name = Some(name.into()); + + if props.contains_key(REST_CATALOG_PROP_URI) { + self.0.uri = props + .get(REST_CATALOG_PROP_URI) + .cloned() + .unwrap_or_default(); + } + + if props.contains_key(REST_CATALOG_PROP_WAREHOUSE) { + self.0.warehouse = props.get(REST_CATALOG_PROP_WAREHOUSE).cloned() + } + + // Collect other remaining properties + self.0.props = props + .into_iter() + .filter(|(k, _)| k != REST_CATALOG_PROP_URI && k != REST_CATALOG_PROP_WAREHOUSE) + .collect(); + + let result = { + if self.0.name.is_none() { + Err(Error::new( + ErrorKind::DataInvalid, + "Catalog name is required", + )) + } else if self.0.uri.is_empty() { + Err(Error::new( + ErrorKind::DataInvalid, + "Catalog uri is required", + )) + } else { + Ok(RestCatalog::new(self.0)) + } + }; + + std::future::ready(result) + } +} + +impl RestCatalogBuilder { + /// Configures the catalog with a custom HTTP client. + pub fn with_client(mut self, client: Client) -> Self { + self.0.client = Some(client); + self + } +} + /// Rest catalog configuration. #[derive(Clone, Debug, TypedBuilder)] -pub struct RestCatalogConfig { +pub(crate) struct RestCatalogConfig { + #[builder(default, setter(strip_option))] + name: Option, + uri: String, #[builder(default, setter(strip_option(fallback = warehouse_opt)))] @@ -251,7 +331,7 @@ pub struct RestCatalog { impl RestCatalog { /// Creates a `RestCatalog` from a [`RestCatalogConfig`]. - pub fn new(config: RestCatalogConfig) -> Self { + fn new(config: RestCatalogConfig) -> Self { Self { user_config: config, ctx: OnceCell::new(), @@ -2604,4 +2684,54 @@ mod tests { config_mock.assert_async().await; register_table_mock.assert_async().await; } + + #[tokio::test] + async fn test_create_rest_catalog() { + let builder = RestCatalogBuilder::default().with_client(Client::new()); + + let catalog = builder + .load( + "test", + HashMap::from([ + ( + REST_CATALOG_PROP_URI.to_string(), + "http://localhost:8080".to_string(), + ), + ("a".to_string(), "b".to_string()), + ]), + ) + .await; + + assert!(catalog.is_ok()); + + let catalog_config = catalog.unwrap().user_config; + assert_eq!(catalog_config.name.as_deref(), Some("test")); + assert_eq!(catalog_config.uri, "http://localhost:8080"); + assert_eq!(catalog_config.warehouse, None); + assert!(catalog_config.client.is_some()); + + assert_eq!(catalog_config.props.get("a"), Some(&"b".to_string())); + assert!(!catalog_config.props.contains_key(REST_CATALOG_PROP_URI)); + } + + #[tokio::test] + async fn test_create_rest_catalog_no_uri() { + let builder = RestCatalogBuilder::default(); + + let catalog = builder + .load( + "test", + HashMap::from([( + REST_CATALOG_PROP_WAREHOUSE.to_string(), + "s3://warehouse".to_string(), + )]), + ) + .await; + + assert!(catalog.is_err()); + if let Err(err) = catalog { + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert_eq!(err.message(), "Catalog uri is required"); + } + } } diff --git a/crates/catalog/rest/src/lib.rs b/crates/catalog/rest/src/lib.rs index f94ee87815..70cdeaabd0 100644 --- a/crates/catalog/rest/src/lib.rs +++ b/crates/catalog/rest/src/lib.rs @@ -16,6 +16,38 @@ // under the License. //! Iceberg REST API implementation. +//! +//! To build a rest catalog with configurations +//! # Example +//! +//! ```rust, no_run +//! use std::collections::HashMap; +//! +//! use iceberg::CatalogBuilder; +//! use iceberg_catalog_rest::{ +//! REST_CATALOG_PROP_URI, REST_CATALOG_PROP_WAREHOUSE, RestCatalogBuilder, +//! }; +//! +//! #[tokio::main] +//! async fn main() { +//! let catalog = RestCatalogBuilder::default() +//! .load( +//! "rest", +//! HashMap::from([ +//! ( +//! REST_CATALOG_PROP_URI.to_string(), +//! "http://localhost:8181".to_string(), +//! ), +//! ( +//! REST_CATALOG_PROP_WAREHOUSE.to_string(), +//! "s3://warehouse".to_string(), +//! ), +//! ]), +//! ) +//! .await +//! .unwrap(); +//! } +//! ``` #![deny(missing_docs)] diff --git a/crates/catalog/rest/tests/rest_catalog_test.rs b/crates/catalog/rest/tests/rest_catalog_test.rs index 393b243537..7ce76ec344 100644 --- a/crates/catalog/rest/tests/rest_catalog_test.rs +++ b/crates/catalog/rest/tests/rest_catalog_test.rs @@ -24,8 +24,8 @@ use std::sync::RwLock; use ctor::{ctor, dtor}; use iceberg::spec::{FormatVersion, NestedField, PrimitiveType, Schema, Type}; use iceberg::transaction::{ApplyTransactionAction, Transaction}; -use iceberg::{Catalog, Namespace, NamespaceIdent, TableCreation, TableIdent}; -use iceberg_catalog_rest::{RestCatalog, RestCatalogConfig}; +use iceberg::{Catalog, CatalogBuilder, Namespace, NamespaceIdent, TableCreation, TableIdent}; +use iceberg_catalog_rest::{REST_CATALOG_PROP_URI, RestCatalog, RestCatalogBuilder}; use iceberg_test_utils::docker::DockerCompose; use iceberg_test_utils::{normalize_test_name, set_up}; use port_scanner::scan_port_addr; @@ -67,10 +67,16 @@ async fn get_catalog() -> RestCatalog { sleep(std::time::Duration::from_millis(1000)).await; } - let config = RestCatalogConfig::builder() - .uri(format!("http://{}", rest_socket_addr)) - .build(); - RestCatalog::new(config) + RestCatalogBuilder::default() + .load( + "rest", + HashMap::from([( + REST_CATALOG_PROP_URI.to_string(), + format!("http://{}", rest_socket_addr), + )]), + ) + .await + .unwrap() } #[tokio::test] diff --git a/crates/examples/src/oss_backend.rs b/crates/examples/src/oss_backend.rs index c5c955f717..8f0b866d23 100644 --- a/crates/examples/src/oss_backend.rs +++ b/crates/examples/src/oss_backend.rs @@ -18,8 +18,8 @@ use std::collections::HashMap; use futures::stream::StreamExt; -use iceberg::{Catalog, NamespaceIdent, TableIdent}; -use iceberg_catalog_rest::{RestCatalog, RestCatalogConfig}; +use iceberg::{Catalog, CatalogBuilder, NamespaceIdent, TableIdent}; +use iceberg_catalog_rest::{REST_CATALOG_PROP_URI, RestCatalogBuilder}; // Configure these values according to your environment @@ -42,24 +42,27 @@ static OSS_ACCESS_KEY_SECRET: &str = "99999999999999999999999999999999"; #[tokio::main] async fn main() { // Create the REST iceberg catalog. - let config = RestCatalogConfig::builder() - .uri(REST_URI.to_string()) - .props(HashMap::from([ - ( - iceberg::io::OSS_ENDPOINT.to_string(), - OSS_ENDPOINT.to_string(), - ), - ( - iceberg::io::OSS_ACCESS_KEY_ID.to_string(), - OSS_ACCESS_KEY_ID.to_string(), - ), - ( - iceberg::io::OSS_ACCESS_KEY_SECRET.to_string(), - OSS_ACCESS_KEY_SECRET.to_string(), - ), - ])) - .build(); - let catalog = RestCatalog::new(config); + let catalog = RestCatalogBuilder::default() + .load( + "rest", + HashMap::from([ + (REST_CATALOG_PROP_URI.to_string(), REST_URI.to_string()), + ( + iceberg::io::OSS_ENDPOINT.to_string(), + OSS_ENDPOINT.to_string(), + ), + ( + iceberg::io::OSS_ACCESS_KEY_ID.to_string(), + OSS_ACCESS_KEY_ID.to_string(), + ), + ( + iceberg::io::OSS_ACCESS_KEY_SECRET.to_string(), + OSS_ACCESS_KEY_SECRET.to_string(), + ), + ]), + ) + .await + .unwrap(); // Create the table identifier. let namespace_ident = NamespaceIdent::from_vec(vec![NAMESPACE.to_string()]).unwrap(); diff --git a/crates/examples/src/rest_catalog_namespace.rs b/crates/examples/src/rest_catalog_namespace.rs index 04cad54fab..27196a2696 100644 --- a/crates/examples/src/rest_catalog_namespace.rs +++ b/crates/examples/src/rest_catalog_namespace.rs @@ -17,8 +17,8 @@ use std::collections::HashMap; -use iceberg::{Catalog, NamespaceIdent}; -use iceberg_catalog_rest::{RestCatalog, RestCatalogConfig}; +use iceberg::{Catalog, CatalogBuilder, NamespaceIdent}; +use iceberg_catalog_rest::{REST_CATALOG_PROP_URI, RestCatalogBuilder}; static REST_URI: &str = "http://localhost:8181"; @@ -33,10 +33,13 @@ static REST_URI: &str = "http://localhost:8181"; async fn main() { // ANCHOR: create_catalog // Create the REST iceberg catalog. - let config = RestCatalogConfig::builder() - .uri(REST_URI.to_string()) - .build(); - let catalog = RestCatalog::new(config); + let catalog = RestCatalogBuilder::default() + .load( + "rest", + HashMap::from([(REST_CATALOG_PROP_URI.to_string(), REST_URI.to_string())]), + ) + .await + .unwrap(); // ANCHOR_END: create_catalog // ANCHOR: list_all_namespace diff --git a/crates/examples/src/rest_catalog_table.rs b/crates/examples/src/rest_catalog_table.rs index 9597576580..306d886ae6 100644 --- a/crates/examples/src/rest_catalog_table.rs +++ b/crates/examples/src/rest_catalog_table.rs @@ -18,8 +18,8 @@ use std::collections::HashMap; use iceberg::spec::{NestedField, PrimitiveType, Schema, Type}; -use iceberg::{Catalog, NamespaceIdent, TableCreation, TableIdent}; -use iceberg_catalog_rest::{RestCatalog, RestCatalogConfig}; +use iceberg::{Catalog, CatalogBuilder, NamespaceIdent, TableCreation, TableIdent}; +use iceberg_catalog_rest::{REST_CATALOG_PROP_URI, RestCatalogBuilder}; static REST_URI: &str = "http://localhost:8181"; static NAMESPACE: &str = "default"; @@ -35,10 +35,13 @@ static TABLE_NAME: &str = "t1"; #[tokio::main] async fn main() { // Create the REST iceberg catalog. - let config = RestCatalogConfig::builder() - .uri(REST_URI.to_string()) - .build(); - let catalog = RestCatalog::new(config); + let catalog = RestCatalogBuilder::default() + .load( + "rest", + HashMap::from([(REST_CATALOG_PROP_URI.to_string(), REST_URI.to_string())]), + ) + .await + .unwrap(); // ANCHOR: create_table // Create the table identifier. diff --git a/crates/integration_tests/src/lib.rs b/crates/integration_tests/src/lib.rs index 1d2d5dc1ec..422cd55863 100644 --- a/crates/integration_tests/src/lib.rs +++ b/crates/integration_tests/src/lib.rs @@ -18,7 +18,7 @@ use std::collections::HashMap; use iceberg::io::{S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_REGION, S3_SECRET_ACCESS_KEY}; -use iceberg_catalog_rest::RestCatalogConfig; +use iceberg_catalog_rest::REST_CATALOG_PROP_URI; use iceberg_test_utils::docker::DockerCompose; use iceberg_test_utils::{normalize_test_name, set_up}; @@ -26,7 +26,7 @@ const REST_CATALOG_PORT: u16 = 8181; pub struct TestFixture { pub _docker_compose: DockerCompose, - pub catalog_config: RestCatalogConfig, + pub catalog_config: HashMap, } pub fn set_test_fixture(func: &str) -> TestFixture { @@ -43,18 +43,19 @@ pub fn set_test_fixture(func: &str) -> TestFixture { let rest_catalog_ip = docker_compose.get_container_ip("rest"); let minio_ip = docker_compose.get_container_ip("minio"); - let catalog_config = RestCatalogConfig::builder() - .uri(format!("http://{}:{}", rest_catalog_ip, REST_CATALOG_PORT)) - .props(HashMap::from([ - ( - S3_ENDPOINT.to_string(), - format!("http://{}:{}", minio_ip, 9000), - ), - (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), - (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), - (S3_REGION.to_string(), "us-east-1".to_string()), - ])) - .build(); + let catalog_config = HashMap::from([ + ( + REST_CATALOG_PROP_URI.to_string(), + format!("http://{}:{}", rest_catalog_ip, REST_CATALOG_PORT), + ), + ( + S3_ENDPOINT.to_string(), + format!("http://{}:{}", minio_ip, 9000), + ), + (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), + (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), + (S3_REGION.to_string(), "us-east-1".to_string()), + ]); TestFixture { _docker_compose: docker_compose, diff --git a/crates/integration_tests/tests/shared_tests/append_data_file_test.rs b/crates/integration_tests/tests/shared_tests/append_data_file_test.rs index dd54ac98f4..4550db1513 100644 --- a/crates/integration_tests/tests/shared_tests/append_data_file_test.rs +++ b/crates/integration_tests/tests/shared_tests/append_data_file_test.rs @@ -28,8 +28,8 @@ use iceberg::writer::file_writer::location_generator::{ DefaultFileNameGenerator, DefaultLocationGenerator, }; use iceberg::writer::{IcebergWriter, IcebergWriterBuilder}; -use iceberg::{Catalog, TableCreation}; -use iceberg_catalog_rest::RestCatalog; +use iceberg::{Catalog, CatalogBuilder, TableCreation}; +use iceberg_catalog_rest::RestCatalogBuilder; use parquet::arrow::arrow_reader::ArrowReaderOptions; use parquet::file::properties::WriterProperties; @@ -39,7 +39,10 @@ use crate::shared_tests::{random_ns, test_schema}; #[tokio::test] async fn test_append_data_file() { let fixture = get_shared_containers(); - let rest_catalog = RestCatalog::new(fixture.catalog_config.clone()); + let rest_catalog = RestCatalogBuilder::default() + .load("rest", fixture.catalog_config.clone()) + .await + .unwrap(); let ns = random_ns().await; let schema = test_schema(); diff --git a/crates/integration_tests/tests/shared_tests/append_partition_data_file_test.rs b/crates/integration_tests/tests/shared_tests/append_partition_data_file_test.rs index 36a1c3643f..2c16f0f0df 100644 --- a/crates/integration_tests/tests/shared_tests/append_partition_data_file_test.rs +++ b/crates/integration_tests/tests/shared_tests/append_partition_data_file_test.rs @@ -30,8 +30,8 @@ use iceberg::writer::file_writer::location_generator::{ DefaultFileNameGenerator, DefaultLocationGenerator, }; use iceberg::writer::{IcebergWriter, IcebergWriterBuilder}; -use iceberg::{Catalog, TableCreation}; -use iceberg_catalog_rest::RestCatalog; +use iceberg::{Catalog, CatalogBuilder, TableCreation}; +use iceberg_catalog_rest::RestCatalogBuilder; use parquet::file::properties::WriterProperties; use crate::get_shared_containers; @@ -40,7 +40,10 @@ use crate::shared_tests::{random_ns, test_schema}; #[tokio::test] async fn test_append_partition_data_file() { let fixture = get_shared_containers(); - let rest_catalog = RestCatalog::new(fixture.catalog_config.clone()); + let rest_catalog = RestCatalogBuilder::default() + .load("rest", fixture.catalog_config.clone()) + .await + .unwrap(); let ns = random_ns().await; let schema = test_schema(); diff --git a/crates/integration_tests/tests/shared_tests/conflict_commit_test.rs b/crates/integration_tests/tests/shared_tests/conflict_commit_test.rs index 551116bf91..9ca4e215e1 100644 --- a/crates/integration_tests/tests/shared_tests/conflict_commit_test.rs +++ b/crates/integration_tests/tests/shared_tests/conflict_commit_test.rs @@ -28,8 +28,8 @@ use iceberg::writer::file_writer::location_generator::{ DefaultFileNameGenerator, DefaultLocationGenerator, }; use iceberg::writer::{IcebergWriter, IcebergWriterBuilder}; -use iceberg::{Catalog, TableCreation}; -use iceberg_catalog_rest::RestCatalog; +use iceberg::{Catalog, CatalogBuilder, TableCreation}; +use iceberg_catalog_rest::RestCatalogBuilder; use parquet::file::properties::WriterProperties; use crate::get_shared_containers; @@ -38,7 +38,10 @@ use crate::shared_tests::{random_ns, test_schema}; #[tokio::test] async fn test_append_data_file_conflict() { let fixture = get_shared_containers(); - let rest_catalog = RestCatalog::new(fixture.catalog_config.clone()); + let rest_catalog = RestCatalogBuilder::default() + .load("rest", fixture.catalog_config.clone()) + .await + .unwrap(); let ns = random_ns().await; let schema = test_schema(); diff --git a/crates/integration_tests/tests/shared_tests/datafusion.rs b/crates/integration_tests/tests/shared_tests/datafusion.rs index badb6496fc..81bbb5f54c 100644 --- a/crates/integration_tests/tests/shared_tests/datafusion.rs +++ b/crates/integration_tests/tests/shared_tests/datafusion.rs @@ -24,8 +24,8 @@ use datafusion::assert_batches_eq; use datafusion::catalog::TableProvider; use datafusion::error::DataFusionError; use datafusion::prelude::SessionContext; -use iceberg::{Catalog, TableIdent}; -use iceberg_catalog_rest::RestCatalog; +use iceberg::{Catalog, CatalogBuilder, TableIdent}; +use iceberg_catalog_rest::RestCatalogBuilder; use iceberg_datafusion::IcebergTableProvider; use parquet::arrow::PARQUET_FIELD_ID_META_KEY; @@ -34,7 +34,10 @@ use crate::get_shared_containers; #[tokio::test] async fn test_basic_queries() -> Result<(), DataFusionError> { let fixture = get_shared_containers(); - let rest_catalog = RestCatalog::new(fixture.catalog_config.clone()); + let rest_catalog = RestCatalogBuilder::default() + .load("rest", fixture.catalog_config.clone()) + .await + .unwrap(); let table = rest_catalog .load_table(&TableIdent::from_strs(["default", "types_test"]).unwrap()) diff --git a/crates/integration_tests/tests/shared_tests/mod.rs b/crates/integration_tests/tests/shared_tests/mod.rs index feb1c4e585..065b14d5da 100644 --- a/crates/integration_tests/tests/shared_tests/mod.rs +++ b/crates/integration_tests/tests/shared_tests/mod.rs @@ -18,8 +18,8 @@ use std::collections::HashMap; use iceberg::spec::{NestedField, PrimitiveType, Schema, Type}; -use iceberg::{Catalog, Namespace, NamespaceIdent}; -use iceberg_catalog_rest::RestCatalog; +use iceberg::{Catalog, CatalogBuilder, Namespace, NamespaceIdent}; +use iceberg_catalog_rest::RestCatalogBuilder; use crate::get_shared_containers; @@ -33,7 +33,10 @@ mod scan_all_type; pub async fn random_ns() -> Namespace { let fixture = get_shared_containers(); - let rest_catalog = RestCatalog::new(fixture.catalog_config.clone()); + let rest_catalog = RestCatalogBuilder::default() + .load("rest", fixture.catalog_config.clone()) + .await + .unwrap(); let ns = Namespace::with_properties( NamespaceIdent::from_strs([uuid::Uuid::new_v4().to_string()]).unwrap(), diff --git a/crates/integration_tests/tests/shared_tests/read_evolved_schema.rs b/crates/integration_tests/tests/shared_tests/read_evolved_schema.rs index 94e2fbf9c1..fe8a488fa3 100644 --- a/crates/integration_tests/tests/shared_tests/read_evolved_schema.rs +++ b/crates/integration_tests/tests/shared_tests/read_evolved_schema.rs @@ -21,8 +21,8 @@ use arrow_array::{Decimal128Array, Float64Array, Int64Array, StringArray}; use futures::TryStreamExt; use iceberg::expr::Reference; use iceberg::spec::Datum; -use iceberg::{Catalog, TableIdent}; -use iceberg_catalog_rest::RestCatalog; +use iceberg::{Catalog, CatalogBuilder, TableIdent}; +use iceberg_catalog_rest::RestCatalogBuilder; use ordered_float::OrderedFloat; use crate::get_shared_containers; @@ -30,7 +30,10 @@ use crate::get_shared_containers; #[tokio::test] async fn test_evolved_schema() { let fixture = get_shared_containers(); - let rest_catalog = RestCatalog::new(fixture.catalog_config.clone()); + let rest_catalog = RestCatalogBuilder::default() + .load("rest", fixture.catalog_config.clone()) + .await + .unwrap(); let table = rest_catalog .load_table(&TableIdent::from_strs(["default", "test_rename_column"]).unwrap()) diff --git a/crates/integration_tests/tests/shared_tests/read_positional_deletes.rs b/crates/integration_tests/tests/shared_tests/read_positional_deletes.rs index f631586edc..565f8ba427 100644 --- a/crates/integration_tests/tests/shared_tests/read_positional_deletes.rs +++ b/crates/integration_tests/tests/shared_tests/read_positional_deletes.rs @@ -18,15 +18,18 @@ //! Integration tests for rest catalog. use futures::TryStreamExt; -use iceberg::{Catalog, TableIdent}; -use iceberg_catalog_rest::RestCatalog; +use iceberg::{Catalog, CatalogBuilder, TableIdent}; +use iceberg_catalog_rest::RestCatalogBuilder; use crate::get_shared_containers; #[tokio::test] async fn test_read_table_with_positional_deletes() { let fixture = get_shared_containers(); - let rest_catalog = RestCatalog::new(fixture.catalog_config.clone()); + let rest_catalog = RestCatalogBuilder::default() + .load("rest", fixture.catalog_config.clone()) + .await + .unwrap(); let table = rest_catalog .load_table( diff --git a/crates/integration_tests/tests/shared_tests/scan_all_type.rs b/crates/integration_tests/tests/shared_tests/scan_all_type.rs index 51f4093927..4ddaa8138a 100644 --- a/crates/integration_tests/tests/shared_tests/scan_all_type.rs +++ b/crates/integration_tests/tests/shared_tests/scan_all_type.rs @@ -40,8 +40,8 @@ use iceberg::writer::file_writer::location_generator::{ DefaultFileNameGenerator, DefaultLocationGenerator, }; use iceberg::writer::{IcebergWriter, IcebergWriterBuilder}; -use iceberg::{Catalog, TableCreation}; -use iceberg_catalog_rest::RestCatalog; +use iceberg::{Catalog, CatalogBuilder, TableCreation}; +use iceberg_catalog_rest::RestCatalogBuilder; use parquet::arrow::PARQUET_FIELD_ID_META_KEY; use parquet::file::properties::WriterProperties; use uuid::Uuid; @@ -52,7 +52,10 @@ use crate::shared_tests::random_ns; #[tokio::test] async fn test_scan_all_type() { let fixture = get_shared_containers(); - let rest_catalog = RestCatalog::new(fixture.catalog_config.clone()); + let rest_catalog = RestCatalogBuilder::default() + .load("rest", fixture.catalog_config.clone()) + .await + .unwrap(); let ns = random_ns().await; let schema = Schema::builder() diff --git a/crates/integrations/playground/Cargo.toml b/crates/integrations/playground/Cargo.toml index ae66bb5417..8ba983d058 100644 --- a/crates/integrations/playground/Cargo.toml +++ b/crates/integrations/playground/Cargo.toml @@ -33,6 +33,7 @@ datafusion = { workspace = true } datafusion-cli = { workspace = true } dirs = { workspace = true } fs-err = { workspace = true } +iceberg = { workspace = true } iceberg-catalog-rest = { workspace = true } iceberg-datafusion = { workspace = true } mimalloc = { workspace = true } diff --git a/crates/integrations/playground/src/catalog.rs b/crates/integrations/playground/src/catalog.rs index e9450f13fe..f7fd14ca84 100644 --- a/crates/integrations/playground/src/catalog.rs +++ b/crates/integrations/playground/src/catalog.rs @@ -23,7 +23,8 @@ use std::sync::Arc; use anyhow::anyhow; use datafusion::catalog::{CatalogProvider, CatalogProviderList}; use fs_err::read_to_string; -use iceberg_catalog_rest::{RestCatalog, RestCatalogConfig}; +use iceberg::CatalogBuilder; +use iceberg_catalog_rest::RestCatalogBuilder; use iceberg_datafusion::IcebergCatalogProvider; use toml::{Table as TomlTable, Value}; @@ -87,44 +88,19 @@ impl IcebergCatalogList { .as_table() .ok_or_else(|| anyhow::anyhow!("config is not table for catalog {name}"))?; - let uri = catalog_config - .get("uri") - .ok_or_else(|| anyhow::anyhow!("uri not found for catalog {name}"))? - .as_str() - .ok_or_else(|| anyhow::anyhow!("uri is not string"))?; - - let warehouse = catalog_config - .get("warehouse") - .ok_or_else(|| anyhow::anyhow!("warehouse not found for catalog {name}"))? - .as_str() - .ok_or_else(|| anyhow::anyhow!("warehouse is not string for catalog {name}"))?; - - let props_table = catalog_config - .get("props") - .ok_or_else(|| anyhow::anyhow!("props not found for catalog {name}"))? - .as_table() - .ok_or_else(|| anyhow::anyhow!("props is not table for catalog {name}"))?; - - let mut props = HashMap::with_capacity(props_table.len()); - for (key, value) in props_table { + // parse all config into props + let mut props = HashMap::with_capacity(catalog_config.len()); + for (key, value) in catalog_config { let value_str = value .as_str() .ok_or_else(|| anyhow::anyhow!("props {key} is not string"))?; props.insert(key.to_string(), value_str.to_string()); } - - let rest_catalog_config = RestCatalogConfig::builder() - .uri(uri.to_string()) - .warehouse(warehouse.to_string()) - .props(props) - .build(); + let catalog = RestCatalogBuilder::default().load(name, props).await?; Ok(( name.to_string(), - Arc::new( - IcebergCatalogProvider::try_new(Arc::new(RestCatalog::new(rest_catalog_config))) - .await?, - ), + Arc::new(IcebergCatalogProvider::try_new(Arc::new(catalog)).await?), )) } }