Skip to content

feat: builtin descriptions & examples #2178

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 14 commits into from
Dec 4, 2023
Merged

feat: builtin descriptions & examples #2178

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
9f91f6a
wip
universalmind303 Nov 28, 2023
45e7189
wip
universalmind303 Nov 29, 2023
8047bb5
add more docs & examples
universalmind303 Nov 29, 2023
c55140f
add simple slts
universalmind303 Nov 29, 2023
11e5b35
clippy
universalmind303 Nov 29, 2023
a192656
update bad comment string
universalmind303 Nov 29, 2023
5095073
fix merge
universalmind303 Nov 30, 2023
05c2f74
address some comments
universalmind303 Nov 30, 2023
90dbfc7
address some comments
universalmind303 Nov 30, 2023
db2a2a2
address some more comments and some refactoring
universalmind303 Nov 30, 2023
b8a65d7
clippy
universalmind303 Dec 1, 2023
8796467
merge conflicts
universalmind303 Dec 1, 2023
a128027
pr feedback
universalmind303 Dec 4, 2023
de78bdb
add docs for to_hex
universalmind303 Dec 4, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions crates/catalog/src/session_catalog.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -446,6 +446,8 @@ impl TempCatalog {
builtin: false,
external: false,
is_temp: true,
sql_example: None,
description: None,
},
options: TableOptions::Internal(TableOptionsInternal { columns }),
tunnel_id: None,
Expand Down Expand Up @@ -487,6 +489,8 @@ impl TempCatalog {
builtin: false,
external: false,
is_temp: true,
sql_example: None,
description: None,
},
options: TableOptions::Internal(TableOptionsInternal {
columns: Vec::new(),
Expand Down
37 changes: 2 additions & 35 deletions crates/datafusion_ext/src/functions.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,53 +1,20 @@
use std::collections::HashMap;
use std::fmt;
use std::fmt::Display;
use std::sync::Arc;
use std::fmt::{self, Display};

use crate::errors::{ExtensionError, Result};
use crate::vars::SessionVars;
use async_trait::async_trait;
use catalog::session_catalog::SessionCatalog;
use datafusion::arrow::datatypes::{Field, Fields};
use datafusion::datasource::TableProvider;
use datafusion::execution::context::SessionState;
use datafusion::logical_expr::Signature;
use datafusion::prelude::SessionContext;
use datafusion::scalar::ScalarValue;
use decimal::Decimal128;
use protogen::metastore::types::catalog::{EntryType, RuntimePreference};
use protogen::metastore::types::catalog::EntryType;
use protogen::rpcsrv::types::func_param_value::{
FuncParamValue as ProtoFuncParamValue, FuncParamValueArrayVariant,
FuncParamValueEnum as ProtoFuncParamValueEnum,
};

#[async_trait]
pub trait TableFunc: Sync + Send {
/// The name for this table function. This name will be used when looking up
/// function implementations.
fn name(&self) -> &str;
fn runtime_preference(&self) -> RuntimePreference;
fn detect_runtime(
&self,
_args: &[FuncParamValue],
_parent: RuntimePreference,
) -> Result<RuntimePreference> {
Ok(self.runtime_preference())
}

/// Return a table provider using the provided args.
async fn create_provider(
&self,
ctx: &dyn TableFuncContextProvider,
args: Vec<FuncParamValue>,
opts: HashMap<String, FuncParamValue>,
) -> Result<Arc<dyn TableProvider>>;
/// Return the signature for this function.
/// Defaults to None.
// TODO: Remove the default impl once we have `signature` implemented for all functions
fn signature(&self) -> Option<Signature> {
None
}
}
pub trait TableFuncContextProvider: Sync + Send {
/// Get a reference to the session catalog.
fn get_session_catalog(&self) -> &SessionCatalog;
Expand Down
2 changes: 2 additions & 0 deletions crates/datasources/src/native/access.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -363,6 +363,8 @@ mod tests {
builtin: false,
external: false,
is_temp: false,
description: None,
sql_example: None,
},
options: TableOptions::Internal(TableOptionsInternal {
columns: vec![InternalColumnDefinition {
Expand Down
100 changes: 32 additions & 68 deletions crates/metastore/src/database.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,7 @@ use once_cell::sync::Lazy;
use pgrepr::oid::FIRST_AVAILABLE_ID;
use protogen::metastore::types::catalog::{
CatalogEntry, CatalogState, CredentialsEntry, DatabaseEntry, DeploymentMetadata, EntryMeta,
EntryType, FunctionEntry, FunctionType, RuntimePreference, SchemaEntry, SourceAccessMode,
TableEntry, TunnelEntry, ViewEntry,
EntryType, SchemaEntry, SourceAccessMode, TableEntry, TunnelEntry, ViewEntry,
};
use protogen::metastore::types::options::{
DatabaseOptions, DatabaseOptionsInternal, TableOptions, TunnelOptions,
Expand All @@ -17,7 +16,7 @@ use sqlbuiltins::builtins::{
BuiltinDatabase, BuiltinSchema, BuiltinTable, BuiltinView, DATABASE_DEFAULT, DEFAULT_SCHEMA,
FIRST_NON_SCHEMA_ID,
};
use sqlbuiltins::functions::{BUILTIN_AGGREGATE_FUNCS, BUILTIN_SCALAR_FUNCS, BUILTIN_TABLE_FUNCS};
use sqlbuiltins::functions::{BUILTIN_FUNCS, BUILTIN_TABLE_FUNCS};
use sqlbuiltins::validation::{
validate_database_tunnel_support, validate_object_name, validate_table_tunnel_support,
};
Expand Down Expand Up @@ -670,6 +669,8 @@ impl State {
builtin: false,
external: true,
is_temp: false,
sql_example: None,
description: None,
},
options: create_database.options,
tunnel_id,
Expand Down Expand Up @@ -700,6 +701,8 @@ impl State {
builtin: false,
external: false,
is_temp: false,
sql_example: None,
description: None,
},
options: create_tunnel.options,
};
Expand Down Expand Up @@ -735,6 +738,8 @@ impl State {
builtin: false,
external: false,
is_temp: false,
sql_example: None,
description: None,
},
options: create_credentials.options,
comment: create_credentials.comment,
Expand Down Expand Up @@ -766,6 +771,8 @@ impl State {
builtin: false,
external: false,
is_temp: false,
sql_example: None,
description: None,
},
options: create_credential.options,
comment: create_credential.comment,
Expand Down Expand Up @@ -798,6 +805,8 @@ impl State {
builtin: false,
external: false,
is_temp: false,
sql_example: None,
description: None,
},
};
self.entries.insert(oid, CatalogEntry::Schema(ent))?;
Expand All @@ -821,6 +830,8 @@ impl State {
builtin: false,
external: false,
is_temp: false,
sql_example: None,
description: None,
},
sql: create_view.sql,
columns: create_view.columns,
Expand Down Expand Up @@ -851,6 +862,8 @@ impl State {
builtin: false,
external: false,
is_temp: false,
sql_example: None,
description: None,
},
options: TableOptions::Internal(create_table.options),
tunnel_id: None,
Expand Down Expand Up @@ -892,6 +905,8 @@ impl State {
builtin: false,
external: true,
is_temp: false,
sql_example: None,
description: None,
},
options: create_ext.options,
tunnel_id,
Expand Down Expand Up @@ -1170,6 +1185,8 @@ impl BuiltinCatalog {
builtin: true,
external: false,
is_temp: false,
sql_example: None,
description: None,
},
options: DatabaseOptions::Internal(DatabaseOptionsInternal {}),
tunnel_id: None,
Expand All @@ -1192,6 +1209,8 @@ impl BuiltinCatalog {
builtin: true,
external: false,
is_temp: false,
sql_example: None,
description: None,
},
}),
);
Expand All @@ -1215,6 +1234,8 @@ impl BuiltinCatalog {
builtin: true,
external: false,
is_temp: false,
sql_example: None,
description: None,
},
options: TableOptions::new_internal(table.columns.clone()),
tunnel_id: None,
Expand Down Expand Up @@ -1245,6 +1266,8 @@ impl BuiltinCatalog {
builtin: true,
external: false,
is_temp: false,
sql_example: None,
description: None,
},
sql: view.sql.to_string(),
columns: Vec::new(),
Expand All @@ -1264,24 +1287,10 @@ impl BuiltinCatalog {
let schema_id = schema_names
.get(DEFAULT_SCHEMA)
.ok_or_else(|| MetastoreError::MissingNamedSchema(DEFAULT_SCHEMA.to_string()))?;
let mut entry = func.as_function_entry(oid, *schema_id);
entry.runtime_preference = func.runtime_preference();

entries.insert(
oid,
CatalogEntry::Function(FunctionEntry {
meta: EntryMeta {
entry_type: EntryType::Function,
id: oid,
parent: *schema_id,
name: func.name().to_string(),
builtin: true,
external: false,
is_temp: false,
},
func_type: FunctionType::TableReturning,
runtime_preference: func.runtime_preference(),
signature: func.signature(),
}),
);
entries.insert(oid, CatalogEntry::Function(entry));
schema_objects
.get_mut(schema_id)
.unwrap()
Expand All @@ -1290,66 +1299,21 @@ impl BuiltinCatalog {

oid += 1;
}

for func in BUILTIN_SCALAR_FUNCS.iter_funcs() {
// Put them all in the default schema.
let schema_id = schema_names
.get(DEFAULT_SCHEMA)
.ok_or_else(|| MetastoreError::MissingNamedSchema(DEFAULT_SCHEMA.to_string()))?;

entries.insert(
oid,
CatalogEntry::Function(FunctionEntry {
meta: EntryMeta {
entry_type: EntryType::Function,
id: oid,
parent: *schema_id,
name: func.to_string(),
builtin: true,
external: false,
is_temp: false,
},
func_type: FunctionType::Scalar,
runtime_preference: RuntimePreference::Unspecified,
signature: Some(func.signature()),
}),
);
schema_objects
.get_mut(schema_id)
.unwrap()
.functions
.insert(func.to_string(), oid);

oid += 1;
}
for func in BUILTIN_AGGREGATE_FUNCS.iter_funcs() {
for func in BUILTIN_FUNCS.iter_funcs() {
// Put them all in the default schema.
let schema_id = schema_names
.get(DEFAULT_SCHEMA)
.ok_or_else(|| MetastoreError::MissingNamedSchema(DEFAULT_SCHEMA.to_string()))?;

entries.insert(
oid,
CatalogEntry::Function(FunctionEntry {
meta: EntryMeta {
entry_type: EntryType::Function,
id: oid,
parent: *schema_id,
name: func.to_string(),
builtin: true,
external: false,
is_temp: false,
},
func_type: FunctionType::Aggregate,
runtime_preference: RuntimePreference::Unspecified,
signature: Some(func.signature()),
}),
CatalogEntry::Function(func.as_function_entry(oid, *schema_id)),
);
schema_objects
.get_mut(schema_id)
.unwrap()
.functions
.insert(func.to_string().to_ascii_uppercase(), oid);
.insert(func.name().to_string(), oid);

oid += 1;
}
Expand Down
6 changes: 5 additions & 1 deletion crates/protogen/proto/metastore/catalog.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,11 @@ message EntryMeta {
// Temp objects should never be persisted.
bool is_temp = 7;

// next: 8
// Optional sql example string.
optional string sql_example = 8;
// Optional description string
optional string description = 9;
// next: 10
}

// Defines what kind of access is allowed on the data source.
Expand Down
6 changes: 6 additions & 0 deletions crates/protogen/src/metastore/types/catalog.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -266,6 +266,8 @@ pub struct EntryMeta {
pub builtin: bool,
pub external: bool,
pub is_temp: bool,
pub sql_example: Option<String>,
pub description: Option<String>,
}

impl From<EntryMeta> for catalog::EntryMeta {
Expand All @@ -279,6 +281,8 @@ impl From<EntryMeta> for catalog::EntryMeta {
builtin: value.builtin,
external: value.external,
is_temp: value.is_temp,
sql_example: value.sql_example,
description: value.description,
}
}
}
Expand All @@ -294,6 +298,8 @@ impl TryFrom<catalog::EntryMeta> for EntryMeta {
builtin: value.builtin,
external: value.external,
is_temp: value.is_temp,
sql_example: value.sql_example,
description: value.description,
})
}
}
Expand Down
Loading