diff --git a/datafusion-postgres/export_pg_catalog_arrow.sh b/datafusion-postgres/export_pg_catalog_arrow.sh index 10ac861..8ea42b6 100755 --- a/datafusion-postgres/export_pg_catalog_arrow.sh +++ b/datafusion-postgres/export_pg_catalog_arrow.sh @@ -173,7 +173,7 @@ def pg_type_to_arrow_type(pg_type, is_nullable=True): 'character varying': pa.string(), 'character': pa.string(), 'name': pa.string(), - 'oid': pa.uint32(), # OIDs are unsigned + 'oid': pa.int32(), # OIDs are unsigned 'regproc': pa.string(), 'regtype': pa.string(), 'regclass': pa.string(), diff --git a/datafusion-postgres/pg_catalog_arrow_exports/pg_aggregate.feather b/datafusion-postgres/pg_catalog_arrow_exports/pg_aggregate.feather index 7e69410..68de2ee 100644 Binary files a/datafusion-postgres/pg_catalog_arrow_exports/pg_aggregate.feather and b/datafusion-postgres/pg_catalog_arrow_exports/pg_aggregate.feather differ diff --git a/datafusion-postgres/pg_catalog_arrow_exports/pg_am.feather b/datafusion-postgres/pg_catalog_arrow_exports/pg_am.feather index 372445a..1eb3406 100644 Binary files a/datafusion-postgres/pg_catalog_arrow_exports/pg_am.feather and b/datafusion-postgres/pg_catalog_arrow_exports/pg_am.feather differ diff --git a/datafusion-postgres/pg_catalog_arrow_exports/pg_amop.feather b/datafusion-postgres/pg_catalog_arrow_exports/pg_amop.feather index 8919bc5..6027908 100644 Binary files a/datafusion-postgres/pg_catalog_arrow_exports/pg_amop.feather and b/datafusion-postgres/pg_catalog_arrow_exports/pg_amop.feather differ diff --git a/datafusion-postgres/pg_catalog_arrow_exports/pg_amproc.feather b/datafusion-postgres/pg_catalog_arrow_exports/pg_amproc.feather index f9149d4..94fac52 100644 Binary files a/datafusion-postgres/pg_catalog_arrow_exports/pg_amproc.feather and b/datafusion-postgres/pg_catalog_arrow_exports/pg_amproc.feather differ diff --git a/datafusion-postgres/pg_catalog_arrow_exports/pg_cast.feather b/datafusion-postgres/pg_catalog_arrow_exports/pg_cast.feather index 60d6285..9edd104 100644 Binary files a/datafusion-postgres/pg_catalog_arrow_exports/pg_cast.feather and b/datafusion-postgres/pg_catalog_arrow_exports/pg_cast.feather differ diff --git a/datafusion-postgres/pg_catalog_arrow_exports/pg_collation.feather b/datafusion-postgres/pg_catalog_arrow_exports/pg_collation.feather index c42fcee..bfd5b37 100644 Binary files a/datafusion-postgres/pg_catalog_arrow_exports/pg_collation.feather and b/datafusion-postgres/pg_catalog_arrow_exports/pg_collation.feather differ diff --git a/datafusion-postgres/pg_catalog_arrow_exports/pg_conversion.feather b/datafusion-postgres/pg_catalog_arrow_exports/pg_conversion.feather index 308f777..7118783 100644 Binary files a/datafusion-postgres/pg_catalog_arrow_exports/pg_conversion.feather and b/datafusion-postgres/pg_catalog_arrow_exports/pg_conversion.feather differ diff --git a/datafusion-postgres/pg_catalog_arrow_exports/pg_language.feather b/datafusion-postgres/pg_catalog_arrow_exports/pg_language.feather index aad59be..b1a124b 100644 Binary files a/datafusion-postgres/pg_catalog_arrow_exports/pg_language.feather and b/datafusion-postgres/pg_catalog_arrow_exports/pg_language.feather differ diff --git a/datafusion-postgres/pg_catalog_arrow_exports/pg_opclass.feather b/datafusion-postgres/pg_catalog_arrow_exports/pg_opclass.feather index c353565..3784878 100644 Binary files a/datafusion-postgres/pg_catalog_arrow_exports/pg_opclass.feather and b/datafusion-postgres/pg_catalog_arrow_exports/pg_opclass.feather differ diff --git a/datafusion-postgres/pg_catalog_arrow_exports/pg_operator.feather b/datafusion-postgres/pg_catalog_arrow_exports/pg_operator.feather index 245fc95..b41a920 100644 Binary files a/datafusion-postgres/pg_catalog_arrow_exports/pg_operator.feather and b/datafusion-postgres/pg_catalog_arrow_exports/pg_operator.feather differ diff --git a/datafusion-postgres/pg_catalog_arrow_exports/pg_opfamily.feather b/datafusion-postgres/pg_catalog_arrow_exports/pg_opfamily.feather index 05c8806..1e38454 100644 Binary files a/datafusion-postgres/pg_catalog_arrow_exports/pg_opfamily.feather and b/datafusion-postgres/pg_catalog_arrow_exports/pg_opfamily.feather differ diff --git a/datafusion-postgres/pg_catalog_arrow_exports/pg_proc.feather b/datafusion-postgres/pg_catalog_arrow_exports/pg_proc.feather index d6bbfdf..11fed4b 100644 Binary files a/datafusion-postgres/pg_catalog_arrow_exports/pg_proc.feather and b/datafusion-postgres/pg_catalog_arrow_exports/pg_proc.feather differ diff --git a/datafusion-postgres/pg_catalog_arrow_exports/pg_range.feather b/datafusion-postgres/pg_catalog_arrow_exports/pg_range.feather index c7ad9f1..01e8d4f 100644 Binary files a/datafusion-postgres/pg_catalog_arrow_exports/pg_range.feather and b/datafusion-postgres/pg_catalog_arrow_exports/pg_range.feather differ diff --git a/datafusion-postgres/pg_catalog_arrow_exports/pg_ts_config.feather b/datafusion-postgres/pg_catalog_arrow_exports/pg_ts_config.feather index bb99403..5fb8ea6 100644 Binary files a/datafusion-postgres/pg_catalog_arrow_exports/pg_ts_config.feather and b/datafusion-postgres/pg_catalog_arrow_exports/pg_ts_config.feather differ diff --git a/datafusion-postgres/pg_catalog_arrow_exports/pg_ts_dict.feather b/datafusion-postgres/pg_catalog_arrow_exports/pg_ts_dict.feather index 2cd8bbf..839c290 100644 Binary files a/datafusion-postgres/pg_catalog_arrow_exports/pg_ts_dict.feather and b/datafusion-postgres/pg_catalog_arrow_exports/pg_ts_dict.feather differ diff --git a/datafusion-postgres/pg_catalog_arrow_exports/pg_ts_parser.feather b/datafusion-postgres/pg_catalog_arrow_exports/pg_ts_parser.feather index 1253c92..46fc55d 100644 Binary files a/datafusion-postgres/pg_catalog_arrow_exports/pg_ts_parser.feather and b/datafusion-postgres/pg_catalog_arrow_exports/pg_ts_parser.feather differ diff --git a/datafusion-postgres/pg_catalog_arrow_exports/pg_ts_template.feather b/datafusion-postgres/pg_catalog_arrow_exports/pg_ts_template.feather index 6526971..8b93d2c 100644 Binary files a/datafusion-postgres/pg_catalog_arrow_exports/pg_ts_template.feather and b/datafusion-postgres/pg_catalog_arrow_exports/pg_ts_template.feather differ diff --git a/datafusion-postgres/pg_catalog_arrow_exports/pg_type.feather b/datafusion-postgres/pg_catalog_arrow_exports/pg_type.feather index d2a78b2..31910cb 100644 Binary files a/datafusion-postgres/pg_catalog_arrow_exports/pg_type.feather and b/datafusion-postgres/pg_catalog_arrow_exports/pg_type.feather differ diff --git a/datafusion-postgres/src/pg_catalog/format_type.rs b/datafusion-postgres/src/pg_catalog/format_type.rs index cfcda4d..6bf8e84 100644 --- a/datafusion-postgres/src/pg_catalog/format_type.rs +++ b/datafusion-postgres/src/pg_catalog/format_type.rs @@ -2,10 +2,13 @@ use std::sync::Arc; use datafusion::{ arrow::{ - array::{Array, StringBuilder}, - datatypes::DataType, + array::{Array, PrimitiveArray, StringBuilder}, + datatypes::{DataType, Int32Type}, + }, + common::{ + cast::{as_int32_array, as_int64_array}, + DataFusionError, }, - common::{cast::as_int32_array, DataFusionError}, logical_expr::{ ColumnarValue, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, TypeSignature, Volatility, @@ -14,12 +17,32 @@ use datafusion::{ pub(crate) fn format_type_impl(args: &[ColumnarValue]) -> Result { let args = ColumnarValue::values_to_arrays(args)?; - let type_oids = as_int32_array(&args[0])?; + let type_oids = match &args[0].data_type() { + DataType::Int32 => as_int32_array(&args[0])?, + DataType::Int64 => { + let oid_i64 = as_int64_array(&args[0])?; + &PrimitiveArray::::from_iter(oid_i64.iter().map(|v| v.map(|x| x as i32))) + } + _ => { + return Err(DataFusionError::Internal(format!( + "Unexpected oid type in format_type: {}", + args[0].data_type() + ))) + } + }; - let typemods = if args.len() > 1 { - Some(as_int32_array(&args[1])?) - } else { - None + let typemods = match &args[1].data_type() { + DataType::Int32 => as_int32_array(&args[1])?, + DataType::Int64 => { + let oid_i64 = as_int64_array(&args[1])?; + &PrimitiveArray::::from_iter(oid_i64.iter().map(|v| v.map(|x| x as i32))) + } + _ => { + return Err(DataFusionError::Internal(format!( + "Unexpected oid type in format_type: {}", + args[1].data_type() + ))) + } }; let mut result = StringBuilder::new(); @@ -31,9 +54,11 @@ pub(crate) fn format_type_impl(args: &[ColumnarValue]) -> Result PgAttributeTable { /// Map DataFusion data types to PostgreSQL type information fn datafusion_to_pg_type(data_type: &DataType) -> (i32, i16, bool, &'static str, &'static str) { match data_type { - DataType::Boolean => (16, 1, true, "c", "p"), // bool - DataType::Int8 => (18, 1, true, "c", "p"), // char - DataType::Int16 => (21, 2, true, "s", "p"), // int2 - DataType::Int32 => (23, 4, true, "i", "p"), // int4 - DataType::Int64 => (20, 8, true, "d", "p"), // int8 - DataType::UInt8 => (21, 2, true, "s", "p"), // Treat as int2 - DataType::UInt16 => (23, 4, true, "i", "p"), // Treat as int4 - DataType::UInt32 => (20, 8, true, "d", "p"), // Treat as int8 - DataType::UInt64 => (1700, -1, false, "i", "m"), // Treat as numeric - DataType::Float32 => (700, 4, true, "i", "p"), // float4 - DataType::Float64 => (701, 8, true, "d", "p"), // float8 - DataType::Utf8 => (25, -1, false, "i", "x"), // text + DataType::Boolean => (16, 1, true, "c", "p"), // bool + DataType::Int8 => (18, 1, true, "c", "p"), // char + DataType::Int16 => (21, 2, true, "s", "p"), // int2 + DataType::Int32 => (23, 4, true, "i", "p"), // int4 + DataType::Int64 => (20, 8, true, "d", "p"), // int8 + DataType::UInt8 => (18, 2, true, "s", "p"), // char + DataType::UInt16 => (21, 4, true, "i", "p"), // int2 + DataType::UInt32 => (23, 8, true, "d", "p"), // int4 + DataType::UInt64 => (20, -1, false, "i", "m"), // int8 + DataType::Float32 => (700, 4, true, "i", "p"), // float4 + DataType::Float64 => (701, 8, true, "d", "p"), // float8 + DataType::Utf8 => (25, -1, false, "i", "x"), // text DataType::LargeUtf8 => (25, -1, false, "i", "x"), // text - DataType::Binary => (17, -1, false, "i", "x"), // bytea + DataType::Binary => (17, -1, false, "i", "x"), // bytea DataType::LargeBinary => (17, -1, false, "i", "x"), // bytea - DataType::Date32 => (1082, 4, true, "i", "p"), // date - DataType::Date64 => (1082, 4, true, "i", "p"), // date + DataType::Date32 => (1082, 4, true, "i", "p"), // date + DataType::Date64 => (1082, 4, true, "i", "p"), // date DataType::Time32(_) => (1083, 8, true, "d", "p"), // time DataType::Time64(_) => (1083, 8, true, "d", "p"), // time DataType::Timestamp(_, _) => (1114, 8, true, "d", "p"), // timestamp DataType::Decimal128(_, _) => (1700, -1, false, "i", "m"), // numeric DataType::Decimal256(_, _) => (1700, -1, false, "i", "m"), // numeric - _ => (25, -1, false, "i", "x"), // Default to text for unknown types + _ => (25, -1, false, "i", "x"), // Default to text for unknown types } } }