chore: rework the arrow integration tests

This now relies on the features explicitly supported by the kernel crate
rather than released arrow versions which may or may not be supported

Author: rtyler

Cargo.toml

@@ -20,7 +20,7 @@ license = "Apache-2.0"
 repository = "https://github.com/delta-io/delta-kernel-rs"
 readme = "README.md"
 rust-version = "1.80"
-version = "0.7.0"
+version = "0.6.1"
 
 [workspace.dependencies]
 object_store = { version = ">=0.11, <0.12" }

feature-tests/Cargo.toml

@@ -12,7 +12,7 @@ version.workspace = true
 release = false
 
 [dependencies]
-delta_kernel = { path = "../kernel" }
+delta_kernel = { path = "../kernel", features = ["arrow_53"] }
 
 [features]
 default-engine = [ "delta_kernel/default-engine" ]

ffi/Cargo.toml

@@ -25,7 +25,7 @@ delta_kernel = { path = "../kernel", default-features = false, features = [
   "arrow",
   "developer-visibility",
 ] }
-delta_kernel_ffi_macros = { path = "../ffi-proc-macros", version = "0.7.0" }
+delta_kernel_ffi_macros = { path = "../ffi-proc-macros", version = "0.6.1" }
 
 [build-dependencies]
 cbindgen = "0.27.0"

integration-tests/Cargo.toml

@@ -6,19 +6,4 @@ edition = "2021"
 [workspace]
 
 [dependencies]
-arrow = "=53.0.0"
-delta_kernel = { path = "../kernel", features = ["arrow-conversion", "arrow-expression", "default-engine", "sync-engine"] }
-
-[patch.'file:///../kernel']
-arrow = "=53.0.0"
-arrow-arith = "=53.0.0"
-arrow-array = "=53.0.0"
-arrow-buffer = "=53.0.0"
-arrow-cast = "=53.0.0"
-arrow-data = "=53.0.0"
-arrow-ord = "=53.0.0"
-arrow-json = "=53.0.0"
-arrow-select = "=53.0.0"
-arrow-schema = "=53.0.0"
-parquet = "=53.0.0"
-object_store = "=0.11.1"
+delta_kernel = { path = "../kernel", features = ["default-engine", "sync-engine"] }

integration-tests/src/main.rs

@@ -1,15 +1,16 @@
-fn create_arrow_schema() -> arrow::datatypes::Schema {
-    use arrow::datatypes::{DataType, Field, Schema};
+use delta_kernel::arrow::datatypes::{DataType, Field, Schema};
+
+fn create_arrow_schema() -> Schema {
     let field_a = Field::new("a", DataType::Int64, false);
     let field_b = Field::new("b", DataType::Boolean, false);
     Schema::new(vec![field_a, field_b])
 }
 
 fn create_kernel_schema() -> delta_kernel::schema::Schema {
-    use delta_kernel::schema::{DataType, Schema, StructField};
+    use delta_kernel::schema::{DataType, StructField};
     let field_a = StructField::not_null("a", DataType::LONG);
     let field_b = StructField::not_null("b", DataType::BOOLEAN);
-    Schema::new(vec![field_a, field_b])
+    delta_kernel::schema::Schema::new(vec![field_a, field_b])
 }
 
 fn main() {

integration-tests/test-all-arrow-versions.sh

@@ -2,38 +2,25 @@
 
 set -eu -o pipefail
 
-is_version_le() {
-    [  "$1" = "$(echo -e "$1\n$2" | sort -V | head -n1)" ]
-}
-
-is_version_lt() {
-  if [ "$1" = "$2" ]
-  then
-    return 1
-  else
-    is_version_le "$1" "$2"
-  fi
-}
-
 test_arrow_version() {
   ARROW_VERSION="$1"
   echo "== Testing version $ARROW_VERSION =="
-  sed -i'' -e "s/\(arrow[^\"]*=[^\"]*\).*/\1\"=$ARROW_VERSION\"/" Cargo.toml
-  sed -i'' -e "s/\(parquet[^\"]*\).*/\1\"=$ARROW_VERSION\"/" Cargo.toml
   cargo clean
   rm -f Cargo.lock
   cargo update
   cat Cargo.toml
-  cargo run
+  cargo run --features ${ARROW_VERSION}
 }
 
-MIN_ARROW_VER="53.0.0"
-MAX_ARROW_VER="54.0.0"
+FEATURES=$(cat ../kernel/Cargo.toml | grep -e ^arrow_ | awk '{ print $1 }' | sort -u)
 
-for ARROW_VERSION in $(curl -s https://crates.io/api/v1/crates/arrow | jq -r '.versions[].num' | tr -d '\r')
+
+echo "[features]" >> Cargo.toml
+
+for ARROW_VERSION in ${FEATURES}
 do
-  if ! is_version_lt "$ARROW_VERSION" "$MIN_ARROW_VER" && is_version_lt "$ARROW_VERSION" "$MAX_ARROW_VER"
-  then
-    test_arrow_version "$ARROW_VERSION"
-  fi
+  echo "${ARROW_VERSION} = [\"delta_kernel/${ARROW_VERSION}\"]" >> Cargo.toml
+  test_arrow_version $ARROW_VERSION
 done
+
+git checkout Cargo.toml

kernel/Cargo.toml

@@ -39,7 +39,7 @@ uuid = "1.10.0"
 z85 = "3.0.5"
 
 # bring in our derive macros
-delta_kernel_derive = { path = "../derive-macros", version = "0.7.0" }
+delta_kernel_derive = { path = "../derive-macros", version = "0.6.1" }
 
 # used for developer-visibility
 visibility = "0.1.1"
@@ -77,14 +77,10 @@ walkdir = { workspace = true, optional = true }
 [features]
 # The default version to be expected
 arrow = ["arrow_53"]
-# The default version to be expected
-parquet = ["parquet_53"]
 
-arrow_53 = ["dep:arrow_53", "parquet_53"]
-parquet_53 = ["dep:parquet_53"]
+arrow_53 = ["dep:arrow_53", "dep:parquet_53"]
 
-arrow_54 = ["dep:arrow_54", "parquet_54"]
-parquet_54 = ["dep:parquet_54"]
+arrow_54 = ["dep:arrow_54", "dep:parquet_54"]
 
 arrow-conversion = []
 arrow-expression = []

kernel/examples/read-table-changes/src/main.rs

@@ -1,8 +1,8 @@
 use std::{collections::HashMap, sync::Arc};
 
 use clap::Parser;
-use delta_kernel::arrow::{compute::filter_record_batch, util::pretty::print_batches};
 use delta_kernel::arrow::array::RecordBatch;
+use delta_kernel::arrow::{compute::filter_record_batch, util::pretty::print_batches};
 use delta_kernel::engine::arrow_data::ArrowEngineData;
 use delta_kernel::engine::default::executor::tokio::TokioBackgroundExecutor;
 use delta_kernel::engine::default::DefaultEngine;

kernel/src/arrow.rs

@@ -1,4 +1,4 @@
-//! This module exists to help re-export the version of arrow used by default-gengine and other
+//! This module exists to help re-export the version of arrow used by default-engine and other
 //! parts of kernel that need arrow
 
 #[cfg(all(feature = "arrow_53", feature = "arrow_54"))]

kernel/src/parquet.rs

@@ -1,4 +1,4 @@
-//! This module exists to help re-export the version of arrow used by default-gengine and other
+//! This module exists to help re-export the version of arrow used by default-engine and other
 //! parts of kernel that need arrow
 
 #[cfg(all(feature = "arrow_53", feature = "arrow_54"))]