Upgrade to datafusion 38

Cargo.toml

@@ -37,13 +37,13 @@ substrait = ["dep:datafusion-substrait"]
 tokio = { version = "1.35", features = ["macros", "rt", "rt-multi-thread", "sync"] }
 rand = "0.8"
 pyo3 = { version = "0.20", features = ["extension-module", "abi3", "abi3-py38"] }
-datafusion = { version = "37.1.0", features = ["pyarrow", "avro", "unicode_expressions"] }
-datafusion-common = { version = "37.1.0", features = ["pyarrow"] }
-datafusion-expr = "37.1.0"
-datafusion-functions-array = "37.1.0"
-datafusion-optimizer = "37.1.0"
-datafusion-sql = "37.1.0"
-datafusion-substrait = { version = "37.1.0", optional = true }
+datafusion = { version = "38.0.0", features = ["pyarrow", "avro", "unicode_expressions"] }
+datafusion-common = { version = "38.0.0", features = ["pyarrow"] }
+datafusion-expr = "38.0.0"
+datafusion-functions-array = "38.0.0"
+datafusion-optimizer = "38.0.0"
+datafusion-sql = "38.0.0"
+datafusion-substrait = { version = "38.0.0", optional = true }
 prost = "0.12"
 prost-types = "0.12"
 uuid = { version = "1.8", features = ["v4"] }

datafusion/__init__.py

@@ -37,7 +37,6 @@
 )
 
 from .common import (
-    DFField,
     DFSchema,
 )
 
@@ -64,8 +63,6 @@
     IsNotFalse,
     IsNotUnknown,
     Negative,
-    ScalarFunction,
-    BuiltinScalarFunction,
     InList,
     Exists,
     Subquery,

datafusion/tests/test_dataframe.py

@@ -730,9 +730,9 @@ def test_describe(df):
             "max",
             "median",
         ],
-        "a": [3.0, 3.0, 2.0, 1.0, 1.0, 3.0, 2.0],
-        "b": [3.0, 3.0, 5.0, 1.0, 4.0, 6.0, 5.0],
-        "c": [3.0, 3.0, 7.0, 1.7320508075688772, 5.0, 8.0, 8.0],
+        "a": [3.0, 0.0, 2.0, 1.0, 1.0, 3.0, 2.0],
+        "b": [3.0, 0.0, 5.0, 1.0, 4.0, 6.0, 5.0],
+        "c": [3.0, 0.0, 7.0, 1.7320508075688772, 5.0, 8.0, 8.0],
     }