Merge branch 'main' into enterprise-changes

Author: nikhilsinhaparseable

src/cli.rs

@@ -270,6 +270,14 @@ pub struct Options {
     )]
     pub row_group_size: usize,
 
+    #[arg(
+        long,
+        env = "P_EXECUTION_BATCH_SIZE",
+        default_value = "20000",
+        help = "batch size for query execution"
+    )]
+    pub execution_batch_size: usize,
+
     #[arg(
         long = "compression-algo",
         env = "P_PARQUET_COMPRESSION_ALGO",

src/query/mod.rs

@@ -111,7 +111,9 @@ impl Query {
         let mut config = SessionConfig::default()
             .with_parquet_pruning(true)
             .with_prefer_existing_sort(true)
-            .with_batch_size(1000000);
+            //batch size has been made configurable via environment variable
+            //default value is 20000
+            .with_batch_size(PARSEABLE.options.execution_batch_size);
 
         // Pushdown filters allows DF to push the filters as far down in the plan as possible
         // and thus, reducing the number of rows decoded