/Users/andrewlamb/Software/arrow-rs/arrow-select/src/coalesce/generic.rs
Line | Count | Source |
1 | | // Licensed to the Apache Software Foundation (ASF) under one |
2 | | // or more contributor license agreements. See the NOTICE file |
3 | | // distributed with this work for additional information |
4 | | // regarding copyright ownership. The ASF licenses this file |
5 | | // to you under the Apache License, Version 2.0 (the |
6 | | // "License"); you may not use this file except in compliance |
7 | | // with the License. You may obtain a copy of the License at |
8 | | // |
9 | | // http://www.apache.org/licenses/LICENSE-2.0 |
10 | | // |
11 | | // Unless required by applicable law or agreed to in writing, |
12 | | // software distributed under the License is distributed on an |
13 | | // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
14 | | // KIND, either express or implied. See the License for the |
15 | | // specific language governing permissions and limitations |
16 | | // under the License. |
17 | | |
18 | | use super::InProgressArray; |
19 | | use crate::concat::concat; |
20 | | use arrow_array::ArrayRef; |
21 | | use arrow_schema::ArrowError; |
22 | | |
23 | | /// Generic implementation for [`InProgressArray`] that works with any type of |
24 | | /// array. |
25 | | /// |
26 | | /// Internally, this buffers arrays and then calls other kernels such as |
27 | | /// [`concat`] to produce the final array. |
28 | | /// |
29 | | /// [`concat`]: crate::concat::concat |
30 | | #[derive(Debug)] |
31 | | pub(crate) struct GenericInProgressArray { |
32 | | /// The current source |
33 | | source: Option<ArrayRef>, |
34 | | /// The buffered array slices |
35 | | buffered_arrays: Vec<ArrayRef>, |
36 | | } |
37 | | |
38 | | impl GenericInProgressArray { |
39 | | /// Create a new `GenericInProgressArray` |
40 | 0 | pub(crate) fn new() -> Self { |
41 | 0 | Self { |
42 | 0 | source: None, |
43 | 0 | buffered_arrays: vec![], |
44 | 0 | } |
45 | 0 | } |
46 | | } |
47 | | impl InProgressArray for GenericInProgressArray { |
48 | 0 | fn set_source(&mut self, source: Option<ArrayRef>) { |
49 | 0 | self.source = source |
50 | 0 | } |
51 | | |
52 | 0 | fn copy_rows(&mut self, offset: usize, len: usize) -> Result<(), ArrowError> { |
53 | 0 | let source = self.source.as_ref().ok_or_else(|| { |
54 | 0 | ArrowError::InvalidArgumentError( |
55 | 0 | "Internal Error: GenericInProgressArray: source not set".to_string(), |
56 | 0 | ) |
57 | 0 | })?; |
58 | 0 | let array = source.slice(offset, len); |
59 | 0 | self.buffered_arrays.push(array); |
60 | 0 | Ok(()) |
61 | 0 | } |
62 | | |
63 | 0 | fn finish(&mut self) -> Result<ArrayRef, ArrowError> { |
64 | | // Concatenate all buffered arrays into a single array, which uses 2x |
65 | | // peak memory |
66 | 0 | let array = concat( |
67 | 0 | &self |
68 | 0 | .buffered_arrays |
69 | 0 | .iter() |
70 | 0 | .map(|array| array.as_ref()) |
71 | 0 | .collect::<Vec<_>>(), |
72 | 0 | )?; |
73 | 0 | self.buffered_arrays.clear(); |
74 | 0 | Ok(array) |
75 | 0 | } |
76 | | } |