Coverage Report

Created: 2025-11-17 14:14

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/Users/andrewlamb/Software/arrow-rs/arrow-buffer/src/buffer/scalar.rs
Line
Count
Source
1
// Licensed to the Apache Software Foundation (ASF) under one
2
// or more contributor license agreements.  See the NOTICE file
3
// distributed with this work for additional information
4
// regarding copyright ownership.  The ASF licenses this file
5
// to you under the Apache License, Version 2.0 (the
6
// "License"); you may not use this file except in compliance
7
// with the License.  You may obtain a copy of the License at
8
//
9
//   http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing,
12
// software distributed under the License is distributed on an
13
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
// KIND, either express or implied.  See the License for the
15
// specific language governing permissions and limitations
16
// under the License.
17
18
use crate::alloc::Deallocation;
19
use crate::buffer::Buffer;
20
use crate::native::ArrowNativeType;
21
use crate::{BufferBuilder, MutableBuffer, OffsetBuffer};
22
use std::fmt::Formatter;
23
use std::marker::PhantomData;
24
use std::ops::Deref;
25
26
/// A strongly-typed [`Buffer`] supporting zero-copy cloning and slicing
27
///
28
/// The easiest way to think about `ScalarBuffer<T>` is being equivalent to a `Arc<Vec<T>>`,
29
/// with the following differences:
30
///
31
/// - slicing and cloning is O(1).
32
/// - support for external allocated memory (e.g. via FFI).
33
///
34
/// See [`Buffer`] for more low-level memory management details.
35
///
36
/// # Example: Convert to/from Vec (without copies)
37
///
38
/// (See [`Buffer::from_vec`] and [`Buffer::into_vec`] for a lower level API)
39
/// ```
40
/// # use arrow_buffer::ScalarBuffer;
41
/// // Zero-copy conversion from Vec
42
/// let buffer = ScalarBuffer::from(vec![1, 2, 3]);
43
/// assert_eq!(&buffer, &[1, 2, 3]);
44
/// // convert the buffer back to Vec without copy assuming:
45
/// // 1. the inner buffer is not sliced
46
/// // 2. the inner buffer uses standard allocation
47
/// // 3. there are no other references to the inner buffer
48
/// let vec: Vec<i32> = buffer.into();
49
/// assert_eq!(&vec, &[1, 2, 3]);
50
/// ```
51
///
52
/// # Example: Zero copy slicing
53
/// ```
54
/// # use arrow_buffer::ScalarBuffer;
55
/// let buffer = ScalarBuffer::from(vec![1, 2, 3]);
56
/// assert_eq!(&buffer, &[1, 2, 3]);
57
/// // Zero-copy slicing
58
/// let sliced = buffer.slice(1, 2);
59
/// assert_eq!(&sliced, &[2, 3]);
60
/// // Original buffer is unchanged
61
/// assert_eq!(&buffer, &[1, 2, 3]);
62
/// // converting the sliced buffer back to Vec incurs a copy
63
/// let vec: Vec<i32> = sliced.into();
64
/// ```
65
#[derive(Clone, Default)]
66
pub struct ScalarBuffer<T: ArrowNativeType> {
67
    /// Underlying data buffer
68
    buffer: Buffer,
69
    phantom: PhantomData<T>,
70
}
71
72
impl<T: ArrowNativeType> std::fmt::Debug for ScalarBuffer<T> {
73
0
    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
74
0
        f.debug_tuple("ScalarBuffer").field(&self.as_ref()).finish()
75
0
    }
76
}
77
78
impl<T: ArrowNativeType> ScalarBuffer<T> {
79
    /// Create a new [`ScalarBuffer`] from a [`Buffer`], and an `offset`
80
    /// and `length` in units of `T`
81
    ///
82
    /// # Panics
83
    ///
84
    /// This method will panic if
85
    ///
86
    /// * `offset` or `len` would result in overflow
87
    /// * `buffer` is not aligned to a multiple of `std::mem::align_of::<T>`
88
    /// * `bytes` is not large enough for the requested slice
89
485k
    pub fn new(buffer: Buffer, offset: usize, len: usize) -> Self {
90
485k
        let size = std::mem::size_of::<T>();
91
485k
        let byte_offset = offset.checked_mul(size).expect("offset overflow");
92
485k
        let byte_len = len.checked_mul(size).expect("length overflow");
93
485k
        buffer.slice_with_length(byte_offset, byte_len).into()
94
485k
    }
95
96
    /// Unsafe function to create a new [`ScalarBuffer`] from a [`Buffer`].
97
    /// Only use for testing purpose.
98
    ///
99
    /// # Safety
100
    ///
101
    /// This function is unsafe because it does not check if the `buffer` is aligned
102
    pub unsafe fn new_unchecked(buffer: Buffer) -> Self {
103
        Self {
104
            buffer,
105
            phantom: Default::default(),
106
        }
107
    }
108
109
    /// Free up unused memory.
110
0
    pub fn shrink_to_fit(&mut self) {
111
0
        self.buffer.shrink_to_fit();
112
0
    }
113
114
    /// Returns a zero-copy slice of this buffer with length `len` and starting at `offset`
115
160k
    pub fn slice(&self, offset: usize, len: usize) -> Self {
116
160k
        Self::new(self.buffer.clone(), offset, len)
117
160k
    }
118
119
    /// Returns the inner [`Buffer`]
120
17
    pub fn inner(&self) -> &Buffer {
121
17
        &self.buffer
122
17
    }
123
124
    /// Returns the inner [`Buffer`], consuming self
125
484k
    pub fn into_inner(self) -> Buffer {
126
484k
        self.buffer
127
484k
    }
128
129
    /// Returns true if this [`ScalarBuffer`] is equal to `other`, using pointer comparisons
130
    /// to determine buffer equality. This is cheaper than `PartialEq::eq` but may
131
    /// return false when the arrays are logically equal
132
    #[inline]
133
2
    pub fn ptr_eq(&self, other: &Self) -> bool {
134
2
        self.buffer.ptr_eq(&other.buffer)
135
2
    }
136
137
    /// Returns the number of elements in the buffer
138
488k
    pub fn len(&self) -> usize {
139
488k
        self.buffer.len() / std::mem::size_of::<T>()
140
488k
    }
141
142
    /// Returns if the buffer is empty
143
41
    pub fn is_empty(&self) -> bool {
144
41
        self.len() == 0
145
41
    }
146
}
147
148
impl<T: ArrowNativeType> Deref for ScalarBuffer<T> {
149
    type Target = [T];
150
151
    #[inline]
152
1.78M
    fn deref(&self) -> &Self::Target {
153
        // SAFETY: Verified alignment in From<Buffer>
154
        unsafe {
155
1.78M
            std::slice::from_raw_parts(
156
1.78M
                self.buffer.as_ptr() as *const T,
157
1.78M
                self.buffer.len() / std::mem::size_of::<T>(),
158
1.78M
            )
159
        }
160
1.78M
    }
161
}
162
163
impl<T: ArrowNativeType> AsRef<[T]> for ScalarBuffer<T> {
164
    #[inline]
165
80.2k
    fn as_ref(&self) -> &[T] {
166
80.2k
        self
167
80.2k
    }
168
}
169
170
impl<T: ArrowNativeType> From<MutableBuffer> for ScalarBuffer<T> {
171
2
    fn from(value: MutableBuffer) -> Self {
172
2
        Buffer::from(value).into()
173
2
    }
174
}
175
176
impl<T: ArrowNativeType> From<Buffer> for ScalarBuffer<T> {
177
565k
    fn from(buffer: Buffer) -> Self {
178
565k
        let align = std::mem::align_of::<T>();
179
565k
        let is_aligned = buffer.as_ptr().align_offset(align) == 0;
180
181
565k
        match buffer.deallocation() {
182
565k
            Deallocation::Standard(_) => assert!(
183
565k
                is_aligned,
184
0
                "Memory pointer is not aligned with the specified scalar type"
185
            ),
186
0
            Deallocation::Custom(_, _) => assert!(
187
0
                is_aligned,
188
0
                "Memory pointer from external source (e.g, FFI) is not aligned with the specified scalar type. Before importing buffer through FFI, please make sure the allocation is aligned."
189
            ),
190
        }
191
192
565k
        Self {
193
565k
            buffer,
194
565k
            phantom: Default::default(),
195
565k
        }
196
565k
    }
197
}
198
199
impl<T: ArrowNativeType> From<OffsetBuffer<T>> for ScalarBuffer<T> {
200
    fn from(value: OffsetBuffer<T>) -> Self {
201
        value.into_inner()
202
    }
203
}
204
205
impl<T: ArrowNativeType> From<Vec<T>> for ScalarBuffer<T> {
206
606
    fn from(value: Vec<T>) -> Self {
207
606
        Self {
208
606
            buffer: Buffer::from_vec(value),
209
606
            phantom: Default::default(),
210
606
        }
211
606
    }
212
}
213
214
impl<T: ArrowNativeType> From<ScalarBuffer<T>> for Vec<T> {
215
    fn from(value: ScalarBuffer<T>) -> Self {
216
        value
217
            .buffer
218
            .into_vec()
219
            .unwrap_or_else(|buffer| buffer.typed_data::<T>().into())
220
    }
221
}
222
223
impl<T: ArrowNativeType> From<BufferBuilder<T>> for ScalarBuffer<T> {
224
31
    fn from(mut value: BufferBuilder<T>) -> Self {
225
31
        let len = value.len();
226
31
        Self::new(value.finish(), 0, len)
227
31
    }
228
}
229
230
impl<T: ArrowNativeType> FromIterator<T> for ScalarBuffer<T> {
231
    #[inline]
232
91
    fn from_iter<I: IntoIterator<Item = T>>(iter: I) -> Self {
233
91
        iter.into_iter().collect::<Vec<_>>().into()
234
91
    }
235
}
236
237
impl<'a, T: ArrowNativeType> IntoIterator for &'a ScalarBuffer<T> {
238
    type Item = &'a T;
239
    type IntoIter = std::slice::Iter<'a, T>;
240
241
80.1k
    fn into_iter(self) -> Self::IntoIter {
242
80.1k
        self.as_ref().iter()
243
80.1k
    }
244
}
245
246
impl<T: ArrowNativeType, S: AsRef<[T]> + ?Sized> PartialEq<S> for ScalarBuffer<T> {
247
7
    fn eq(&self, other: &S) -> bool {
248
7
        self.as_ref().eq(other.as_ref())
249
7
    }
250
}
251
252
impl<T: ArrowNativeType, const N: usize> PartialEq<ScalarBuffer<T>> for [T; N] {
253
9
    fn eq(&self, other: &ScalarBuffer<T>) -> bool {
254
9
        self.as_ref().eq(other.as_ref())
255
9
    }
256
}
257
258
impl<T: ArrowNativeType> PartialEq<ScalarBuffer<T>> for [T] {
259
    fn eq(&self, other: &ScalarBuffer<T>) -> bool {
260
        self.as_ref().eq(other.as_ref())
261
    }
262
}
263
264
impl<T: ArrowNativeType> PartialEq<ScalarBuffer<T>> for Vec<T> {
265
    fn eq(&self, other: &ScalarBuffer<T>) -> bool {
266
        self.as_slice().eq(other.as_ref())
267
    }
268
}
269
270
/// If T implements Eq, then so does ScalarBuffer.
271
impl<T: ArrowNativeType + Eq> Eq for ScalarBuffer<T> {}
272
273
#[cfg(test)]
274
mod tests {
275
    use std::{ptr::NonNull, sync::Arc};
276
277
    use super::*;
278
279
    #[test]
280
    fn test_basic() {
281
        let expected = [0_i32, 1, 2];
282
        let buffer = Buffer::from_iter(expected.iter().cloned());
283
        let typed = ScalarBuffer::<i32>::new(buffer.clone(), 0, 3);
284
        assert_eq!(*typed, expected);
285
286
        let typed = ScalarBuffer::<i32>::new(buffer.clone(), 1, 2);
287
        assert_eq!(*typed, expected[1..]);
288
289
        let typed = ScalarBuffer::<i32>::new(buffer.clone(), 1, 0);
290
        assert!(typed.is_empty());
291
292
        let typed = ScalarBuffer::<i32>::new(buffer, 3, 0);
293
        assert!(typed.is_empty());
294
    }
295
296
    #[test]
297
    fn test_debug() {
298
        let buffer = ScalarBuffer::from(vec![1, 2, 3]);
299
        assert_eq!(format!("{buffer:?}"), "ScalarBuffer([1, 2, 3])");
300
    }
301
302
    #[test]
303
    #[should_panic(expected = "Memory pointer is not aligned with the specified scalar type")]
304
    fn test_unaligned() {
305
        let expected = [0_i32, 1, 2];
306
        let buffer = Buffer::from_iter(expected.iter().cloned());
307
        let buffer = buffer.slice(1);
308
        ScalarBuffer::<i32>::new(buffer, 0, 2);
309
    }
310
311
    #[test]
312
    #[should_panic(expected = "the offset of the new Buffer cannot exceed the existing length")]
313
    fn test_length_out_of_bounds() {
314
        let buffer = Buffer::from_iter([0_i32, 1, 2]);
315
        ScalarBuffer::<i32>::new(buffer, 1, 3);
316
    }
317
318
    #[test]
319
    #[should_panic(expected = "the offset of the new Buffer cannot exceed the existing length")]
320
    fn test_offset_out_of_bounds() {
321
        let buffer = Buffer::from_iter([0_i32, 1, 2]);
322
        ScalarBuffer::<i32>::new(buffer, 4, 0);
323
    }
324
325
    #[test]
326
    #[should_panic(expected = "offset overflow")]
327
    fn test_length_overflow() {
328
        let buffer = Buffer::from_iter([0_i32, 1, 2]);
329
        ScalarBuffer::<i32>::new(buffer, usize::MAX, 1);
330
    }
331
332
    #[test]
333
    #[should_panic(expected = "offset overflow")]
334
    fn test_start_overflow() {
335
        let buffer = Buffer::from_iter([0_i32, 1, 2]);
336
        ScalarBuffer::<i32>::new(buffer, usize::MAX / 4 + 1, 0);
337
    }
338
339
    #[test]
340
    #[should_panic(expected = "length overflow")]
341
    fn test_end_overflow() {
342
        let buffer = Buffer::from_iter([0_i32, 1, 2]);
343
        ScalarBuffer::<i32>::new(buffer, 0, usize::MAX / 4 + 1);
344
    }
345
346
    #[test]
347
    fn convert_from_buffer_builder() {
348
        let input = vec![1, 2, 3, 4];
349
        let buffer_builder = BufferBuilder::from(input.clone());
350
        let scalar_buffer = ScalarBuffer::from(buffer_builder);
351
        assert_eq!(scalar_buffer.as_ref(), input);
352
    }
353
354
    #[test]
355
    fn into_vec() {
356
        let input = vec![1u8, 2, 3, 4];
357
358
        // No copy
359
        let input_buffer = Buffer::from_vec(input.clone());
360
        let input_ptr = input_buffer.as_ptr();
361
        let input_len = input_buffer.len();
362
        let scalar_buffer = ScalarBuffer::<u8>::new(input_buffer, 0, input_len);
363
        let vec = Vec::from(scalar_buffer);
364
        assert_eq!(vec.as_slice(), input.as_slice());
365
        assert_eq!(vec.as_ptr(), input_ptr);
366
367
        // Custom allocation - makes a copy
368
        let mut input_clone = input.clone();
369
        let input_ptr = NonNull::new(input_clone.as_mut_ptr()).unwrap();
370
        let dealloc = Arc::new(());
371
        let buffer =
372
            unsafe { Buffer::from_custom_allocation(input_ptr, input_clone.len(), dealloc as _) };
373
        let scalar_buffer = ScalarBuffer::<u8>::new(buffer, 0, input.len());
374
        let vec = Vec::from(scalar_buffer);
375
        assert_eq!(vec, input.as_slice());
376
        assert_ne!(vec.as_ptr(), input_ptr.as_ptr());
377
378
        // Offset - makes a copy
379
        let input_buffer = Buffer::from_vec(input.clone());
380
        let input_ptr = input_buffer.as_ptr();
381
        let input_len = input_buffer.len();
382
        let scalar_buffer = ScalarBuffer::<u8>::new(input_buffer, 1, input_len - 1);
383
        let vec = Vec::from(scalar_buffer);
384
        assert_eq!(vec.as_slice(), &input[1..]);
385
        assert_ne!(vec.as_ptr(), input_ptr);
386
387
        // Inner buffer Arc ref count != 0 - makes a copy
388
        let buffer = Buffer::from_slice_ref(input.as_slice());
389
        let scalar_buffer = ScalarBuffer::<u8>::new(buffer, 0, input.len());
390
        let vec = Vec::from(scalar_buffer);
391
        assert_eq!(vec, input.as_slice());
392
        assert_ne!(vec.as_ptr(), input.as_ptr());
393
    }
394
395
    #[test]
396
    fn scalar_buffer_impl_eq() {
397
        fn are_equal<T: Eq>(a: &T, b: &T) -> bool {
398
            a.eq(b)
399
        }
400
401
        assert!(
402
            are_equal(
403
                &ScalarBuffer::<i16>::from(vec![23]),
404
                &ScalarBuffer::<i16>::from(vec![23])
405
            ),
406
            "ScalarBuffer should implement Eq if the inner type does"
407
        );
408
    }
409
}