rearrange

Author: alamb

datafusion/physical-expr/src/string_map.rs

@@ -15,16 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
-//! Specialized HashMap that maps Strings/LargeStrings to values
-//!
-//! This is a specialized HashMap that is optimized for storing and emitting ArrowArrays
-//! as efficiently as possible by minimizing copying of the string values themselves
-//! both when inserting and when emitting the final array.
-//!
-//! Note it can be used as a HashSet by specifing the value type as `()`.
-//!
-//! This is used by the special `COUNT DISTINCT` string aggregate function to store the distinct values
-//! and by the `GROUP BY` operator to store the distinct values for each group when they are single strings
+//! [`ArrowStringMap`] Specialized HashMap that stores values from
+//! Strings/LargeStrings to values.
 
 use ahash::RandomState;
 use arrow_array::cast::AsArray;
@@ -37,63 +29,19 @@ use std::mem;
 use std::ops::Range;
 use std::sync::Arc;
 
-/// Maximum size of a string that can be inlined in the hash table
-const SHORT_STRING_LEN: usize = mem::size_of::<usize>();
-
-/// Entry that is stored in a `ArrowStringHashSet` that represents a string
-/// that is either stored inline or in the buffer
-///
-/// This helps the case where there are many short (less than 8 bytes) strings
-/// that are the same (e.g. "MA", "CA", "NY", "TX", etc)
-///
-/// ```text
-///                                                                ┌──────────────────┐
-///                                                  ─ ─ ─ ─ ─ ─ ─▶│...               │
-///                                                 │              │TheQuickBrownFox  │
-///                                                                │...               │
-///                                                 │              │                  │
-///                                                                └──────────────────┘
-///                                                 │               buffer of u8
-///
-///                                                 │
-///                        ┌────────────────┬───────────────┬───────────────┐
-///  Storing               │                │ starting byte │  length, in   │
-///  "TheQuickBrownFox"    │   hash value   │   offset in   │  bytes (not   │
-///  (long string)         │                │    buffer     │  characters)  │
-///                        └────────────────┴───────────────┴───────────────┘
-///                              8 bytes          8 bytes       4 or 8
-///
-///
-///                         ┌───────────────┬─┬─┬─┬─┬─┬─┬─┬─┬───────────────┐
-/// Storing "foobar"        │               │ │ │ │ │ │ │ │ │  length, in   │
-/// (short string)          │  hash value   │?│?│f│o│o│b│a│r│  bytes (not   │
-///                         │               │ │ │ │ │ │ │ │ │  characters)  │
-///                         └───────────────┴─┴─┴─┴─┴─┴─┴─┴─┴───────────────┘
-///                              8 bytes         8 bytes        4 or 8
-/// ```
-#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)]
-struct Entry {
-    /// hash of the string value (stored to avoid recomputing it in hash table
-    /// check)
-    hash: u64,
-    /// if len =< SHORT_STRING_LEN: the string data inlined
-    /// if len > SHORT_STRING_LEN, the offset of where the data starts
-    offset_or_inline: usize,
-    /// length of the string, in bytes
-    len: usize,
-}
-
-impl Entry {
-    /// returns self.offset..self.offset + self.len
-    fn range(&self) -> Range<usize> {
-        self.offset_or_inline..self.offset_or_inline + self.len
-    }
-}
-
 /// HashSet optimized for storing `String` and `LargeString` values
 /// and producing the final set as a GenericStringArray with minimal copies.
 ///
-/// Equivalent to `HashSet<String>` but with better performance for arrow data.
+/// This is a specialized HashMap that is optimized for storing and emitting
+/// ArrowArrays as efficiently as possible by minimizing copying of the string
+/// values themselves both when inserting and when emitting the final array.
+///
+/// Note it can be used as a HashSet by specifying the value type as `()`.
+///
+/// This is used by the special `COUNT DISTINCT` string aggregate function to
+/// store the distinct values and by the `GROUP BY` operator to store the
+/// distinct values for each group when they are single strings/// Equivalent to
+/// `HashSet<String>` but with better performance for arrow data.
 struct ArrowStringMap<O> {
     /// Underlying hash set for each distinct string
     map: hashbrown::raw::RawTable<Entry>,
@@ -275,6 +223,59 @@ impl<O: OffsetSizeTrait> Debug for ArrowStringMap<O> {
     }
 }
 
+/// Maximum size of a string that can be inlined in the hash table
+const SHORT_STRING_LEN: usize = mem::size_of::<usize>();
+
+/// Entry that is stored in a `ArrowStringHashSet` that represents a string
+/// that is either stored inline or in the buffer
+///
+/// This helps the case where there are many short (less than 8 bytes) strings
+/// that are the same (e.g. "MA", "CA", "NY", "TX", etc)
+///
+/// ```text
+///                                                                ┌──────────────────┐
+///                                                  ─ ─ ─ ─ ─ ─ ─▶│...               │
+///                                                 │              │TheQuickBrownFox  │
+///                                                                │...               │
+///                                                 │              │                  │
+///                                                                └──────────────────┘
+///                                                 │               buffer of u8
+///
+///                                                 │
+///                        ┌────────────────┬───────────────┬───────────────┐
+///  Storing               │                │ starting byte │  length, in   │
+///  "TheQuickBrownFox"    │   hash value   │   offset in   │  bytes (not   │
+///  (long string)         │                │    buffer     │  characters)  │
+///                        └────────────────┴───────────────┴───────────────┘
+///                              8 bytes          8 bytes       4 or 8
+///
+///
+///                         ┌───────────────┬─┬─┬─┬─┬─┬─┬─┬─┬───────────────┐
+/// Storing "foobar"        │               │ │ │ │ │ │ │ │ │  length, in   │
+/// (short string)          │  hash value   │?│?│f│o│o│b│a│r│  bytes (not   │
+///                         │               │ │ │ │ │ │ │ │ │  characters)  │
+///                         └───────────────┴─┴─┴─┴─┴─┴─┴─┴─┴───────────────┘
+///                              8 bytes         8 bytes        4 or 8
+/// ```
+#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)]
+struct Entry {
+    /// hash of the string value (stored to avoid recomputing it in hash table
+    /// check)
+    hash: u64,
+    /// if len =< SHORT_STRING_LEN: the string data inlined
+    /// if len > SHORT_STRING_LEN, the offset of where the data starts
+    offset_or_inline: usize,
+    /// length of the string, in bytes
+    len: usize,
+}
+
+impl Entry {
+    /// returns self.offset..self.offset + self.len
+    fn range(&self) -> Range<usize> {
+        self.offset_or_inline..self.offset_or_inline + self.len
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;