document reader transform (WIP)

Author: MarinPostma

milli/src/documents/mod.rs

@@ -1,3 +1,9 @@
+/// The documents module defines an intermediary document format that milli uses for indexation, and
+/// provides an API to easily build and read such documents.
+///
+/// The `DocumentBuilder` interface allows to write batches of documents to a writer, that can
+/// later be read by milli using the `DocumentsReader` interface.
+
 mod reader;
 mod builder;
 mod serde;

milli/src/update/index_documents/mod.rs

@@ -1,7 +1,7 @@
 use std::borrow::Cow;
 use std::collections::HashSet;
 use std::fs::File;
-use std::io::{self, BufRead, BufReader, Seek, SeekFrom};
+use std::io::{self, BufRead, BufReader, Seek, SeekFrom, Read};
 use std::num::{NonZeroU32, NonZeroUsize};
 use std::result::Result as StdResult;
 use std::str;
@@ -329,7 +329,7 @@ impl<'t, 'u, 'i, 'a> IndexDocuments<'t, 'u, 'i, 'a> {
 
     pub fn execute<R, F>(self, reader: R, progress_callback: F) -> Result<DocumentAdditionResult>
     where
-        R: io::Read,
+        R: Read + Seek,
         F: Fn(UpdateIndexingStep, u64) + Sync,
     {
         let mut reader = BufReader::new(reader);
@@ -358,14 +358,7 @@ impl<'t, 'u, 'i, 'a> IndexDocuments<'t, 'u, 'i, 'a> {
             autogenerate_docids: self.autogenerate_docids,
         };
 
-        let output = match self.update_format {
-            UpdateFormat::Csv => transform.output_from_csv(reader, &progress_callback)?,
-            UpdateFormat::Json => transform.output_from_json(reader, &progress_callback)?,
-            UpdateFormat::JsonStream => {
-                transform.output_from_json_stream(reader, &progress_callback)?
-            }
-        };
-
+        let output = transform.read_documents(reader, progress_callback)?;
         let nb_documents = output.documents_count;
 
         info!("Update transformed in {:.02?}", before_transform.elapsed());