fix: Upgrade rayon and jwalk to get rid of intermediate vec (#163)

Author: untitaker

Cargo.lock

@@ -17,7 +17,7 @@ pulldown-cmark = "0.8.0"
 blake3 = "1.0.0"
 
 html5gum = "0.5.3"
-jwalk = "0.6.0"
+jwalk = "0.7.0"
 patricia_tree = "0.3.1"
 bumpalo = { version = "3.11.1", features = ["collections"] }
 percent-encoding = "2.1.0"

Cargo.toml

@@ -326,10 +326,7 @@ fn test_document_href() {
         Path::new("public/platforms/python/troubleshooting.html"),
     );
 
-    assert_eq!(
-        doc.href(),
-        Href("platforms/python/troubleshooting.html")
-    );
+    assert_eq!(doc.href(), Href("platforms/python/troubleshooting.html"));
 }
 
 #[test]

src/html/mod.rs

@@ -352,8 +352,8 @@ struct HtmlResult<C> {
 
 fn walk_files(
     base_path: &Path,
-) -> Result<impl ParallelIterator<Item = jwalk::DirEntry<((), bool)>>, Error> {
-    let entries = WalkDirGeneric::<((), bool)>::new(base_path)
+) -> impl ParallelIterator<Item = Result<jwalk::DirEntry<((), bool)>, jwalk::Error>> {
+    WalkDirGeneric::<((), bool)>::new(base_path)
         .sort(true) // helps branch predictor (?)
         .skip_hidden(false)
         .process_read_dir(|_, _, _, children| {
@@ -364,6 +364,7 @@ fn walk_files(
             }
         })
         .into_iter()
+        .par_bridge()
         .filter_map(|entry_result| {
             if let Ok(entry) = entry_result {
                 if let Some(err) = entry.read_children_error {
@@ -379,24 +380,18 @@ fn walk_files(
                 Some(entry_result)
             }
         })
-        // XXX: cannot use par_bridge because of https://github.com/rayon-rs/rayon/issues/690
-        .collect::<Result<Vec<_>, _>>()?;
-
-    // Minimize amount of LinkCollector instances created. This impacts parallelism but
-    // `LinkCollector::merge` is rather slow.
-    let min_len = entries.len() / rayon::current_num_threads();
-    Ok(entries.into_par_iter().with_min_len(min_len))
 }
 
 fn extract_html_links<C: LinkCollector<P::Paragraph>, P: ParagraphWalker>(
     base_path: &Path,
     check_anchors: bool,
     get_paragraphs: bool,
 ) -> Result<HtmlResult<C>, Error> {
-    let result: Result<_, Error> = walk_files(base_path)?
+    let result: Result<_, Error> = walk_files(base_path)
         .try_fold(
             || (DocumentBuffers::default(), C::new(), 0, 0),
             |(mut doc_buf, mut collector, mut documents_count, mut file_count), entry| {
+                let entry = entry?;
                 let path = entry.path();
                 let document = Document::new(base_path, &path);
 
@@ -458,8 +453,9 @@ type MarkdownResult<P> = BTreeMap<P, Vec<(DocumentSource, usize)>>;
 fn extract_markdown_paragraphs<P: ParagraphWalker>(
     sources_path: &Path,
 ) -> Result<MarkdownResult<P::Paragraph>, Error> {
-    let results: Vec<Result<_, Error>> = walk_files(sources_path)?
+    let results: Vec<Result<_, Error>> = walk_files(sources_path)
         .try_fold(Vec::new, |mut paragraphs, entry| {
+            let entry = entry?;
             let source = DocumentSource::new(entry.path());
 
             if !source