Lucene search backend

CHANGELOG.md

@@ -7,7 +7,17 @@ In case, there is no issue present, the pull request implementing the feature is
 
 Note that this project **does not** adhere to [Semantic Versioning](http://semver.org/).
 
-## [Unreleased]
+## [version6] - Big changes
+
+The branch `version6` is intended to collect improvements, where multiple pull requests build on each other.
+From the user perspective, the combination of the pull request needs to be tested.
+(We created the branch to support lowering the amount of open pull requests)
+
+
+
+
+
+## [Unreleased] - version 5.7 - not yet released
 
 ### Added
 
@@ -840,6 +850,7 @@ The changelog of JabRef 4.x is available at the [v4.3.1 tag](https://github.com/
 The changelog of JabRef 3.x is available at the [v3.8.2 tag](https://github.com/JabRef/jabref/blob/v3.8.2/CHANGELOG.md).
 The changelog of JabRef 2.11 and all previous versions is available as [text file in the v2.11.1 tag](https://github.com/JabRef/jabref/blob/v2.11.1/CHANGELOG).
 
+[version6]: https://github.com/JabRef/jabref/compare/main...version6
 [Unreleased]: https://github.com/JabRef/jabref/compare/v5.6...HEAD
 [5.6]: https://github.com/JabRef/jabref/compare/v5.5...v5.6
 [5.5]: https://github.com/JabRef/jabref/compare/v5.4...v5.5

src/main/java/org/jabref/gui/LibraryTab.java

@@ -48,7 +48,7 @@
 import org.jabref.logic.l10n.Localization;
 import org.jabref.logic.pdf.FileAnnotationCache;
 import org.jabref.logic.pdf.search.indexing.IndexingTaskManager;
-import org.jabref.logic.pdf.search.indexing.PdfIndexer;
+import org.jabref.logic.pdf.search.indexing.LuceneIndexer;
 import org.jabref.logic.search.SearchQuery;
 import org.jabref.logic.shared.DatabaseLocation;
 import org.jabref.logic.util.UpdateField;
@@ -855,22 +855,23 @@ public void listen(EntriesRemovedEvent removedEntriesEvent) {
     private class IndexUpdateListener {
 
         public IndexUpdateListener() {
-            if (preferencesService.getFilePreferences().shouldFulltextIndexLinkedFiles()) {
-                try {
-                    indexingTaskManager.updateIndex(PdfIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences()), bibDatabaseContext);
-                } catch (IOException e) {
-                    LOGGER.error("Cannot access lucene index", e);
+            try {
+                indexingTaskManager.manageFulltextIndexAccordingToPrefs(LuceneIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences()));
+                if (preferencesService.getFilePreferences().shouldFulltextIndexLinkedFiles()) {
+                        indexingTaskManager.updateIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences()));
                 }
+            } catch (IOException e) {
+                LOGGER.error("Cannot access lucene index", e);
             }
         }
 
         @Subscribe
         public void listen(EntriesAddedEvent addedEntryEvent) {
             if (preferencesService.getFilePreferences().shouldFulltextIndexLinkedFiles()) {
                 try {
-                    PdfIndexer pdfIndexer = PdfIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences());
+                    LuceneIndexer luceneIndexer = LuceneIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences());
                     for (BibEntry addedEntry : addedEntryEvent.getBibEntries()) {
-                        indexingTaskManager.addToIndex(pdfIndexer, addedEntry, bibDatabaseContext);
+                        indexingTaskManager.addToIndex(luceneIndexer, addedEntry);
                     }
                 } catch (IOException e) {
                     LOGGER.error("Cannot access lucene index", e);
@@ -882,9 +883,9 @@ public void listen(EntriesAddedEvent addedEntryEvent) {
         public void listen(EntriesRemovedEvent removedEntriesEvent) {
             if (preferencesService.getFilePreferences().shouldFulltextIndexLinkedFiles()) {
                 try {
-                    PdfIndexer pdfIndexer = PdfIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences());
+                    LuceneIndexer luceneIndexer = LuceneIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences());
                     for (BibEntry removedEntry : removedEntriesEvent.getBibEntries()) {
-                        indexingTaskManager.removeFromIndex(pdfIndexer, removedEntry);
+                        indexingTaskManager.removeFromIndex(luceneIndexer, removedEntry);
                     }
                 } catch (IOException e) {
                     LOGGER.error("Cannot access lucene index", e);
@@ -895,18 +896,15 @@ public void listen(EntriesRemovedEvent removedEntriesEvent) {
         @Subscribe
         public void listen(FieldChangedEvent fieldChangedEvent) {
             if (preferencesService.getFilePreferences().shouldFulltextIndexLinkedFiles()) {
-                if (fieldChangedEvent.getField().equals(StandardField.FILE)) {
-                    List<LinkedFile> oldFileList = FileFieldParser.parse(fieldChangedEvent.getOldValue());
-                    List<LinkedFile> newFileList = FileFieldParser.parse(fieldChangedEvent.getNewValue());
-
-                    List<LinkedFile> addedFiles = new ArrayList<>(newFileList);
-                    addedFiles.remove(oldFileList);
-                    List<LinkedFile> removedFiles = new ArrayList<>(oldFileList);
-                    removedFiles.remove(newFileList);
-
+                for (BibEntry bibEntry : fieldChangedEvent.getBibEntries()) {
                     try {
-                        indexingTaskManager.addToIndex(PdfIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences()), fieldChangedEvent.getBibEntry(), addedFiles, bibDatabaseContext);
-                        indexingTaskManager.removeFromIndex(PdfIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences()), fieldChangedEvent.getBibEntry(), removedFiles);
+                        List<LinkedFile> removedFiles = new ArrayList<>();
+                        if (fieldChangedEvent.getField().equals(StandardField.FILE)) {
+                            List<LinkedFile> oldFileList = FileFieldParser.parse(fieldChangedEvent.getOldValue());
+                            List<LinkedFile> newFileList = FileFieldParser.parse(fieldChangedEvent.getNewValue());
+                            removedFiles.remove(newFileList);
+                        }
+                        indexingTaskManager.updateIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences()), bibEntry, removedFiles);
                     } catch (IOException e) {
                         LOGGER.warn("I/O error when writing lucene index", e);
                     }

src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java

@@ -28,7 +28,7 @@
 import org.jabref.logic.l10n.Localization;
 import org.jabref.model.entry.BibEntry;
 import org.jabref.model.entry.LinkedFile;
-import org.jabref.model.pdf.search.PdfSearchResults;
+import org.jabref.model.pdf.search.LuceneSearchResults;
 import org.jabref.model.pdf.search.SearchResult;
 import org.jabref.model.search.rules.SearchRules;
 import org.jabref.preferences.PreferencesService;
@@ -83,7 +83,7 @@ protected void bindToEntry(BibEntry entry) {
             documentViewerView = new DocumentViewerView();
         }
         this.entry = entry;
-        PdfSearchResults searchResults = stateManager.activeSearchQueryProperty().get().get().getRule().getFulltextResults(stateManager.activeSearchQueryProperty().get().get().getQuery(), entry);
+        LuceneSearchResults searchResults = stateManager.activeSearchQueryProperty().get().get().getRule().getLuceneResults(stateManager.activeSearchQueryProperty().get().get().getQuery(), entry);
 
         content.getChildren().clear();
 

src/main/java/org/jabref/gui/externalfiles/ExternalFilesEntryLinker.java

@@ -12,7 +12,7 @@
 import org.jabref.logic.cleanup.MoveFilesCleanup;
 import org.jabref.logic.cleanup.RenamePdfCleanup;
 import org.jabref.logic.pdf.search.indexing.IndexingTaskManager;
-import org.jabref.logic.pdf.search.indexing.PdfIndexer;
+import org.jabref.logic.pdf.search.indexing.LuceneIndexer;
 import org.jabref.logic.util.io.FileUtil;
 import org.jabref.model.database.BibDatabaseContext;
 import org.jabref.model.entry.BibEntry;
@@ -81,7 +81,7 @@ public void moveFilesToFileDirAndAddToEntry(BibEntry entry, List<Path> files, In
         }
 
         try {
-            indexingTaskManager.addToIndex(PdfIndexer.of(bibDatabaseContext, filePreferences), entry, bibDatabaseContext);
+            indexingTaskManager.addToIndex(LuceneIndexer.of(bibDatabaseContext, filePreferences), entry);
         } catch (IOException e) {
             LOGGER.error("Could not access Fulltext-Index", e);
         }
@@ -99,7 +99,7 @@ public void copyFilesToFileDirAndAddToEntry(BibEntry entry, List<Path> files, In
         }
 
         try {
-            indexingTaskManager.addToIndex(PdfIndexer.of(bibDatabaseContext, filePreferences), entry, bibDatabaseContext);
+            indexingTaskManager.addToIndex(LuceneIndexer.of(bibDatabaseContext, filePreferences), entry);
         } catch (IOException e) {
             LOGGER.error("Could not access Fulltext-Index", e);
         }

src/main/java/org/jabref/gui/maintable/BibEntryTableViewModel.java

@@ -3,6 +3,7 @@
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -13,9 +14,15 @@
 import javafx.beans.Observable;
 import javafx.beans.binding.Binding;
 import javafx.beans.binding.Bindings;
+import javafx.beans.property.FloatProperty;
+import javafx.beans.property.ReadOnlyFloatProperty;
 import javafx.beans.property.ReadOnlyStringWrapper;
+import javafx.beans.property.SimpleFloatProperty;
 import javafx.beans.property.StringProperty;
 import javafx.beans.value.ObservableValue;
+import javafx.collections.FXCollections;
+import javafx.collections.ListChangeListener;
+import javafx.collections.ObservableList;
 
 import org.jabref.gui.specialfields.SpecialFieldValueViewModel;
 import org.jabref.gui.util.uithreadaware.UiThreadBinding;
@@ -29,6 +36,7 @@
 import org.jabref.model.entry.field.StandardField;
 import org.jabref.model.groups.AbstractGroup;
 import org.jabref.model.groups.GroupTreeNode;
+import org.jabref.model.pdf.search.SearchResult;
 
 import com.tobiasdiez.easybind.EasyBind;
 import com.tobiasdiez.easybind.EasyBinding;
@@ -44,6 +52,8 @@ public class BibEntryTableViewModel {
     private final EasyBinding<Map<Field, String>> linkedIdentifiers;
     private final Binding<List<AbstractGroup>> matchedGroups;
     private final BibDatabaseContext bibDatabaseContext;
+    private ObservableList<SearchResult> searchResults = FXCollections.observableArrayList();
+    private FloatProperty searchScore = new SimpleFloatProperty();
 
     public BibEntryTableViewModel(BibEntry entry, BibDatabaseContext bibDatabaseContext, ObservableValue<MainTableFieldValueFormatter> fieldValueFormatter) {
         this.entry = entry;
@@ -53,6 +63,10 @@ public BibEntryTableViewModel(BibEntry entry, BibDatabaseContext bibDatabaseCont
         this.linkedIdentifiers = createLinkedIdentifiersBinding(entry);
         this.matchedGroups = createMatchedGroupsBinding(bibDatabaseContext, entry);
         this.bibDatabaseContext = bibDatabaseContext;
+
+        searchResults.addListener((ListChangeListener<? super SearchResult>) (change) -> {
+            searchScore.setValue(searchResults.stream().map(SearchResult::getLuceneScore).max(Comparator.comparing(Float::valueOf)).orElseGet(() -> (float) 0.0));
+        });
     }
 
     private static EasyBinding<Map<Field, String>> createLinkedIdentifiersBinding(BibEntry entry) {
@@ -132,4 +146,16 @@ public ObservableValue<String> getFields(OrFields fields) {
     public StringProperty bibDatabaseContextProperty() {
         return new ReadOnlyStringWrapper(bibDatabaseContext.getDatabasePath().map(Path::toString).orElse(""));
     }
+
+    public void resetSearchResults() {
+        searchResults.clear();
+    }
+
+    public void addSearchResults(List<SearchResult> results) {
+        searchResults.addAll(results);
+    }
+
+    public ReadOnlyFloatProperty getSearchScore() {
+        return searchScore;
+    }
 }

src/main/java/org/jabref/gui/maintable/MainTable.java

@@ -148,12 +148,30 @@ public MainTable(MainTableDataModel model,
         }
         */
 
+        // always sort by score first. If no search is ongoing, it will be equal for all columns.
+        ListChangeListener<? super TableColumn<BibEntryTableViewModel, ?>> scoreSortOderPrioritizer = new ListChangeListener<TableColumn<BibEntryTableViewModel, ?>>() {
+            @Override
+            public void onChanged(Change<? extends TableColumn<BibEntryTableViewModel, ?>> c) {
+                getSortOrder().removeListener(this);
+                getSortOrder().removeAll(getColumns().get(0));
+                getSortOrder().add(0, getColumns().get(0));
+                getSortOrder().addListener(this);
+            }
+        };
+
+        // insert score sort order
+        this.getSortOrder().add(0, this.getColumns().get(0));
         mainTablePreferences.getColumnPreferences().getColumnSortOrder().forEach(columnModel ->
                 this.getColumns().stream()
                     .map(column -> (MainTableColumn<?>) column)
                     .filter(column -> column.getModel().equals(columnModel))
+                    .filter(column -> !column.getModel().getType().equals(MainTableColumnModel.Type.SCORE))
                     .findFirst()
                     .ifPresent(column -> this.getSortOrder().add(column)));
+        this.getSortOrder().addListener(scoreSortOderPrioritizer);
+
+        // Is this always called after the search is done?
+        stateManager.activeSearchQueryProperty().addListener((observable, oldValue, newValue) -> sort());
 
         if (mainTablePreferences.getResizeColumnsToFit()) {
             this.setColumnResizePolicy(new SmartConstrainedResizePolicy());

src/main/java/org/jabref/gui/maintable/MainTableColumnFactory.java

@@ -81,6 +81,8 @@ public MainTableColumnFactory(BibDatabaseContext database,
     public List<TableColumn<BibEntryTableViewModel, ?>> createColumns() {
         List<TableColumn<BibEntryTableViewModel, ?>> columns = new ArrayList<>();
 
+        columns.add(createScoreColumn(new MainTableColumnModel(MainTableColumnModel.Type.SCORE)));
+
         columnPreferences.getColumns().forEach(column -> {
 
             switch (column.getType()) {
@@ -133,6 +135,28 @@ public static void setExactWidth(TableColumn<?, ?> column, double width) {
         column.setMaxWidth(width);
     }
 
+    /**
+     * Creates a column with the search score
+     */
+    private TableColumn<BibEntryTableViewModel, String> createScoreColumn(MainTableColumnModel columnModel) {
+        TableColumn<BibEntryTableViewModel, String> column = new MainTableColumn<>(columnModel);
+        Node header = new Text("Score");
+        header.getStyleClass().add("mainTable-header");
+        Tooltip.install(header, new Tooltip(MainTableColumnModel.Type.SCORE.getDisplayName()));
+        column.setGraphic(header);
+        column.setStyle("-fx-alignment: CENTER-RIGHT;");
+        column.setCellValueFactory(cellData ->
+                cellData.getValue().getSearchScore().asString());
+        new ValueTableCellFactory<BibEntryTableViewModel, String>()
+                .withText(text -> text)
+                .install(column);
+        column.setSortable(true);
+        column.setSortType(TableColumn.SortType.DESCENDING);
+        column.visibleProperty().bind(stateManager.activeSearchQueryProperty().isPresent());
+        column.setReorderable(false);
+        return column;
+    }
+
     /**
      * Creates a column with a continous number
      */

src/main/java/org/jabref/gui/maintable/MainTableColumnModel.java

@@ -30,6 +30,7 @@ public class MainTableColumnModel {
     private static final Logger LOGGER = LoggerFactory.getLogger(MainTableColumnModel.class);
 
     public enum Type {
+        SCORE("search_score", Localization.lang("Search score")),
         INDEX("index", Localization.lang("Index")),
         EXTRAFILE("extrafile", Localization.lang("File type")),
         FILES("files", Localization.lang("Linked files")),

src/main/java/org/jabref/gui/maintable/MainTableDataModel.java

@@ -1,9 +1,13 @@
 package org.jabref.gui.maintable;
 
+import java.io.IOException;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
+import java.util.function.Predicate;
 
 import javafx.beans.binding.Bindings;
+import javafx.beans.binding.ObjectBinding;
 import javafx.beans.property.IntegerProperty;
 import javafx.beans.property.ObjectProperty;
 import javafx.beans.property.SimpleIntegerProperty;
@@ -16,10 +20,12 @@
 import org.jabref.gui.groups.GroupViewMode;
 import org.jabref.gui.groups.GroupsPreferences;
 import org.jabref.gui.util.BindingsHelper;
+import org.jabref.logic.pdf.search.retrieval.LuceneSearcher;
 import org.jabref.logic.search.SearchQuery;
 import org.jabref.model.database.BibDatabaseContext;
 import org.jabref.model.entry.BibEntry;
 import org.jabref.model.groups.GroupTreeNode;
+import org.jabref.model.pdf.search.SearchResult;
 import org.jabref.model.search.matchers.MatcherSet;
 import org.jabref.model.search.matchers.MatcherSets;
 import org.jabref.preferences.PreferencesService;
@@ -46,9 +52,11 @@ public MainTableDataModel(BibDatabaseContext context, PreferencesService prefere
                 new BibEntryTableViewModel(entry, bibDatabaseContext, fieldValueFormatter));
 
         entriesFiltered = new FilteredList<>(entriesViewModel);
-        entriesFiltered.predicateProperty().bind(
-                EasyBind.combine(stateManager.activeGroupProperty(), stateManager.activeSearchQueryProperty(), (groups, query) -> entry -> isMatched(groups, query, entry))
-        );
+        ObjectBinding<Predicate<BibEntryTableViewModel>> filter = Bindings.createObjectBinding(
+                () -> entry -> isMatchedByGroup(stateManager.activeGroupProperty(), entry), stateManager.activeGroupProperty());
+        entriesFiltered.predicateProperty().bind(filter);
+
+        stateManager.activeSearchQueryProperty().addListener((observable, oldValue, newValue) -> doSearch(newValue));
 
         IntegerProperty resultSize = new SimpleIntegerProperty();
         resultSize.bind(Bindings.size(entriesFiltered));
@@ -57,13 +65,26 @@ public MainTableDataModel(BibDatabaseContext context, PreferencesService prefere
         entriesSorted = new SortedList<>(entriesFiltered);
     }
 
-    private boolean isMatched(ObservableList<GroupTreeNode> groups, Optional<SearchQuery> query, BibEntryTableViewModel entry) {
-        return isMatchedByGroup(groups, entry) && isMatchedBySearch(query, entry);
+    private void doSearch(Optional<SearchQuery> query) {
+        for (BibEntryTableViewModel entry : entriesFiltered) {
+            entry.resetSearchResults();
+        }
+        if (query.isPresent()) {
+            String searchString = query.get().getQuery();
+            try {
+                LuceneSearcher searcher = LuceneSearcher.of(bibDatabaseContext);
+                Map<BibEntry, List<SearchResult>> results = searcher.search(query.get());
+                for (Map.Entry<BibEntry, List<SearchResult>> result : results.entrySet()) {
+                    getTableViewModelForEntry(result.getKey()).ifPresent(bibEntryTableViewModel -> bibEntryTableViewModel.addSearchResults(result.getValue()));
+                }
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+        }
     }
 
-    private boolean isMatchedBySearch(Optional<SearchQuery> query, BibEntryTableViewModel entry) {
-        return query.map(matcher -> matcher.isMatch(entry.getEntry()))
-                    .orElse(true);
+    private Optional<BibEntryTableViewModel> getTableViewModelForEntry(BibEntry entry) {
+        return entriesSorted.stream().filter(viewModel -> viewModel.getEntry().equals(entry)).findFirst();
     }
 
     private boolean isMatchedByGroup(ObservableList<GroupTreeNode> groups, BibEntryTableViewModel entry) {