From 972bac72521da6f800c437c204c2b66ad9022715 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Sat, 9 Jul 2022 11:38:53 +0200 Subject: [PATCH 001/256] Cleanup index when opening a library A user might have deleted a link to a fulltext-file in the source tex file and we need to reflect that change in the index as well. --- src/main/java/org/jabref/gui/LibraryTab.java | 2 +- .../RebuildFulltextSearchIndexAction.java | 2 +- .../search/indexing/IndexingTaskManager.java | 12 +++++- .../logic/pdf/search/indexing/PdfIndexer.java | 38 +++++++++++++++---- 4 files changed, 42 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/jabref/gui/LibraryTab.java b/src/main/java/org/jabref/gui/LibraryTab.java index c62741b37c1..31e64458016 100644 --- a/src/main/java/org/jabref/gui/LibraryTab.java +++ b/src/main/java/org/jabref/gui/LibraryTab.java @@ -857,7 +857,7 @@ private class IndexUpdateListener { public IndexUpdateListener() { if (preferencesService.getFilePreferences().shouldFulltextIndexLinkedFiles()) { try { - indexingTaskManager.addToIndex(PdfIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences()), bibDatabaseContext); + indexingTaskManager.updateIndex(PdfIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences()), bibDatabaseContext); } catch (IOException e) { LOGGER.error("Cannot access lucene index", e); } diff --git a/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java b/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java index 2de75512b39..90d4fe5f1bd 100644 --- a/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java +++ b/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java @@ -69,7 +69,7 @@ private void rebuildIndex() { } try { currentLibraryTab.get().getIndexingTaskManager().createIndex(PdfIndexer.of(databaseContext, filePreferences)); - currentLibraryTab.get().getIndexingTaskManager().addToIndex(PdfIndexer.of(databaseContext, filePreferences), databaseContext); + currentLibraryTab.get().getIndexingTaskManager().updateIndex(PdfIndexer.of(databaseContext, filePreferences), databaseContext); } catch (IOException e) { dialogService.notify(Localization.lang("Failed to access fulltext search index")); LOGGER.error("Failed to access fulltext search index", e); diff --git a/src/main/java/org/jabref/logic/pdf/search/indexing/IndexingTaskManager.java b/src/main/java/org/jabref/logic/pdf/search/indexing/IndexingTaskManager.java index 24e897ae926..d189003ca39 100644 --- a/src/main/java/org/jabref/logic/pdf/search/indexing/IndexingTaskManager.java +++ b/src/main/java/org/jabref/logic/pdf/search/indexing/IndexingTaskManager.java @@ -2,6 +2,7 @@ import java.util.List; import java.util.Queue; +import java.util.Set; import java.util.concurrent.ConcurrentLinkedQueue; import org.jabref.gui.util.BackgroundTask; @@ -88,12 +89,19 @@ public void createIndex(PdfIndexer indexer) { enqueueTask(() -> indexer.createIndex()); } - public void addToIndex(PdfIndexer indexer, BibDatabaseContext databaseContext) { + public void updateIndex(PdfIndexer indexer, BibDatabaseContext databaseContext) { + Set pathsToRemove = indexer.getListOfFilePaths(); for (BibEntry entry : databaseContext.getEntries()) { for (LinkedFile file : entry.getFiles()) { + System.out.println("Adding file " + file.getLink()); enqueueTask(() -> indexer.addToIndex(entry, file, databaseContext)); + pathsToRemove.remove(file.getLink()); } } + for (String pathToRemove : pathsToRemove) { + System.out.println("Removing file " + pathToRemove); + enqueueTask(() -> indexer.removeFromIndex(pathToRemove)); + } } public void addToIndex(PdfIndexer indexer, BibEntry entry, BibDatabaseContext databaseContext) { @@ -108,7 +116,7 @@ public void addToIndex(PdfIndexer indexer, BibEntry entry, List link public void removeFromIndex(PdfIndexer indexer, BibEntry entry, List linkedFiles) { for (LinkedFile file : linkedFiles) { - enqueueTask(() -> indexer.removeFromIndex(entry, file)); + enqueueTask(() -> indexer.removeFromIndex(file.getLink())); } } diff --git a/src/main/java/org/jabref/logic/pdf/search/indexing/PdfIndexer.java b/src/main/java/org/jabref/logic/pdf/search/indexing/PdfIndexer.java index 3b91be44f71..975cf83b1ec 100644 --- a/src/main/java/org/jabref/logic/pdf/search/indexing/PdfIndexer.java +++ b/src/main/java/org/jabref/logic/pdf/search/indexing/PdfIndexer.java @@ -4,8 +4,10 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; +import java.util.HashSet; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.concurrent.TimeUnit; import org.jabref.gui.LibraryTab; @@ -25,6 +27,8 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.Term; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; @@ -110,19 +114,16 @@ public void addToIndex(BibEntry entry, LinkedFile linkedFile, BibDatabaseContext } /** - * Removes a pdf file linked to one entry in the database from the index + * Removes a pdf file identified by its path from the index * - * @param entry the entry the file is linked to - * @param linkedFile the link to the file to be removed + * @param linkedFilePath the path to the file to be removed */ - public void removeFromIndex(BibEntry entry, LinkedFile linkedFile) { + public void removeFromIndex(String linkedFilePath) { try (IndexWriter indexWriter = new IndexWriter( directoryToIndex, new IndexWriterConfig( new EnglishStemAnalyzer()).setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND))) { - if (!entry.getFiles().isEmpty()) { - indexWriter.deleteDocuments(new Term(SearchFieldConstants.PATH, linkedFile.getLink())); - } + indexWriter.deleteDocuments(new Term(SearchFieldConstants.PATH, linkedFilePath)); indexWriter.commit(); } catch (IOException e) { LOGGER.warn("Could not initialize the IndexWriter!", e); @@ -145,7 +146,7 @@ public void removeFromIndex(BibEntry entry) { */ public void removeFromIndex(BibEntry entry, List linkedFiles) { for (LinkedFile linkedFile : linkedFiles) { - removeFromIndex(entry, linkedFile); + removeFromIndex(linkedFile.getLink()); } } @@ -224,4 +225,25 @@ private void writeToIndex(BibEntry entry, LinkedFile linkedFile) { LOGGER.warn("Could not add the document {} to the index!", linkedFile.getLink(), e); } } + + /** + * Lists the paths of all the files that are stored in the index + * + * @return all file paths + */ + public Set getListOfFilePaths() { + Set paths = new HashSet<>(); + try (IndexReader reader = DirectoryReader.open(directoryToIndex)) { + IndexSearcher searcher = new IndexSearcher(reader); + MatchAllDocsQuery query = new MatchAllDocsQuery(); + TopDocs allDocs = searcher.search(query, Integer.MAX_VALUE); + for (ScoreDoc scoreDoc : allDocs.scoreDocs) { + Document doc = reader.document(scoreDoc.doc); + paths.add(doc.getField(SearchFieldConstants.PATH).stringValue()); + } + } catch (IOException e) { + return paths; + } + return paths; + } } From dd81c9ee1cc331bd41ad63266921e676916e423e Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Sat, 9 Jul 2022 11:42:26 +0200 Subject: [PATCH 002/256] Changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 36ef36dedad..c9c60549ad2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,7 @@ Note that this project **does not** adhere to [Semantic Versioning](http://semve - We fixed an issue where removing several groups deletes only one of them. [#8390](https://github.com/JabRef/jabref/issues/8390) - We fixed an issue where the Sidepane (groups, web search and open office) width is not remembered after restarting JabRef. [#8907](https://github.com/JabRef/jabref/issues/8907) - We fixed a bug where switching between themes will cause an error/exception. [#8939](https://github.com/JabRef/jabref/pull/8939) +- We fixed a bug where files that were deleted in the source bibtex file were kept in the index. [8962](https://github.com/JabRef/jabref/pull/8962) ### Removed From f0c7039f3eb16a4f847a1faef333e53b7f134101 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Sat, 9 Jul 2022 11:43:09 +0200 Subject: [PATCH 003/256] Remove debug output --- .../jabref/logic/pdf/search/indexing/IndexingTaskManager.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/org/jabref/logic/pdf/search/indexing/IndexingTaskManager.java b/src/main/java/org/jabref/logic/pdf/search/indexing/IndexingTaskManager.java index d189003ca39..8ebc6bddd04 100644 --- a/src/main/java/org/jabref/logic/pdf/search/indexing/IndexingTaskManager.java +++ b/src/main/java/org/jabref/logic/pdf/search/indexing/IndexingTaskManager.java @@ -93,13 +93,11 @@ public void updateIndex(PdfIndexer indexer, BibDatabaseContext databaseContext) Set pathsToRemove = indexer.getListOfFilePaths(); for (BibEntry entry : databaseContext.getEntries()) { for (LinkedFile file : entry.getFiles()) { - System.out.println("Adding file " + file.getLink()); enqueueTask(() -> indexer.addToIndex(entry, file, databaseContext)); pathsToRemove.remove(file.getLink()); } } for (String pathToRemove : pathsToRemove) { - System.out.println("Removing file " + pathToRemove); enqueueTask(() -> indexer.removeFromIndex(pathToRemove)); } } From 4e491467e318c0f5025175f2ae6a732399fafdc4 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Sat, 9 Jul 2022 13:25:31 +0200 Subject: [PATCH 004/256] Refractor to prepare for full lucene-search-backend --- src/main/java/org/jabref/gui/LibraryTab.java | 28 ++----- .../ExternalFilesEntryLinker.java | 6 +- .../RebuildFulltextSearchIndexAction.java | 6 +- .../search/indexing/IndexingTaskManager.java | 31 ++++--- .../{PdfIndexer.java => LuceneIndexer.java} | 82 +++++-------------- .../pdf/search/SearchFieldConstants.java | 15 ++-- src/main/resources/l10n/JabRef_en.properties | 2 +- ...ndexerTest.java => LuceneIndexerTest.java} | 36 +++++--- .../pdf/search/retrieval/PdfSearcherTest.java | 8 +- 9 files changed, 88 insertions(+), 126 deletions(-) rename src/main/java/org/jabref/logic/pdf/search/indexing/{PdfIndexer.java => LuceneIndexer.java} (76%) rename src/test/java/org/jabref/logic/pdf/search/indexing/{PdfIndexerTest.java => LuceneIndexerTest.java} (87%) diff --git a/src/main/java/org/jabref/gui/LibraryTab.java b/src/main/java/org/jabref/gui/LibraryTab.java index 31e64458016..a5939ccf711 100644 --- a/src/main/java/org/jabref/gui/LibraryTab.java +++ b/src/main/java/org/jabref/gui/LibraryTab.java @@ -44,11 +44,10 @@ import org.jabref.logic.autosaveandbackup.BackupManager; import org.jabref.logic.citationstyle.CitationStyleCache; import org.jabref.logic.importer.ParserResult; -import org.jabref.logic.importer.util.FileFieldParser; import org.jabref.logic.l10n.Localization; import org.jabref.logic.pdf.FileAnnotationCache; import org.jabref.logic.pdf.search.indexing.IndexingTaskManager; -import org.jabref.logic.pdf.search.indexing.PdfIndexer; +import org.jabref.logic.pdf.search.indexing.LuceneIndexer; import org.jabref.logic.search.SearchQuery; import org.jabref.logic.shared.DatabaseLocation; import org.jabref.logic.util.UpdateField; @@ -60,13 +59,11 @@ import org.jabref.model.database.event.EntriesAddedEvent; import org.jabref.model.database.event.EntriesRemovedEvent; import org.jabref.model.entry.BibEntry; -import org.jabref.model.entry.LinkedFile; import org.jabref.model.entry.event.EntriesEventSource; import org.jabref.model.entry.event.EntryChangedEvent; import org.jabref.model.entry.event.FieldChangedEvent; import org.jabref.model.entry.field.Field; import org.jabref.model.entry.field.FieldFactory; -import org.jabref.model.entry.field.StandardField; import org.jabref.preferences.PreferencesService; import com.google.common.eventbus.Subscribe; @@ -857,7 +854,7 @@ private class IndexUpdateListener { public IndexUpdateListener() { if (preferencesService.getFilePreferences().shouldFulltextIndexLinkedFiles()) { try { - indexingTaskManager.updateIndex(PdfIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences()), bibDatabaseContext); + indexingTaskManager.updateIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences())); } catch (IOException e) { LOGGER.error("Cannot access lucene index", e); } @@ -868,9 +865,9 @@ public IndexUpdateListener() { public void listen(EntriesAddedEvent addedEntryEvent) { if (preferencesService.getFilePreferences().shouldFulltextIndexLinkedFiles()) { try { - PdfIndexer pdfIndexer = PdfIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences()); + LuceneIndexer luceneIndexer = LuceneIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences()); for (BibEntry addedEntry : addedEntryEvent.getBibEntries()) { - indexingTaskManager.addToIndex(pdfIndexer, addedEntry, bibDatabaseContext); + indexingTaskManager.addToIndex(luceneIndexer, addedEntry); } } catch (IOException e) { LOGGER.error("Cannot access lucene index", e); @@ -882,9 +879,9 @@ public void listen(EntriesAddedEvent addedEntryEvent) { public void listen(EntriesRemovedEvent removedEntriesEvent) { if (preferencesService.getFilePreferences().shouldFulltextIndexLinkedFiles()) { try { - PdfIndexer pdfIndexer = PdfIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences()); + LuceneIndexer luceneIndexer = LuceneIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences()); for (BibEntry removedEntry : removedEntriesEvent.getBibEntries()) { - indexingTaskManager.removeFromIndex(pdfIndexer, removedEntry); + indexingTaskManager.removeFromIndex(luceneIndexer, removedEntry); } } catch (IOException e) { LOGGER.error("Cannot access lucene index", e); @@ -895,18 +892,9 @@ public void listen(EntriesRemovedEvent removedEntriesEvent) { @Subscribe public void listen(FieldChangedEvent fieldChangedEvent) { if (preferencesService.getFilePreferences().shouldFulltextIndexLinkedFiles()) { - if (fieldChangedEvent.getField().equals(StandardField.FILE)) { - List oldFileList = FileFieldParser.parse(fieldChangedEvent.getOldValue()); - List newFileList = FileFieldParser.parse(fieldChangedEvent.getNewValue()); - - List addedFiles = new ArrayList<>(newFileList); - addedFiles.remove(oldFileList); - List removedFiles = new ArrayList<>(oldFileList); - removedFiles.remove(newFileList); - + for (BibEntry bibEntry : fieldChangedEvent.getBibEntries()) { try { - indexingTaskManager.addToIndex(PdfIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences()), fieldChangedEvent.getBibEntry(), addedFiles, bibDatabaseContext); - indexingTaskManager.removeFromIndex(PdfIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences()), fieldChangedEvent.getBibEntry(), removedFiles); + indexingTaskManager.updateIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences()), bibEntry); } catch (IOException e) { LOGGER.warn("I/O error when writing lucene index", e); } diff --git a/src/main/java/org/jabref/gui/externalfiles/ExternalFilesEntryLinker.java b/src/main/java/org/jabref/gui/externalfiles/ExternalFilesEntryLinker.java index b62748a4959..9fe9a218903 100644 --- a/src/main/java/org/jabref/gui/externalfiles/ExternalFilesEntryLinker.java +++ b/src/main/java/org/jabref/gui/externalfiles/ExternalFilesEntryLinker.java @@ -12,7 +12,7 @@ import org.jabref.logic.cleanup.MoveFilesCleanup; import org.jabref.logic.cleanup.RenamePdfCleanup; import org.jabref.logic.pdf.search.indexing.IndexingTaskManager; -import org.jabref.logic.pdf.search.indexing.PdfIndexer; +import org.jabref.logic.pdf.search.indexing.LuceneIndexer; import org.jabref.logic.util.io.FileUtil; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; @@ -81,7 +81,7 @@ public void moveFilesToFileDirAndAddToEntry(BibEntry entry, List files, In } try { - indexingTaskManager.addToIndex(PdfIndexer.of(bibDatabaseContext, filePreferences), entry, bibDatabaseContext); + indexingTaskManager.addToIndex(LuceneIndexer.of(bibDatabaseContext, filePreferences), entry); } catch (IOException e) { LOGGER.error("Could not access Fulltext-Index", e); } @@ -99,7 +99,7 @@ public void copyFilesToFileDirAndAddToEntry(BibEntry entry, List files, In } try { - indexingTaskManager.addToIndex(PdfIndexer.of(bibDatabaseContext, filePreferences), entry, bibDatabaseContext); + indexingTaskManager.addToIndex(LuceneIndexer.of(bibDatabaseContext, filePreferences), entry); } catch (IOException e) { LOGGER.error("Could not access Fulltext-Index", e); } diff --git a/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java b/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java index 90d4fe5f1bd..2d1c081cea7 100644 --- a/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java +++ b/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java @@ -9,7 +9,7 @@ import org.jabref.gui.actions.SimpleCommand; import org.jabref.gui.util.BackgroundTask; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.pdf.search.indexing.PdfIndexer; +import org.jabref.logic.pdf.search.indexing.LuceneIndexer; import org.jabref.model.database.BibDatabaseContext; import org.jabref.preferences.FilePreferences; @@ -68,8 +68,8 @@ private void rebuildIndex() { return; } try { - currentLibraryTab.get().getIndexingTaskManager().createIndex(PdfIndexer.of(databaseContext, filePreferences)); - currentLibraryTab.get().getIndexingTaskManager().updateIndex(PdfIndexer.of(databaseContext, filePreferences), databaseContext); + currentLibraryTab.get().getIndexingTaskManager().createIndex(LuceneIndexer.of(databaseContext, filePreferences)); + currentLibraryTab.get().getIndexingTaskManager().updateIndex(LuceneIndexer.of(databaseContext, filePreferences)); } catch (IOException e) { dialogService.notify(Localization.lang("Failed to access fulltext search index")); LOGGER.error("Failed to access fulltext search index", e); diff --git a/src/main/java/org/jabref/logic/pdf/search/indexing/IndexingTaskManager.java b/src/main/java/org/jabref/logic/pdf/search/indexing/IndexingTaskManager.java index 8ebc6bddd04..dd3d3fe3402 100644 --- a/src/main/java/org/jabref/logic/pdf/search/indexing/IndexingTaskManager.java +++ b/src/main/java/org/jabref/logic/pdf/search/indexing/IndexingTaskManager.java @@ -9,12 +9,11 @@ import org.jabref.gui.util.DefaultTaskExecutor; import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.l10n.Localization; -import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.LinkedFile; /** - * Wrapper around {@link PdfIndexer} to execute all operations in the background. + * Wrapper around {@link LuceneIndexer} to execute all operations in the background. */ public class IndexingTaskManager extends BackgroundTask { @@ -55,7 +54,7 @@ protected Void call() throws Exception { private void updateProgress() { DefaultTaskExecutor.runInJavaFXThread(() -> { - updateMessage(Localization.lang("%0 of %1 linked files added to the index", numOfIndexedFiles, numOfIndexedFiles + taskQueue.size())); + updateMessage(Localization.lang("%0 of %1 entries added to the index", numOfIndexedFiles, numOfIndexedFiles + taskQueue.size())); updateProgress(numOfIndexedFiles, numOfIndexedFiles + taskQueue.size()); }); } @@ -85,15 +84,15 @@ public AutoCloseable blockNewTasks() { }; } - public void createIndex(PdfIndexer indexer) { + public void createIndex(LuceneIndexer indexer) { enqueueTask(() -> indexer.createIndex()); } - public void updateIndex(PdfIndexer indexer, BibDatabaseContext databaseContext) { + public void updateIndex(LuceneIndexer indexer) { Set pathsToRemove = indexer.getListOfFilePaths(); - for (BibEntry entry : databaseContext.getEntries()) { + for (BibEntry entry : indexer.getDatabaseContext().getEntries()) { + enqueueTask(() -> indexer.addToIndex(entry)); for (LinkedFile file : entry.getFiles()) { - enqueueTask(() -> indexer.addToIndex(entry, file, databaseContext)); pathsToRemove.remove(file.getLink()); } } @@ -102,24 +101,22 @@ public void updateIndex(PdfIndexer indexer, BibDatabaseContext databaseContext) } } - public void addToIndex(PdfIndexer indexer, BibEntry entry, BibDatabaseContext databaseContext) { - enqueueTask(() -> addToIndex(indexer, entry, entry.getFiles(), databaseContext)); + public void addToIndex(LuceneIndexer indexer, BibEntry entry) { + enqueueTask(() -> addToIndex(indexer, entry)); } - public void addToIndex(PdfIndexer indexer, BibEntry entry, List linkedFiles, BibDatabaseContext databaseContext) { + public void removeFromIndex(LuceneIndexer indexer, List linkedFiles) { for (LinkedFile file : linkedFiles) { - enqueueTask(() -> indexer.addToIndex(entry, file, databaseContext)); + enqueueTask(() -> indexer.removeFromIndex(file.getLink())); } } - public void removeFromIndex(PdfIndexer indexer, BibEntry entry, List linkedFiles) { - for (LinkedFile file : linkedFiles) { - enqueueTask(() -> indexer.removeFromIndex(file.getLink())); - } + public void removeFromIndex(LuceneIndexer indexer, BibEntry entry) { + enqueueTask(() -> removeFromIndex(indexer, entry)); } - public void removeFromIndex(PdfIndexer indexer, BibEntry entry) { - enqueueTask(() -> removeFromIndex(indexer, entry, entry.getFiles())); + public void updateIndex(LuceneIndexer indexer, BibEntry entry) { + enqueueTask(() -> updateIndex(indexer, entry)); } public void updateDatabaseName(String name) { diff --git a/src/main/java/org/jabref/logic/pdf/search/indexing/PdfIndexer.java b/src/main/java/org/jabref/logic/pdf/search/indexing/LuceneIndexer.java similarity index 76% rename from src/main/java/org/jabref/logic/pdf/search/indexing/PdfIndexer.java rename to src/main/java/org/jabref/logic/pdf/search/indexing/LuceneIndexer.java index 975cf83b1ec..a747cf73c41 100644 --- a/src/main/java/org/jabref/logic/pdf/search/indexing/PdfIndexer.java +++ b/src/main/java/org/jabref/logic/pdf/search/indexing/LuceneIndexer.java @@ -39,7 +39,7 @@ /** * Indexes the text of PDF files and adds it into the lucene search index. */ -public class PdfIndexer { +public class LuceneIndexer { private static final Logger LOGGER = LoggerFactory.getLogger(LibraryTab.class); @@ -48,13 +48,18 @@ public class PdfIndexer { private final FilePreferences filePreferences; - public PdfIndexer(Directory indexDirectory, FilePreferences filePreferences) { - this.directoryToIndex = indexDirectory; + public LuceneIndexer(BibDatabaseContext databaseContext, FilePreferences filePreferences) throws IOException { + this.databaseContext = databaseContext; + this.directoryToIndex = new NIOFSDirectory(databaseContext.getFulltextIndexPath()); this.filePreferences = filePreferences; } - public static PdfIndexer of(BibDatabaseContext databaseContext, FilePreferences filePreferences) throws IOException { - return new PdfIndexer(new NIOFSDirectory(databaseContext.getFulltextIndexPath()), filePreferences); + public static LuceneIndexer of(BibDatabaseContext databaseContext, FilePreferences filePreferences) throws IOException { + return new LuceneIndexer(databaseContext, filePreferences); + } + + public BibDatabaseContext getDatabaseContext() { + return databaseContext; } /** @@ -70,45 +75,14 @@ public void createIndex() { } } - public void addToIndex(BibDatabaseContext databaseContext) { - for (BibEntry entry : databaseContext.getEntries()) { - addToIndex(entry, databaseContext); - } - } - /** * Adds all the pdf files linked to one entry in the database to an existing (or new) Lucene search index * * @param entry a bibtex entry to link the pdf files to - * @param databaseContext the associated BibDatabaseContext - */ - public void addToIndex(BibEntry entry, BibDatabaseContext databaseContext) { - addToIndex(entry, entry.getFiles(), databaseContext); - } - - /** - * Adds a list of pdf files linked to one entry in the database to an existing (or new) Lucene search index - * - * @param entry a bibtex entry to link the pdf files to - * @param databaseContext the associated BibDatabaseContext - */ - public void addToIndex(BibEntry entry, List linkedFiles, BibDatabaseContext databaseContext) { - for (LinkedFile linkedFile : linkedFiles) { - addToIndex(entry, linkedFile, databaseContext); - } - } - - /** - * Adds a pdf file linked to one entry in the database to an existing (or new) Lucene search index - * - * @param entry a bibtex entry - * @param linkedFile the link to the pdf files */ - public void addToIndex(BibEntry entry, LinkedFile linkedFile, BibDatabaseContext databaseContext) { - if (databaseContext != null) { - this.databaseContext = databaseContext; - } - if (!entry.getFiles().isEmpty()) { + public void addToIndex(BibEntry entry) { + // write bib fields + for (LinkedFile linkedFile : entry.getFiles()) { writeToIndex(entry, linkedFile); } } @@ -130,22 +104,13 @@ public void removeFromIndex(String linkedFilePath) { } } - /** - * Removes all files linked to a bib-entry from the index - * - * @param entry the entry documents are linked to - */ - public void removeFromIndex(BibEntry entry) { - removeFromIndex(entry, entry.getFiles()); - } - /** * Removes a list of files linked to a bib-entry from the index * * @param entry the entry documents are linked to */ - public void removeFromIndex(BibEntry entry, List linkedFiles) { - for (LinkedFile linkedFile : linkedFiles) { + public void removeFromIndex(BibEntry entry) { + for (LinkedFile linkedFile : entry.getFiles()) { removeFromIndex(linkedFile.getLink()); } } @@ -163,18 +128,6 @@ public void flushIndex() { } } - /** - * Writes all files linked to an entry to the index if the files are not yet in the index or the files on the fs are - * newer than the one in the index. - * - * @param entry the entry associated with the file - */ - private void writeToIndex(BibEntry entry) { - for (LinkedFile linkedFile : entry.getFiles()) { - writeToIndex(entry, linkedFile); - } - } - /** * Writes the file to the index if the file is not yet in the index or the file on the fs is newer than the one in * the index. @@ -226,6 +179,11 @@ private void writeToIndex(BibEntry entry, LinkedFile linkedFile) { } } + public void updateIndex(BibEntry entry) { + // For fields: if hash matches, do nothing. If not, re-index + // For files: check for missing files to index and indexed files to delete + } + /** * Lists the paths of all the files that are stored in the index * diff --git a/src/main/java/org/jabref/model/pdf/search/SearchFieldConstants.java b/src/main/java/org/jabref/model/pdf/search/SearchFieldConstants.java index a42baead1b6..51cfc4f327d 100644 --- a/src/main/java/org/jabref/model/pdf/search/SearchFieldConstants.java +++ b/src/main/java/org/jabref/model/pdf/search/SearchFieldConstants.java @@ -2,13 +2,16 @@ public class SearchFieldConstants { - public static final String PATH = "path"; - public static final String CONTENT = "content"; - public static final String PAGE_NUMBER = "pageNumber"; - public static final String ANNOTATIONS = "annotations"; - public static final String MODIFIED = "modified"; + public static final String BIB_FIELDS_PREFIX = "bfp_"; + public static final String FILE_FIELDS_PREFIX = "f_"; + + public static final String PATH = FILE_FIELDS_PREFIX + "path"; + public static final String CONTENT = FILE_FIELDS_PREFIX + "content"; + public static final String PAGE_NUMBER = FILE_FIELDS_PREFIX + "pageNumber"; + public static final String ANNOTATIONS = FILE_FIELDS_PREFIX + "annotations"; + public static final String MODIFIED = FILE_FIELDS_PREFIX + "modified"; public static final String[] PDF_FIELDS = new String[]{PATH, CONTENT, PAGE_NUMBER, MODIFIED, ANNOTATIONS}; - public static final String VERSION = "lucene92"; + public static final String VERSION = "lucene92_jabref0"; } diff --git a/src/main/resources/l10n/JabRef_en.properties b/src/main/resources/l10n/JabRef_en.properties index 32c2d62ae61..e8a7c2a1dcc 100644 --- a/src/main/resources/l10n/JabRef_en.properties +++ b/src/main/resources/l10n/JabRef_en.properties @@ -468,7 +468,7 @@ I\ Agree=I Agree Indexing\ pdf\ files=Indexing pdf files Indexing\ for\ %0=Indexing for %0 -%0\ of\ %1\ linked\ files\ added\ to\ the\ index=%0 of %1 linked files added to the index +%0\ of\ %1\ entries\ added\ to\ the\ index=%0 of %1 entries added to the index Invalid\ citation\ key=Invalid citation key diff --git a/src/test/java/org/jabref/logic/pdf/search/indexing/PdfIndexerTest.java b/src/test/java/org/jabref/logic/pdf/search/indexing/LuceneIndexerTest.java similarity index 87% rename from src/test/java/org/jabref/logic/pdf/search/indexing/PdfIndexerTest.java rename to src/test/java/org/jabref/logic/pdf/search/indexing/LuceneIndexerTest.java index 629bb293778..e5e90192985 100644 --- a/src/test/java/org/jabref/logic/pdf/search/indexing/PdfIndexerTest.java +++ b/src/test/java/org/jabref/logic/pdf/search/indexing/LuceneIndexerTest.java @@ -25,9 +25,9 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -public class PdfIndexerTest { +public class LuceneIndexerTest { - private PdfIndexer indexer; + private LuceneIndexer indexer; private BibDatabase database; private BibDatabaseContext context = mock(BibDatabaseContext.class); @@ -42,7 +42,7 @@ public void setUp(@TempDir Path indexDir) throws IOException { when(context.getFulltextIndexPath()).thenReturn(indexDir); when(context.getDatabase()).thenReturn(database); when(context.getEntries()).thenReturn(database.getEntries()); - this.indexer = PdfIndexer.of(context, filePreferences); + this.indexer = LuceneIndexer.of(context, filePreferences); } @Test @@ -54,7 +54,9 @@ public void exampleThesisIndex() throws IOException { // when indexer.createIndex(); - indexer.addToIndex(context); + for (BibEntry bibEntry : context.getEntries()) { + indexer.addToIndex(bibEntry); + } // then try (IndexReader reader = DirectoryReader.open(new NIOFSDirectory(context.getFulltextIndexPath()))) { @@ -71,7 +73,9 @@ public void dontIndexNonPdf() throws IOException { // when indexer.createIndex(); - indexer.addToIndex(context); + for (BibEntry bibEntry : context.getEntries()) { + indexer.addToIndex(bibEntry); + } // then try (IndexReader reader = DirectoryReader.open(new NIOFSDirectory(context.getFulltextIndexPath()))) { @@ -88,7 +92,9 @@ public void dontIndexOnlineLinks() throws IOException { // when indexer.createIndex(); - indexer.addToIndex(context); + for (BibEntry bibEntry : context.getEntries()) { + indexer.addToIndex(bibEntry); + } // then try (IndexReader reader = DirectoryReader.open(new NIOFSDirectory(context.getFulltextIndexPath()))) { @@ -106,7 +112,9 @@ public void exampleThesisIndexWithKey() throws IOException { // when indexer.createIndex(); - indexer.addToIndex(context); + for (BibEntry bibEntry : context.getEntries()) { + indexer.addToIndex(bibEntry); + } // then try (IndexReader reader = DirectoryReader.open(new NIOFSDirectory(context.getFulltextIndexPath()))) { @@ -124,7 +132,9 @@ public void metaDataIndex() throws IOException { // when indexer.createIndex(); - indexer.addToIndex(context); + for (BibEntry bibEntry : context.getEntries()) { + indexer.addToIndex(bibEntry); + } // then try (IndexReader reader = DirectoryReader.open(new NIOFSDirectory(context.getFulltextIndexPath()))) { @@ -141,7 +151,9 @@ public void testFlushIndex() throws IOException { database.insertEntry(entry); indexer.createIndex(); - indexer.addToIndex(context); + for (BibEntry bibEntry : context.getEntries()) { + indexer.addToIndex(bibEntry); + } // index actually exists try (IndexReader reader = DirectoryReader.open(new NIOFSDirectory(context.getFulltextIndexPath()))) { assertEquals(33, reader.numDocs()); @@ -164,7 +176,9 @@ public void exampleThesisIndexAppendMetaData() throws IOException { exampleThesis.setFiles(Collections.singletonList(new LinkedFile("Example Thesis", "thesis-example.pdf", StandardFileType.PDF.getName()))); database.insertEntry(exampleThesis); indexer.createIndex(); - indexer.addToIndex(context); + for (BibEntry bibEntry : context.getEntries()) { + indexer.addToIndex(bibEntry); + } // index with first entry try (IndexReader reader = DirectoryReader.open(new NIOFSDirectory(context.getFulltextIndexPath()))) { @@ -176,7 +190,7 @@ public void exampleThesisIndexAppendMetaData() throws IOException { metadata.setFiles(Collections.singletonList(new LinkedFile("Metadata file", "metaData.pdf", StandardFileType.PDF.getName()))); // when - indexer.addToIndex(metadata, null); + indexer.addToIndex(metadata); // then try (IndexReader reader = DirectoryReader.open(new NIOFSDirectory(context.getFulltextIndexPath()))) { diff --git a/src/test/java/org/jabref/logic/pdf/search/retrieval/PdfSearcherTest.java b/src/test/java/org/jabref/logic/pdf/search/retrieval/PdfSearcherTest.java index e005c271ff1..cfd014a8a34 100644 --- a/src/test/java/org/jabref/logic/pdf/search/retrieval/PdfSearcherTest.java +++ b/src/test/java/org/jabref/logic/pdf/search/retrieval/PdfSearcherTest.java @@ -4,7 +4,7 @@ import java.nio.file.Path; import java.util.Collections; -import org.jabref.logic.pdf.search.indexing.PdfIndexer; +import org.jabref.logic.pdf.search.indexing.LuceneIndexer; import org.jabref.logic.util.StandardFileType; import org.jabref.model.database.BibDatabase; import org.jabref.model.database.BibDatabaseContext; @@ -53,11 +53,13 @@ public void setUp(@TempDir Path indexDir) throws IOException { exampleThesis.setCitationKey("ExampleThesis"); database.insertEntry(exampleThesis); - PdfIndexer indexer = PdfIndexer.of(context, filePreferences); + LuceneIndexer indexer = LuceneIndexer.of(context, filePreferences); search = PdfSearcher.of(context); indexer.createIndex(); - indexer.addToIndex(context); + for (BibEntry bibEntry : context.getEntries()) { + indexer.addToIndex(bibEntry); + } } @Test From 4c40de628f066d9b87fb94fc8c2719012aedf7ba Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Sat, 9 Jul 2022 14:32:21 +0200 Subject: [PATCH 005/256] Fix infinite recursion --- .../pdf/search/indexing/IndexingTaskManager.java | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/jabref/logic/pdf/search/indexing/IndexingTaskManager.java b/src/main/java/org/jabref/logic/pdf/search/indexing/IndexingTaskManager.java index dd3d3fe3402..1082065adc7 100644 --- a/src/main/java/org/jabref/logic/pdf/search/indexing/IndexingTaskManager.java +++ b/src/main/java/org/jabref/logic/pdf/search/indexing/IndexingTaskManager.java @@ -1,6 +1,5 @@ package org.jabref.logic.pdf.search.indexing; -import java.util.List; import java.util.Queue; import java.util.Set; import java.util.concurrent.ConcurrentLinkedQueue; @@ -102,21 +101,15 @@ public void updateIndex(LuceneIndexer indexer) { } public void addToIndex(LuceneIndexer indexer, BibEntry entry) { - enqueueTask(() -> addToIndex(indexer, entry)); - } - - public void removeFromIndex(LuceneIndexer indexer, List linkedFiles) { - for (LinkedFile file : linkedFiles) { - enqueueTask(() -> indexer.removeFromIndex(file.getLink())); - } + enqueueTask(() -> indexer.addToIndex(entry)); } public void removeFromIndex(LuceneIndexer indexer, BibEntry entry) { - enqueueTask(() -> removeFromIndex(indexer, entry)); + enqueueTask(() -> indexer.removeFromIndex(entry)); } public void updateIndex(LuceneIndexer indexer, BibEntry entry) { - enqueueTask(() -> updateIndex(indexer, entry)); + enqueueTask(() -> indexer.updateIndex(entry)); } public void updateDatabaseName(String name) { From 59047f1e7d6f5e26061735fa99f5913a7e55a4b2 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Sat, 9 Jul 2022 17:24:06 +0200 Subject: [PATCH 006/256] Implement lucene search backend --- src/main/java/org/jabref/gui/LibraryTab.java | 11 ++- .../FulltextSearchResultsTab.java | 4 +- .../search/indexing/IndexingTaskManager.java | 11 ++- .../pdf/search/indexing/LuceneIndexer.java | 68 ++++++++++++++++--- .../{PdfSearcher.java => LuceneSearcher.java} | 27 ++++---- .../model/database/BibDatabaseContext.java | 5 +- .../java/org/jabref/model/entry/BibEntry.java | 10 +++ ...hResults.java => LuceneSearchResults.java} | 6 +- .../pdf/search/SearchFieldConstants.java | 8 ++- .../jabref/model/pdf/search/SearchResult.java | 60 +++++++++------- .../search/rules/ContainsBasedSearchRule.java | 2 +- .../search/rules/FullTextSearchRule.java | 20 +++--- .../search/rules/GrammarBasedSearchRule.java | 14 ++-- .../search/rules/RegexBasedSearchRule.java | 2 +- .../jabref/model/search/rules/SearchRule.java | 4 +- ...rcherTest.java => LuceneSearcherTest.java} | 20 +++--- 16 files changed, 185 insertions(+), 87 deletions(-) rename src/main/java/org/jabref/logic/pdf/search/retrieval/{PdfSearcher.java => LuceneSearcher.java} (71%) rename src/main/java/org/jabref/model/pdf/search/{PdfSearchResults.java => LuceneSearchResults.java} (91%) rename src/test/java/org/jabref/logic/pdf/search/retrieval/{PdfSearcherTest.java => LuceneSearcherTest.java} (87%) diff --git a/src/main/java/org/jabref/gui/LibraryTab.java b/src/main/java/org/jabref/gui/LibraryTab.java index a5939ccf711..25f3c5675cc 100644 --- a/src/main/java/org/jabref/gui/LibraryTab.java +++ b/src/main/java/org/jabref/gui/LibraryTab.java @@ -44,6 +44,7 @@ import org.jabref.logic.autosaveandbackup.BackupManager; import org.jabref.logic.citationstyle.CitationStyleCache; import org.jabref.logic.importer.ParserResult; +import org.jabref.logic.importer.util.FileFieldParser; import org.jabref.logic.l10n.Localization; import org.jabref.logic.pdf.FileAnnotationCache; import org.jabref.logic.pdf.search.indexing.IndexingTaskManager; @@ -59,11 +60,13 @@ import org.jabref.model.database.event.EntriesAddedEvent; import org.jabref.model.database.event.EntriesRemovedEvent; import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.LinkedFile; import org.jabref.model.entry.event.EntriesEventSource; import org.jabref.model.entry.event.EntryChangedEvent; import org.jabref.model.entry.event.FieldChangedEvent; import org.jabref.model.entry.field.Field; import org.jabref.model.entry.field.FieldFactory; +import org.jabref.model.entry.field.StandardField; import org.jabref.preferences.PreferencesService; import com.google.common.eventbus.Subscribe; @@ -894,7 +897,13 @@ public void listen(FieldChangedEvent fieldChangedEvent) { if (preferencesService.getFilePreferences().shouldFulltextIndexLinkedFiles()) { for (BibEntry bibEntry : fieldChangedEvent.getBibEntries()) { try { - indexingTaskManager.updateIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences()), bibEntry); + List removedFiles = new ArrayList<>(); + if (fieldChangedEvent.getField().equals(StandardField.FILE)) { + List oldFileList = FileFieldParser.parse(fieldChangedEvent.getOldValue()); + List newFileList = FileFieldParser.parse(fieldChangedEvent.getNewValue()); + removedFiles.remove(newFileList); + } + indexingTaskManager.updateIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences()), bibEntry, removedFiles); } catch (IOException e) { LOGGER.warn("I/O error when writing lucene index", e); } diff --git a/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java b/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java index 8fe5c146b5b..f50e7c6e3ec 100644 --- a/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java @@ -28,7 +28,7 @@ import org.jabref.logic.l10n.Localization; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.LinkedFile; -import org.jabref.model.pdf.search.PdfSearchResults; +import org.jabref.model.pdf.search.LuceneSearchResults; import org.jabref.model.pdf.search.SearchResult; import org.jabref.model.search.rules.SearchRules; import org.jabref.preferences.PreferencesService; @@ -83,7 +83,7 @@ protected void bindToEntry(BibEntry entry) { documentViewerView = new DocumentViewerView(); } this.entry = entry; - PdfSearchResults searchResults = stateManager.activeSearchQueryProperty().get().get().getRule().getFulltextResults(stateManager.activeSearchQueryProperty().get().get().getQuery(), entry); + LuceneSearchResults searchResults = stateManager.activeSearchQueryProperty().get().get().getRule().getLuceneResults(stateManager.activeSearchQueryProperty().get().get().getQuery(), entry); content.getChildren().clear(); diff --git a/src/main/java/org/jabref/logic/pdf/search/indexing/IndexingTaskManager.java b/src/main/java/org/jabref/logic/pdf/search/indexing/IndexingTaskManager.java index 1082065adc7..487ba3a45de 100644 --- a/src/main/java/org/jabref/logic/pdf/search/indexing/IndexingTaskManager.java +++ b/src/main/java/org/jabref/logic/pdf/search/indexing/IndexingTaskManager.java @@ -1,8 +1,10 @@ package org.jabref.logic.pdf.search.indexing; +import java.util.List; import java.util.Queue; import java.util.Set; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.stream.Collectors; import org.jabref.gui.util.BackgroundTask; import org.jabref.gui.util.DefaultTaskExecutor; @@ -89,8 +91,10 @@ public void createIndex(LuceneIndexer indexer) { public void updateIndex(LuceneIndexer indexer) { Set pathsToRemove = indexer.getListOfFilePaths(); + Set hashesOfEntriesToRemove = indexer.getDatabaseContext().getEntries().stream().map(BibEntry::updateAndGetIndexHash).collect(Collectors.toSet()); for (BibEntry entry : indexer.getDatabaseContext().getEntries()) { enqueueTask(() -> indexer.addToIndex(entry)); + hashesOfEntriesToRemove.removeIf(hash -> Integer.valueOf(entry.getLastIndexHash()).equals(hash)); for (LinkedFile file : entry.getFiles()) { pathsToRemove.remove(file.getLink()); } @@ -98,6 +102,9 @@ public void updateIndex(LuceneIndexer indexer) { for (String pathToRemove : pathsToRemove) { enqueueTask(() -> indexer.removeFromIndex(pathToRemove)); } + for (int hashToRemove : hashesOfEntriesToRemove) { + enqueueTask(() -> indexer.removeFromIndex(hashToRemove)); + } } public void addToIndex(LuceneIndexer indexer, BibEntry entry) { @@ -108,8 +115,8 @@ public void removeFromIndex(LuceneIndexer indexer, BibEntry entry) { enqueueTask(() -> indexer.removeFromIndex(entry)); } - public void updateIndex(LuceneIndexer indexer, BibEntry entry) { - enqueueTask(() -> indexer.updateIndex(entry)); + public void updateIndex(LuceneIndexer indexer, BibEntry entry, List removedFiles) { + enqueueTask(() -> indexer.updateIndex(entry, removedFiles)); } public void updateDatabaseName(String name) { diff --git a/src/main/java/org/jabref/logic/pdf/search/indexing/LuceneIndexer.java b/src/main/java/org/jabref/logic/pdf/search/indexing/LuceneIndexer.java index a747cf73c41..b41f5f365aa 100644 --- a/src/main/java/org/jabref/logic/pdf/search/indexing/LuceneIndexer.java +++ b/src/main/java/org/jabref/logic/pdf/search/indexing/LuceneIndexer.java @@ -6,6 +6,7 @@ import java.nio.file.attribute.BasicFileAttributes; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -15,11 +16,14 @@ import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.LinkedFile; +import org.jabref.model.entry.field.Field; import org.jabref.model.pdf.search.EnglishStemAnalyzer; import org.jabref.model.pdf.search.SearchFieldConstants; import org.jabref.preferences.FilePreferences; import org.apache.lucene.document.Document; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexNotFoundException; import org.apache.lucene.index.IndexReader; @@ -44,7 +48,7 @@ public class LuceneIndexer { private static final Logger LOGGER = LoggerFactory.getLogger(LibraryTab.class); private final Directory directoryToIndex; - private BibDatabaseContext databaseContext; + private final BibDatabaseContext databaseContext; private final FilePreferences filePreferences; @@ -81,9 +85,26 @@ public void createIndex() { * @param entry a bibtex entry to link the pdf files to */ public void addToIndex(BibEntry entry) { - // write bib fields - for (LinkedFile linkedFile : entry.getFiles()) { - writeToIndex(entry, linkedFile); + writeBibFieldsToIndex(entry); + for (LinkedFile file : entry.getFiles()) { + writeFileToIndex(entry, file); + } + } + + /** + * Removes an entry identified by its hash from the index + * + * @param hash the hash to be removed + */ + public void removeFromIndex(int hash) { + try (IndexWriter indexWriter = new IndexWriter( + directoryToIndex, + new IndexWriterConfig( + new EnglishStemAnalyzer()).setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND))) { + indexWriter.deleteDocuments(new Term(SearchFieldConstants.BIB_ENTRY_ID_HASH, String.valueOf(hash))); + indexWriter.commit(); + } catch (IOException e) { + LOGGER.warn("Could not initialize the IndexWriter!", e); } } @@ -128,6 +149,25 @@ public void flushIndex() { } } + private void writeBibFieldsToIndex(BibEntry bibEntry) { + try { + try (IndexWriter indexWriter = new IndexWriter(directoryToIndex, + new IndexWriterConfig( + new EnglishStemAnalyzer()).setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND))) { + Document document = new Document(); + document.add(new StringField(SearchFieldConstants.BIB_ENTRY_ID_HASH, String.valueOf(bibEntry.getLastIndexHash()), org.apache.lucene.document.Field.Store.YES)); + for (Map.Entry field : bibEntry.getFieldMap().entrySet()) { + document.add(new TextField(SearchFieldConstants.BIB_FIELDS_PREFIX + field.getKey().getName(), field.getValue(), org.apache.lucene.document.Field.Store.YES)); + SearchFieldConstants.all_searchable_fields.add(SearchFieldConstants.BIB_FIELDS_PREFIX + field.getKey().getName()); + } + indexWriter.addDocument(document); + indexWriter.commit(); + } + } catch (IOException e) { + LOGGER.warn("Could not add an entry to the index!", e); + } + } + /** * Writes the file to the index if the file is not yet in the index or the file on the fs is newer than the one in * the index. @@ -135,7 +175,7 @@ public void flushIndex() { * @param entry the entry associated with the file * @param linkedFile the file to write to the index */ - private void writeToIndex(BibEntry entry, LinkedFile linkedFile) { + private void writeFileToIndex(BibEntry entry, LinkedFile linkedFile) { if (linkedFile.isOnlineLink() || !StandardFileType.PDF.getName().equals(linkedFile.getFileType())) { return; } @@ -179,9 +219,17 @@ private void writeToIndex(BibEntry entry, LinkedFile linkedFile) { } } - public void updateIndex(BibEntry entry) { - // For fields: if hash matches, do nothing. If not, re-index - // For files: check for missing files to index and indexed files to delete + public void updateIndex(BibEntry entry, List removedFiles) { + int oldHash = entry.getLastIndexHash(); + int newHash = entry.updateAndGetIndexHash(); + if (oldHash == newHash) { + return; + } + addToIndex(entry); + removeFromIndex(oldHash); + for (LinkedFile removedFile : removedFiles) { + removeFromIndex(removedFile.getLink()); + } } /** @@ -197,7 +245,9 @@ public Set getListOfFilePaths() { TopDocs allDocs = searcher.search(query, Integer.MAX_VALUE); for (ScoreDoc scoreDoc : allDocs.scoreDocs) { Document doc = reader.document(scoreDoc.doc); - paths.add(doc.getField(SearchFieldConstants.PATH).stringValue()); + if (doc.getField(SearchFieldConstants.PATH) != null) { + paths.add(doc.getField(SearchFieldConstants.PATH).stringValue()); + } } } catch (IOException e) { return paths; diff --git a/src/main/java/org/jabref/logic/pdf/search/retrieval/PdfSearcher.java b/src/main/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcher.java similarity index 71% rename from src/main/java/org/jabref/logic/pdf/search/retrieval/PdfSearcher.java rename to src/main/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcher.java index 2fd7e54e5b0..27695b16ef9 100644 --- a/src/main/java/org/jabref/logic/pdf/search/retrieval/PdfSearcher.java +++ b/src/main/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcher.java @@ -8,7 +8,8 @@ import org.jabref.gui.LibraryTab; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.pdf.search.EnglishStemAnalyzer; -import org.jabref.model.pdf.search.PdfSearchResults; +import org.jabref.model.pdf.search.LuceneSearchResults; +import org.jabref.model.pdf.search.SearchFieldConstants; import org.jabref.model.pdf.search.SearchResult; import org.jabref.model.strings.StringUtil; @@ -25,20 +26,18 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.jabref.model.pdf.search.SearchFieldConstants.PDF_FIELDS; - -public final class PdfSearcher { +public final class LuceneSearcher { private static final Logger LOGGER = LoggerFactory.getLogger(LibraryTab.class); private final Directory indexDirectory; - private PdfSearcher(Directory indexDirectory) { + private LuceneSearcher(Directory indexDirectory) { this.indexDirectory = indexDirectory; } - public static PdfSearcher of(BibDatabaseContext databaseContext) throws IOException { - return new PdfSearcher(new NIOFSDirectory(databaseContext.getFulltextIndexPath())); + public static LuceneSearcher of(BibDatabaseContext databaseContext) throws IOException { + return new LuceneSearcher(new NIOFSDirectory(databaseContext.getFulltextIndexPath())); } /** @@ -48,10 +47,10 @@ public static PdfSearcher of(BibDatabaseContext databaseContext) throws IOExcept * @param maxHits number of maximum search results, must be positive * @return a result set of all documents that have matches in any fields */ - public PdfSearchResults search(final String searchString, final int maxHits) + public LuceneSearchResults search(final String searchString, final int maxHits) throws IOException { if (StringUtil.isBlank(Objects.requireNonNull(searchString, "The search string was null!"))) { - return new PdfSearchResults(); + return new LuceneSearchResults(); } if (maxHits <= 0) { throw new IllegalArgumentException("Must be called with at least 1 maxHits, was" + maxHits); @@ -61,20 +60,22 @@ public PdfSearchResults search(final String searchString, final int maxHits) if (!DirectoryReader.indexExists(indexDirectory)) { LOGGER.debug("Index directory {} does not yet exist", indexDirectory); - return new PdfSearchResults(); + return new LuceneSearchResults(); } try (IndexReader reader = DirectoryReader.open(indexDirectory)) { IndexSearcher searcher = new IndexSearcher(reader); - Query query = new MultiFieldQueryParser(PDF_FIELDS, new EnglishStemAnalyzer()).parse(searchString); + String[] searchable_fields = new String[SearchFieldConstants.all_searchable_fields.size()]; + SearchFieldConstants.all_searchable_fields.toArray(searchable_fields); + Query query = new MultiFieldQueryParser(searchable_fields, new EnglishStemAnalyzer()).parse(searchString); TopDocs results = searcher.search(query, maxHits); for (ScoreDoc scoreDoc : results.scoreDocs) { resultDocs.add(new SearchResult(searcher, query, scoreDoc)); } - return new PdfSearchResults(resultDocs); + return new LuceneSearchResults(resultDocs); } catch (ParseException e) { LOGGER.warn("Could not parse query: '{}'!\n{}", searchString, e.getMessage()); - return new PdfSearchResults(); + return new LuceneSearchResults(); } } } diff --git a/src/main/java/org/jabref/model/database/BibDatabaseContext.java b/src/main/java/org/jabref/model/database/BibDatabaseContext.java index e5824aecc61..ee5e703c20a 100644 --- a/src/main/java/org/jabref/model/database/BibDatabaseContext.java +++ b/src/main/java/org/jabref/model/database/BibDatabaseContext.java @@ -230,8 +230,9 @@ public Path getFulltextIndexPath() { Path appData = getFulltextIndexBasePath(); if (getDatabasePath().isPresent()) { - LOGGER.info("Index path for {} is {}", getDatabasePath().get(), appData); - return appData.resolve(String.valueOf(this.getDatabasePath().get().hashCode())); + Path databasePath = appData.resolve(String.valueOf(this.getDatabasePath().get().hashCode())); + LOGGER.info("Index path for {} is {}", getDatabasePath().get(), databasePath); + return databasePath; } return appData.resolve("unsaved"); diff --git a/src/main/java/org/jabref/model/entry/BibEntry.java b/src/main/java/org/jabref/model/entry/BibEntry.java index 2643e777c84..485ead7c6ab 100644 --- a/src/main/java/org/jabref/model/entry/BibEntry.java +++ b/src/main/java/org/jabref/model/entry/BibEntry.java @@ -57,6 +57,7 @@ public class BibEntry implements Cloneable { public static final EntryType DEFAULT_TYPE = StandardEntryType.Misc; private static final Logger LOGGER = LoggerFactory.getLogger(BibEntry.class); private final SharedBibEntryData sharedBibEntryData; + private int lastIndexHash; /** * Map to store the words in every field @@ -817,6 +818,15 @@ public int hashCode() { return Objects.hash(type.getValue(), fields); } + public int getLastIndexHash() { + return lastIndexHash; + } + + public int updateAndGetIndexHash() { + lastIndexHash = hashCode(); + return lastIndexHash; + } + public void registerListener(Object object) { this.eventBus.register(object); } diff --git a/src/main/java/org/jabref/model/pdf/search/PdfSearchResults.java b/src/main/java/org/jabref/model/pdf/search/LuceneSearchResults.java similarity index 91% rename from src/main/java/org/jabref/model/pdf/search/PdfSearchResults.java rename to src/main/java/org/jabref/model/pdf/search/LuceneSearchResults.java index db38bfe9548..415905ec1cb 100644 --- a/src/main/java/org/jabref/model/pdf/search/PdfSearchResults.java +++ b/src/main/java/org/jabref/model/pdf/search/LuceneSearchResults.java @@ -5,15 +5,15 @@ import java.util.HashMap; import java.util.List; -public final class PdfSearchResults { +public final class LuceneSearchResults { private final List searchResults; - public PdfSearchResults(List search) { + public LuceneSearchResults(List search) { this.searchResults = Collections.unmodifiableList(search); } - public PdfSearchResults() { + public LuceneSearchResults() { this.searchResults = Collections.emptyList(); } diff --git a/src/main/java/org/jabref/model/pdf/search/SearchFieldConstants.java b/src/main/java/org/jabref/model/pdf/search/SearchFieldConstants.java index 51cfc4f327d..3fc4203c09e 100644 --- a/src/main/java/org/jabref/model/pdf/search/SearchFieldConstants.java +++ b/src/main/java/org/jabref/model/pdf/search/SearchFieldConstants.java @@ -1,9 +1,14 @@ package org.jabref.model.pdf.search; +import java.util.Arrays; +import java.util.Set; +import java.util.stream.Collectors; + public class SearchFieldConstants { public static final String BIB_FIELDS_PREFIX = "bfp_"; public static final String FILE_FIELDS_PREFIX = "f_"; + public static final String BIB_ENTRY_ID_HASH = "id"; public static final String PATH = FILE_FIELDS_PREFIX + "path"; public static final String CONTENT = FILE_FIELDS_PREFIX + "content"; @@ -11,7 +16,8 @@ public class SearchFieldConstants { public static final String ANNOTATIONS = FILE_FIELDS_PREFIX + "annotations"; public static final String MODIFIED = FILE_FIELDS_PREFIX + "modified"; - public static final String[] PDF_FIELDS = new String[]{PATH, CONTENT, PAGE_NUMBER, MODIFIED, ANNOTATIONS}; + public static final String[] PDF_FIELDS = new String[]{PATH, CONTENT, ANNOTATIONS}; + public static Set all_searchable_fields = Arrays.stream(PDF_FIELDS).collect(Collectors.toSet()); public static final String VERSION = "lucene92_jabref0"; } diff --git a/src/main/java/org/jabref/model/pdf/search/SearchResult.java b/src/main/java/org/jabref/model/pdf/search/SearchResult.java index 7a79191e94c..63c1a562dc3 100644 --- a/src/main/java/org/jabref/model/pdf/search/SearchResult.java +++ b/src/main/java/org/jabref/model/pdf/search/SearchResult.java @@ -19,6 +19,7 @@ import org.apache.lucene.search.highlight.TextFragment; import static org.jabref.model.pdf.search.SearchFieldConstants.ANNOTATIONS; +import static org.jabref.model.pdf.search.SearchFieldConstants.BIB_ENTRY_ID_HASH; import static org.jabref.model.pdf.search.SearchFieldConstants.CONTENT; import static org.jabref.model.pdf.search.SearchFieldConstants.MODIFIED; import static org.jabref.model.pdf.search.SearchFieldConstants.PAGE_NUMBER; @@ -30,34 +31,44 @@ public final class SearchResult { private final int pageNumber; private final long modified; + private final int hash; private final float luceneScore; private List contentResultStringsHtml; private List annotationsResultStringsHtml; public SearchResult(IndexSearcher searcher, Query query, ScoreDoc scoreDoc) throws IOException { - this.path = getFieldContents(searcher, scoreDoc, PATH); - this.pageNumber = Integer.parseInt(getFieldContents(searcher, scoreDoc, PAGE_NUMBER)); - this.modified = Long.parseLong(getFieldContents(searcher, scoreDoc, MODIFIED)); this.luceneScore = scoreDoc.score; - - String content = getFieldContents(searcher, scoreDoc, CONTENT); - String annotations = getFieldContents(searcher, scoreDoc, ANNOTATIONS); - - Highlighter highlighter = new Highlighter(new SimpleHTMLFormatter("", ""), new QueryScorer(query)); - - try (TokenStream contentStream = new EnglishStemAnalyzer().tokenStream(CONTENT, content)) { - TextFragment[] frags = highlighter.getBestTextFragments(contentStream, content, true, 10); - this.contentResultStringsHtml = Arrays.stream(frags).map(TextFragment::toString).collect(Collectors.toList()); - } catch (InvalidTokenOffsetsException e) { - this.contentResultStringsHtml = List.of(); - } - - try (TokenStream annotationStream = new EnglishStemAnalyzer().tokenStream(ANNOTATIONS, annotations)) { - TextFragment[] frags = highlighter.getBestTextFragments(annotationStream, annotations, true, 10); - this.annotationsResultStringsHtml = Arrays.stream(frags).map(TextFragment::toString).collect(Collectors.toList()); - } catch (InvalidTokenOffsetsException e) { - this.annotationsResultStringsHtml = List.of(); + this.path = getFieldContents(searcher, scoreDoc, PATH); + if (this.path.length() > 0) { + // pdf result + this.pageNumber = Integer.parseInt(getFieldContents(searcher, scoreDoc, PAGE_NUMBER)); + this.modified = Long.parseLong(getFieldContents(searcher, scoreDoc, MODIFIED)); + this.hash = 0; + + String content = getFieldContents(searcher, scoreDoc, CONTENT); + String annotations = getFieldContents(searcher, scoreDoc, ANNOTATIONS); + + Highlighter highlighter = new Highlighter(new SimpleHTMLFormatter("", ""), new QueryScorer(query)); + + try (TokenStream contentStream = new EnglishStemAnalyzer().tokenStream(CONTENT, content)) { + TextFragment[] frags = highlighter.getBestTextFragments(contentStream, content, true, 10); + this.contentResultStringsHtml = Arrays.stream(frags).map(TextFragment::toString).collect(Collectors.toList()); + } catch (InvalidTokenOffsetsException e) { + this.contentResultStringsHtml = List.of(); + } + + try (TokenStream annotationStream = new EnglishStemAnalyzer().tokenStream(ANNOTATIONS, annotations)) { + TextFragment[] frags = highlighter.getBestTextFragments(annotationStream, annotations, true, 10); + this.annotationsResultStringsHtml = Arrays.stream(frags).map(TextFragment::toString).collect(Collectors.toList()); + } catch (InvalidTokenOffsetsException e) { + this.annotationsResultStringsHtml = List.of(); + } + } else { + // Found somewhere in the bib entry + this.hash = Integer.parseInt(getFieldContents(searcher, scoreDoc, BIB_ENTRY_ID_HASH)); + this.pageNumber = -1; + this.modified = -1; } } @@ -69,8 +80,11 @@ private String getFieldContents(IndexSearcher searcher, ScoreDoc scoreDoc, Strin return indexableField.stringValue(); } - public boolean isResultFor(BibEntry entry) { - return entry.getFiles().stream().anyMatch(linkedFile -> path.equals(linkedFile.getLink())); + public float getSearchScoreFor(BibEntry entry) { + if (this.path != null) { + return entry.getFiles().stream().anyMatch(linkedFile -> path.equals(linkedFile.getLink())) ? luceneScore : 0; + } + return entry.getLastIndexHash() == hash ? luceneScore : 0; } public String getPath() { diff --git a/src/main/java/org/jabref/model/search/rules/ContainsBasedSearchRule.java b/src/main/java/org/jabref/model/search/rules/ContainsBasedSearchRule.java index 7162061641a..be074a3e573 100644 --- a/src/main/java/org/jabref/model/search/rules/ContainsBasedSearchRule.java +++ b/src/main/java/org/jabref/model/search/rules/ContainsBasedSearchRule.java @@ -54,6 +54,6 @@ public boolean applyRule(String query, BibEntry bibEntry) { } } - return getFulltextResults(query, bibEntry).numSearchResults() > 0; // Didn't match all words. + return getLuceneResults(query, bibEntry).numSearchResults() > 0; // Didn't match all words. } } diff --git a/src/main/java/org/jabref/model/search/rules/FullTextSearchRule.java b/src/main/java/org/jabref/model/search/rules/FullTextSearchRule.java index 27cfbf65b8b..4ac130e45e9 100644 --- a/src/main/java/org/jabref/model/search/rules/FullTextSearchRule.java +++ b/src/main/java/org/jabref/model/search/rules/FullTextSearchRule.java @@ -8,10 +8,10 @@ import org.jabref.architecture.AllowedToUseLogic; import org.jabref.gui.Globals; -import org.jabref.logic.pdf.search.retrieval.PdfSearcher; +import org.jabref.logic.pdf.search.retrieval.LuceneSearcher; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; -import org.jabref.model.pdf.search.PdfSearchResults; +import org.jabref.model.pdf.search.LuceneSearchResults; import org.jabref.model.pdf.search.SearchResult; import org.slf4j.Logger; @@ -39,7 +39,7 @@ public FullTextSearchRule(EnumSet searchFlags) { this.lastQuery = ""; lastSearchResults = Collections.emptyList(); - databaseContext = Globals.stateManager.getActiveDatabase().orElse(null); + databaseContext = Globals.stateManager.getActiveDatabase().orElse(null); } public EnumSet getSearchFlags() { @@ -47,25 +47,25 @@ public EnumSet getSearchFlags() { } @Override - public PdfSearchResults getFulltextResults(String query, BibEntry bibEntry) { + public LuceneSearchResults getLuceneResults(String query, BibEntry bibEntry) { if (!searchFlags.contains(SearchRules.SearchFlags.FULLTEXT) || databaseContext == null) { - return new PdfSearchResults(); + return new LuceneSearchResults(); } if (!query.equals(this.lastQuery)) { this.lastQuery = query; lastSearchResults = Collections.emptyList(); try { - PdfSearcher searcher = PdfSearcher.of(databaseContext); - PdfSearchResults results = searcher.search(query, 5); + LuceneSearcher searcher = LuceneSearcher.of(databaseContext); + LuceneSearchResults results = searcher.search(query, 5); lastSearchResults = results.getSortedByScore(); } catch (IOException e) { LOGGER.error("Could not retrieve search results!", e); } } - return new PdfSearchResults(lastSearchResults.stream() - .filter(searchResult -> searchResult.isResultFor(bibEntry)) - .collect(Collectors.toList())); + return new LuceneSearchResults(lastSearchResults.stream() + .filter(searchResult -> searchResult.getSearchScoreFor(bibEntry) > 0) + .collect(Collectors.toList())); } } diff --git a/src/main/java/org/jabref/model/search/rules/GrammarBasedSearchRule.java b/src/main/java/org/jabref/model/search/rules/GrammarBasedSearchRule.java index 704b9066cc2..cb0c3e24a3f 100644 --- a/src/main/java/org/jabref/model/search/rules/GrammarBasedSearchRule.java +++ b/src/main/java/org/jabref/model/search/rules/GrammarBasedSearchRule.java @@ -14,13 +14,13 @@ import org.jabref.architecture.AllowedToUseLogic; import org.jabref.gui.Globals; -import org.jabref.logic.pdf.search.retrieval.PdfSearcher; +import org.jabref.logic.pdf.search.retrieval.LuceneSearcher; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.Keyword; import org.jabref.model.entry.field.Field; import org.jabref.model.entry.field.InternalField; -import org.jabref.model.pdf.search.PdfSearchResults; +import org.jabref.model.pdf.search.LuceneSearchResults; import org.jabref.model.pdf.search.SearchResult; import org.jabref.model.search.rules.SearchRules.SearchFlags; import org.jabref.model.strings.StringUtil; @@ -105,8 +105,8 @@ private void init(String query) throws ParseCancellationException { return; } try { - PdfSearcher searcher = PdfSearcher.of(databaseContext); - PdfSearchResults results = searcher.search(query, 5); + LuceneSearcher searcher = LuceneSearcher.of(databaseContext); + LuceneSearchResults results = searcher.search(query, 5); searchResults = results.getSortedByScore(); } catch (IOException e) { LOGGER.error("Could not retrieve search results!", e); @@ -119,13 +119,13 @@ public boolean applyRule(String query, BibEntry bibEntry) { return new BibtexSearchVisitor(searchFlags, bibEntry).visit(tree); } catch (Exception e) { LOGGER.debug("Search failed", e); - return getFulltextResults(query, bibEntry).numSearchResults() > 0; + return getLuceneResults(query, bibEntry).numSearchResults() > 0; } } @Override - public PdfSearchResults getFulltextResults(String query, BibEntry bibEntry) { - return new PdfSearchResults(searchResults.stream().filter(searchResult -> searchResult.isResultFor(bibEntry)).collect(Collectors.toList())); + public LuceneSearchResults getLuceneResults(String query, BibEntry bibEntry) { + return new LuceneSearchResults(searchResults.stream().filter(searchResult -> searchResult.getSearchScoreFor(bibEntry) > 0).collect(Collectors.toList())); } @Override diff --git a/src/main/java/org/jabref/model/search/rules/RegexBasedSearchRule.java b/src/main/java/org/jabref/model/search/rules/RegexBasedSearchRule.java index 3c2a3e001c7..8c607c8dfc0 100644 --- a/src/main/java/org/jabref/model/search/rules/RegexBasedSearchRule.java +++ b/src/main/java/org/jabref/model/search/rules/RegexBasedSearchRule.java @@ -57,6 +57,6 @@ public boolean applyRule(String query, BibEntry bibEntry) { } } } - return getFulltextResults(query, bibEntry).numSearchResults() > 0; + return getLuceneResults(query, bibEntry).numSearchResults() > 0; } } diff --git a/src/main/java/org/jabref/model/search/rules/SearchRule.java b/src/main/java/org/jabref/model/search/rules/SearchRule.java index 1be2b05b342..469a6fc43de 100644 --- a/src/main/java/org/jabref/model/search/rules/SearchRule.java +++ b/src/main/java/org/jabref/model/search/rules/SearchRule.java @@ -1,13 +1,13 @@ package org.jabref.model.search.rules; import org.jabref.model.entry.BibEntry; -import org.jabref.model.pdf.search.PdfSearchResults; +import org.jabref.model.pdf.search.LuceneSearchResults; public interface SearchRule { boolean applyRule(String query, BibEntry bibEntry); - PdfSearchResults getFulltextResults(String query, BibEntry bibEntry); + LuceneSearchResults getLuceneResults(String query, BibEntry bibEntry); boolean validateSearchStrings(String query); } diff --git a/src/test/java/org/jabref/logic/pdf/search/retrieval/PdfSearcherTest.java b/src/test/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcherTest.java similarity index 87% rename from src/test/java/org/jabref/logic/pdf/search/retrieval/PdfSearcherTest.java rename to src/test/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcherTest.java index cfd014a8a34..a328050296f 100644 --- a/src/test/java/org/jabref/logic/pdf/search/retrieval/PdfSearcherTest.java +++ b/src/test/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcherTest.java @@ -11,7 +11,7 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.LinkedFile; import org.jabref.model.entry.types.StandardEntryType; -import org.jabref.model.pdf.search.PdfSearchResults; +import org.jabref.model.pdf.search.LuceneSearchResults; import org.jabref.preferences.FilePreferences; import org.apache.lucene.queryparser.classic.ParseException; @@ -25,9 +25,9 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -public class PdfSearcherTest { +public class LuceneSearcherTest { - private PdfSearcher search; + private LuceneSearcher search; @BeforeEach public void setUp(@TempDir Path indexDir) throws IOException { @@ -54,7 +54,7 @@ public void setUp(@TempDir Path indexDir) throws IOException { database.insertEntry(exampleThesis); LuceneIndexer indexer = LuceneIndexer.of(context, filePreferences); - search = PdfSearcher.of(context); + search = LuceneSearcher.of(context); indexer.createIndex(); for (BibEntry bibEntry : context.getEntries()) { @@ -64,37 +64,37 @@ public void setUp(@TempDir Path indexDir) throws IOException { @Test public void searchForTest() throws IOException, ParseException { - PdfSearchResults result = search.search("test", 10); + LuceneSearchResults result = search.search("test", 10); assertEquals(8, result.numSearchResults()); } @Test public void searchForUniversity() throws IOException, ParseException { - PdfSearchResults result = search.search("University", 10); + LuceneSearchResults result = search.search("University", 10); assertEquals(1, result.numSearchResults()); } @Test public void searchForStopWord() throws IOException, ParseException { - PdfSearchResults result = search.search("and", 10); + LuceneSearchResults result = search.search("and", 10); assertEquals(0, result.numSearchResults()); } @Test public void searchForSecond() throws IOException, ParseException { - PdfSearchResults result = search.search("second", 10); + LuceneSearchResults result = search.search("second", 10); assertEquals(4, result.numSearchResults()); } @Test public void searchForAnnotation() throws IOException, ParseException { - PdfSearchResults result = search.search("annotation", 10); + LuceneSearchResults result = search.search("annotation", 10); assertEquals(2, result.numSearchResults()); } @Test public void searchForEmptyString() throws IOException { - PdfSearchResults result = search.search("", 10); + LuceneSearchResults result = search.search("", 10); assertEquals(0, result.numSearchResults()); } From be31a6274f33e229b89d865437de9e03e8080c3a Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Sat, 9 Jul 2022 18:48:24 +0200 Subject: [PATCH 007/256] Consider fulltext-index-preference --- src/main/java/org/jabref/gui/LibraryTab.java | 11 +++-- .../search/indexing/IndexingTaskManager.java | 10 ++++ .../pdf/search/indexing/LuceneIndexer.java | 49 ++++++++++++++++--- 3 files changed, 58 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/jabref/gui/LibraryTab.java b/src/main/java/org/jabref/gui/LibraryTab.java index 25f3c5675cc..b0705a4dbb6 100644 --- a/src/main/java/org/jabref/gui/LibraryTab.java +++ b/src/main/java/org/jabref/gui/LibraryTab.java @@ -855,12 +855,13 @@ public void listen(EntriesRemovedEvent removedEntriesEvent) { private class IndexUpdateListener { public IndexUpdateListener() { - if (preferencesService.getFilePreferences().shouldFulltextIndexLinkedFiles()) { - try { - indexingTaskManager.updateIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences())); - } catch (IOException e) { - LOGGER.error("Cannot access lucene index", e); + try { + indexingTaskManager.manageFulltextIndexAccordingToPrefs(LuceneIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences())); + if (preferencesService.getFilePreferences().shouldFulltextIndexLinkedFiles()) { + indexingTaskManager.updateIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences())); } + } catch (IOException e) { + LOGGER.error("Cannot access lucene index", e); } } diff --git a/src/main/java/org/jabref/logic/pdf/search/indexing/IndexingTaskManager.java b/src/main/java/org/jabref/logic/pdf/search/indexing/IndexingTaskManager.java index 487ba3a45de..fd9e617e383 100644 --- a/src/main/java/org/jabref/logic/pdf/search/indexing/IndexingTaskManager.java +++ b/src/main/java/org/jabref/logic/pdf/search/indexing/IndexingTaskManager.java @@ -89,6 +89,16 @@ public void createIndex(LuceneIndexer indexer) { enqueueTask(() -> indexer.createIndex()); } + public void manageFulltextIndexAccordingToPrefs(LuceneIndexer indexer) { + indexer.getFilePreferences().fulltextIndexLinkedFilesProperty().addListener((observable, oldValue, newValue) -> { + if (newValue.booleanValue()) { + enqueueTask(() -> indexer.updateIndex()); + } else { + enqueueTask(() -> indexer.deleteLinkedFilesIndex()); + } + }); + } + public void updateIndex(LuceneIndexer indexer) { Set pathsToRemove = indexer.getListOfFilePaths(); Set hashesOfEntriesToRemove = indexer.getDatabaseContext().getEntries().stream().map(BibEntry::updateAndGetIndexHash).collect(Collectors.toSet()); diff --git a/src/main/java/org/jabref/logic/pdf/search/indexing/LuceneIndexer.java b/src/main/java/org/jabref/logic/pdf/search/indexing/LuceneIndexer.java index b41f5f365aa..c36e004ba4f 100644 --- a/src/main/java/org/jabref/logic/pdf/search/indexing/LuceneIndexer.java +++ b/src/main/java/org/jabref/logic/pdf/search/indexing/LuceneIndexer.java @@ -30,6 +30,8 @@ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.Term; +import org.apache.lucene.queryparser.classic.ParseException; +import org.apache.lucene.queryparser.classic.QueryParser; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.ScoreDoc; @@ -176,6 +178,9 @@ private void writeBibFieldsToIndex(BibEntry bibEntry) { * @param linkedFile the file to write to the index */ private void writeFileToIndex(BibEntry entry, LinkedFile linkedFile) { + if (!filePreferences.shouldFulltextIndexLinkedFiles()) { + return; + } if (linkedFile.isOnlineLink() || !StandardFileType.PDF.getName().equals(linkedFile.getFileType())) { return; } @@ -219,16 +224,26 @@ private void writeFileToIndex(BibEntry entry, LinkedFile linkedFile) { } } + public void updateIndex() { + for (BibEntry bibEntry : databaseContext.getEntries()) { + updateIndex(bibEntry, List.of()); + } + } + public void updateIndex(BibEntry entry, List removedFiles) { int oldHash = entry.getLastIndexHash(); int newHash = entry.updateAndGetIndexHash(); - if (oldHash == newHash) { - return; - } - addToIndex(entry); - removeFromIndex(oldHash); - for (LinkedFile removedFile : removedFiles) { - removeFromIndex(removedFile.getLink()); + if (oldHash != newHash) { + addToIndex(entry); + removeFromIndex(oldHash); + for (LinkedFile removedFile : removedFiles) { + removeFromIndex(removedFile.getLink()); + } + } else { + // This only happens when fulltext-indexing was turned off and is now turned back on again + for (LinkedFile linkedFile : entry.getFiles()) { + writeFileToIndex(entry, linkedFile); + } } } @@ -254,4 +269,24 @@ public Set getListOfFilePaths() { } return paths; } + + public void deleteLinkedFilesIndex() { + try (IndexWriter indexWriter = new IndexWriter( + directoryToIndex, + new IndexWriterConfig( + new EnglishStemAnalyzer()).setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND))) { + QueryParser queryParser = new QueryParser(SearchFieldConstants.PATH, new EnglishStemAnalyzer()); + queryParser.setAllowLeadingWildcard(true); + indexWriter.deleteDocuments(queryParser.parse("*")); + indexWriter.commit(); + } catch (IOException e) { + LOGGER.warn("Could not initialize the IndexWriter!", e); + } catch (ParseException e) { + e.printStackTrace(); + } + } + + public FilePreferences getFilePreferences() { + return filePreferences; + } } From e88d5d0570898498cf3ceda40a3ae628918cf83e Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Sat, 9 Jul 2022 18:53:18 +0200 Subject: [PATCH 008/256] Better status-reporting for re-indexing on preference change --- .../logic/pdf/search/indexing/IndexingTaskManager.java | 4 +++- .../org/jabref/logic/pdf/search/indexing/LuceneIndexer.java | 6 ------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/jabref/logic/pdf/search/indexing/IndexingTaskManager.java b/src/main/java/org/jabref/logic/pdf/search/indexing/IndexingTaskManager.java index fd9e617e383..22ca18b6377 100644 --- a/src/main/java/org/jabref/logic/pdf/search/indexing/IndexingTaskManager.java +++ b/src/main/java/org/jabref/logic/pdf/search/indexing/IndexingTaskManager.java @@ -92,7 +92,9 @@ public void createIndex(LuceneIndexer indexer) { public void manageFulltextIndexAccordingToPrefs(LuceneIndexer indexer) { indexer.getFilePreferences().fulltextIndexLinkedFilesProperty().addListener((observable, oldValue, newValue) -> { if (newValue.booleanValue()) { - enqueueTask(() -> indexer.updateIndex()); + for (BibEntry bibEntry : indexer.getDatabaseContext().getEntries()) { + enqueueTask(() -> indexer.updateIndex(bibEntry, List.of())); + } } else { enqueueTask(() -> indexer.deleteLinkedFilesIndex()); } diff --git a/src/main/java/org/jabref/logic/pdf/search/indexing/LuceneIndexer.java b/src/main/java/org/jabref/logic/pdf/search/indexing/LuceneIndexer.java index c36e004ba4f..c7c2832efb1 100644 --- a/src/main/java/org/jabref/logic/pdf/search/indexing/LuceneIndexer.java +++ b/src/main/java/org/jabref/logic/pdf/search/indexing/LuceneIndexer.java @@ -224,12 +224,6 @@ private void writeFileToIndex(BibEntry entry, LinkedFile linkedFile) { } } - public void updateIndex() { - for (BibEntry bibEntry : databaseContext.getEntries()) { - updateIndex(bibEntry, List.of()); - } - } - public void updateIndex(BibEntry entry, List removedFiles) { int oldHash = entry.getLastIndexHash(); int newHash = entry.updateAndGetIndexHash(); From ccad245b7e9fd74fb493d8aed89ade4c380712ec Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Sat, 9 Jul 2022 21:32:45 +0200 Subject: [PATCH 009/256] Fix tests --- .../search/indexing/LuceneIndexerTest.java | 41 +++++++++++-------- .../search/retrieval/LuceneSearcherTest.java | 4 +- 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/src/test/java/org/jabref/logic/pdf/search/indexing/LuceneIndexerTest.java b/src/test/java/org/jabref/logic/pdf/search/indexing/LuceneIndexerTest.java index e5e90192985..afc0fd31675 100644 --- a/src/test/java/org/jabref/logic/pdf/search/indexing/LuceneIndexerTest.java +++ b/src/test/java/org/jabref/logic/pdf/search/indexing/LuceneIndexerTest.java @@ -34,6 +34,7 @@ public class LuceneIndexerTest { @BeforeEach public void setUp(@TempDir Path indexDir) throws IOException { FilePreferences filePreferences = mock(FilePreferences.class); + when(filePreferences.shouldFulltextIndexLinkedFiles()).thenReturn(true); this.database = new BibDatabase(); this.context = mock(BibDatabaseContext.class); @@ -55,12 +56,13 @@ public void exampleThesisIndex() throws IOException { // when indexer.createIndex(); for (BibEntry bibEntry : context.getEntries()) { - indexer.addToIndex(bibEntry); + indexer.addBibFieldsToIndex(bibEntry); + indexer.addLinkedFilesToIndex(bibEntry); } // then try (IndexReader reader = DirectoryReader.open(new NIOFSDirectory(context.getFulltextIndexPath()))) { - assertEquals(33, reader.numDocs()); + assertEquals(34, reader.numDocs()); } } @@ -74,12 +76,13 @@ public void dontIndexNonPdf() throws IOException { // when indexer.createIndex(); for (BibEntry bibEntry : context.getEntries()) { - indexer.addToIndex(bibEntry); + indexer.addBibFieldsToIndex(bibEntry); + indexer.addLinkedFilesToIndex(bibEntry); } // then try (IndexReader reader = DirectoryReader.open(new NIOFSDirectory(context.getFulltextIndexPath()))) { - assertEquals(0, reader.numDocs()); + assertEquals(1, reader.numDocs()); } } @@ -93,12 +96,13 @@ public void dontIndexOnlineLinks() throws IOException { // when indexer.createIndex(); for (BibEntry bibEntry : context.getEntries()) { - indexer.addToIndex(bibEntry); + indexer.addBibFieldsToIndex(bibEntry); + indexer.addLinkedFilesToIndex(bibEntry); } // then try (IndexReader reader = DirectoryReader.open(new NIOFSDirectory(context.getFulltextIndexPath()))) { - assertEquals(0, reader.numDocs()); + assertEquals(1, reader.numDocs()); } } @@ -113,12 +117,13 @@ public void exampleThesisIndexWithKey() throws IOException { // when indexer.createIndex(); for (BibEntry bibEntry : context.getEntries()) { - indexer.addToIndex(bibEntry); + indexer.addBibFieldsToIndex(bibEntry); + indexer.addLinkedFilesToIndex(bibEntry); } // then try (IndexReader reader = DirectoryReader.open(new NIOFSDirectory(context.getFulltextIndexPath()))) { - assertEquals(33, reader.numDocs()); + assertEquals(34, reader.numDocs()); } } @@ -133,12 +138,13 @@ public void metaDataIndex() throws IOException { // when indexer.createIndex(); for (BibEntry bibEntry : context.getEntries()) { - indexer.addToIndex(bibEntry); + indexer.addBibFieldsToIndex(bibEntry); + indexer.addLinkedFilesToIndex(bibEntry); } // then try (IndexReader reader = DirectoryReader.open(new NIOFSDirectory(context.getFulltextIndexPath()))) { - assertEquals(1, reader.numDocs()); + assertEquals(2, reader.numDocs()); } } @@ -152,11 +158,12 @@ public void testFlushIndex() throws IOException { indexer.createIndex(); for (BibEntry bibEntry : context.getEntries()) { - indexer.addToIndex(bibEntry); + indexer.addBibFieldsToIndex(bibEntry); + indexer.addLinkedFilesToIndex(bibEntry); } // index actually exists try (IndexReader reader = DirectoryReader.open(new NIOFSDirectory(context.getFulltextIndexPath()))) { - assertEquals(33, reader.numDocs()); + assertEquals(34, reader.numDocs()); } // when @@ -177,12 +184,13 @@ public void exampleThesisIndexAppendMetaData() throws IOException { database.insertEntry(exampleThesis); indexer.createIndex(); for (BibEntry bibEntry : context.getEntries()) { - indexer.addToIndex(bibEntry); + indexer.addBibFieldsToIndex(bibEntry); + indexer.addLinkedFilesToIndex(bibEntry); } // index with first entry try (IndexReader reader = DirectoryReader.open(new NIOFSDirectory(context.getFulltextIndexPath()))) { - assertEquals(33, reader.numDocs()); + assertEquals(34, reader.numDocs()); } BibEntry metadata = new BibEntry(StandardEntryType.Article); @@ -190,11 +198,12 @@ public void exampleThesisIndexAppendMetaData() throws IOException { metadata.setFiles(Collections.singletonList(new LinkedFile("Metadata file", "metaData.pdf", StandardFileType.PDF.getName()))); // when - indexer.addToIndex(metadata); + indexer.addBibFieldsToIndex(metadata); + indexer.addLinkedFilesToIndex(metadata); // then try (IndexReader reader = DirectoryReader.open(new NIOFSDirectory(context.getFulltextIndexPath()))) { - assertEquals(34, reader.numDocs()); + assertEquals(36, reader.numDocs()); } } } diff --git a/src/test/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcherTest.java b/src/test/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcherTest.java index a328050296f..932e86f0ba4 100644 --- a/src/test/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcherTest.java +++ b/src/test/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcherTest.java @@ -32,6 +32,7 @@ public class LuceneSearcherTest { @BeforeEach public void setUp(@TempDir Path indexDir) throws IOException { FilePreferences filePreferences = mock(FilePreferences.class); + when(filePreferences.shouldFulltextIndexLinkedFiles()).thenReturn(true); // given BibDatabase database = new BibDatabase(); BibDatabaseContext context = mock(BibDatabaseContext.class); @@ -58,7 +59,8 @@ public void setUp(@TempDir Path indexDir) throws IOException { indexer.createIndex(); for (BibEntry bibEntry : context.getEntries()) { - indexer.addToIndex(bibEntry); + indexer.addBibFieldsToIndex(bibEntry); + indexer.addLinkedFilesToIndex(bibEntry); } } From 6ad3c2a3ef35eaaf26d45f5e9b9cac246b34b139 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Sat, 9 Jul 2022 21:33:11 +0200 Subject: [PATCH 010/256] Prioritize BibEntry field changes --- .../search/indexing/IndexingTaskManager.java | 36 +++++++++++-------- .../pdf/search/indexing/LuceneIndexer.java | 26 +++++++------- 2 files changed, 34 insertions(+), 28 deletions(-) diff --git a/src/main/java/org/jabref/logic/pdf/search/indexing/IndexingTaskManager.java b/src/main/java/org/jabref/logic/pdf/search/indexing/IndexingTaskManager.java index 22ca18b6377..c5cb4b6bd5e 100644 --- a/src/main/java/org/jabref/logic/pdf/search/indexing/IndexingTaskManager.java +++ b/src/main/java/org/jabref/logic/pdf/search/indexing/IndexingTaskManager.java @@ -1,9 +1,8 @@ package org.jabref.logic.pdf.search.indexing; import java.util.List; -import java.util.Queue; import java.util.Set; -import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentLinkedDeque; import java.util.stream.Collectors; import org.jabref.gui.util.BackgroundTask; @@ -18,7 +17,7 @@ */ public class IndexingTaskManager extends BackgroundTask { - private final Queue taskQueue = new ConcurrentLinkedQueue<>(); + private final ConcurrentLinkedDeque taskQueue = new ConcurrentLinkedDeque<>(); private TaskExecutor taskExecutor; private int numOfIndexedFiles = 0; @@ -43,7 +42,7 @@ protected Void call() throws Exception { } updateProgress(); while (!taskQueue.isEmpty() && !isCanceled()) { - taskQueue.poll().run(); + taskQueue.pollFirst().run(); numOfIndexedFiles++; updateProgress(); } @@ -60,9 +59,13 @@ private void updateProgress() { }); } - private void enqueueTask(Runnable indexingTask) { + private void enqueueTask(Runnable indexingTask, boolean skipToFront) { if (!isBlockingNewTasks) { - taskQueue.add(indexingTask); + if (skipToFront) { + taskQueue.addFirst(indexingTask); + } else { + taskQueue.addLast(indexingTask); + } // What if already running? synchronized (lock) { if (!isRunning) { @@ -86,17 +89,17 @@ public AutoCloseable blockNewTasks() { } public void createIndex(LuceneIndexer indexer) { - enqueueTask(() -> indexer.createIndex()); + enqueueTask(() -> indexer.createIndex(), true); } public void manageFulltextIndexAccordingToPrefs(LuceneIndexer indexer) { indexer.getFilePreferences().fulltextIndexLinkedFilesProperty().addListener((observable, oldValue, newValue) -> { if (newValue.booleanValue()) { for (BibEntry bibEntry : indexer.getDatabaseContext().getEntries()) { - enqueueTask(() -> indexer.updateIndex(bibEntry, List.of())); + enqueueTask(() -> indexer.updateLinkedFilesInIndex(bibEntry, List.of()), false); } } else { - enqueueTask(() -> indexer.deleteLinkedFilesIndex()); + enqueueTask(() -> indexer.deleteLinkedFilesIndex(), true); } }); } @@ -105,30 +108,33 @@ public void updateIndex(LuceneIndexer indexer) { Set pathsToRemove = indexer.getListOfFilePaths(); Set hashesOfEntriesToRemove = indexer.getDatabaseContext().getEntries().stream().map(BibEntry::updateAndGetIndexHash).collect(Collectors.toSet()); for (BibEntry entry : indexer.getDatabaseContext().getEntries()) { - enqueueTask(() -> indexer.addToIndex(entry)); + enqueueTask(() -> indexer.addBibFieldsToIndex(entry), true); + enqueueTask(() -> indexer.addLinkedFilesToIndex(entry), false); hashesOfEntriesToRemove.removeIf(hash -> Integer.valueOf(entry.getLastIndexHash()).equals(hash)); for (LinkedFile file : entry.getFiles()) { pathsToRemove.remove(file.getLink()); } } for (String pathToRemove : pathsToRemove) { - enqueueTask(() -> indexer.removeFromIndex(pathToRemove)); + enqueueTask(() -> indexer.removeFromIndex(pathToRemove), true); } for (int hashToRemove : hashesOfEntriesToRemove) { - enqueueTask(() -> indexer.removeFromIndex(hashToRemove)); + enqueueTask(() -> indexer.removeFromIndex(hashToRemove), true); } } public void addToIndex(LuceneIndexer indexer, BibEntry entry) { - enqueueTask(() -> indexer.addToIndex(entry)); + enqueueTask(() -> indexer.addBibFieldsToIndex(entry), true); + enqueueTask(() -> indexer.addLinkedFilesToIndex(entry), false); } public void removeFromIndex(LuceneIndexer indexer, BibEntry entry) { - enqueueTask(() -> indexer.removeFromIndex(entry)); + enqueueTask(() -> indexer.removeFromIndex(entry), false); } public void updateIndex(LuceneIndexer indexer, BibEntry entry, List removedFiles) { - enqueueTask(() -> indexer.updateIndex(entry, removedFiles)); + enqueueTask(() -> indexer.updateBibFieldsInIndex(entry), true); + enqueueTask(() -> indexer.updateLinkedFilesInIndex(entry, removedFiles), false); } public void updateDatabaseName(String name) { diff --git a/src/main/java/org/jabref/logic/pdf/search/indexing/LuceneIndexer.java b/src/main/java/org/jabref/logic/pdf/search/indexing/LuceneIndexer.java index c7c2832efb1..ec96de2dfa3 100644 --- a/src/main/java/org/jabref/logic/pdf/search/indexing/LuceneIndexer.java +++ b/src/main/java/org/jabref/logic/pdf/search/indexing/LuceneIndexer.java @@ -86,8 +86,7 @@ public void createIndex() { * * @param entry a bibtex entry to link the pdf files to */ - public void addToIndex(BibEntry entry) { - writeBibFieldsToIndex(entry); + public void addLinkedFilesToIndex(BibEntry entry) { for (LinkedFile file : entry.getFiles()) { writeFileToIndex(entry, file); } @@ -151,7 +150,7 @@ public void flushIndex() { } } - private void writeBibFieldsToIndex(BibEntry bibEntry) { + public void addBibFieldsToIndex(BibEntry bibEntry) { try { try (IndexWriter indexWriter = new IndexWriter(directoryToIndex, new IndexWriterConfig( @@ -224,20 +223,21 @@ private void writeFileToIndex(BibEntry entry, LinkedFile linkedFile) { } } - public void updateIndex(BibEntry entry, List removedFiles) { + public void updateBibFieldsInIndex(BibEntry entry) { int oldHash = entry.getLastIndexHash(); int newHash = entry.updateAndGetIndexHash(); if (oldHash != newHash) { - addToIndex(entry); + addBibFieldsToIndex(entry); removeFromIndex(oldHash); - for (LinkedFile removedFile : removedFiles) { - removeFromIndex(removedFile.getLink()); - } - } else { - // This only happens when fulltext-indexing was turned off and is now turned back on again - for (LinkedFile linkedFile : entry.getFiles()) { - writeFileToIndex(entry, linkedFile); - } + } + } + + public void updateLinkedFilesInIndex(BibEntry entry, List removedFiles) { + for (LinkedFile removedFile : removedFiles) { + removeFromIndex(removedFile.getLink()); + } + for (LinkedFile linkedFile : entry.getFiles()) { + writeFileToIndex(entry, linkedFile); } } From 21927c1af14fe46afdc76465265abc13befbdb3d Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Sun, 10 Jul 2022 10:41:30 +0200 Subject: [PATCH 011/256] Prepare CHANGELOG.md --- CHANGELOG.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c63017de4b8..c2663a4aea5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,17 @@ In case, there is no issue present, the pull request implementing the feature is Note that this project **does not** adhere to [Semantic Versioning](http://semver.org/). -## [Unreleased] +## [version6] - Big changes + +The branch `version6` is intended to collect improvements, where multiple pull requests build on each other. +From the user perspective, the combination of the pull request needs to be tested. +(We created the branch to support lowering the amount of open pull requests) + + + + + +## [Unreleased] - version 5.7 - not yet released ### Added @@ -840,6 +850,7 @@ The changelog of JabRef 4.x is available at the [v4.3.1 tag](https://github.com/ The changelog of JabRef 3.x is available at the [v3.8.2 tag](https://github.com/JabRef/jabref/blob/v3.8.2/CHANGELOG.md). The changelog of JabRef 2.11 and all previous versions is available as [text file in the v2.11.1 tag](https://github.com/JabRef/jabref/blob/v2.11.1/CHANGELOG). +[version6]: https://github.com/JabRef/jabref/compare/main...version6 [Unreleased]: https://github.com/JabRef/jabref/compare/v5.6...HEAD [5.6]: https://github.com/JabRef/jabref/compare/v5.5...v5.6 [5.5]: https://github.com/JabRef/jabref/compare/v5.4...v5.5 From b78d3fe46b3a27a62239bf73ec3b3ad0717435f0 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Sun, 10 Jul 2022 13:44:12 +0200 Subject: [PATCH 012/256] Trigger lucene-search in gui and display scoring --- .../gui/maintable/BibEntryTableViewModel.java | 26 +++++++++++ .../org/jabref/gui/maintable/MainTable.java | 18 ++++++++ .../gui/maintable/MainTableColumnFactory.java | 24 ++++++++++ .../gui/maintable/MainTableColumnModel.java | 1 + .../gui/maintable/MainTableDataModel.java | 37 +++++++++++---- .../search/SearchResultsTableDataModel.java | 1 + .../pdf/search/indexing/LuceneIndexer.java | 2 +- .../pdf/search/retrieval/LuceneSearcher.java | 46 +++++++++++++++++-- .../pdf/search/SearchFieldConstants.java | 5 +- .../jabref/model/pdf/search/SearchResult.java | 13 ++++++ src/main/resources/l10n/JabRef_en.properties | 2 + 11 files changed, 159 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/jabref/gui/maintable/BibEntryTableViewModel.java b/src/main/java/org/jabref/gui/maintable/BibEntryTableViewModel.java index 6fe42cca9b3..99849d0adee 100644 --- a/src/main/java/org/jabref/gui/maintable/BibEntryTableViewModel.java +++ b/src/main/java/org/jabref/gui/maintable/BibEntryTableViewModel.java @@ -3,6 +3,7 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -13,9 +14,15 @@ import javafx.beans.Observable; import javafx.beans.binding.Binding; import javafx.beans.binding.Bindings; +import javafx.beans.property.FloatProperty; +import javafx.beans.property.ReadOnlyFloatProperty; import javafx.beans.property.ReadOnlyStringWrapper; +import javafx.beans.property.SimpleFloatProperty; import javafx.beans.property.StringProperty; import javafx.beans.value.ObservableValue; +import javafx.collections.FXCollections; +import javafx.collections.ListChangeListener; +import javafx.collections.ObservableList; import org.jabref.gui.specialfields.SpecialFieldValueViewModel; import org.jabref.gui.util.uithreadaware.UiThreadBinding; @@ -29,6 +36,7 @@ import org.jabref.model.entry.field.StandardField; import org.jabref.model.groups.AbstractGroup; import org.jabref.model.groups.GroupTreeNode; +import org.jabref.model.pdf.search.SearchResult; import com.tobiasdiez.easybind.EasyBind; import com.tobiasdiez.easybind.EasyBinding; @@ -44,6 +52,8 @@ public class BibEntryTableViewModel { private final EasyBinding> linkedIdentifiers; private final Binding> matchedGroups; private final BibDatabaseContext bibDatabaseContext; + private ObservableList searchResults = FXCollections.observableArrayList(); + private FloatProperty searchScore = new SimpleFloatProperty(); public BibEntryTableViewModel(BibEntry entry, BibDatabaseContext bibDatabaseContext, ObservableValue fieldValueFormatter) { this.entry = entry; @@ -53,6 +63,10 @@ public BibEntryTableViewModel(BibEntry entry, BibDatabaseContext bibDatabaseCont this.linkedIdentifiers = createLinkedIdentifiersBinding(entry); this.matchedGroups = createMatchedGroupsBinding(bibDatabaseContext, entry); this.bibDatabaseContext = bibDatabaseContext; + + searchResults.addListener((ListChangeListener) (change) -> { + searchScore.setValue(searchResults.stream().map(SearchResult::getLuceneScore).max(Comparator.comparing(Float::valueOf)).orElseGet(() -> (float) 0.0)); + }); } private static EasyBinding> createLinkedIdentifiersBinding(BibEntry entry) { @@ -132,4 +146,16 @@ public ObservableValue getFields(OrFields fields) { public StringProperty bibDatabaseContextProperty() { return new ReadOnlyStringWrapper(bibDatabaseContext.getDatabasePath().map(Path::toString).orElse("")); } + + public void resetSearchResults() { + searchResults.clear(); + } + + public void addSearchResults(List results) { + searchResults.addAll(results); + } + + public ReadOnlyFloatProperty getSearchScore() { + return searchScore; + } } diff --git a/src/main/java/org/jabref/gui/maintable/MainTable.java b/src/main/java/org/jabref/gui/maintable/MainTable.java index 51494fe6315..2c6ed434294 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTable.java +++ b/src/main/java/org/jabref/gui/maintable/MainTable.java @@ -147,12 +147,30 @@ public MainTable(MainTableDataModel model, } */ + // always sort by score first. If no search is ongoing, it will be equal for all columns. + ListChangeListener> scoreSortOderPrioritizer = new ListChangeListener>() { + @Override + public void onChanged(Change> c) { + getSortOrder().removeListener(this); + getSortOrder().removeAll(getColumns().get(0)); + getSortOrder().add(0, getColumns().get(0)); + getSortOrder().addListener(this); + } + }; + + // insert score sort order + this.getSortOrder().add(0, this.getColumns().get(0)); mainTablePreferences.getColumnPreferences().getColumnSortOrder().forEach(columnModel -> this.getColumns().stream() .map(column -> (MainTableColumn) column) .filter(column -> column.getModel().equals(columnModel)) + .filter(column -> !column.getModel().getType().equals(MainTableColumnModel.Type.SCORE)) .findFirst() .ifPresent(column -> this.getSortOrder().add(column))); + this.getSortOrder().addListener(scoreSortOderPrioritizer); + + // Is this always called after the search is done? + stateManager.activeSearchQueryProperty().addListener((observable, oldValue, newValue) -> sort()); if (mainTablePreferences.getResizeColumnsToFit()) { this.setColumnResizePolicy(new SmartConstrainedResizePolicy()); diff --git a/src/main/java/org/jabref/gui/maintable/MainTableColumnFactory.java b/src/main/java/org/jabref/gui/maintable/MainTableColumnFactory.java index ece234f9ff4..3a234aaa94b 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableColumnFactory.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableColumnFactory.java @@ -81,6 +81,8 @@ public MainTableColumnFactory(BibDatabaseContext database, public List> createColumns() { List> columns = new ArrayList<>(); + columns.add(createScoreColumn(new MainTableColumnModel(MainTableColumnModel.Type.SCORE))); + columnPreferences.getColumns().forEach(column -> { switch (column.getType()) { @@ -133,6 +135,28 @@ public static void setExactWidth(TableColumn column, double width) { column.setMaxWidth(width); } + /** + * Creates a column with the search score + */ + private TableColumn createScoreColumn(MainTableColumnModel columnModel) { + TableColumn column = new MainTableColumn<>(columnModel); + Node header = new Text("Score"); + header.getStyleClass().add("mainTable-header"); + Tooltip.install(header, new Tooltip(MainTableColumnModel.Type.SCORE.getDisplayName())); + column.setGraphic(header); + column.setStyle("-fx-alignment: CENTER-RIGHT;"); + column.setCellValueFactory(cellData -> + cellData.getValue().getSearchScore().asString()); + new ValueTableCellFactory() + .withText(text -> text) + .install(column); + column.setSortable(true); + column.setSortType(TableColumn.SortType.DESCENDING); + column.visibleProperty().bind(stateManager.activeSearchQueryProperty().isPresent()); + column.setReorderable(false); + return column; + } + /** * Creates a column with a continous number */ diff --git a/src/main/java/org/jabref/gui/maintable/MainTableColumnModel.java b/src/main/java/org/jabref/gui/maintable/MainTableColumnModel.java index 5ff872c1426..76ca8e9dcc7 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableColumnModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableColumnModel.java @@ -30,6 +30,7 @@ public class MainTableColumnModel { private static final Logger LOGGER = LoggerFactory.getLogger(MainTableColumnModel.class); public enum Type { + SCORE("search_score", Localization.lang("Search score")), INDEX("index", Localization.lang("Index")), EXTRAFILE("extrafile", Localization.lang("File type")), FILES("files", Localization.lang("Linked files")), diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index 5210613e4de..93567fd39a3 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -1,9 +1,13 @@ package org.jabref.gui.maintable; +import java.io.IOException; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.function.Predicate; import javafx.beans.binding.Bindings; +import javafx.beans.binding.ObjectBinding; import javafx.beans.property.IntegerProperty; import javafx.beans.property.ObjectProperty; import javafx.beans.property.SimpleIntegerProperty; @@ -16,10 +20,12 @@ import org.jabref.gui.groups.GroupViewMode; import org.jabref.gui.groups.GroupsPreferences; import org.jabref.gui.util.BindingsHelper; +import org.jabref.logic.pdf.search.retrieval.LuceneSearcher; import org.jabref.logic.search.SearchQuery; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.groups.GroupTreeNode; +import org.jabref.model.pdf.search.SearchResult; import org.jabref.model.search.matchers.MatcherSet; import org.jabref.model.search.matchers.MatcherSets; import org.jabref.preferences.PreferencesService; @@ -46,9 +52,11 @@ public MainTableDataModel(BibDatabaseContext context, PreferencesService prefere new BibEntryTableViewModel(entry, bibDatabaseContext, fieldValueFormatter)); entriesFiltered = new FilteredList<>(entriesViewModel); - entriesFiltered.predicateProperty().bind( - EasyBind.combine(stateManager.activeGroupProperty(), stateManager.activeSearchQueryProperty(), (groups, query) -> entry -> isMatched(groups, query, entry)) - ); + ObjectBinding> filter = Bindings.createObjectBinding( + () -> entry -> isMatchedByGroup(stateManager.activeGroupProperty(), entry), stateManager.activeGroupProperty()); + entriesFiltered.predicateProperty().bind(filter); + + stateManager.activeSearchQueryProperty().addListener((observable, oldValue, newValue) -> doSearch(newValue)); IntegerProperty resultSize = new SimpleIntegerProperty(); resultSize.bind(Bindings.size(entriesFiltered)); @@ -57,13 +65,26 @@ public MainTableDataModel(BibDatabaseContext context, PreferencesService prefere entriesSorted = new SortedList<>(entriesFiltered); } - private boolean isMatched(ObservableList groups, Optional query, BibEntryTableViewModel entry) { - return isMatchedByGroup(groups, entry) && isMatchedBySearch(query, entry); + private void doSearch(Optional query) { + for (BibEntryTableViewModel entry : entriesFiltered) { + entry.resetSearchResults(); + } + if (query.isPresent()) { + String searchString = query.get().getQuery(); + try { + LuceneSearcher searcher = LuceneSearcher.of(bibDatabaseContext); + Map> results = searcher.search(query.get()); + for (Map.Entry> result : results.entrySet()) { + getTableViewModelForEntry(result.getKey()).ifPresent(bibEntryTableViewModel -> bibEntryTableViewModel.addSearchResults(result.getValue())); + } + } catch (IOException e) { + e.printStackTrace(); + } + } } - private boolean isMatchedBySearch(Optional query, BibEntryTableViewModel entry) { - return query.map(matcher -> matcher.isMatch(entry.getEntry())) - .orElse(true); + private Optional getTableViewModelForEntry(BibEntry entry) { + return entriesSorted.stream().filter(viewModel -> viewModel.getEntry().equals(entry)).findFirst(); } private boolean isMatchedByGroup(ObservableList groups, BibEntryTableViewModel entry) { diff --git a/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java b/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java index c1aebd29966..fe95d04210f 100644 --- a/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java +++ b/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java @@ -41,6 +41,7 @@ public SearchResultsTableDataModel(BibDatabaseContext bibDatabaseContext, Prefer } entriesFiltered = new FilteredList<>(entriesViewModel); + // don't need this if ranking is in place? entriesFiltered.predicateProperty().bind(EasyBind.map(stateManager.activeSearchQueryProperty(), (query) -> entry -> isMatchedBySearch(query, entry))); // We need to wrap the list since otherwise sorting in the table does not work diff --git a/src/main/java/org/jabref/logic/pdf/search/indexing/LuceneIndexer.java b/src/main/java/org/jabref/logic/pdf/search/indexing/LuceneIndexer.java index ec96de2dfa3..0d3700ca364 100644 --- a/src/main/java/org/jabref/logic/pdf/search/indexing/LuceneIndexer.java +++ b/src/main/java/org/jabref/logic/pdf/search/indexing/LuceneIndexer.java @@ -159,7 +159,7 @@ public void addBibFieldsToIndex(BibEntry bibEntry) { document.add(new StringField(SearchFieldConstants.BIB_ENTRY_ID_HASH, String.valueOf(bibEntry.getLastIndexHash()), org.apache.lucene.document.Field.Store.YES)); for (Map.Entry field : bibEntry.getFieldMap().entrySet()) { document.add(new TextField(SearchFieldConstants.BIB_FIELDS_PREFIX + field.getKey().getName(), field.getValue(), org.apache.lucene.document.Field.Store.YES)); - SearchFieldConstants.all_searchable_fields.add(SearchFieldConstants.BIB_FIELDS_PREFIX + field.getKey().getName()); + SearchFieldConstants.searchableBibFields.add(SearchFieldConstants.BIB_FIELDS_PREFIX + field.getKey().getName()); } indexWriter.addDocument(document); indexWriter.commit(); diff --git a/src/main/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcher.java b/src/main/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcher.java index 27695b16ef9..13286fcb603 100644 --- a/src/main/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcher.java +++ b/src/main/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcher.java @@ -1,16 +1,22 @@ package org.jabref.logic.pdf.search.retrieval; import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Objects; +import java.util.Set; import org.jabref.gui.LibraryTab; +import org.jabref.logic.search.SearchQuery; import org.jabref.model.database.BibDatabaseContext; +import org.jabref.model.entry.BibEntry; import org.jabref.model.pdf.search.EnglishStemAnalyzer; import org.jabref.model.pdf.search.LuceneSearchResults; import org.jabref.model.pdf.search.SearchFieldConstants; import org.jabref.model.pdf.search.SearchResult; +import org.jabref.model.search.rules.SearchRules; import org.jabref.model.strings.StringUtil; import org.apache.lucene.index.DirectoryReader; @@ -30,14 +36,16 @@ public final class LuceneSearcher { private static final Logger LOGGER = LoggerFactory.getLogger(LibraryTab.class); + private final BibDatabaseContext databaseContext; private final Directory indexDirectory; - private LuceneSearcher(Directory indexDirectory) { + private LuceneSearcher(BibDatabaseContext databaseContext, Directory indexDirectory) { + this.databaseContext = databaseContext; this.indexDirectory = indexDirectory; } public static LuceneSearcher of(BibDatabaseContext databaseContext) throws IOException { - return new LuceneSearcher(new NIOFSDirectory(databaseContext.getFulltextIndexPath())); + return new LuceneSearcher(databaseContext, new NIOFSDirectory(databaseContext.getFulltextIndexPath())); } /** @@ -49,6 +57,7 @@ public static LuceneSearcher of(BibDatabaseContext databaseContext) throws IOExc */ public LuceneSearchResults search(final String searchString, final int maxHits) throws IOException { + Thread.dumpStack(); if (StringUtil.isBlank(Objects.requireNonNull(searchString, "The search string was null!"))) { return new LuceneSearchResults(); } @@ -65,8 +74,8 @@ public LuceneSearchResults search(final String searchString, final int maxHits) try (IndexReader reader = DirectoryReader.open(indexDirectory)) { IndexSearcher searcher = new IndexSearcher(reader); - String[] searchable_fields = new String[SearchFieldConstants.all_searchable_fields.size()]; - SearchFieldConstants.all_searchable_fields.toArray(searchable_fields); + String[] searchable_fields = new String[SearchFieldConstants.searchableBibFields.size()]; + SearchFieldConstants.searchableBibFields.toArray(searchable_fields); Query query = new MultiFieldQueryParser(searchable_fields, new EnglishStemAnalyzer()).parse(searchString); TopDocs results = searcher.search(query, maxHits); for (ScoreDoc scoreDoc : results.scoreDocs) { @@ -78,4 +87,33 @@ public LuceneSearchResults search(final String searchString, final int maxHits) return new LuceneSearchResults(); } } + + public HashMap> search(SearchQuery query) { + HashMap> results = new HashMap<>(); + Set fieldsToSearchIn = new HashSet<>(SearchFieldConstants.searchableBibFields); + if (query.getSearchFlags().contains(SearchRules.SearchFlags.FULLTEXT)) { + fieldsToSearchIn.addAll(List.of(SearchFieldConstants.PDF_FIELDS)); + } + try (IndexReader reader = DirectoryReader.open(indexDirectory)) { + IndexSearcher searcher = new IndexSearcher(reader); + String[] fieldsToSearchArray = new String[fieldsToSearchIn.size()]; + fieldsToSearchIn.toArray(fieldsToSearchArray); + Query luceneQuery = new MultiFieldQueryParser(fieldsToSearchArray, new EnglishStemAnalyzer()).parse(query.getQuery()); + TopDocs docs = searcher.search(luceneQuery, Integer.MAX_VALUE); + for (ScoreDoc scoreDoc : docs.scoreDocs) { + SearchResult searchResult = new SearchResult(searcher, luceneQuery, scoreDoc); + for (BibEntry match : searchResult.getMatchingEntries(databaseContext)) { + if (!results.containsKey(match)) { + results.put(match, new LinkedList<>()); + } + results.get(match).add(searchResult); + } + } + } catch (ParseException e) { + LOGGER.warn("Could not parse query: '{}'!\n{}", query.getQuery(), e.getMessage()); + } catch (IOException e) { + LOGGER.warn("Could not open Index at: '{}'!\n{}", indexDirectory, e.getMessage()); + } + return results; + } } diff --git a/src/main/java/org/jabref/model/pdf/search/SearchFieldConstants.java b/src/main/java/org/jabref/model/pdf/search/SearchFieldConstants.java index 3fc4203c09e..91c800c8de0 100644 --- a/src/main/java/org/jabref/model/pdf/search/SearchFieldConstants.java +++ b/src/main/java/org/jabref/model/pdf/search/SearchFieldConstants.java @@ -1,8 +1,7 @@ package org.jabref.model.pdf.search; -import java.util.Arrays; +import java.util.HashSet; import java.util.Set; -import java.util.stream.Collectors; public class SearchFieldConstants { @@ -17,7 +16,7 @@ public class SearchFieldConstants { public static final String MODIFIED = FILE_FIELDS_PREFIX + "modified"; public static final String[] PDF_FIELDS = new String[]{PATH, CONTENT, ANNOTATIONS}; - public static Set all_searchable_fields = Arrays.stream(PDF_FIELDS).collect(Collectors.toSet()); + public static Set searchableBibFields = new HashSet<>(); public static final String VERSION = "lucene92_jabref0"; } diff --git a/src/main/java/org/jabref/model/pdf/search/SearchResult.java b/src/main/java/org/jabref/model/pdf/search/SearchResult.java index 63c1a562dc3..3b1050ae453 100644 --- a/src/main/java/org/jabref/model/pdf/search/SearchResult.java +++ b/src/main/java/org/jabref/model/pdf/search/SearchResult.java @@ -5,7 +5,9 @@ import java.util.List; import java.util.stream.Collectors; +import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.LinkedFile; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.index.IndexableField; @@ -72,6 +74,17 @@ public SearchResult(IndexSearcher searcher, Query query, ScoreDoc scoreDoc) thro } } + public List getMatchingEntries(BibDatabaseContext databaseContext) { + if (this.path.length() > 0) { + return getEntriesWithFile(path, databaseContext); + } + return databaseContext.getEntries().stream().filter(bibEntry -> bibEntry.getLastIndexHash() == this.hash).collect(Collectors.toList()); + } + + private List getEntriesWithFile(String path, BibDatabaseContext databaseContext) { + return databaseContext.getEntries().stream().filter(entry -> entry.getFiles().stream().map(LinkedFile::getLink).anyMatch(link -> link.equals(path))).collect(Collectors.toList()); + } + private String getFieldContents(IndexSearcher searcher, ScoreDoc scoreDoc, String field) throws IOException { IndexableField indexableField = searcher.doc(scoreDoc.doc).getField(field); if (indexableField == null) { diff --git a/src/main/resources/l10n/JabRef_en.properties b/src/main/resources/l10n/JabRef_en.properties index e8a7c2a1dcc..ed7ef64c888 100644 --- a/src/main/resources/l10n/JabRef_en.properties +++ b/src/main/resources/l10n/JabRef_en.properties @@ -789,6 +789,8 @@ Search=Search Search\ expression=Search expression +Search\ score=Search score + Searching\ for\ duplicates...=Searching for duplicates... Searching\ for\ files=Searching for files From d59d5282779b92fd04e27b6eee3cd23274194018 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Sun, 10 Jul 2022 14:52:38 +0200 Subject: [PATCH 013/256] Don't prefix fields that users should be able to search in --- .../jabref/logic/pdf/search/indexing/LuceneIndexer.java | 4 ++-- .../org/jabref/model/pdf/search/SearchFieldConstants.java | 7 +++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/jabref/logic/pdf/search/indexing/LuceneIndexer.java b/src/main/java/org/jabref/logic/pdf/search/indexing/LuceneIndexer.java index 0d3700ca364..add5a83bc46 100644 --- a/src/main/java/org/jabref/logic/pdf/search/indexing/LuceneIndexer.java +++ b/src/main/java/org/jabref/logic/pdf/search/indexing/LuceneIndexer.java @@ -158,8 +158,8 @@ public void addBibFieldsToIndex(BibEntry bibEntry) { Document document = new Document(); document.add(new StringField(SearchFieldConstants.BIB_ENTRY_ID_HASH, String.valueOf(bibEntry.getLastIndexHash()), org.apache.lucene.document.Field.Store.YES)); for (Map.Entry field : bibEntry.getFieldMap().entrySet()) { - document.add(new TextField(SearchFieldConstants.BIB_FIELDS_PREFIX + field.getKey().getName(), field.getValue(), org.apache.lucene.document.Field.Store.YES)); - SearchFieldConstants.searchableBibFields.add(SearchFieldConstants.BIB_FIELDS_PREFIX + field.getKey().getName()); + document.add(new TextField(field.getKey().getName(), field.getValue(), org.apache.lucene.document.Field.Store.YES)); + SearchFieldConstants.searchableBibFields.add(field.getKey().getName()); } indexWriter.addDocument(document); indexWriter.commit(); diff --git a/src/main/java/org/jabref/model/pdf/search/SearchFieldConstants.java b/src/main/java/org/jabref/model/pdf/search/SearchFieldConstants.java index 91c800c8de0..6340f392b57 100644 --- a/src/main/java/org/jabref/model/pdf/search/SearchFieldConstants.java +++ b/src/main/java/org/jabref/model/pdf/search/SearchFieldConstants.java @@ -5,14 +5,13 @@ public class SearchFieldConstants { - public static final String BIB_FIELDS_PREFIX = "bfp_"; public static final String FILE_FIELDS_PREFIX = "f_"; - public static final String BIB_ENTRY_ID_HASH = "id"; + public static final String BIB_ENTRY_ID_HASH = "id_hash"; public static final String PATH = FILE_FIELDS_PREFIX + "path"; - public static final String CONTENT = FILE_FIELDS_PREFIX + "content"; + public static final String CONTENT = "content"; public static final String PAGE_NUMBER = FILE_FIELDS_PREFIX + "pageNumber"; - public static final String ANNOTATIONS = FILE_FIELDS_PREFIX + "annotations"; + public static final String ANNOTATIONS = "annotations"; public static final String MODIFIED = FILE_FIELDS_PREFIX + "modified"; public static final String[] PDF_FIELDS = new String[]{PATH, CONTENT, ANNOTATIONS}; From 006b8280c116b8675e90bd8b6cbfd470904cf35b Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Tue, 12 Jul 2022 16:41:07 +0200 Subject: [PATCH 014/256] Matches in bib-fields should score higher than fulltext-matches --- .../pdf/search/retrieval/LuceneSearcher.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcher.java b/src/main/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcher.java index 13286fcb603..eedc56370dd 100644 --- a/src/main/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcher.java +++ b/src/main/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcher.java @@ -1,12 +1,11 @@ package org.jabref.logic.pdf.search.retrieval; import java.io.IOException; +import java.util.Arrays; import java.util.HashMap; -import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Objects; -import java.util.Set; import org.jabref.gui.LibraryTab; import org.jabref.logic.search.SearchQuery; @@ -90,15 +89,17 @@ public LuceneSearchResults search(final String searchString, final int maxHits) public HashMap> search(SearchQuery query) { HashMap> results = new HashMap<>(); - Set fieldsToSearchIn = new HashSet<>(SearchFieldConstants.searchableBibFields); + HashMap boosts = new HashMap(); + SearchFieldConstants.searchableBibFields.stream().forEach(field -> boosts.put(field, Float.valueOf(4))); + if (query.getSearchFlags().contains(SearchRules.SearchFlags.FULLTEXT)) { - fieldsToSearchIn.addAll(List.of(SearchFieldConstants.PDF_FIELDS)); + Arrays.stream(SearchFieldConstants.PDF_FIELDS).forEach(field -> boosts.put(field, Float.valueOf(1))); } try (IndexReader reader = DirectoryReader.open(indexDirectory)) { IndexSearcher searcher = new IndexSearcher(reader); - String[] fieldsToSearchArray = new String[fieldsToSearchIn.size()]; - fieldsToSearchIn.toArray(fieldsToSearchArray); - Query luceneQuery = new MultiFieldQueryParser(fieldsToSearchArray, new EnglishStemAnalyzer()).parse(query.getQuery()); + String[] fieldsToSearchArray = new String[boosts.size()]; + boosts.keySet().toArray(fieldsToSearchArray); + Query luceneQuery = new MultiFieldQueryParser(fieldsToSearchArray, new EnglishStemAnalyzer(), boosts).parse(query.getQuery()); TopDocs docs = searcher.search(luceneQuery, Integer.MAX_VALUE); for (ScoreDoc scoreDoc : docs.scoreDocs) { SearchResult searchResult = new SearchResult(searcher, luceneQuery, scoreDoc); From b19be229f6cc29b92983714bb48e8f359b4c1223 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Mon, 15 Aug 2022 10:28:32 +0200 Subject: [PATCH 015/256] Refactoring --- .../java/org/jabref/gui/StateManager.java | 6 +++ .../FulltextSearchResultsTab.java | 2 +- .../gui/maintable/BibEntryTableViewModel.java | 33 ++++++------ .../gui/maintable/MainTableColumnFactory.java | 3 +- .../gui/maintable/MainTableDataModel.java | 16 +++--- .../search/SearchResultsTableDataModel.java | 2 +- .../pdf/search/retrieval/LuceneSearcher.java | 54 +++---------------- .../model/pdf/search/LuceneSearchResults.java | 20 +++---- .../search/rules/ContainsBasedSearchRule.java | 5 +- .../search/rules/FullTextSearchRule.java | 30 ----------- .../search/rules/GrammarBasedSearchRule.java | 23 +------- .../search/rules/RegexBasedSearchRule.java | 4 +- .../jabref/model/search/rules/SearchRule.java | 3 -- 13 files changed, 53 insertions(+), 148 deletions(-) diff --git a/src/main/java/org/jabref/gui/StateManager.java b/src/main/java/org/jabref/gui/StateManager.java index a8d93152de9..c5b2b948c1c 100644 --- a/src/main/java/org/jabref/gui/StateManager.java +++ b/src/main/java/org/jabref/gui/StateManager.java @@ -27,6 +27,7 @@ import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.groups.GroupTreeNode; +import org.jabref.model.pdf.search.LuceneSearchResults; import org.jabref.model.util.OptionalUtil; import com.tobiasdiez.easybind.EasyBind; @@ -53,6 +54,7 @@ public class StateManager { private final ObservableList selectedEntries = FXCollections.observableArrayList(); private final ObservableMap> selectedGroups = FXCollections.observableHashMap(); private final OptionalObjectProperty activeSearchQuery = OptionalObjectProperty.empty(); + private final ObservableMap searchResults = FXCollections.observableHashMap(); private final ObservableMap searchResultMap = FXCollections.observableHashMap(); private final OptionalObjectProperty focusOwner = OptionalObjectProperty.empty(); private final ObservableList>> backgroundTasks = FXCollections.observableArrayList(task -> new Observable[] {task.getValue().progressProperty(), task.getValue().runningProperty()}); @@ -172,4 +174,8 @@ public DialogWindowState getDialogWindowState(String className) { public void setDialogWindowState(String className, DialogWindowState state) { dialogWindowStates.put(className, state); } + + public ObservableMap getSearchResults() { + return searchResults; + } } diff --git a/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java b/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java index f50e7c6e3ec..acadf39a1dd 100644 --- a/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java @@ -83,7 +83,7 @@ protected void bindToEntry(BibEntry entry) { documentViewerView = new DocumentViewerView(); } this.entry = entry; - LuceneSearchResults searchResults = stateManager.activeSearchQueryProperty().get().get().getRule().getLuceneResults(stateManager.activeSearchQueryProperty().get().get().getQuery(), entry); + LuceneSearchResults searchResults = stateManager.getSearchResults().get(entry); content.getChildren().clear(); diff --git a/src/main/java/org/jabref/gui/maintable/BibEntryTableViewModel.java b/src/main/java/org/jabref/gui/maintable/BibEntryTableViewModel.java index 99849d0adee..29124d5cbe6 100644 --- a/src/main/java/org/jabref/gui/maintable/BibEntryTableViewModel.java +++ b/src/main/java/org/jabref/gui/maintable/BibEntryTableViewModel.java @@ -3,7 +3,6 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -15,15 +14,13 @@ import javafx.beans.binding.Binding; import javafx.beans.binding.Bindings; import javafx.beans.property.FloatProperty; -import javafx.beans.property.ReadOnlyFloatProperty; import javafx.beans.property.ReadOnlyStringWrapper; import javafx.beans.property.SimpleFloatProperty; import javafx.beans.property.StringProperty; import javafx.beans.value.ObservableValue; -import javafx.collections.FXCollections; -import javafx.collections.ListChangeListener; -import javafx.collections.ObservableList; +import javafx.collections.MapChangeListener; +import org.jabref.gui.StateManager; import org.jabref.gui.specialfields.SpecialFieldValueViewModel; import org.jabref.gui.util.uithreadaware.UiThreadBinding; import org.jabref.logic.importer.util.FileFieldParser; @@ -36,7 +33,7 @@ import org.jabref.model.entry.field.StandardField; import org.jabref.model.groups.AbstractGroup; import org.jabref.model.groups.GroupTreeNode; -import org.jabref.model.pdf.search.SearchResult; +import org.jabref.model.pdf.search.LuceneSearchResults; import com.tobiasdiez.easybind.EasyBind; import com.tobiasdiez.easybind.EasyBinding; @@ -52,10 +49,10 @@ public class BibEntryTableViewModel { private final EasyBinding> linkedIdentifiers; private final Binding> matchedGroups; private final BibDatabaseContext bibDatabaseContext; - private ObservableList searchResults = FXCollections.observableArrayList(); - private FloatProperty searchScore = new SimpleFloatProperty(); - public BibEntryTableViewModel(BibEntry entry, BibDatabaseContext bibDatabaseContext, ObservableValue fieldValueFormatter) { + private final FloatProperty searchScore = new SimpleFloatProperty(); + + public BibEntryTableViewModel(BibEntry entry, BibDatabaseContext bibDatabaseContext, ObservableValue fieldValueFormatter, StateManager stateManager) { this.entry = entry; this.fieldValueFormatter = fieldValueFormatter; @@ -64,8 +61,12 @@ public BibEntryTableViewModel(BibEntry entry, BibDatabaseContext bibDatabaseCont this.matchedGroups = createMatchedGroupsBinding(bibDatabaseContext, entry); this.bibDatabaseContext = bibDatabaseContext; - searchResults.addListener((ListChangeListener) (change) -> { - searchScore.setValue(searchResults.stream().map(SearchResult::getLuceneScore).max(Comparator.comparing(Float::valueOf)).orElseGet(() -> (float) 0.0)); + stateManager.getSearchResults().addListener((MapChangeListener) change -> { + if (stateManager.getSearchResults().containsKey(entry)) { + searchScore.set(stateManager.getSearchResults().get(entry).getSearchScore()); + } else { + searchScore.set(0); + } }); } @@ -147,15 +148,11 @@ public StringProperty bibDatabaseContextProperty() { return new ReadOnlyStringWrapper(bibDatabaseContext.getDatabasePath().map(Path::toString).orElse("")); } - public void resetSearchResults() { - searchResults.clear(); - } - - public void addSearchResults(List results) { - searchResults.addAll(results); + public float getSearchScore() { + return searchScore.get(); } - public ReadOnlyFloatProperty getSearchScore() { + public FloatProperty searchScoreProperty() { return searchScore; } } diff --git a/src/main/java/org/jabref/gui/maintable/MainTableColumnFactory.java b/src/main/java/org/jabref/gui/maintable/MainTableColumnFactory.java index 3a234aaa94b..202ebe416f8 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableColumnFactory.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableColumnFactory.java @@ -145,8 +145,7 @@ private TableColumn createScoreColumn(MainTableC Tooltip.install(header, new Tooltip(MainTableColumnModel.Type.SCORE.getDisplayName())); column.setGraphic(header); column.setStyle("-fx-alignment: CENTER-RIGHT;"); - column.setCellValueFactory(cellData -> - cellData.getValue().getSearchScore().asString()); + column.setCellValueFactory(cellData -> cellData.getValue().searchScoreProperty().asString()); new ValueTableCellFactory() .withText(text -> text) .install(column); diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index 93567fd39a3..da8667325a5 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -25,7 +25,7 @@ import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.groups.GroupTreeNode; -import org.jabref.model.pdf.search.SearchResult; +import org.jabref.model.pdf.search.LuceneSearchResults; import org.jabref.model.search.matchers.MatcherSet; import org.jabref.model.search.matchers.MatcherSets; import org.jabref.preferences.PreferencesService; @@ -39,6 +39,7 @@ public class MainTableDataModel { private final PreferencesService preferencesService; private final GroupsPreferences groupsPreferences; private final BibDatabaseContext bibDatabaseContext; + private final StateManager stateManager; public MainTableDataModel(BibDatabaseContext context, PreferencesService preferencesService, StateManager stateManager) { this.preferencesService = preferencesService; @@ -49,7 +50,7 @@ public MainTableDataModel(BibDatabaseContext context, PreferencesService prefere ObservableList allEntries = BindingsHelper.forUI(context.getDatabase().getEntries()); ObservableList entriesViewModel = EasyBind.mapBacked(allEntries, entry -> - new BibEntryTableViewModel(entry, bibDatabaseContext, fieldValueFormatter)); + new BibEntryTableViewModel(entry, bibDatabaseContext, fieldValueFormatter, stateManager)); entriesFiltered = new FilteredList<>(entriesViewModel); ObjectBinding> filter = Bindings.createObjectBinding( @@ -57,6 +58,7 @@ public MainTableDataModel(BibDatabaseContext context, PreferencesService prefere entriesFiltered.predicateProperty().bind(filter); stateManager.activeSearchQueryProperty().addListener((observable, oldValue, newValue) -> doSearch(newValue)); + this.stateManager = stateManager; IntegerProperty resultSize = new SimpleIntegerProperty(); resultSize.bind(Bindings.size(entriesFiltered)); @@ -66,17 +68,13 @@ public MainTableDataModel(BibDatabaseContext context, PreferencesService prefere } private void doSearch(Optional query) { - for (BibEntryTableViewModel entry : entriesFiltered) { - entry.resetSearchResults(); - } if (query.isPresent()) { String searchString = query.get().getQuery(); try { LuceneSearcher searcher = LuceneSearcher.of(bibDatabaseContext); - Map> results = searcher.search(query.get()); - for (Map.Entry> result : results.entrySet()) { - getTableViewModelForEntry(result.getKey()).ifPresent(bibEntryTableViewModel -> bibEntryTableViewModel.addSearchResults(result.getValue())); - } + Map results = searcher.search(query.get()); + stateManager.getSearchResults().clear(); + stateManager.getSearchResults().putAll(results); } catch (IOException e) { e.printStackTrace(); } diff --git a/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java b/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java index fe95d04210f..a5ae019f574 100644 --- a/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java +++ b/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java @@ -36,7 +36,7 @@ public SearchResultsTableDataModel(BibDatabaseContext bibDatabaseContext, Prefer ObservableList entriesViewModel = FXCollections.observableArrayList(); for (BibDatabaseContext context : stateManager.getOpenDatabases()) { ObservableList entriesForDb = context.getDatabase().getEntries(); - List viewModelForDb = EasyBind.mapBacked(entriesForDb, entry -> new BibEntryTableViewModel(entry, context, fieldValueFormatter)); + List viewModelForDb = EasyBind.mapBacked(entriesForDb, entry -> new BibEntryTableViewModel(entry, context, fieldValueFormatter, stateManager)); entriesViewModel.addAll(viewModelForDb); } diff --git a/src/main/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcher.java b/src/main/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcher.java index eedc56370dd..b71390d5b41 100644 --- a/src/main/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcher.java +++ b/src/main/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcher.java @@ -3,9 +3,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Objects; import org.jabref.gui.LibraryTab; import org.jabref.logic.search.SearchQuery; @@ -16,7 +13,6 @@ import org.jabref.model.pdf.search.SearchFieldConstants; import org.jabref.model.pdf.search.SearchResult; import org.jabref.model.search.rules.SearchRules; -import org.jabref.model.strings.StringUtil; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; @@ -50,47 +46,13 @@ public static LuceneSearcher of(BibDatabaseContext databaseContext) throws IOExc /** * Search for results matching a query in the Lucene search index * - * @param searchString a pattern to search for matching entries in the index, must not be null - * @param maxHits number of maximum search results, must be positive - * @return a result set of all documents that have matches in any fields + * @param query query to search for + * @return a result map of all entries that have matches in any fields */ - public LuceneSearchResults search(final String searchString, final int maxHits) - throws IOException { - Thread.dumpStack(); - if (StringUtil.isBlank(Objects.requireNonNull(searchString, "The search string was null!"))) { - return new LuceneSearchResults(); - } - if (maxHits <= 0) { - throw new IllegalArgumentException("Must be called with at least 1 maxHits, was" + maxHits); - } - - List resultDocs = new LinkedList<>(); - - if (!DirectoryReader.indexExists(indexDirectory)) { - LOGGER.debug("Index directory {} does not yet exist", indexDirectory); - return new LuceneSearchResults(); - } - - try (IndexReader reader = DirectoryReader.open(indexDirectory)) { - IndexSearcher searcher = new IndexSearcher(reader); - String[] searchable_fields = new String[SearchFieldConstants.searchableBibFields.size()]; - SearchFieldConstants.searchableBibFields.toArray(searchable_fields); - Query query = new MultiFieldQueryParser(searchable_fields, new EnglishStemAnalyzer()).parse(searchString); - TopDocs results = searcher.search(query, maxHits); - for (ScoreDoc scoreDoc : results.scoreDocs) { - resultDocs.add(new SearchResult(searcher, query, scoreDoc)); - } - return new LuceneSearchResults(resultDocs); - } catch (ParseException e) { - LOGGER.warn("Could not parse query: '{}'!\n{}", searchString, e.getMessage()); - return new LuceneSearchResults(); - } - } - - public HashMap> search(SearchQuery query) { - HashMap> results = new HashMap<>(); - HashMap boosts = new HashMap(); - SearchFieldConstants.searchableBibFields.stream().forEach(field -> boosts.put(field, Float.valueOf(4))); + public HashMap search(SearchQuery query) { + HashMap results = new HashMap<>(); + HashMap boosts = new HashMap<>(); + SearchFieldConstants.searchableBibFields.forEach(field -> boosts.put(field, Float.valueOf(4))); if (query.getSearchFlags().contains(SearchRules.SearchFlags.FULLTEXT)) { Arrays.stream(SearchFieldConstants.PDF_FIELDS).forEach(field -> boosts.put(field, Float.valueOf(1))); @@ -105,9 +67,9 @@ public HashMap> search(SearchQuery query) { SearchResult searchResult = new SearchResult(searcher, luceneQuery, scoreDoc); for (BibEntry match : searchResult.getMatchingEntries(databaseContext)) { if (!results.containsKey(match)) { - results.put(match, new LinkedList<>()); + results.put(match, new LuceneSearchResults()); } - results.get(match).add(searchResult); + results.get(match).addResult(searchResult); } } } catch (ParseException e) { diff --git a/src/main/java/org/jabref/model/pdf/search/LuceneSearchResults.java b/src/main/java/org/jabref/model/pdf/search/LuceneSearchResults.java index 415905ec1cb..4e6e73803c7 100644 --- a/src/main/java/org/jabref/model/pdf/search/LuceneSearchResults.java +++ b/src/main/java/org/jabref/model/pdf/search/LuceneSearchResults.java @@ -2,20 +2,14 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; +import java.util.LinkedList; import java.util.List; public final class LuceneSearchResults { - private final List searchResults; - - public LuceneSearchResults(List search) { - this.searchResults = Collections.unmodifiableList(search); - } - - public LuceneSearchResults() { - this.searchResults = Collections.emptyList(); - } + private final List searchResults = new LinkedList<>(); public List getSortedByScore() { List sortedList = new ArrayList<>(searchResults); @@ -44,4 +38,12 @@ public HashMap> getSearchResultsByPath() { public int numSearchResults() { return this.searchResults.size(); } + + public void addResult(SearchResult result) { + this.searchResults.add(result); + } + + public float getSearchScore() { + return this.searchResults.stream().map(SearchResult::getLuceneScore).max(Comparator.comparing(Float::floatValue)).orElse(Float.valueOf(0)); + } } diff --git a/src/main/java/org/jabref/model/search/rules/ContainsBasedSearchRule.java b/src/main/java/org/jabref/model/search/rules/ContainsBasedSearchRule.java index be074a3e573..0e8e11cfdba 100644 --- a/src/main/java/org/jabref/model/search/rules/ContainsBasedSearchRule.java +++ b/src/main/java/org/jabref/model/search/rules/ContainsBasedSearchRule.java @@ -5,7 +5,6 @@ import java.util.List; import java.util.Locale; -import org.jabref.architecture.AllowedToUseLogic; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.Field; import org.jabref.model.search.rules.SearchRules.SearchFlags; @@ -14,7 +13,6 @@ /** * Search rule for a search based on String.contains() */ -@AllowedToUseLogic("Because access to the lucene index is needed") public class ContainsBasedSearchRule extends FullTextSearchRule { public ContainsBasedSearchRule(EnumSet searchFlags) { @@ -53,7 +51,6 @@ public boolean applyRule(String query, BibEntry bibEntry) { return true; } } - - return getLuceneResults(query, bibEntry).numSearchResults() > 0; // Didn't match all words. + return false; } } diff --git a/src/main/java/org/jabref/model/search/rules/FullTextSearchRule.java b/src/main/java/org/jabref/model/search/rules/FullTextSearchRule.java index 4ac130e45e9..2b34872fe66 100644 --- a/src/main/java/org/jabref/model/search/rules/FullTextSearchRule.java +++ b/src/main/java/org/jabref/model/search/rules/FullTextSearchRule.java @@ -1,17 +1,11 @@ package org.jabref.model.search.rules; -import java.io.IOException; import java.util.Collections; import java.util.EnumSet; import java.util.List; -import java.util.stream.Collectors; -import org.jabref.architecture.AllowedToUseLogic; import org.jabref.gui.Globals; -import org.jabref.logic.pdf.search.retrieval.LuceneSearcher; import org.jabref.model.database.BibDatabaseContext; -import org.jabref.model.entry.BibEntry; -import org.jabref.model.pdf.search.LuceneSearchResults; import org.jabref.model.pdf.search.SearchResult; import org.slf4j.Logger; @@ -22,7 +16,6 @@ *

* Some kind of caching of the full text search results is implemented. */ -@AllowedToUseLogic("Because access to the lucene index is needed") public abstract class FullTextSearchRule implements SearchRule { private static final Logger LOGGER = LoggerFactory.getLogger(FullTextSearchRule.class); @@ -45,27 +38,4 @@ public FullTextSearchRule(EnumSet searchFlags) { public EnumSet getSearchFlags() { return searchFlags; } - - @Override - public LuceneSearchResults getLuceneResults(String query, BibEntry bibEntry) { - if (!searchFlags.contains(SearchRules.SearchFlags.FULLTEXT) || databaseContext == null) { - return new LuceneSearchResults(); - } - - if (!query.equals(this.lastQuery)) { - this.lastQuery = query; - lastSearchResults = Collections.emptyList(); - try { - LuceneSearcher searcher = LuceneSearcher.of(databaseContext); - LuceneSearchResults results = searcher.search(query, 5); - lastSearchResults = results.getSortedByScore(); - } catch (IOException e) { - LOGGER.error("Could not retrieve search results!", e); - } - } - - return new LuceneSearchResults(lastSearchResults.stream() - .filter(searchResult -> searchResult.getSearchScoreFor(bibEntry) > 0) - .collect(Collectors.toList())); - } } diff --git a/src/main/java/org/jabref/model/search/rules/GrammarBasedSearchRule.java b/src/main/java/org/jabref/model/search/rules/GrammarBasedSearchRule.java index cb0c3e24a3f..fddae7ee48a 100644 --- a/src/main/java/org/jabref/model/search/rules/GrammarBasedSearchRule.java +++ b/src/main/java/org/jabref/model/search/rules/GrammarBasedSearchRule.java @@ -1,6 +1,5 @@ package org.jabref.model.search.rules; -import java.io.IOException; import java.util.ArrayList; import java.util.EnumSet; import java.util.List; @@ -12,15 +11,12 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; -import org.jabref.architecture.AllowedToUseLogic; import org.jabref.gui.Globals; -import org.jabref.logic.pdf.search.retrieval.LuceneSearcher; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.Keyword; import org.jabref.model.entry.field.Field; import org.jabref.model.entry.field.InternalField; -import org.jabref.model.pdf.search.LuceneSearchResults; import org.jabref.model.pdf.search.SearchResult; import org.jabref.model.search.rules.SearchRules.SearchFlags; import org.jabref.model.strings.StringUtil; @@ -44,7 +40,6 @@ *

* This class implements the "Advanced Search Mode" described in the help */ -@AllowedToUseLogic("Because access to the lucene index is needed") public class GrammarBasedSearchRule implements SearchRule { private static final Logger LOGGER = LoggerFactory.getLogger(GrammarBasedSearchRule.class); @@ -100,17 +95,6 @@ private void init(String query) throws ParseCancellationException { parser.setErrorHandler(new BailErrorStrategy()); // ParseCancelationException on parse errors tree = parser.start(); this.query = query; - - if (!searchFlags.contains(SearchRules.SearchFlags.FULLTEXT) || (databaseContext == null)) { - return; - } - try { - LuceneSearcher searcher = LuceneSearcher.of(databaseContext); - LuceneSearchResults results = searcher.search(query, 5); - searchResults = results.getSortedByScore(); - } catch (IOException e) { - LOGGER.error("Could not retrieve search results!", e); - } } @Override @@ -119,15 +103,10 @@ public boolean applyRule(String query, BibEntry bibEntry) { return new BibtexSearchVisitor(searchFlags, bibEntry).visit(tree); } catch (Exception e) { LOGGER.debug("Search failed", e); - return getLuceneResults(query, bibEntry).numSearchResults() > 0; + return false; } } - @Override - public LuceneSearchResults getLuceneResults(String query, BibEntry bibEntry) { - return new LuceneSearchResults(searchResults.stream().filter(searchResult -> searchResult.getSearchScoreFor(bibEntry) > 0).collect(Collectors.toList())); - } - @Override public boolean validateSearchStrings(String query) { try { diff --git a/src/main/java/org/jabref/model/search/rules/RegexBasedSearchRule.java b/src/main/java/org/jabref/model/search/rules/RegexBasedSearchRule.java index 8c607c8dfc0..70ba0a35a90 100644 --- a/src/main/java/org/jabref/model/search/rules/RegexBasedSearchRule.java +++ b/src/main/java/org/jabref/model/search/rules/RegexBasedSearchRule.java @@ -6,7 +6,6 @@ import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; -import org.jabref.architecture.AllowedToUseLogic; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.Field; import org.jabref.model.search.rules.SearchRules.SearchFlags; @@ -18,7 +17,6 @@ /** * Search rule for regex-based search. */ -@AllowedToUseLogic("Because access to the lucene index is needed") public class RegexBasedSearchRule extends FullTextSearchRule { private static final Logger LOGGER = LoggerFactory.getLogger(RegexBasedSearchRule.class); @@ -57,6 +55,6 @@ public boolean applyRule(String query, BibEntry bibEntry) { } } } - return getLuceneResults(query, bibEntry).numSearchResults() > 0; + return false; } } diff --git a/src/main/java/org/jabref/model/search/rules/SearchRule.java b/src/main/java/org/jabref/model/search/rules/SearchRule.java index 469a6fc43de..ffc4dced699 100644 --- a/src/main/java/org/jabref/model/search/rules/SearchRule.java +++ b/src/main/java/org/jabref/model/search/rules/SearchRule.java @@ -1,13 +1,10 @@ package org.jabref.model.search.rules; import org.jabref.model.entry.BibEntry; -import org.jabref.model.pdf.search.LuceneSearchResults; public interface SearchRule { boolean applyRule(String query, BibEntry bibEntry); - LuceneSearchResults getLuceneResults(String query, BibEntry bibEntry); - boolean validateSearchStrings(String query); } From b89714a567848a4fc4b237a6cec148ab2990b667 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Mon, 15 Aug 2022 12:10:17 +0200 Subject: [PATCH 016/256] Change file-icon if search yielded a fulltext-result --- .../gui/maintable/MainTableColumnFactory.java | 4 +++- .../jabref/gui/maintable/columns/FileColumn.java | 15 ++++++++++++--- .../model/pdf/search/LuceneSearchResults.java | 4 ++++ .../org/jabref/model/pdf/search/SearchResult.java | 7 +++++++ 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/jabref/gui/maintable/MainTableColumnFactory.java b/src/main/java/org/jabref/gui/maintable/MainTableColumnFactory.java index 202ebe416f8..a89cde55236 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableColumnFactory.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableColumnFactory.java @@ -258,7 +258,8 @@ private TableColumn> createFilesColumn( database, externalFileTypes, dialogService, - preferencesService); + preferencesService, + stateManager); } /** @@ -270,6 +271,7 @@ private TableColumn> createExtraFileCol externalFileTypes, dialogService, preferencesService, + stateManager, columnModel.getQualifier()); } diff --git a/src/main/java/org/jabref/gui/maintable/columns/FileColumn.java b/src/main/java/org/jabref/gui/maintable/columns/FileColumn.java index c325637af32..dc69ea4d45c 100644 --- a/src/main/java/org/jabref/gui/maintable/columns/FileColumn.java +++ b/src/main/java/org/jabref/gui/maintable/columns/FileColumn.java @@ -12,6 +12,7 @@ import org.jabref.gui.DialogService; import org.jabref.gui.Globals; +import org.jabref.gui.StateManager; import org.jabref.gui.externalfiletype.ExternalFileType; import org.jabref.gui.externalfiletype.ExternalFileTypes; import org.jabref.gui.fieldeditors.LinkedFileViewModel; @@ -36,6 +37,7 @@ public class FileColumn extends MainTableColumn> { private final DialogService dialogService; private final BibDatabaseContext database; private final PreferencesService preferencesService; + private final StateManager stateManager; /** * Creates a joined column for all the linked files. @@ -44,12 +46,14 @@ public FileColumn(MainTableColumnModel model, BibDatabaseContext database, ExternalFileTypes externalFileTypes, DialogService dialogService, - PreferencesService preferencesService) { + PreferencesService preferencesService, + StateManager stateManager) { super(model); this.externalFileTypes = Objects.requireNonNull(externalFileTypes); this.database = Objects.requireNonNull(database); this.dialogService = dialogService; this.preferencesService = preferencesService; + this.stateManager = stateManager; setCommonSettings(); @@ -84,12 +88,14 @@ public FileColumn(MainTableColumnModel model, ExternalFileTypes externalFileTypes, DialogService dialogService, PreferencesService preferencesService, + StateManager stateManager, String fileType) { super(model); this.externalFileTypes = Objects.requireNonNull(externalFileTypes); this.database = Objects.requireNonNull(database); this.dialogService = dialogService; this.preferencesService = preferencesService; + this.stateManager = stateManager; setCommonSettings(); @@ -99,7 +105,7 @@ public FileColumn(MainTableColumnModel model, .getGraphicNode()); new ValueTableCellFactory>() - .withGraphic(linkedFiles -> createFileIcon(linkedFiles.stream().filter(linkedFile -> + .withGraphic((entry, linkedFiles) -> createFileIcon(entry, linkedFiles.stream().filter(linkedFile -> linkedFile.getFileType().equalsIgnoreCase(fileType)).collect(Collectors.toList()))) .install(this); } @@ -143,7 +149,10 @@ private ContextMenu createFileMenu(BibEntryTableViewModel entry, List linkedFiles) { + private Node createFileIcon(BibEntryTableViewModel entry, List linkedFiles) { + if (linkedFiles.size() > 0 && stateManager.getSearchResults().containsKey(entry.getEntry()) && stateManager.getSearchResults().get(entry.getEntry()).hasFulltextResults()) { + return IconTheme.JabRefIcons.FILE_SEARCH.getGraphicNode(); + } if (linkedFiles.size() > 1) { return IconTheme.JabRefIcons.FILE_MULTIPLE.getGraphicNode(); } else if (linkedFiles.size() == 1) { diff --git a/src/main/java/org/jabref/model/pdf/search/LuceneSearchResults.java b/src/main/java/org/jabref/model/pdf/search/LuceneSearchResults.java index 4e6e73803c7..36738440228 100644 --- a/src/main/java/org/jabref/model/pdf/search/LuceneSearchResults.java +++ b/src/main/java/org/jabref/model/pdf/search/LuceneSearchResults.java @@ -46,4 +46,8 @@ public void addResult(SearchResult result) { public float getSearchScore() { return this.searchResults.stream().map(SearchResult::getLuceneScore).max(Comparator.comparing(Float::floatValue)).orElse(Float.valueOf(0)); } + + public boolean hasFulltextResults() { + return this.searchResults.stream().map(SearchResult::hasFulltextResults).anyMatch(Boolean::valueOf); + } } diff --git a/src/main/java/org/jabref/model/pdf/search/SearchResult.java b/src/main/java/org/jabref/model/pdf/search/SearchResult.java index 3b1050ae453..6ea93b0dac8 100644 --- a/src/main/java/org/jabref/model/pdf/search/SearchResult.java +++ b/src/main/java/org/jabref/model/pdf/search/SearchResult.java @@ -34,6 +34,7 @@ public final class SearchResult { private final int pageNumber; private final long modified; private final int hash; + private final boolean hasFulltextResults; private final float luceneScore; private List contentResultStringsHtml; @@ -50,6 +51,7 @@ public SearchResult(IndexSearcher searcher, Query query, ScoreDoc scoreDoc) thro String content = getFieldContents(searcher, scoreDoc, CONTENT); String annotations = getFieldContents(searcher, scoreDoc, ANNOTATIONS); + this.hasFulltextResults = !(content.isEmpty() && annotations.isEmpty()); Highlighter highlighter = new Highlighter(new SimpleHTMLFormatter("", ""), new QueryScorer(query)); @@ -71,6 +73,7 @@ public SearchResult(IndexSearcher searcher, Query query, ScoreDoc scoreDoc) thro this.hash = Integer.parseInt(getFieldContents(searcher, scoreDoc, BIB_ENTRY_ID_HASH)); this.pageNumber = -1; this.modified = -1; + this.hasFulltextResults = false; } } @@ -123,4 +126,8 @@ public List getAnnotationsResultStringsHtml() { public int getPageNumber() { return pageNumber; } + + public boolean hasFulltextResults() { + return this.hasFulltextResults; + } } From a9f0e7547340a9e86655a09214f2a222f8745574 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Mon, 15 Aug 2022 13:45:51 +0200 Subject: [PATCH 017/256] Update index even if fulltext-index is disabled --- src/main/java/org/jabref/gui/LibraryTab.java | 58 ++++++++------------ 1 file changed, 24 insertions(+), 34 deletions(-) diff --git a/src/main/java/org/jabref/gui/LibraryTab.java b/src/main/java/org/jabref/gui/LibraryTab.java index 659b06c9312..69b1f1620ae 100644 --- a/src/main/java/org/jabref/gui/LibraryTab.java +++ b/src/main/java/org/jabref/gui/LibraryTab.java @@ -346,9 +346,7 @@ public void updateTabTitle(boolean isChanged) { setTooltip(new Tooltip(toolTipText.toString())); }); - if (preferencesService.getFilePreferences().shouldFulltextIndexLinkedFiles()) { - indexingTaskManager.updateDatabaseName(tabTitle.toString()); - } + indexingTaskManager.updateDatabaseName(tabTitle.toString()); } private List collectAllDatabasePaths() { @@ -867,9 +865,7 @@ private class IndexUpdateListener { public IndexUpdateListener() { try { indexingTaskManager.manageFulltextIndexAccordingToPrefs(LuceneIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences())); - if (preferencesService.getFilePreferences().shouldFulltextIndexLinkedFiles()) { - indexingTaskManager.updateIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences())); - } + indexingTaskManager.updateIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences())); } catch (IOException e) { LOGGER.error("Cannot access lucene index", e); } @@ -877,47 +873,41 @@ public IndexUpdateListener() { @Subscribe public void listen(EntriesAddedEvent addedEntryEvent) { - if (preferencesService.getFilePreferences().shouldFulltextIndexLinkedFiles()) { - try { - LuceneIndexer luceneIndexer = LuceneIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences()); - for (BibEntry addedEntry : addedEntryEvent.getBibEntries()) { - indexingTaskManager.addToIndex(luceneIndexer, addedEntry); - } - } catch (IOException e) { - LOGGER.error("Cannot access lucene index", e); + try { + LuceneIndexer luceneIndexer = LuceneIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences()); + for (BibEntry addedEntry : addedEntryEvent.getBibEntries()) { + indexingTaskManager.addToIndex(luceneIndexer, addedEntry); } + } catch (IOException e) { + LOGGER.error("Cannot access lucene index", e); } } @Subscribe public void listen(EntriesRemovedEvent removedEntriesEvent) { - if (preferencesService.getFilePreferences().shouldFulltextIndexLinkedFiles()) { - try { - LuceneIndexer luceneIndexer = LuceneIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences()); - for (BibEntry removedEntry : removedEntriesEvent.getBibEntries()) { - indexingTaskManager.removeFromIndex(luceneIndexer, removedEntry); - } - } catch (IOException e) { - LOGGER.error("Cannot access lucene index", e); + try { + LuceneIndexer luceneIndexer = LuceneIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences()); + for (BibEntry removedEntry : removedEntriesEvent.getBibEntries()) { + indexingTaskManager.removeFromIndex(luceneIndexer, removedEntry); } + } catch (IOException e) { + LOGGER.error("Cannot access lucene index", e); } } @Subscribe public void listen(FieldChangedEvent fieldChangedEvent) { - if (preferencesService.getFilePreferences().shouldFulltextIndexLinkedFiles()) { - for (BibEntry bibEntry : fieldChangedEvent.getBibEntries()) { - try { - List removedFiles = new ArrayList<>(); - if (fieldChangedEvent.getField().equals(StandardField.FILE)) { - List oldFileList = FileFieldParser.parse(fieldChangedEvent.getOldValue()); - List newFileList = FileFieldParser.parse(fieldChangedEvent.getNewValue()); - removedFiles.remove(newFileList); - } - indexingTaskManager.updateIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences()), bibEntry, removedFiles); - } catch (IOException e) { - LOGGER.warn("I/O error when writing lucene index", e); + for (BibEntry bibEntry : fieldChangedEvent.getBibEntries()) { + try { + List removedFiles = new ArrayList<>(); + if (fieldChangedEvent.getField().equals(StandardField.FILE)) { + List oldFileList = FileFieldParser.parse(fieldChangedEvent.getOldValue()); + List newFileList = FileFieldParser.parse(fieldChangedEvent.getNewValue()); + removedFiles.remove(newFileList); } + indexingTaskManager.updateIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences()), bibEntry, removedFiles); + } catch (IOException e) { + LOGGER.warn("I/O error when writing lucene index", e); } } } From 750346f734705fb3eae0de3e9531a296b8ad34c3 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Mon, 15 Aug 2022 14:48:58 +0200 Subject: [PATCH 018/256] Adapt global search --- .../java/org/jabref/gui/StateManager.java | 9 +++---- .../gui/maintable/MainTableDataModel.java | 12 +++------- .../jabref/gui/search/SearchResultsTable.java | 24 +++++++++++++++++++ .../search/SearchResultsTableDataModel.java | 3 +-- 4 files changed, 31 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/jabref/gui/StateManager.java b/src/main/java/org/jabref/gui/StateManager.java index af3968fa36e..1c09095453f 100644 --- a/src/main/java/org/jabref/gui/StateManager.java +++ b/src/main/java/org/jabref/gui/StateManager.java @@ -58,7 +58,6 @@ public class StateManager { private final ObservableMap> selectedGroups = FXCollections.observableHashMap(); private final OptionalObjectProperty activeSearchQuery = OptionalObjectProperty.empty(); private final ObservableMap searchResults = FXCollections.observableHashMap(); - private final ObservableMap searchResultMap = FXCollections.observableHashMap(); private final OptionalObjectProperty focusOwner = OptionalObjectProperty.empty(); private final ObservableList>> backgroundTasks = FXCollections.observableArrayList(task -> new Observable[] {task.getValue().progressProperty(), task.getValue().runningProperty()}); private final EasyBinding anyTaskRunning = EasyBind.reduce(backgroundTasks, tasks -> tasks.map(Pair::getValue).anyMatch(Task::isRunning)); @@ -93,12 +92,10 @@ public OptionalObjectProperty activeSearchQueryProperty() { return activeSearchQuery; } - public void setActiveSearchResultSize(BibDatabaseContext database, IntegerProperty resultSize) { - searchResultMap.put(database, resultSize); - } - public IntegerProperty getSearchResultSize() { - return searchResultMap.getOrDefault(activeDatabase.getValue().orElse(new BibDatabaseContext()), new SimpleIntegerProperty(0)); + IntegerProperty size = new SimpleIntegerProperty(); + size.bind(Bindings.size(searchResults)); + return size; } public ReadOnlyListProperty activeGroupProperty() { diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index a2fb3348bcf..f3950183f63 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -5,10 +5,7 @@ import java.util.Map; import java.util.Optional; -import javafx.beans.binding.Bindings; -import javafx.beans.property.IntegerProperty; import javafx.beans.property.ObjectProperty; -import javafx.beans.property.SimpleIntegerProperty; import javafx.beans.property.SimpleObjectProperty; import javafx.collections.ObservableList; import javafx.collections.transformation.FilteredList; @@ -60,9 +57,6 @@ public MainTableDataModel(BibDatabaseContext context, PreferencesService prefere ); stateManager.activeSearchQueryProperty().addListener((observable, oldValue, newValue) -> doSearch(newValue)); - IntegerProperty resultSize = new SimpleIntegerProperty(); - resultSize.bind(Bindings.size(entriesFiltered)); - stateManager.setActiveSearchResultSize(context, resultSize); // We need to wrap the list since otherwise sorting in the table does not work entriesSorted = new SortedList<>(entriesFiltered); } @@ -71,10 +65,10 @@ private void doSearch(Optional query) { if (query.isPresent()) { String searchString = query.get().getQuery(); try { - LuceneSearcher searcher = LuceneSearcher.of(bibDatabaseContext); - Map results = searcher.search(query.get()); stateManager.getSearchResults().clear(); - stateManager.getSearchResults().putAll(results); + for (BibDatabaseContext context : stateManager.getOpenDatabases()) { + stateManager.getSearchResults().putAll(LuceneSearcher.of(context).search(query.get())); + } } catch (IOException e) { e.printStackTrace(); } diff --git a/src/main/java/org/jabref/gui/search/SearchResultsTable.java b/src/main/java/org/jabref/gui/search/SearchResultsTable.java index a0f3ac20ea6..40bf771abe0 100644 --- a/src/main/java/org/jabref/gui/search/SearchResultsTable.java +++ b/src/main/java/org/jabref/gui/search/SearchResultsTable.java @@ -4,6 +4,7 @@ import javax.swing.undo.UndoManager; +import javafx.collections.ListChangeListener; import javafx.scene.control.SelectionMode; import javafx.scene.control.TableColumn; import javafx.scene.control.TableView; @@ -13,6 +14,7 @@ import org.jabref.gui.maintable.BibEntryTableViewModel; import org.jabref.gui.maintable.MainTable; import org.jabref.gui.maintable.MainTableColumnFactory; +import org.jabref.gui.maintable.MainTableColumnModel; import org.jabref.gui.maintable.MainTablePreferences; import org.jabref.gui.maintable.SmartConstrainedResizePolicy; import org.jabref.gui.maintable.columns.LibraryColumn; @@ -45,6 +47,22 @@ public SearchResultsTable(SearchResultsTableDataModel model, } this.getColumns().addAll(allCols); + TableColumn scoreColumn = this.getColumns().stream().filter(c -> c instanceof MainTableColumn) + .map(c -> ((MainTableColumn) c)) + .filter(c -> c.getModel().getType() == MainTableColumnModel.Type.SCORE) + .findFirst().orElse(null); + + // always sort by score first. If no search is ongoing, it will be equal for all columns. + ListChangeListener> scoreSortOderPrioritizer = new ListChangeListener>() { + @Override + public void onChanged(Change> c) { + getSortOrder().removeListener(this); + getSortOrder().removeAll(scoreColumn); + getSortOrder().add(0, scoreColumn); + getSortOrder().addListener(this); + } + }; + this.getSortOrder().clear(); preferencesService.getSearchDialogColumnPreferences().getColumnSortOrder().forEach(columnModel -> this.getColumns().stream() @@ -52,6 +70,11 @@ public SearchResultsTable(SearchResultsTableDataModel model, .filter(column -> column.getModel().equals(columnModel)) .findFirst() .ifPresent(column -> this.getSortOrder().add(column))); + // insert score sort order + if (scoreColumn != null) { + this.getSortOrder().add(0, scoreColumn); + this.getSortOrder().addListener(scoreSortOderPrioritizer); + } if (mainTablePreferences.getResizeColumnsToFit()) { this.setColumnResizePolicy(new SmartConstrainedResizePolicy()); @@ -61,6 +84,7 @@ public SearchResultsTable(SearchResultsTableDataModel model, this.setItems(model.getEntriesFilteredAndSorted()); // Enable sorting model.getEntriesFilteredAndSorted().comparatorProperty().bind(this.comparatorProperty()); + this.sort(); this.getStylesheets().add(MainTable.class.getResource("MainTable.css").toExternalForm()); diff --git a/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java b/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java index a5ae019f574..e95a8ff03fe 100644 --- a/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java +++ b/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java @@ -41,8 +41,7 @@ public SearchResultsTableDataModel(BibDatabaseContext bibDatabaseContext, Prefer } entriesFiltered = new FilteredList<>(entriesViewModel); - // don't need this if ranking is in place? - entriesFiltered.predicateProperty().bind(EasyBind.map(stateManager.activeSearchQueryProperty(), (query) -> entry -> isMatchedBySearch(query, entry))); + entriesFiltered.predicateProperty().bind(EasyBind.map(stateManager.activeSearchQueryProperty(), (query) -> entry -> stateManager.getSearchResults().containsKey(entry.getEntry()))); // We need to wrap the list since otherwise sorting in the table does not work entriesSorted = new SortedList<>(entriesFiltered); From 00a7110374ad79bb8f5d4dcd98c65813c4dfc829 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Mon, 15 Aug 2022 15:17:15 +0200 Subject: [PATCH 019/256] Improved keyword indexing --- src/main/java/org/jabref/gui/JabRefFrame.java | 2 +- src/main/java/org/jabref/gui/LibraryTab.java | 10 ++++---- .../jabref/gui/entryeditor/EntryEditor.java | 2 +- .../ExternalFilesEntryLinker.java | 9 ++++--- .../gui/externalfiles/ImportHandler.java | 2 +- .../org/jabref/gui/preview/PreviewPanel.java | 2 +- .../RebuildFulltextSearchIndexAction.java | 9 ++++--- .../pdf/search/indexing/LuceneIndexer.java | 24 +++++++++++++++---- 8 files changed, 40 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/jabref/gui/JabRefFrame.java b/src/main/java/org/jabref/gui/JabRefFrame.java index 4d07c12aca2..c28211fb5b8 100644 --- a/src/main/java/org/jabref/gui/JabRefFrame.java +++ b/src/main/java/org/jabref/gui/JabRefFrame.java @@ -886,7 +886,7 @@ private MenuBar createMenu() { new SeparatorMenuItem(), - factory.createMenuItem(StandardActions.REBUILD_FULLTEXT_SEARCH_INDEX, new RebuildFulltextSearchIndexAction(stateManager, this::getCurrentLibraryTab, dialogService, prefs.getFilePreferences())) + factory.createMenuItem(StandardActions.REBUILD_FULLTEXT_SEARCH_INDEX, new RebuildFulltextSearchIndexAction(stateManager, this::getCurrentLibraryTab, dialogService, prefs, prefs.getFilePreferences())) ); SidePaneType webSearchPane = SidePaneType.WEB_SEARCH; SidePaneType groupsPane = SidePaneType.GROUPS; diff --git a/src/main/java/org/jabref/gui/LibraryTab.java b/src/main/java/org/jabref/gui/LibraryTab.java index 69b1f1620ae..51ba737351d 100644 --- a/src/main/java/org/jabref/gui/LibraryTab.java +++ b/src/main/java/org/jabref/gui/LibraryTab.java @@ -864,8 +864,8 @@ private class IndexUpdateListener { public IndexUpdateListener() { try { - indexingTaskManager.manageFulltextIndexAccordingToPrefs(LuceneIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences())); - indexingTaskManager.updateIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences())); + indexingTaskManager.manageFulltextIndexAccordingToPrefs(LuceneIndexer.of(bibDatabaseContext, preferencesService, preferencesService.getFilePreferences())); + indexingTaskManager.updateIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService, preferencesService.getFilePreferences())); } catch (IOException e) { LOGGER.error("Cannot access lucene index", e); } @@ -874,7 +874,7 @@ public IndexUpdateListener() { @Subscribe public void listen(EntriesAddedEvent addedEntryEvent) { try { - LuceneIndexer luceneIndexer = LuceneIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences()); + LuceneIndexer luceneIndexer = LuceneIndexer.of(bibDatabaseContext, preferencesService, preferencesService.getFilePreferences()); for (BibEntry addedEntry : addedEntryEvent.getBibEntries()) { indexingTaskManager.addToIndex(luceneIndexer, addedEntry); } @@ -886,7 +886,7 @@ public void listen(EntriesAddedEvent addedEntryEvent) { @Subscribe public void listen(EntriesRemovedEvent removedEntriesEvent) { try { - LuceneIndexer luceneIndexer = LuceneIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences()); + LuceneIndexer luceneIndexer = LuceneIndexer.of(bibDatabaseContext, preferencesService, preferencesService.getFilePreferences()); for (BibEntry removedEntry : removedEntriesEvent.getBibEntries()) { indexingTaskManager.removeFromIndex(luceneIndexer, removedEntry); } @@ -905,7 +905,7 @@ public void listen(FieldChangedEvent fieldChangedEvent) { List newFileList = FileFieldParser.parse(fieldChangedEvent.getNewValue()); removedFiles.remove(newFileList); } - indexingTaskManager.updateIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences()), bibEntry, removedFiles); + indexingTaskManager.updateIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService, preferencesService.getFilePreferences()), bibEntry, removedFiles); } catch (IOException e) { LOGGER.warn("I/O error when writing lucene index", e); } diff --git a/src/main/java/org/jabref/gui/entryeditor/EntryEditor.java b/src/main/java/org/jabref/gui/entryeditor/EntryEditor.java index b5f2d72ca39..509e7bd5872 100644 --- a/src/main/java/org/jabref/gui/entryeditor/EntryEditor.java +++ b/src/main/java/org/jabref/gui/entryeditor/EntryEditor.java @@ -121,7 +121,7 @@ public EntryEditor(LibraryTab libraryTab) { .load(); this.entryEditorPreferences = preferencesService.getEntryEditorPreferences(); - this.fileLinker = new ExternalFilesEntryLinker(preferencesService.getFilePreferences(), databaseContext); + this.fileLinker = new ExternalFilesEntryLinker(preferencesService, preferencesService.getFilePreferences(), databaseContext); EasyBind.subscribe(tabbed.getSelectionModel().selectedItemProperty(), tab -> { EntryEditorTab activeTab = (EntryEditorTab) tab; diff --git a/src/main/java/org/jabref/gui/externalfiles/ExternalFilesEntryLinker.java b/src/main/java/org/jabref/gui/externalfiles/ExternalFilesEntryLinker.java index dc309231bd7..b9fb53d1530 100644 --- a/src/main/java/org/jabref/gui/externalfiles/ExternalFilesEntryLinker.java +++ b/src/main/java/org/jabref/gui/externalfiles/ExternalFilesEntryLinker.java @@ -18,6 +18,7 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.LinkedFile; import org.jabref.preferences.FilePreferences; +import org.jabref.preferences.PreferencesService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,12 +27,14 @@ public class ExternalFilesEntryLinker { private static final Logger LOGGER = LoggerFactory.getLogger(ExternalFilesEntryLinker.class); + private final PreferencesService preferencesService; private final FilePreferences filePreferences; private final BibDatabaseContext bibDatabaseContext; private final MoveFilesCleanup moveFilesCleanup; private final RenamePdfCleanup renameFilesCleanup; - public ExternalFilesEntryLinker(FilePreferences filePreferences, BibDatabaseContext bibDatabaseContext) { + public ExternalFilesEntryLinker(PreferencesService preferencesService, FilePreferences filePreferences, BibDatabaseContext bibDatabaseContext) { + this.preferencesService = preferencesService; this.filePreferences = filePreferences; this.bibDatabaseContext = bibDatabaseContext; this.moveFilesCleanup = new MoveFilesCleanup(bibDatabaseContext, filePreferences); @@ -79,7 +82,7 @@ public void moveFilesToFileDirAndAddToEntry(BibEntry entry, List files, In } try { - indexingTaskManager.addToIndex(LuceneIndexer.of(bibDatabaseContext, filePreferences), entry); + indexingTaskManager.addToIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService, filePreferences), entry); } catch (IOException e) { LOGGER.error("Could not access Fulltext-Index", e); } @@ -97,7 +100,7 @@ public void copyFilesToFileDirAndAddToEntry(BibEntry entry, List files, In } try { - indexingTaskManager.addToIndex(LuceneIndexer.of(bibDatabaseContext, filePreferences), entry); + indexingTaskManager.addToIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService, filePreferences), entry); } catch (IOException e) { LOGGER.error("Could not access Fulltext-Index", e); } diff --git a/src/main/java/org/jabref/gui/externalfiles/ImportHandler.java b/src/main/java/org/jabref/gui/externalfiles/ImportHandler.java index f8ab4c2354c..0c59b8b7803 100644 --- a/src/main/java/org/jabref/gui/externalfiles/ImportHandler.java +++ b/src/main/java/org/jabref/gui/externalfiles/ImportHandler.java @@ -78,7 +78,7 @@ public ImportHandler(BibDatabaseContext database, this.dialogService = dialogService; this.importFormatReader = importFormatReader; - this.linker = new ExternalFilesEntryLinker(preferencesService.getFilePreferences(), database); + this.linker = new ExternalFilesEntryLinker(preferencesService, preferencesService.getFilePreferences(), database); this.contentImporter = new ExternalFilesContentImporter( preferencesService.getGeneralPreferences(), preferencesService.getImporterPreferences(), diff --git a/src/main/java/org/jabref/gui/preview/PreviewPanel.java b/src/main/java/org/jabref/gui/preview/PreviewPanel.java index 94cf57cd1e3..d79cdf72324 100644 --- a/src/main/java/org/jabref/gui/preview/PreviewPanel.java +++ b/src/main/java/org/jabref/gui/preview/PreviewPanel.java @@ -59,7 +59,7 @@ public PreviewPanel(BibDatabaseContext database, this.stateManager = stateManager; this.previewPreferences = preferences.getPreviewPreferences(); this.indexingTaskManager = indexingTaskManager; - this.fileLinker = new ExternalFilesEntryLinker(preferences.getFilePreferences(), database); + this.fileLinker = new ExternalFilesEntryLinker(preferences, preferences.getFilePreferences(), database); PreviewPreferences previewPreferences = preferences.getPreviewPreferences(); previewView = new PreviewViewer(database, dialogService, stateManager, themeManager); diff --git a/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java b/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java index 2d1c081cea7..3ed71df7ac5 100644 --- a/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java +++ b/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java @@ -12,6 +12,7 @@ import org.jabref.logic.pdf.search.indexing.LuceneIndexer; import org.jabref.model.database.BibDatabaseContext; import org.jabref.preferences.FilePreferences; +import org.jabref.preferences.PreferencesService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,16 +26,18 @@ public class RebuildFulltextSearchIndexAction extends SimpleCommand { private final StateManager stateManager; private final GetCurrentLibraryTab currentLibraryTab; private final DialogService dialogService; + private final PreferencesService preferencesService; private final FilePreferences filePreferences; private BibDatabaseContext databaseContext; private boolean shouldContinue = true; - public RebuildFulltextSearchIndexAction(StateManager stateManager, GetCurrentLibraryTab currentLibraryTab, DialogService dialogService, FilePreferences filePreferences) { + public RebuildFulltextSearchIndexAction(StateManager stateManager, GetCurrentLibraryTab currentLibraryTab, DialogService dialogService, PreferencesService preferences, FilePreferences filePreferences) { this.stateManager = stateManager; this.currentLibraryTab = currentLibraryTab; this.dialogService = dialogService; + this.preferencesService = preferences; this.filePreferences = filePreferences; this.executable.bind(needsDatabase(stateManager)); @@ -68,8 +71,8 @@ private void rebuildIndex() { return; } try { - currentLibraryTab.get().getIndexingTaskManager().createIndex(LuceneIndexer.of(databaseContext, filePreferences)); - currentLibraryTab.get().getIndexingTaskManager().updateIndex(LuceneIndexer.of(databaseContext, filePreferences)); + currentLibraryTab.get().getIndexingTaskManager().createIndex(LuceneIndexer.of(databaseContext, preferencesService, filePreferences)); + currentLibraryTab.get().getIndexingTaskManager().updateIndex(LuceneIndexer.of(databaseContext, preferencesService, filePreferences)); } catch (IOException e) { dialogService.notify(Localization.lang("Failed to access fulltext search index")); LOGGER.error("Failed to access fulltext search index", e); diff --git a/src/main/java/org/jabref/logic/pdf/search/indexing/LuceneIndexer.java b/src/main/java/org/jabref/logic/pdf/search/indexing/LuceneIndexer.java index add5a83bc46..dbe7f00b2b1 100644 --- a/src/main/java/org/jabref/logic/pdf/search/indexing/LuceneIndexer.java +++ b/src/main/java/org/jabref/logic/pdf/search/indexing/LuceneIndexer.java @@ -15,11 +15,15 @@ import org.jabref.logic.util.StandardFileType; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.Keyword; +import org.jabref.model.entry.KeywordList; import org.jabref.model.entry.LinkedFile; import org.jabref.model.entry.field.Field; +import org.jabref.model.entry.field.StandardField; import org.jabref.model.pdf.search.EnglishStemAnalyzer; import org.jabref.model.pdf.search.SearchFieldConstants; import org.jabref.preferences.FilePreferences; +import org.jabref.preferences.PreferencesService; import org.apache.lucene.document.Document; import org.apache.lucene.document.StringField; @@ -52,16 +56,18 @@ public class LuceneIndexer { private final Directory directoryToIndex; private final BibDatabaseContext databaseContext; + private final PreferencesService preferences; private final FilePreferences filePreferences; - public LuceneIndexer(BibDatabaseContext databaseContext, FilePreferences filePreferences) throws IOException { + public LuceneIndexer(BibDatabaseContext databaseContext, PreferencesService preferences, FilePreferences filePreferences) throws IOException { this.databaseContext = databaseContext; this.directoryToIndex = new NIOFSDirectory(databaseContext.getFulltextIndexPath()); + this.preferences = preferences; this.filePreferences = filePreferences; } - public static LuceneIndexer of(BibDatabaseContext databaseContext, FilePreferences filePreferences) throws IOException { - return new LuceneIndexer(databaseContext, filePreferences); + public static LuceneIndexer of(BibDatabaseContext databaseContext, PreferencesService preferences, FilePreferences filePreferences) throws IOException { + return new LuceneIndexer(databaseContext, preferences, filePreferences); } public BibDatabaseContext getDatabaseContext() { @@ -158,8 +164,16 @@ public void addBibFieldsToIndex(BibEntry bibEntry) { Document document = new Document(); document.add(new StringField(SearchFieldConstants.BIB_ENTRY_ID_HASH, String.valueOf(bibEntry.getLastIndexHash()), org.apache.lucene.document.Field.Store.YES)); for (Map.Entry field : bibEntry.getFieldMap().entrySet()) { - document.add(new TextField(field.getKey().getName(), field.getValue(), org.apache.lucene.document.Field.Store.YES)); - SearchFieldConstants.searchableBibFields.add(field.getKey().getName()); + if (field.getKey() == StandardField.KEYWORDS) { + System.out.println("Indexing keywords " + field.getValue()); + KeywordList keywords = KeywordList.parse(field.getValue(), preferences.getKeywordDelimiter()); + for (Keyword keyword : keywords) { + document.add(new StringField(field.getKey().getName(), keyword.toString(), org.apache.lucene.document.Field.Store.YES)); + } + } else { + document.add(new TextField(field.getKey().getName(), field.getValue(), org.apache.lucene.document.Field.Store.YES)); + SearchFieldConstants.searchableBibFields.add(field.getKey().getName()); + } } indexWriter.addDocument(document); indexWriter.commit(); From 60c9ed3ed9ffdf3f5f2d2046d7dbd9388c5e0f99 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Mon, 15 Aug 2022 15:17:51 +0200 Subject: [PATCH 020/256] Remove debug output --- .../java/org/jabref/logic/pdf/search/indexing/LuceneIndexer.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/jabref/logic/pdf/search/indexing/LuceneIndexer.java b/src/main/java/org/jabref/logic/pdf/search/indexing/LuceneIndexer.java index dbe7f00b2b1..7fd60bcde63 100644 --- a/src/main/java/org/jabref/logic/pdf/search/indexing/LuceneIndexer.java +++ b/src/main/java/org/jabref/logic/pdf/search/indexing/LuceneIndexer.java @@ -165,7 +165,6 @@ public void addBibFieldsToIndex(BibEntry bibEntry) { document.add(new StringField(SearchFieldConstants.BIB_ENTRY_ID_HASH, String.valueOf(bibEntry.getLastIndexHash()), org.apache.lucene.document.Field.Store.YES)); for (Map.Entry field : bibEntry.getFieldMap().entrySet()) { if (field.getKey() == StandardField.KEYWORDS) { - System.out.println("Indexing keywords " + field.getValue()); KeywordList keywords = KeywordList.parse(field.getValue(), preferences.getKeywordDelimiter()); for (Keyword keyword : keywords) { document.add(new StringField(field.getKey().getName(), keyword.toString(), org.apache.lucene.document.Field.Store.YES)); From c71b7496e30c44e096a2ae087e24e4d8c209e4f7 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Mon, 15 Aug 2022 15:41:21 +0200 Subject: [PATCH 021/256] Improve global search Make the results-counter react on the correct observables and actually filter out results with score == 0 --- src/main/java/org/jabref/gui/StateManager.java | 8 -------- src/main/java/org/jabref/gui/search/GlobalSearchBar.java | 9 ++++++--- .../logic/pdf/search/retrieval/LuceneSearcher.java | 8 +++++--- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/jabref/gui/StateManager.java b/src/main/java/org/jabref/gui/StateManager.java index 1c09095453f..d6bd4571f2c 100644 --- a/src/main/java/org/jabref/gui/StateManager.java +++ b/src/main/java/org/jabref/gui/StateManager.java @@ -7,11 +7,9 @@ import javafx.beans.Observable; import javafx.beans.binding.Bindings; -import javafx.beans.property.IntegerProperty; import javafx.beans.property.ObjectProperty; import javafx.beans.property.ReadOnlyListProperty; import javafx.beans.property.ReadOnlyListWrapper; -import javafx.beans.property.SimpleIntegerProperty; import javafx.beans.property.SimpleObjectProperty; import javafx.collections.FXCollections; import javafx.collections.ObservableList; @@ -92,12 +90,6 @@ public OptionalObjectProperty activeSearchQueryProperty() { return activeSearchQuery; } - public IntegerProperty getSearchResultSize() { - IntegerProperty size = new SimpleIntegerProperty(); - size.bind(Bindings.size(searchResults)); - return size; - } - public ReadOnlyListProperty activeGroupProperty() { return activeGroups.getReadOnlyProperty(); } diff --git a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java index d42bcd34cb7..c469068eea9 100644 --- a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java +++ b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java @@ -14,6 +14,7 @@ import javafx.beans.binding.BooleanBinding; import javafx.beans.property.BooleanProperty; import javafx.beans.property.SimpleBooleanProperty; +import javafx.collections.MapChangeListener; import javafx.css.PseudoClass; import javafx.event.Event; import javafx.geometry.Insets; @@ -59,6 +60,8 @@ import org.jabref.logic.l10n.Localization; import org.jabref.logic.search.SearchQuery; import org.jabref.model.entry.Author; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.pdf.search.LuceneSearchResults; import org.jabref.model.search.rules.SearchRules; import org.jabref.preferences.PreferencesService; import org.jabref.preferences.SearchPreferences; @@ -191,12 +194,12 @@ public GlobalSearchBar(JabRefFrame frame, StateManager stateManager, Preferences }, query -> setSearchTerm(query.map(SearchQuery::getQuery).orElse(""))); - this.stateManager.activeSearchQueryProperty().addListener((obs, oldvalue, newValue) -> newValue.ifPresent(this::updateSearchResultsForQuery)); - this.stateManager.activeDatabaseProperty().addListener((obs, oldValue, newValue) -> stateManager.activeSearchQueryProperty().get().ifPresent(this::updateSearchResultsForQuery)); + this.stateManager.activeSearchQueryProperty().addListener((obs, oldValue, newValue) -> newValue.ifPresent(this::updateSearchResultsForQuery)); + this.stateManager.getSearchResults().addListener((MapChangeListener) map -> stateManager.activeSearchQueryProperty().get().ifPresent(this::updateSearchResultsForQuery)); } private void updateSearchResultsForQuery(SearchQuery query) { - updateResults(this.stateManager.getSearchResultSize().intValue(), SearchDescribers.getSearchDescriberFor(query).getDescription(), + updateResults(this.stateManager.getSearchResults().size(), SearchDescribers.getSearchDescriberFor(query).getDescription(), query.isGrammarBasedSearch()); } diff --git a/src/main/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcher.java b/src/main/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcher.java index b71390d5b41..8503c4fcf6f 100644 --- a/src/main/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcher.java +++ b/src/main/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcher.java @@ -66,10 +66,12 @@ public HashMap search(SearchQuery query) { for (ScoreDoc scoreDoc : docs.scoreDocs) { SearchResult searchResult = new SearchResult(searcher, luceneQuery, scoreDoc); for (BibEntry match : searchResult.getMatchingEntries(databaseContext)) { - if (!results.containsKey(match)) { - results.put(match, new LuceneSearchResults()); + if (searchResult.getLuceneScore() > 0) { + if (!results.containsKey(match)) { + results.put(match, new LuceneSearchResults()); + } + results.get(match).addResult(searchResult); } - results.get(match).addResult(searchResult); } } } catch (ParseException e) { From 9e7a41b31273dc1fc64a3221d65e5364065724b8 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Mon, 15 Aug 2022 15:50:35 +0200 Subject: [PATCH 022/256] Checkstyle --- src/main/java/org/jabref/gui/maintable/MainTableDataModel.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index f3950183f63..8857728c122 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -2,7 +2,6 @@ import java.io.IOException; import java.util.List; -import java.util.Map; import java.util.Optional; import javafx.beans.property.ObjectProperty; @@ -20,7 +19,6 @@ import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.groups.GroupTreeNode; -import org.jabref.model.pdf.search.LuceneSearchResults; import org.jabref.model.search.matchers.MatcherSet; import org.jabref.model.search.matchers.MatcherSets; import org.jabref.preferences.PreferencesService; From 6ead73b169b4422ef4e40170824e6c4b7e6227a5 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Mon, 15 Aug 2022 16:20:27 +0200 Subject: [PATCH 023/256] Don't listen for query-update for group-filtering --- src/main/java/org/jabref/gui/maintable/MainTableDataModel.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index 8857728c122..b3f3b1ada83 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -49,9 +49,8 @@ public MainTableDataModel(BibDatabaseContext context, PreferencesService prefere entriesFiltered = new FilteredList<>(entriesViewModel); entriesFiltered.predicateProperty().bind( EasyBind.combine(stateManager.activeGroupProperty(), - stateManager.activeSearchQueryProperty(), groupsPreferences.groupViewModeProperty(), - (groups, query, groupViewMode) -> entry -> isMatchedByGroup(groups, entry)) + (groups, groupViewMode) -> entry -> isMatchedByGroup(groups, entry)) ); stateManager.activeSearchQueryProperty().addListener((observable, oldValue, newValue) -> doSearch(newValue)); From 5e4a7806d09a5b8a990bd0592f98d719a6f1d99a Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Mon, 15 Aug 2022 16:57:59 +0200 Subject: [PATCH 024/256] Switch SearchGroups to Lucene --- .../gui/maintable/MainTableDataModel.java | 12 ++++- .../org/jabref/logic/search/SearchQuery.java | 6 +-- .../org/jabref/model/groups/SearchGroup.java | 16 ++++++- .../jabref/model/pdf/search/SearchResult.java | 4 +- .../jabref/model/search/GroupSearchQuery.java | 48 ++----------------- 5 files changed, 34 insertions(+), 52 deletions(-) diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index b3f3b1ada83..891e9fd75d0 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -19,6 +19,7 @@ import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.groups.GroupTreeNode; +import org.jabref.model.groups.SearchGroup; import org.jabref.model.search.matchers.MatcherSet; import org.jabref.model.search.matchers.MatcherSets; import org.jabref.preferences.PreferencesService; @@ -50,7 +51,12 @@ public MainTableDataModel(BibDatabaseContext context, PreferencesService prefere entriesFiltered.predicateProperty().bind( EasyBind.combine(stateManager.activeGroupProperty(), groupsPreferences.groupViewModeProperty(), - (groups, groupViewMode) -> entry -> isMatchedByGroup(groups, entry)) + (groups, groupViewMode) -> { + return entry -> { + updateSearchGroups(); + return isMatchedByGroup(groups, entry); + }; + }) ); stateManager.activeSearchQueryProperty().addListener((observable, oldValue, newValue) -> doSearch(newValue)); @@ -58,6 +64,10 @@ public MainTableDataModel(BibDatabaseContext context, PreferencesService prefere entriesSorted = new SortedList<>(entriesFiltered); } + private void updateSearchGroups() { + stateManager.getSelectedGroup(bibDatabaseContext).stream().map(GroupTreeNode::getGroup).filter(g -> g instanceof SearchGroup).map(g -> ((SearchGroup) g)).forEach(g -> g.updateMatches(bibDatabaseContext)); + } + private void doSearch(Optional query) { if (query.isPresent()) { String searchString = query.get().getQuery(); diff --git a/src/main/java/org/jabref/logic/search/SearchQuery.java b/src/main/java/org/jabref/logic/search/SearchQuery.java index 4dc9fd40a58..b64ce4903bb 100644 --- a/src/main/java/org/jabref/logic/search/SearchQuery.java +++ b/src/main/java/org/jabref/logic/search/SearchQuery.java @@ -57,9 +57,9 @@ String format(String regex) { abstract String format(String regex); } - private final String query; - private EnumSet searchFlags; - private final SearchRule rule; + protected final String query; + protected EnumSet searchFlags; + protected final SearchRule rule; public SearchQuery(String query, EnumSet searchFlags) { this.query = Objects.requireNonNull(query); diff --git a/src/main/java/org/jabref/model/groups/SearchGroup.java b/src/main/java/org/jabref/model/groups/SearchGroup.java index c1df365bef2..43a034b5ded 100644 --- a/src/main/java/org/jabref/model/groups/SearchGroup.java +++ b/src/main/java/org/jabref/model/groups/SearchGroup.java @@ -1,8 +1,13 @@ package org.jabref.model.groups; +import java.io.IOException; +import java.nio.file.Path; import java.util.EnumSet; import java.util.Objects; +import java.util.Set; +import org.jabref.logic.pdf.search.retrieval.LuceneSearcher; +import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.search.GroupSearchQuery; import org.jabref.model.search.rules.SearchRules.SearchFlags; @@ -17,6 +22,7 @@ public class SearchGroup extends AbstractGroup { private static final Logger LOGGER = LoggerFactory.getLogger(SearchGroup.class); + private Set matches = Set.of(); private final GroupSearchQuery query; public SearchGroup(String name, GroupHierarchyType context, String searchExpression, EnumSet searchFlags) { @@ -45,7 +51,7 @@ public boolean equals(Object o) { @Override public boolean contains(BibEntry entry) { - return query.isMatch(entry); + return matches.contains(entry); } public EnumSet getSearchFlags() { @@ -79,4 +85,12 @@ public boolean isDynamic() { public int hashCode() { return Objects.hash(getName(), getHierarchicalContext(), getSearchExpression(), getSearchFlags()); } + + public void updateMatches(BibDatabaseContext context) { + try { + this.matches = LuceneSearcher.of(context).search(query).keySet(); + } catch (IOException e) { + LOGGER.warn("Could not open Index for: '{}'!\n{}", context.getDatabasePath().orElse(Path.of("unsaved")).toAbsolutePath(), e.getMessage()); + } + } } diff --git a/src/main/java/org/jabref/model/pdf/search/SearchResult.java b/src/main/java/org/jabref/model/pdf/search/SearchResult.java index 6ea93b0dac8..b2e4a29ec18 100644 --- a/src/main/java/org/jabref/model/pdf/search/SearchResult.java +++ b/src/main/java/org/jabref/model/pdf/search/SearchResult.java @@ -37,8 +37,8 @@ public final class SearchResult { private final boolean hasFulltextResults; private final float luceneScore; - private List contentResultStringsHtml; - private List annotationsResultStringsHtml; + private List contentResultStringsHtml = List.of(); + private List annotationsResultStringsHtml = List.of(); public SearchResult(IndexSearcher searcher, Query query, ScoreDoc scoreDoc) throws IOException { this.luceneScore = scoreDoc.score; diff --git a/src/main/java/org/jabref/model/search/GroupSearchQuery.java b/src/main/java/org/jabref/model/search/GroupSearchQuery.java index 90056d9b0f7..0c0becf589f 100644 --- a/src/main/java/org/jabref/model/search/GroupSearchQuery.java +++ b/src/main/java/org/jabref/model/search/GroupSearchQuery.java @@ -1,29 +1,15 @@ package org.jabref.model.search; import java.util.EnumSet; -import java.util.Objects; +import org.jabref.logic.search.SearchQuery; import org.jabref.model.entry.BibEntry; -import org.jabref.model.search.rules.SearchRule; -import org.jabref.model.search.rules.SearchRules; import org.jabref.model.search.rules.SearchRules.SearchFlags; -public class GroupSearchQuery implements SearchMatcher { - - private final String query; - private final EnumSet searchFlags; - private final SearchRule rule; +public class GroupSearchQuery extends SearchQuery { public GroupSearchQuery(String query, EnumSet searchFlags) { - this.query = Objects.requireNonNull(query); - this.searchFlags = searchFlags; - this.rule = Objects.requireNonNull(getSearchRule()); - } - - @Override - public String toString() { - return String.format("\"%s\" (%s, %s)", query, getCaseSensitiveDescription(), - getRegularExpressionDescription()); + super(query, searchFlags); } @Override @@ -31,35 +17,7 @@ public boolean isMatch(BibEntry entry) { return this.getRule().applyRule(query, entry); } - private SearchRule getSearchRule() { - return SearchRules.getSearchRuleByQuery(query, searchFlags); - } - - private String getCaseSensitiveDescription() { - if (searchFlags.contains(SearchRules.SearchFlags.CASE_SENSITIVE)) { - return "case sensitive"; - } else { - return "case insensitive"; - } - } - - private String getRegularExpressionDescription() { - if (searchFlags.contains(SearchRules.SearchFlags.REGULAR_EXPRESSION)) { - return "regular expression"; - } else { - return "plain text"; - } - } - - public SearchRule getRule() { - return rule; - } - public String getSearchExpression() { return query; } - - public EnumSet getSearchFlags() { - return searchFlags; - } } From 1c69125a8dc0d6c248d649d37bfabe9f0b359f0f Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Mon, 15 Aug 2022 17:15:10 +0200 Subject: [PATCH 025/256] Implement regex-string-wrapper --- .../jabref/logic/pdf/search/retrieval/LuceneSearcher.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcher.java b/src/main/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcher.java index 8503c4fcf6f..57dba449077 100644 --- a/src/main/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcher.java +++ b/src/main/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcher.java @@ -61,7 +61,13 @@ public HashMap search(SearchQuery query) { IndexSearcher searcher = new IndexSearcher(reader); String[] fieldsToSearchArray = new String[boosts.size()]; boosts.keySet().toArray(fieldsToSearchArray); - Query luceneQuery = new MultiFieldQueryParser(fieldsToSearchArray, new EnglishStemAnalyzer(), boosts).parse(query.getQuery()); + String queryString = query.getQuery(); + if (query.getSearchFlags().contains(SearchRules.SearchFlags.REGULAR_EXPRESSION)) { + if (queryString.length() > 0 && !(queryString.startsWith("/") && queryString.endsWith("/"))) { + queryString = "/" + queryString + "/"; + } + } + Query luceneQuery = new MultiFieldQueryParser(fieldsToSearchArray, new EnglishStemAnalyzer(), boosts).parse(queryString); TopDocs docs = searcher.search(luceneQuery, Integer.MAX_VALUE); for (ScoreDoc scoreDoc : docs.scoreDocs) { SearchResult searchResult = new SearchResult(searcher, luceneQuery, scoreDoc); From fe26dd196a96dd3bacbf5781e419e530b5010381 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Mon, 15 Aug 2022 17:26:44 +0200 Subject: [PATCH 026/256] Remove case-sensitive search preference Lucene cannot do case-sensitive search --- .../jabref/gui/groups/GroupDialogView.java | 10 ------- .../gui/groups/GroupDialogViewModel.java | 5 ---- .../jabref/gui/search/GlobalSearchBar.java | 15 +---------- ...tainsAndRegexBasedSearchRuleDescriber.java | 10 ------- .../GrammarBasedSearchRuleDescriber.java | 3 --- .../logic/exporter/GroupSerializer.java | 2 +- .../logic/importer/util/GroupsParser.java | 4 +-- .../org/jabref/logic/search/SearchQuery.java | 27 +++---------------- .../search/rules/ContainsBasedSearchRule.java | 8 ------ .../search/rules/GrammarBasedSearchRule.java | 5 ++-- .../search/rules/RegexBasedSearchRule.java | 4 +-- .../model/search/rules/SearchRules.java | 2 +- .../jabref/preferences/JabRefPreferences.java | 4 --- .../jabref/preferences/SearchPreferences.java | 9 +------ 14 files changed, 12 insertions(+), 96 deletions(-) diff --git a/src/main/java/org/jabref/gui/groups/GroupDialogView.java b/src/main/java/org/jabref/gui/groups/GroupDialogView.java index 218c64379f0..9ea122616af 100644 --- a/src/main/java/org/jabref/gui/groups/GroupDialogView.java +++ b/src/main/java/org/jabref/gui/groups/GroupDialogView.java @@ -74,7 +74,6 @@ public class GroupDialogView extends BaseDialog { @FXML private CheckBox keywordGroupRegex; @FXML private TextField searchGroupSearchTerm; - @FXML private CheckBox searchGroupCaseSensitive; @FXML private CheckBox searchGroupRegex; @FXML private RadioButton autoGroupKeywordsOption; @@ -154,15 +153,6 @@ public void initialize() { keywordGroupRegex.selectedProperty().bindBidirectional(viewModel.keywordGroupRegexProperty()); searchGroupSearchTerm.textProperty().bindBidirectional(viewModel.searchGroupSearchTermProperty()); - searchGroupCaseSensitive.selectedProperty().addListener((observable, oldValue, newValue) -> { - EnumSet searchFlags = viewModel.searchFlagsProperty().get(); - if (newValue) { - searchFlags.add(SearchRules.SearchFlags.CASE_SENSITIVE); - } else { - searchFlags.remove(SearchRules.SearchFlags.CASE_SENSITIVE); - } - viewModel.searchFlagsProperty().set(searchFlags); - }); searchGroupRegex.selectedProperty().addListener((observable, oldValue, newValue) -> { EnumSet searchFlags = viewModel.searchFlagsProperty().get(); if (newValue) { diff --git a/src/main/java/org/jabref/gui/groups/GroupDialogViewModel.java b/src/main/java/org/jabref/gui/groups/GroupDialogViewModel.java index 0a1de76104c..b5743b4bc1f 100644 --- a/src/main/java/org/jabref/gui/groups/GroupDialogViewModel.java +++ b/src/main/java/org/jabref/gui/groups/GroupDialogViewModel.java @@ -49,7 +49,6 @@ import org.jabref.model.groups.TexGroup; import org.jabref.model.groups.WordKeywordGroup; import org.jabref.model.metadata.MetaData; -import org.jabref.model.search.rules.SearchRules; import org.jabref.model.search.rules.SearchRules.SearchFlags; import org.jabref.model.strings.StringUtil; import org.jabref.preferences.PreferencesService; @@ -199,10 +198,6 @@ private void setupValidation() { searchRegexValidator = new FunctionBasedValidator<>( searchGroupSearchTermProperty, input -> { - if (!searchFlagsProperty.getValue().contains(SearchRules.SearchFlags.CASE_SENSITIVE)) { - return true; - } - if (StringUtil.isNullOrEmpty(input)) { return false; } diff --git a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java index c469068eea9..2dac810324b 100644 --- a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java +++ b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java @@ -89,7 +89,6 @@ public class GlobalSearchBar extends HBox { private static final PseudoClass CLASS_RESULTS_FOUND = PseudoClass.getPseudoClass("emptyResult"); private final CustomTextField searchField = SearchTextField.create(); - private final ToggleButton caseSensitiveButton; private final ToggleButton regularExpressionButton; private final ToggleButton fulltextButton; private final Button openGlobalSearchButton; @@ -141,7 +140,6 @@ public GlobalSearchBar(JabRefFrame frame, StateManager stateManager, Preferences ClipBoardManager.addX11Support(searchField); regularExpressionButton = IconTheme.JabRefIcons.REG_EX.asToggleButton(); - caseSensitiveButton = IconTheme.JabRefIcons.CASE_SENSITIVE.asToggleButton(); fulltextButton = IconTheme.JabRefIcons.FULLTEXT.asToggleButton(); openGlobalSearchButton = IconTheme.JabRefIcons.OPEN_GLOBAL_SEARCH.asButton(); keepSearchString = IconTheme.JabRefIcons.KEEP_SEARCH_STRING.asToggleButton(); @@ -150,7 +148,6 @@ public GlobalSearchBar(JabRefFrame frame, StateManager stateManager, Preferences BooleanBinding focusedOrActive = searchField.focusedProperty() .or(regularExpressionButton.focusedProperty()) - .or(caseSensitiveButton.focusedProperty()) .or(fulltextButton.focusedProperty()) .or(keepSearchString.focusedProperty()) .or(searchField.textProperty() @@ -158,14 +155,12 @@ public GlobalSearchBar(JabRefFrame frame, StateManager stateManager, Preferences regularExpressionButton.visibleProperty().unbind(); regularExpressionButton.visibleProperty().bind(focusedOrActive); - caseSensitiveButton.visibleProperty().unbind(); - caseSensitiveButton.visibleProperty().bind(focusedOrActive); fulltextButton.visibleProperty().unbind(); fulltextButton.visibleProperty().bind(focusedOrActive); keepSearchString.visibleProperty().unbind(); keepSearchString.visibleProperty().bind(focusedOrActive); - StackPane modifierButtons = new StackPane(new HBox(regularExpressionButton, caseSensitiveButton, fulltextButton, keepSearchString)); + StackPane modifierButtons = new StackPane(new HBox(regularExpressionButton, fulltextButton, keepSearchString)); modifierButtons.setAlignment(Pos.CENTER); searchField.setRight(new HBox(searchField.getRight(), modifierButtons)); searchField.getStyleClass().add("search-field"); @@ -212,14 +207,6 @@ private void initSearchModifierButtons() { performSearch(); }); - caseSensitiveButton.setSelected(searchPreferences.isCaseSensitive()); - caseSensitiveButton.setTooltip(new Tooltip(Localization.lang("Case sensitive"))); - initSearchModifierButton(caseSensitiveButton); - caseSensitiveButton.setOnAction(event -> { - searchPreferences.setSearchFlag(SearchRules.SearchFlags.CASE_SENSITIVE, caseSensitiveButton.isSelected()); - performSearch(); - }); - fulltextButton.setSelected(searchPreferences.isFulltext()); fulltextButton.setTooltip(new Tooltip(Localization.lang("Fulltext search"))); initSearchModifierButton(fulltextButton); diff --git a/src/main/java/org/jabref/gui/search/rules/describer/ContainsAndRegexBasedSearchRuleDescriber.java b/src/main/java/org/jabref/gui/search/rules/describer/ContainsAndRegexBasedSearchRuleDescriber.java index d411c263c7c..00ff67b8387 100644 --- a/src/main/java/org/jabref/gui/search/rules/describer/ContainsAndRegexBasedSearchRuleDescriber.java +++ b/src/main/java/org/jabref/gui/search/rules/describer/ContainsAndRegexBasedSearchRuleDescriber.java @@ -40,18 +40,8 @@ public TextFlow getDescription() { } } - textList.add(getCaseSensitiveDescription()); - TextFlow searchDescription = new TextFlow(); searchDescription.getChildren().setAll(textList); return searchDescription; } - - private Text getCaseSensitiveDescription() { - if (searchFlags.contains(SearchRules.SearchFlags.CASE_SENSITIVE)) { - return TooltipTextUtil.createText(String.format(" (%s). ", Localization.lang("case sensitive")), TooltipTextUtil.TextType.NORMAL); - } else { - return TooltipTextUtil.createText(String.format(" (%s). ", Localization.lang("case insensitive")), TooltipTextUtil.TextType.NORMAL); - } - } } diff --git a/src/main/java/org/jabref/gui/search/rules/describer/GrammarBasedSearchRuleDescriber.java b/src/main/java/org/jabref/gui/search/rules/describer/GrammarBasedSearchRuleDescriber.java index 416aa8dd618..e49c5702101 100644 --- a/src/main/java/org/jabref/gui/search/rules/describer/GrammarBasedSearchRuleDescriber.java +++ b/src/main/java/org/jabref/gui/search/rules/describer/GrammarBasedSearchRuleDescriber.java @@ -40,9 +40,6 @@ public TextFlow getDescription() { textFlow.getChildren().add(TooltipTextUtil.createText(String.format("%s ", Localization.lang("This search contains entries in which")), TooltipTextUtil.TextType.NORMAL)); textFlow.getChildren().addAll(descriptionSearchBaseVisitor.visit(parseTree)); textFlow.getChildren().add(TooltipTextUtil.createText(". ", TooltipTextUtil.TextType.NORMAL)); - textFlow.getChildren().add(TooltipTextUtil.createText(searchFlags.contains(SearchRules.SearchFlags.CASE_SENSITIVE) ? Localization - .lang("The search is case sensitive.") : - Localization.lang("The search is case insensitive."), TooltipTextUtil.TextType.NORMAL)); return textFlow; } diff --git a/src/main/java/org/jabref/logic/exporter/GroupSerializer.java b/src/main/java/org/jabref/logic/exporter/GroupSerializer.java index d9ae19ad5ef..05dd1ba04d9 100644 --- a/src/main/java/org/jabref/logic/exporter/GroupSerializer.java +++ b/src/main/java/org/jabref/logic/exporter/GroupSerializer.java @@ -70,7 +70,7 @@ private String serializeSearchGroup(SearchGroup group) { sb.append(MetadataSerializationConfiguration.GROUP_UNIT_SEPARATOR); sb.append(StringUtil.quote(group.getSearchExpression(), MetadataSerializationConfiguration.GROUP_UNIT_SEPARATOR, MetadataSerializationConfiguration.GROUP_QUOTE_CHAR)); sb.append(MetadataSerializationConfiguration.GROUP_UNIT_SEPARATOR); - sb.append(StringUtil.booleanToBinaryString(group.getSearchFlags().contains(SearchRules.SearchFlags.CASE_SENSITIVE))); + sb.append(StringUtil.booleanToBinaryString(false)); sb.append(MetadataSerializationConfiguration.GROUP_UNIT_SEPARATOR); sb.append(StringUtil.booleanToBinaryString(group.getSearchFlags().contains(SearchRules.SearchFlags.REGULAR_EXPRESSION))); sb.append(MetadataSerializationConfiguration.GROUP_UNIT_SEPARATOR); diff --git a/src/main/java/org/jabref/logic/importer/util/GroupsParser.java b/src/main/java/org/jabref/logic/importer/util/GroupsParser.java index da08aae3abc..4a2f04b3fd4 100644 --- a/src/main/java/org/jabref/logic/importer/util/GroupsParser.java +++ b/src/main/java/org/jabref/logic/importer/util/GroupsParser.java @@ -278,9 +278,7 @@ private static AbstractGroup searchGroupFromString(String s) { int context = Integer.parseInt(tok.nextToken()); String expression = StringUtil.unquote(tok.nextToken(), MetadataSerializationConfiguration.GROUP_QUOTE_CHAR); EnumSet searchFlags = EnumSet.noneOf(SearchFlags.class); - if (Integer.parseInt(tok.nextToken()) == 1) { - searchFlags.add(SearchRules.SearchFlags.CASE_SENSITIVE); - } + tok.nextToken(); // This used to be the flag for CASE_SENSITIVE search. Skip it for backwards-compatibility if (Integer.parseInt(tok.nextToken()) == 1) { searchFlags.add(SearchRules.SearchFlags.REGULAR_EXPRESSION); } diff --git a/src/main/java/org/jabref/logic/search/SearchQuery.java b/src/main/java/org/jabref/logic/search/SearchQuery.java index b64ce4903bb..d8112f15688 100644 --- a/src/main/java/org/jabref/logic/search/SearchQuery.java +++ b/src/main/java/org/jabref/logic/search/SearchQuery.java @@ -69,7 +69,7 @@ public SearchQuery(String query, EnumSet searchFlags) { @Override public String toString() { - return String.format("\"%s\" (%s, %s)", getQuery(), getCaseSensitiveDescription(), getRegularExpressionDescription()); + return String.format("\"%s\" (%s)", getQuery(), getRegularExpressionDescription()); } @Override @@ -85,14 +85,6 @@ public boolean isContainsBasedSearch() { return rule instanceof ContainsBasedSearchRule; } - private String getCaseSensitiveDescription() { - if (searchFlags.contains(SearchRules.SearchFlags.CASE_SENSITIVE)) { - return "case sensitive"; - } else { - return "case insensitive"; - } - } - private String getRegularExpressionDescription() { if (searchFlags.contains(SearchRules.SearchFlags.REGULAR_EXPRESSION)) { return "regular expression"; @@ -102,20 +94,11 @@ private String getRegularExpressionDescription() { } public String localize() { - return String.format("\"%s\" (%s, %s)", + return String.format("\"%s\" (%s)", getQuery(), - getLocalizedCaseSensitiveDescription(), getLocalizedRegularExpressionDescription()); } - private String getLocalizedCaseSensitiveDescription() { - if (searchFlags.contains(SearchRules.SearchFlags.CASE_SENSITIVE)) { - return Localization.lang("case sensitive"); - } else { - return Localization.lang("case insensitive"); - } - } - private String getLocalizedRegularExpressionDescription() { if (searchFlags.contains(SearchRules.SearchFlags.REGULAR_EXPRESSION)) { return Localization.lang("regular expression"); @@ -184,11 +167,7 @@ private Optional joinWordsToPattern(EscapeMode escapeMode) { } String searchPattern = joiner.collect(Collectors.joining(")|(", "(", ")")); - if (searchFlags.contains(SearchRules.SearchFlags.CASE_SENSITIVE)) { - return Optional.of(Pattern.compile(searchPattern)); - } else { - return Optional.of(Pattern.compile(searchPattern, Pattern.CASE_INSENSITIVE)); - } + return Optional.of(Pattern.compile(searchPattern, Pattern.CASE_INSENSITIVE)); } public SearchRule getRule() { diff --git a/src/main/java/org/jabref/model/search/rules/ContainsBasedSearchRule.java b/src/main/java/org/jabref/model/search/rules/ContainsBasedSearchRule.java index 0e8e11cfdba..9d0232eca1e 100644 --- a/src/main/java/org/jabref/model/search/rules/ContainsBasedSearchRule.java +++ b/src/main/java/org/jabref/model/search/rules/ContainsBasedSearchRule.java @@ -3,7 +3,6 @@ import java.util.EnumSet; import java.util.Iterator; import java.util.List; -import java.util.Locale; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.Field; @@ -27,17 +26,10 @@ public boolean validateSearchStrings(String query) { @Override public boolean applyRule(String query, BibEntry bibEntry) { String searchString = query; - if (!searchFlags.contains(SearchRules.SearchFlags.CASE_SENSITIVE)) { - searchString = searchString.toLowerCase(Locale.ROOT); - } - List unmatchedWords = new SentenceAnalyzer(searchString).getWords(); for (Field fieldKey : bibEntry.getFields()) { String formattedFieldContent = StringUtil.stripAccents(bibEntry.getLatexFreeField(fieldKey).get()); - if (!searchFlags.contains(SearchRules.SearchFlags.CASE_SENSITIVE)) { - formattedFieldContent = formattedFieldContent.toLowerCase(Locale.ROOT); - } Iterator unmatchedWordsIterator = unmatchedWords.iterator(); while (unmatchedWordsIterator.hasNext()) { diff --git a/src/main/java/org/jabref/model/search/rules/GrammarBasedSearchRule.java b/src/main/java/org/jabref/model/search/rules/GrammarBasedSearchRule.java index fddae7ee48a..207801e0a3a 100644 --- a/src/main/java/org/jabref/model/search/rules/GrammarBasedSearchRule.java +++ b/src/main/java/org/jabref/model/search/rules/GrammarBasedSearchRule.java @@ -145,9 +145,8 @@ public static class Comparator { public Comparator(String field, String value, ComparisonOperator operator, EnumSet searchFlags) { this.operator = operator; - int option = searchFlags.contains(SearchRules.SearchFlags.CASE_SENSITIVE) ? 0 : Pattern.CASE_INSENSITIVE; - this.fieldPattern = Pattern.compile(searchFlags.contains(SearchRules.SearchFlags.REGULAR_EXPRESSION) ? StringUtil.stripAccents(field) : "\\Q" + StringUtil.stripAccents(field) + "\\E", option); - this.valuePattern = Pattern.compile(searchFlags.contains(SearchRules.SearchFlags.REGULAR_EXPRESSION) ? StringUtil.stripAccents(value) : "\\Q" + StringUtil.stripAccents(value) + "\\E", option); + this.fieldPattern = Pattern.compile(searchFlags.contains(SearchRules.SearchFlags.REGULAR_EXPRESSION) ? StringUtil.stripAccents(field) : "\\Q" + StringUtil.stripAccents(field) + "\\E", Pattern.CASE_INSENSITIVE); + this.valuePattern = Pattern.compile(searchFlags.contains(SearchRules.SearchFlags.REGULAR_EXPRESSION) ? StringUtil.stripAccents(value) : "\\Q" + StringUtil.stripAccents(value) + "\\E", Pattern.CASE_INSENSITIVE); } public boolean compare(BibEntry entry) { diff --git a/src/main/java/org/jabref/model/search/rules/RegexBasedSearchRule.java b/src/main/java/org/jabref/model/search/rules/RegexBasedSearchRule.java index 70ba0a35a90..f90d89842ac 100644 --- a/src/main/java/org/jabref/model/search/rules/RegexBasedSearchRule.java +++ b/src/main/java/org/jabref/model/search/rules/RegexBasedSearchRule.java @@ -28,7 +28,7 @@ public RegexBasedSearchRule(EnumSet searchFlags) { @Override public boolean validateSearchStrings(String query) { try { - Pattern.compile(query, searchFlags.contains(SearchRules.SearchFlags.CASE_SENSITIVE) ? 0 : Pattern.CASE_INSENSITIVE); + Pattern.compile(query, Pattern.CASE_INSENSITIVE); } catch (PatternSyntaxException ex) { return false; } @@ -39,7 +39,7 @@ public boolean validateSearchStrings(String query) { public boolean applyRule(String query, BibEntry bibEntry) { Pattern pattern; try { - pattern = Pattern.compile(StringUtil.stripAccents(query), searchFlags.contains(SearchRules.SearchFlags.CASE_SENSITIVE) ? 0 : Pattern.CASE_INSENSITIVE); + pattern = Pattern.compile(StringUtil.stripAccents(query), Pattern.CASE_INSENSITIVE); } catch (PatternSyntaxException ex) { LOGGER.debug("Could not compile regex {}", query, ex); return false; diff --git a/src/main/java/org/jabref/model/search/rules/SearchRules.java b/src/main/java/org/jabref/model/search/rules/SearchRules.java index 71c35a7e33e..2f4c01392a4 100644 --- a/src/main/java/org/jabref/model/search/rules/SearchRules.java +++ b/src/main/java/org/jabref/model/search/rules/SearchRules.java @@ -44,6 +44,6 @@ static SearchRule getSearchRule(EnumSet searchFlags) { } public enum SearchFlags { - CASE_SENSITIVE, REGULAR_EXPRESSION, FULLTEXT, KEEP_SEARCH_STRING; + REGULAR_EXPRESSION, FULLTEXT, KEEP_SEARCH_STRING; } } diff --git a/src/main/java/org/jabref/preferences/JabRefPreferences.java b/src/main/java/org/jabref/preferences/JabRefPreferences.java index b954e883683..758e2ef5d7e 100644 --- a/src/main/java/org/jabref/preferences/JabRefPreferences.java +++ b/src/main/java/org/jabref/preferences/JabRefPreferences.java @@ -248,7 +248,6 @@ public class JabRefPreferences implements PreferencesService { public static final String MAIN_FILE_DIRECTORY = "fileDirectory"; public static final String SEARCH_DISPLAY_MODE = "searchDisplayMode"; - public static final String SEARCH_CASE_SENSITIVE = "caseSensitiveSearch"; public static final String SEARCH_REG_EXP = "regExpSearch"; public static final String SEARCH_FULLTEXT = "fulltextSearch"; public static final String SEARCH_KEEP_SEARCH_STRING = "keepSearchString"; @@ -474,7 +473,6 @@ private JabRefPreferences() { Localization.setLanguage(getLanguage()); defaults.put(SEARCH_DISPLAY_MODE, SearchDisplayMode.FILTER.toString()); - defaults.put(SEARCH_CASE_SENSITIVE, Boolean.FALSE); defaults.put(SEARCH_REG_EXP, Boolean.FALSE); defaults.put(SEARCH_FULLTEXT, Boolean.TRUE); defaults.put(SEARCH_KEEP_SEARCH_STRING, Boolean.FALSE); @@ -2638,7 +2636,6 @@ public SearchPreferences getSearchPreferences() { searchPreferences = new SearchPreferences( searchDisplayMode, - getBoolean(SEARCH_CASE_SENSITIVE), getBoolean(SEARCH_REG_EXP), getBoolean(SEARCH_FULLTEXT), getBoolean(SEARCH_KEEP_SEARCH_STRING), @@ -2646,7 +2643,6 @@ public SearchPreferences getSearchPreferences() { EasyBind.listen(searchPreferences.searchDisplayModeProperty(), (obs, oldValue, newValue) -> put(SEARCH_DISPLAY_MODE, Objects.requireNonNull(searchPreferences.getSearchDisplayMode()).toString())); searchPreferences.getObservableSearchFlags().addListener((SetChangeListener) c -> { - putBoolean(SEARCH_CASE_SENSITIVE, searchPreferences.getObservableSearchFlags().contains(SearchRules.SearchFlags.CASE_SENSITIVE)); putBoolean(SEARCH_REG_EXP, searchPreferences.getObservableSearchFlags().contains(SearchRules.SearchFlags.REGULAR_EXPRESSION)); putBoolean(SEARCH_FULLTEXT, searchPreferences.getObservableSearchFlags().contains(SearchRules.SearchFlags.FULLTEXT)); putBoolean(SEARCH_KEEP_SEARCH_STRING, searchPreferences.getObservableSearchFlags().contains(SearchRules.SearchFlags.KEEP_SEARCH_STRING)); diff --git a/src/main/java/org/jabref/preferences/SearchPreferences.java b/src/main/java/org/jabref/preferences/SearchPreferences.java index 10273c0a892..43fb2f5f1e8 100644 --- a/src/main/java/org/jabref/preferences/SearchPreferences.java +++ b/src/main/java/org/jabref/preferences/SearchPreferences.java @@ -18,14 +18,11 @@ public class SearchPreferences { private final ObservableSet searchFlags; private final BooleanProperty keepWindowOnTop; - public SearchPreferences(SearchDisplayMode searchDisplayMode, boolean isCaseSensitive, boolean isRegularExpression, boolean isFulltext, boolean isKeepSearchString, boolean keepWindowOnTop) { + public SearchPreferences(SearchDisplayMode searchDisplayMode, boolean isRegularExpression, boolean isFulltext, boolean isKeepSearchString, boolean keepWindowOnTop) { this.searchDisplayMode = new SimpleObjectProperty<>(searchDisplayMode); this.keepWindowOnTop = new SimpleBooleanProperty(keepWindowOnTop); searchFlags = FXCollections.observableSet(EnumSet.noneOf(SearchFlags.class)); - if (isCaseSensitive) { - searchFlags.add(SearchFlags.CASE_SENSITIVE); - } if (isRegularExpression) { searchFlags.add(SearchFlags.REGULAR_EXPRESSION); } @@ -68,10 +65,6 @@ public void setSearchDisplayMode(SearchDisplayMode searchDisplayMode) { this.searchDisplayMode.set(searchDisplayMode); } - public boolean isCaseSensitive() { - return searchFlags.contains(SearchFlags.CASE_SENSITIVE); - } - public void setSearchFlag(SearchFlags flag, boolean value) { if (searchFlags.contains(flag) && !value) { searchFlags.remove(flag); From d759c24b7f590772822ee38b1994d07620d23724 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Mon, 15 Aug 2022 17:46:53 +0200 Subject: [PATCH 027/256] Remove case-sensitive button from group dialog --- src/main/java/org/jabref/gui/groups/GroupDialog.fxml | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/jabref/gui/groups/GroupDialog.fxml b/src/main/java/org/jabref/gui/groups/GroupDialog.fxml index 3d12f80abbc..a4936953876 100644 --- a/src/main/java/org/jabref/gui/groups/GroupDialog.fxml +++ b/src/main/java/org/jabref/gui/groups/GroupDialog.fxml @@ -124,7 +124,6 @@ - From 35e97155685629e0c5eeee4bec373bc02ec137c7 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Mon, 15 Aug 2022 17:46:27 +0200 Subject: [PATCH 028/256] Update tooltip --- src/main/java/org/jabref/gui/search/GlobalSearchBar.java | 2 +- src/main/resources/l10n/JabRef_en.properties | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java index 2dac810324b..4f477761ebf 100644 --- a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java +++ b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java @@ -349,7 +349,7 @@ private void updateResults(int matched, TextFlow description, boolean grammarBas private void setSearchFieldHintTooltip(TextFlow description) { if (preferencesService.getGeneralPreferences().shouldShowAdvancedHints()) { - String genericDescription = Localization.lang("Hint:\n\nTo search all fields for Smith, enter:\nsmith\n\nTo search the field author for Smith and the field title for electrical, enter:\nauthor=Smith and title=electrical"); + String genericDescription = Localization.lang("Hint:\n\nTo search all fields for Smith, enter:\nsmith\n\nTo search the field author for Smith and the field title for electrical, enter:\nauthor:Smith AND title:electrical"); List genericDescriptionTexts = TooltipTextUtil.createTextsFromHtml(genericDescription); if (description == null) { diff --git a/src/main/resources/l10n/JabRef_en.properties b/src/main/resources/l10n/JabRef_en.properties index c50961fdfba..71aec8b1ecb 100644 --- a/src/main/resources/l10n/JabRef_en.properties +++ b/src/main/resources/l10n/JabRef_en.properties @@ -2097,7 +2097,7 @@ Hierarchical\ keyword\ delimiter=Hierarchical keyword delimiter Escape\ ampersands=Escape ampersands Escape\ dollar\ sign=Escape dollar sign -Hint\:\n\nTo\ search\ all\ fields\ for\ Smith,\ enter\:\nsmith\n\nTo\ search\ the\ field\ author\ for\ Smith\ and\ the\ field\ title\ for\ electrical,\ enter\:\nauthor\=Smith\ and\ title\=electrical=Hint:\n\nTo search all fields for Smith, enter:\nsmith\n\nTo search the field author for Smith and the field title for electrical, enter:\nauthor=Smith and title=electrical +Hint\:\n\nTo\ search\ all\ fields\ for\ Smith,\ enter\:\nsmith\n\nTo\ search\ the\ field\ author\ for\ Smith\ and\ the\ field\ title\ for\ electrical,\ enter\:\nauthor\:Smith\ AND\ title\:electrical=Hint:\n\nTo search all fields for Smith, enter:\nsmith\n\nTo search the field author for Smith and the field title for electrical, enter:\nauthor:Smith AND title:electrical Copied\ '%0'\ to\ clipboard.=Copied '%0' to clipboard. This\ operation\ requires\ an\ open\ library.=This operation requires an open library. From e54b4e8ac3e6e1c2454ba882541311d5d6a21878 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Mon, 15 Aug 2022 18:38:21 +0200 Subject: [PATCH 029/256] Move lucene-logic out of pdf package --- src/main/java/org/jabref/gui/LibraryTab.java | 4 ++-- .../java/org/jabref/gui/entryeditor/DeprecatedFieldsTab.java | 2 +- src/main/java/org/jabref/gui/entryeditor/FieldsEditorTab.java | 2 +- .../java/org/jabref/gui/entryeditor/OptionalFields2Tab.java | 2 +- .../java/org/jabref/gui/entryeditor/OptionalFieldsTab.java | 2 +- .../org/jabref/gui/entryeditor/OptionalFieldsTabBase.java | 2 +- src/main/java/org/jabref/gui/entryeditor/OtherFieldsTab.java | 2 +- src/main/java/org/jabref/gui/entryeditor/PreviewTab.java | 2 +- .../java/org/jabref/gui/entryeditor/RequiredFieldsTab.java | 2 +- .../java/org/jabref/gui/entryeditor/UserDefinedFieldsTab.java | 2 +- .../jabref/gui/externalfiles/ExternalFilesEntryLinker.java | 4 ++-- .../java/org/jabref/gui/maintable/MainTableDataModel.java | 2 +- src/main/java/org/jabref/gui/preview/PreviewPanel.java | 2 +- .../jabref/gui/search/RebuildFulltextSearchIndexAction.java | 2 +- .../logic/{pdf => }/search/indexing/DocumentReader.java | 2 +- .../logic/{pdf => }/search/indexing/IndexingTaskManager.java | 2 +- .../jabref/logic/{pdf => }/search/indexing/LuceneIndexer.java | 2 +- .../logic/{pdf => }/search/retrieval/LuceneSearcher.java | 2 +- src/main/java/org/jabref/model/groups/SearchGroup.java | 2 +- .../jabref/logic/pdf/search/indexing/DocumentReaderTest.java | 1 + .../jabref/logic/pdf/search/indexing/LuceneIndexerTest.java | 1 + .../jabref/logic/pdf/search/retrieval/LuceneSearcherTest.java | 3 ++- 22 files changed, 25 insertions(+), 22 deletions(-) rename src/main/java/org/jabref/logic/{pdf => }/search/indexing/DocumentReader.java (99%) rename src/main/java/org/jabref/logic/{pdf => }/search/indexing/IndexingTaskManager.java (99%) rename src/main/java/org/jabref/logic/{pdf => }/search/indexing/LuceneIndexer.java (99%) rename src/main/java/org/jabref/logic/{pdf => }/search/retrieval/LuceneSearcher.java (98%) diff --git a/src/main/java/org/jabref/gui/LibraryTab.java b/src/main/java/org/jabref/gui/LibraryTab.java index 51ba737351d..71eb0071dc2 100644 --- a/src/main/java/org/jabref/gui/LibraryTab.java +++ b/src/main/java/org/jabref/gui/LibraryTab.java @@ -48,8 +48,8 @@ import org.jabref.logic.importer.util.FileFieldParser; import org.jabref.logic.l10n.Localization; import org.jabref.logic.pdf.FileAnnotationCache; -import org.jabref.logic.pdf.search.indexing.IndexingTaskManager; -import org.jabref.logic.pdf.search.indexing.LuceneIndexer; +import org.jabref.logic.search.indexing.IndexingTaskManager; +import org.jabref.logic.search.indexing.LuceneIndexer; import org.jabref.logic.search.SearchQuery; import org.jabref.logic.shared.DatabaseLocation; import org.jabref.logic.util.UpdateField; diff --git a/src/main/java/org/jabref/gui/entryeditor/DeprecatedFieldsTab.java b/src/main/java/org/jabref/gui/entryeditor/DeprecatedFieldsTab.java index d5bc20da9e6..e3164bcd331 100644 --- a/src/main/java/org/jabref/gui/entryeditor/DeprecatedFieldsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/DeprecatedFieldsTab.java @@ -17,7 +17,7 @@ import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.journals.JournalAbbreviationRepository; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.pdf.search.indexing.IndexingTaskManager; +import org.jabref.logic.search.indexing.IndexingTaskManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.BibEntryType; diff --git a/src/main/java/org/jabref/gui/entryeditor/FieldsEditorTab.java b/src/main/java/org/jabref/gui/entryeditor/FieldsEditorTab.java index 49fc1d8715b..09ef2cb184a 100644 --- a/src/main/java/org/jabref/gui/entryeditor/FieldsEditorTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/FieldsEditorTab.java @@ -33,7 +33,7 @@ import org.jabref.gui.theme.ThemeManager; import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.journals.JournalAbbreviationRepository; -import org.jabref.logic.pdf.search.indexing.IndexingTaskManager; +import org.jabref.logic.search.indexing.IndexingTaskManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.Field; diff --git a/src/main/java/org/jabref/gui/entryeditor/OptionalFields2Tab.java b/src/main/java/org/jabref/gui/entryeditor/OptionalFields2Tab.java index 97c38da58dd..3edc308d82a 100644 --- a/src/main/java/org/jabref/gui/entryeditor/OptionalFields2Tab.java +++ b/src/main/java/org/jabref/gui/entryeditor/OptionalFields2Tab.java @@ -9,7 +9,7 @@ import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.journals.JournalAbbreviationRepository; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.pdf.search.indexing.IndexingTaskManager; +import org.jabref.logic.search.indexing.IndexingTaskManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntryTypesManager; import org.jabref.preferences.PreferencesService; diff --git a/src/main/java/org/jabref/gui/entryeditor/OptionalFieldsTab.java b/src/main/java/org/jabref/gui/entryeditor/OptionalFieldsTab.java index 93a6ff4b6f1..66268251e62 100644 --- a/src/main/java/org/jabref/gui/entryeditor/OptionalFieldsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/OptionalFieldsTab.java @@ -9,7 +9,7 @@ import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.journals.JournalAbbreviationRepository; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.pdf.search.indexing.IndexingTaskManager; +import org.jabref.logic.search.indexing.IndexingTaskManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntryTypesManager; import org.jabref.preferences.PreferencesService; diff --git a/src/main/java/org/jabref/gui/entryeditor/OptionalFieldsTabBase.java b/src/main/java/org/jabref/gui/entryeditor/OptionalFieldsTabBase.java index af1a5f9f0b6..f948fe34e85 100644 --- a/src/main/java/org/jabref/gui/entryeditor/OptionalFieldsTabBase.java +++ b/src/main/java/org/jabref/gui/entryeditor/OptionalFieldsTabBase.java @@ -16,7 +16,7 @@ import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.journals.JournalAbbreviationRepository; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.pdf.search.indexing.IndexingTaskManager; +import org.jabref.logic.search.indexing.IndexingTaskManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.BibEntryType; diff --git a/src/main/java/org/jabref/gui/entryeditor/OtherFieldsTab.java b/src/main/java/org/jabref/gui/entryeditor/OtherFieldsTab.java index 0fe1281c077..e39e6c1fa9f 100644 --- a/src/main/java/org/jabref/gui/entryeditor/OtherFieldsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/OtherFieldsTab.java @@ -19,7 +19,7 @@ import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.journals.JournalAbbreviationRepository; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.pdf.search.indexing.IndexingTaskManager; +import org.jabref.logic.search.indexing.IndexingTaskManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.BibEntryType; diff --git a/src/main/java/org/jabref/gui/entryeditor/PreviewTab.java b/src/main/java/org/jabref/gui/entryeditor/PreviewTab.java index 4212506b8f6..e109b854e9f 100644 --- a/src/main/java/org/jabref/gui/entryeditor/PreviewTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/PreviewTab.java @@ -6,7 +6,7 @@ import org.jabref.gui.preview.PreviewPanel; import org.jabref.gui.theme.ThemeManager; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.pdf.search.indexing.IndexingTaskManager; +import org.jabref.logic.search.indexing.IndexingTaskManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.preferences.PreferencesService; diff --git a/src/main/java/org/jabref/gui/entryeditor/RequiredFieldsTab.java b/src/main/java/org/jabref/gui/entryeditor/RequiredFieldsTab.java index 3451f2c8f53..1ea0bbb0438 100644 --- a/src/main/java/org/jabref/gui/entryeditor/RequiredFieldsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/RequiredFieldsTab.java @@ -16,7 +16,7 @@ import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.journals.JournalAbbreviationRepository; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.pdf.search.indexing.IndexingTaskManager; +import org.jabref.logic.search.indexing.IndexingTaskManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.BibEntryType; diff --git a/src/main/java/org/jabref/gui/entryeditor/UserDefinedFieldsTab.java b/src/main/java/org/jabref/gui/entryeditor/UserDefinedFieldsTab.java index aa4e65b48e3..d467a5855f3 100644 --- a/src/main/java/org/jabref/gui/entryeditor/UserDefinedFieldsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/UserDefinedFieldsTab.java @@ -12,7 +12,7 @@ import org.jabref.gui.theme.ThemeManager; import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.journals.JournalAbbreviationRepository; -import org.jabref.logic.pdf.search.indexing.IndexingTaskManager; +import org.jabref.logic.search.indexing.IndexingTaskManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.Field; diff --git a/src/main/java/org/jabref/gui/externalfiles/ExternalFilesEntryLinker.java b/src/main/java/org/jabref/gui/externalfiles/ExternalFilesEntryLinker.java index b9fb53d1530..34e62fa949b 100644 --- a/src/main/java/org/jabref/gui/externalfiles/ExternalFilesEntryLinker.java +++ b/src/main/java/org/jabref/gui/externalfiles/ExternalFilesEntryLinker.java @@ -11,8 +11,8 @@ import org.jabref.gui.externalfiletype.UnknownExternalFileType; import org.jabref.logic.cleanup.MoveFilesCleanup; import org.jabref.logic.cleanup.RenamePdfCleanup; -import org.jabref.logic.pdf.search.indexing.IndexingTaskManager; -import org.jabref.logic.pdf.search.indexing.LuceneIndexer; +import org.jabref.logic.search.indexing.IndexingTaskManager; +import org.jabref.logic.search.indexing.LuceneIndexer; import org.jabref.logic.util.io.FileUtil; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index 891e9fd75d0..7656c70624c 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -14,7 +14,7 @@ import org.jabref.gui.groups.GroupViewMode; import org.jabref.gui.groups.GroupsPreferences; import org.jabref.gui.util.BindingsHelper; -import org.jabref.logic.pdf.search.retrieval.LuceneSearcher; +import org.jabref.logic.search.retrieval.LuceneSearcher; import org.jabref.logic.search.SearchQuery; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; diff --git a/src/main/java/org/jabref/gui/preview/PreviewPanel.java b/src/main/java/org/jabref/gui/preview/PreviewPanel.java index d79cdf72324..822725cbc9d 100644 --- a/src/main/java/org/jabref/gui/preview/PreviewPanel.java +++ b/src/main/java/org/jabref/gui/preview/PreviewPanel.java @@ -24,7 +24,7 @@ import org.jabref.gui.keyboard.KeyBindingRepository; import org.jabref.gui.theme.ThemeManager; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.pdf.search.indexing.IndexingTaskManager; +import org.jabref.logic.search.indexing.IndexingTaskManager; import org.jabref.logic.preview.PreviewLayout; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; diff --git a/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java b/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java index 3ed71df7ac5..85b3f3a2bc0 100644 --- a/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java +++ b/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java @@ -9,7 +9,7 @@ import org.jabref.gui.actions.SimpleCommand; import org.jabref.gui.util.BackgroundTask; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.pdf.search.indexing.LuceneIndexer; +import org.jabref.logic.search.indexing.LuceneIndexer; import org.jabref.model.database.BibDatabaseContext; import org.jabref.preferences.FilePreferences; import org.jabref.preferences.PreferencesService; diff --git a/src/main/java/org/jabref/logic/pdf/search/indexing/DocumentReader.java b/src/main/java/org/jabref/logic/search/indexing/DocumentReader.java similarity index 99% rename from src/main/java/org/jabref/logic/pdf/search/indexing/DocumentReader.java rename to src/main/java/org/jabref/logic/search/indexing/DocumentReader.java index 89db6837ce9..e3ce895e2b3 100644 --- a/src/main/java/org/jabref/logic/pdf/search/indexing/DocumentReader.java +++ b/src/main/java/org/jabref/logic/search/indexing/DocumentReader.java @@ -1,4 +1,4 @@ -package org.jabref.logic.pdf.search.indexing; +package org.jabref.logic.search.indexing; import java.io.IOException; import java.nio.file.Files; diff --git a/src/main/java/org/jabref/logic/pdf/search/indexing/IndexingTaskManager.java b/src/main/java/org/jabref/logic/search/indexing/IndexingTaskManager.java similarity index 99% rename from src/main/java/org/jabref/logic/pdf/search/indexing/IndexingTaskManager.java rename to src/main/java/org/jabref/logic/search/indexing/IndexingTaskManager.java index c5cb4b6bd5e..405a18fc4ba 100644 --- a/src/main/java/org/jabref/logic/pdf/search/indexing/IndexingTaskManager.java +++ b/src/main/java/org/jabref/logic/search/indexing/IndexingTaskManager.java @@ -1,4 +1,4 @@ -package org.jabref.logic.pdf.search.indexing; +package org.jabref.logic.search.indexing; import java.util.List; import java.util.Set; diff --git a/src/main/java/org/jabref/logic/pdf/search/indexing/LuceneIndexer.java b/src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java similarity index 99% rename from src/main/java/org/jabref/logic/pdf/search/indexing/LuceneIndexer.java rename to src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java index 7fd60bcde63..45fc3ea86f8 100644 --- a/src/main/java/org/jabref/logic/pdf/search/indexing/LuceneIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java @@ -1,4 +1,4 @@ -package org.jabref.logic.pdf.search.indexing; +package org.jabref.logic.search.indexing; import java.io.IOException; import java.nio.file.Files; diff --git a/src/main/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcher.java b/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java similarity index 98% rename from src/main/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcher.java rename to src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java index 57dba449077..d967dbc9a9a 100644 --- a/src/main/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcher.java +++ b/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java @@ -1,4 +1,4 @@ -package org.jabref.logic.pdf.search.retrieval; +package org.jabref.logic.search.retrieval; import java.io.IOException; import java.util.Arrays; diff --git a/src/main/java/org/jabref/model/groups/SearchGroup.java b/src/main/java/org/jabref/model/groups/SearchGroup.java index 43a034b5ded..2ba8c902b68 100644 --- a/src/main/java/org/jabref/model/groups/SearchGroup.java +++ b/src/main/java/org/jabref/model/groups/SearchGroup.java @@ -6,7 +6,7 @@ import java.util.Objects; import java.util.Set; -import org.jabref.logic.pdf.search.retrieval.LuceneSearcher; +import org.jabref.logic.search.retrieval.LuceneSearcher; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.search.GroupSearchQuery; diff --git a/src/test/java/org/jabref/logic/pdf/search/indexing/DocumentReaderTest.java b/src/test/java/org/jabref/logic/pdf/search/indexing/DocumentReaderTest.java index 27cdb271844..6215192d65d 100644 --- a/src/test/java/org/jabref/logic/pdf/search/indexing/DocumentReaderTest.java +++ b/src/test/java/org/jabref/logic/pdf/search/indexing/DocumentReaderTest.java @@ -7,6 +7,7 @@ import java.util.Optional; import java.util.stream.Stream; +import org.jabref.logic.search.indexing.DocumentReader; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.LinkedFile; diff --git a/src/test/java/org/jabref/logic/pdf/search/indexing/LuceneIndexerTest.java b/src/test/java/org/jabref/logic/pdf/search/indexing/LuceneIndexerTest.java index afc0fd31675..15edef099aa 100644 --- a/src/test/java/org/jabref/logic/pdf/search/indexing/LuceneIndexerTest.java +++ b/src/test/java/org/jabref/logic/pdf/search/indexing/LuceneIndexerTest.java @@ -5,6 +5,7 @@ import java.util.Collections; import java.util.Optional; +import org.jabref.logic.search.indexing.LuceneIndexer; import org.jabref.logic.util.StandardFileType; import org.jabref.model.database.BibDatabase; import org.jabref.model.database.BibDatabaseContext; diff --git a/src/test/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcherTest.java b/src/test/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcherTest.java index 932e86f0ba4..20a9700a694 100644 --- a/src/test/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcherTest.java +++ b/src/test/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcherTest.java @@ -4,7 +4,8 @@ import java.nio.file.Path; import java.util.Collections; -import org.jabref.logic.pdf.search.indexing.LuceneIndexer; +import org.jabref.logic.search.indexing.LuceneIndexer; +import org.jabref.logic.search.retrieval.LuceneSearcher; import org.jabref.logic.util.StandardFileType; import org.jabref.model.database.BibDatabase; import org.jabref.model.database.BibDatabaseContext; From af4d7f4090d9c8bd119cbc722566f0e6e8e56def Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Tue, 16 Aug 2022 16:59:06 +0200 Subject: [PATCH 030/256] Add floating mode --- .../java/org/jabref/gui/icon/IconTheme.java | 1 + .../gui/maintable/MainTableDataModel.java | 18 ++++++++++++++++-- .../org/jabref/gui/search/GlobalSearchBar.java | 15 ++++++++++++++- .../jabref/model/search/rules/SearchRules.java | 2 +- .../jabref/preferences/JabRefPreferences.java | 4 ++++ .../jabref/preferences/SearchPreferences.java | 9 ++++++++- src/main/resources/l10n/JabRef_en.properties | 2 ++ 7 files changed, 46 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/jabref/gui/icon/IconTheme.java b/src/main/java/org/jabref/gui/icon/IconTheme.java index 80eb1824a9b..2d0791e2024 100644 --- a/src/main/java/org/jabref/gui/icon/IconTheme.java +++ b/src/main/java/org/jabref/gui/icon/IconTheme.java @@ -282,6 +282,7 @@ public enum JabRefIcons implements JabRefIcon { CASE_SENSITIVE(MaterialDesignA.ALPHABETICAL), REG_EX(MaterialDesignR.REGEX), FULLTEXT(MaterialDesignF.FILE_EYE), + FLOATING_MODE(MaterialDesignF.FILTER), CONSOLE(MaterialDesignC.CONSOLE), FORUM(MaterialDesignF.FORUM), FACEBOOK(MaterialDesignF.FACEBOOK), diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index 7656c70624c..12013ba7859 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -22,6 +22,7 @@ import org.jabref.model.groups.SearchGroup; import org.jabref.model.search.matchers.MatcherSet; import org.jabref.model.search.matchers.MatcherSets; +import org.jabref.model.search.rules.SearchRules; import org.jabref.preferences.PreferencesService; import com.tobiasdiez.easybind.EasyBind; @@ -50,11 +51,12 @@ public MainTableDataModel(BibDatabaseContext context, PreferencesService prefere entriesFiltered = new FilteredList<>(entriesViewModel); entriesFiltered.predicateProperty().bind( EasyBind.combine(stateManager.activeGroupProperty(), + stateManager.activeSearchQueryProperty(), groupsPreferences.groupViewModeProperty(), - (groups, groupViewMode) -> { + (groups, query, groupViewMode) -> { return entry -> { updateSearchGroups(); - return isMatchedByGroup(groups, entry); + return isMatched(groups, query, entry); }; }) ); @@ -86,6 +88,18 @@ private Optional getTableViewModelForEntry(BibEntry entr return entriesSorted.stream().filter(viewModel -> viewModel.getEntry().equals(entry)).findFirst(); } + private boolean isMatched(ObservableList groups, Optional query, BibEntryTableViewModel entry) { + return isMatchedByGroup(groups, entry) && isMatchedBySearch(query, entry); + } + + private boolean isMatchedBySearch(Optional query, BibEntryTableViewModel entry) { + if (query.isPresent() && query.get().getSearchFlags().contains(SearchRules.SearchFlags.FLOATING_SEARCH)) { + return true; + } + return query.map(matcher -> matcher.isMatch(entry.getEntry())) + .orElse(true); + } + private boolean isMatchedByGroup(ObservableList groups, BibEntryTableViewModel entry) { return createGroupMatcher(groups) .map(matcher -> matcher.isMatch(entry.getEntry())) diff --git a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java index 4f477761ebf..3fc3884e414 100644 --- a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java +++ b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java @@ -93,6 +93,7 @@ public class GlobalSearchBar extends HBox { private final ToggleButton fulltextButton; private final Button openGlobalSearchButton; private final ToggleButton keepSearchString; + private final ToggleButton floatingModeButton; // private final Button searchModeButton; private final Tooltip searchFieldTooltip = new Tooltip(); private final Label currentResults = new Label(""); @@ -143,6 +144,7 @@ public GlobalSearchBar(JabRefFrame frame, StateManager stateManager, Preferences fulltextButton = IconTheme.JabRefIcons.FULLTEXT.asToggleButton(); openGlobalSearchButton = IconTheme.JabRefIcons.OPEN_GLOBAL_SEARCH.asButton(); keepSearchString = IconTheme.JabRefIcons.KEEP_SEARCH_STRING.asToggleButton(); + floatingModeButton = IconTheme.JabRefIcons.FLOATING_MODE.asToggleButton(); initSearchModifierButtons(); @@ -150,6 +152,7 @@ public GlobalSearchBar(JabRefFrame frame, StateManager stateManager, Preferences .or(regularExpressionButton.focusedProperty()) .or(fulltextButton.focusedProperty()) .or(keepSearchString.focusedProperty()) + .or(floatingModeButton.focusedProperty()) .or(searchField.textProperty() .isNotEmpty()); @@ -159,8 +162,10 @@ public GlobalSearchBar(JabRefFrame frame, StateManager stateManager, Preferences fulltextButton.visibleProperty().bind(focusedOrActive); keepSearchString.visibleProperty().unbind(); keepSearchString.visibleProperty().bind(focusedOrActive); + floatingModeButton.visibleProperty().unbind(); + floatingModeButton.visibleProperty().bind(focusedOrActive); - StackPane modifierButtons = new StackPane(new HBox(regularExpressionButton, fulltextButton, keepSearchString)); + StackPane modifierButtons = new StackPane(new HBox(regularExpressionButton, fulltextButton, keepSearchString, floatingModeButton)); modifierButtons.setAlignment(Pos.CENTER); searchField.setRight(new HBox(searchField.getRight(), modifierButtons)); searchField.getStyleClass().add("search-field"); @@ -223,6 +228,14 @@ private void initSearchModifierButtons() { performSearch(); }); + floatingModeButton.setSelected(!searchPreferences.isFloatingMode()); + floatingModeButton.setTooltip(new Tooltip(Localization.lang("Floating search"))); + initSearchModifierButton(floatingModeButton); + floatingModeButton.setOnAction(event -> { + searchPreferences.setSearchFlag(SearchRules.SearchFlags.FLOATING_SEARCH, !floatingModeButton.isSelected()); + performSearch(); + }); + openGlobalSearchButton.disableProperty().bindBidirectional(globalSearchActive); openGlobalSearchButton.setTooltip(new Tooltip(Localization.lang("Search across libraries in a new window"))); initSearchModifierButton(openGlobalSearchButton); diff --git a/src/main/java/org/jabref/model/search/rules/SearchRules.java b/src/main/java/org/jabref/model/search/rules/SearchRules.java index 2f4c01392a4..b0b7b7542af 100644 --- a/src/main/java/org/jabref/model/search/rules/SearchRules.java +++ b/src/main/java/org/jabref/model/search/rules/SearchRules.java @@ -44,6 +44,6 @@ static SearchRule getSearchRule(EnumSet searchFlags) { } public enum SearchFlags { - REGULAR_EXPRESSION, FULLTEXT, KEEP_SEARCH_STRING; + REGULAR_EXPRESSION, FULLTEXT, KEEP_SEARCH_STRING, FLOATING_SEARCH; } } diff --git a/src/main/java/org/jabref/preferences/JabRefPreferences.java b/src/main/java/org/jabref/preferences/JabRefPreferences.java index 758e2ef5d7e..0a55855b0b2 100644 --- a/src/main/java/org/jabref/preferences/JabRefPreferences.java +++ b/src/main/java/org/jabref/preferences/JabRefPreferences.java @@ -252,6 +252,7 @@ public class JabRefPreferences implements PreferencesService { public static final String SEARCH_FULLTEXT = "fulltextSearch"; public static final String SEARCH_KEEP_SEARCH_STRING = "keepSearchString"; public static final String SEARCH_KEEP_GLOBAL_WINDOW_ON_TOP = "keepOnTop"; + public static final String SEARCH_FLOATING_MODE = "floatingSearch"; public static final String GENERATE_KEY_ON_IMPORT = "generateKeyOnImport"; public static final String GROBID_ENABLED = "grobidEnabled"; @@ -477,6 +478,7 @@ private JabRefPreferences() { defaults.put(SEARCH_FULLTEXT, Boolean.TRUE); defaults.put(SEARCH_KEEP_SEARCH_STRING, Boolean.FALSE); defaults.put(SEARCH_KEEP_GLOBAL_WINDOW_ON_TOP, Boolean.TRUE); + defaults.put(SEARCH_FLOATING_MODE, Boolean.FALSE); defaults.put(GENERATE_KEY_ON_IMPORT, Boolean.TRUE); defaults.put(GROBID_ENABLED, Boolean.FALSE); @@ -2639,6 +2641,7 @@ public SearchPreferences getSearchPreferences() { getBoolean(SEARCH_REG_EXP), getBoolean(SEARCH_FULLTEXT), getBoolean(SEARCH_KEEP_SEARCH_STRING), + getBoolean(SEARCH_FLOATING_MODE), getBoolean(SEARCH_KEEP_GLOBAL_WINDOW_ON_TOP)); EasyBind.listen(searchPreferences.searchDisplayModeProperty(), (obs, oldValue, newValue) -> put(SEARCH_DISPLAY_MODE, Objects.requireNonNull(searchPreferences.getSearchDisplayMode()).toString())); @@ -2646,6 +2649,7 @@ public SearchPreferences getSearchPreferences() { putBoolean(SEARCH_REG_EXP, searchPreferences.getObservableSearchFlags().contains(SearchRules.SearchFlags.REGULAR_EXPRESSION)); putBoolean(SEARCH_FULLTEXT, searchPreferences.getObservableSearchFlags().contains(SearchRules.SearchFlags.FULLTEXT)); putBoolean(SEARCH_KEEP_SEARCH_STRING, searchPreferences.getObservableSearchFlags().contains(SearchRules.SearchFlags.KEEP_SEARCH_STRING)); + putBoolean(SEARCH_FLOATING_MODE, searchPreferences.getObservableSearchFlags().contains(SearchRules.SearchFlags.FLOATING_SEARCH)); }); EasyBind.listen(searchPreferences.keepWindowOnTopProperty(), (obs, oldValue, newValue) -> putBoolean(SEARCH_KEEP_GLOBAL_WINDOW_ON_TOP, searchPreferences.shouldKeepWindowOnTop())); diff --git a/src/main/java/org/jabref/preferences/SearchPreferences.java b/src/main/java/org/jabref/preferences/SearchPreferences.java index 43fb2f5f1e8..40d8d6f16c4 100644 --- a/src/main/java/org/jabref/preferences/SearchPreferences.java +++ b/src/main/java/org/jabref/preferences/SearchPreferences.java @@ -18,7 +18,7 @@ public class SearchPreferences { private final ObservableSet searchFlags; private final BooleanProperty keepWindowOnTop; - public SearchPreferences(SearchDisplayMode searchDisplayMode, boolean isRegularExpression, boolean isFulltext, boolean isKeepSearchString, boolean keepWindowOnTop) { + public SearchPreferences(SearchDisplayMode searchDisplayMode, boolean isRegularExpression, boolean isFulltext, boolean isKeepSearchString, boolean isFloatingMode, boolean keepWindowOnTop) { this.searchDisplayMode = new SimpleObjectProperty<>(searchDisplayMode); this.keepWindowOnTop = new SimpleBooleanProperty(keepWindowOnTop); @@ -32,6 +32,9 @@ public SearchPreferences(SearchDisplayMode searchDisplayMode, boolean isRegularE if (isKeepSearchString) { searchFlags.add(SearchFlags.KEEP_SEARCH_STRING); } + if (isFloatingMode) { + searchFlags.add(SearchFlags.FLOATING_SEARCH); + } } public SearchPreferences(SearchDisplayMode searchDisplayMode, EnumSet searchFlags, boolean keepWindowOnTop) { @@ -85,6 +88,10 @@ public boolean shouldKeepSearchString() { return searchFlags.contains(SearchFlags.KEEP_SEARCH_STRING); } + public boolean isFloatingMode() { + return searchFlags.contains(SearchFlags.FLOATING_SEARCH); + } + public boolean shouldKeepWindowOnTop() { return keepWindowOnTop.get(); } diff --git a/src/main/resources/l10n/JabRef_en.properties b/src/main/resources/l10n/JabRef_en.properties index 71aec8b1ecb..b15b581e974 100644 --- a/src/main/resources/l10n/JabRef_en.properties +++ b/src/main/resources/l10n/JabRef_en.properties @@ -388,6 +388,8 @@ Fulltext\ search=Fulltext search Fulltext\ for=Fulltext for +Floating\ search=Floating search + Further\ information\ about\ Mr.\ DLib\ for\ JabRef\ users.=Further information about Mr. DLib for JabRef users. General=General From 9fdcfd7883439dc8849c458326ce0205b75b18f7 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Tue, 16 Aug 2022 17:41:43 +0200 Subject: [PATCH 031/256] Make sorting by score optional --- .../java/org/jabref/gui/icon/IconTheme.java | 1 + .../org/jabref/gui/maintable/MainTable.java | 23 +++++++++++++++++-- .../jabref/gui/search/GlobalSearchBar.java | 15 +++++++++++- .../model/search/rules/SearchRules.java | 2 +- .../jabref/preferences/JabRefPreferences.java | 4 ++++ .../jabref/preferences/SearchPreferences.java | 11 +++++++-- src/main/resources/l10n/JabRef_en.properties | 2 ++ 7 files changed, 52 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/jabref/gui/icon/IconTheme.java b/src/main/java/org/jabref/gui/icon/IconTheme.java index 2d0791e2024..6eedbdd70fe 100644 --- a/src/main/java/org/jabref/gui/icon/IconTheme.java +++ b/src/main/java/org/jabref/gui/icon/IconTheme.java @@ -283,6 +283,7 @@ public enum JabRefIcons implements JabRefIcon { REG_EX(MaterialDesignR.REGEX), FULLTEXT(MaterialDesignF.FILE_EYE), FLOATING_MODE(MaterialDesignF.FILTER), + SORT_BY_SCORE(MaterialDesignS.SORT_VARIANT), CONSOLE(MaterialDesignC.CONSOLE), FORUM(MaterialDesignF.FORUM), FACEBOOK(MaterialDesignF.FACEBOOK), diff --git a/src/main/java/org/jabref/gui/maintable/MainTable.java b/src/main/java/org/jabref/gui/maintable/MainTable.java index 7da04ead2e4..6fc306557db 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTable.java +++ b/src/main/java/org/jabref/gui/maintable/MainTable.java @@ -12,6 +12,7 @@ import javax.swing.undo.UndoManager; import javafx.collections.ListChangeListener; +import javafx.collections.SetChangeListener; import javafx.scene.control.SelectionMode; import javafx.scene.control.TableColumn; import javafx.scene.control.TableRow; @@ -50,6 +51,7 @@ import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.database.event.EntriesAddedEvent; import org.jabref.model.entry.BibEntry; +import org.jabref.model.search.rules.SearchRules; import org.jabref.preferences.PreferencesService; import com.google.common.eventbus.Subscribe; @@ -63,6 +65,7 @@ public class MainTable extends TableView { private final LibraryTab libraryTab; private final DialogService dialogService; private final StateManager stateManager; + private final PreferencesService preferencesService; private final BibDatabaseContext database; private final MainTableDataModel model; @@ -86,6 +89,7 @@ public MainTable(MainTableDataModel model, this.libraryTab = libraryTab; this.dialogService = dialogService; this.stateManager = stateManager; + this.preferencesService = preferencesService; this.database = Objects.requireNonNull(database); this.model = model; this.clipBoardManager = clipBoardManager; @@ -160,12 +164,20 @@ public MainTable(MainTableDataModel model, @Override public void onChanged(Change> c) { getSortOrder().removeListener(this); - getSortOrder().removeAll(getColumns().get(0)); - getSortOrder().add(0, getColumns().get(0)); + updateSortOrder(); getSortOrder().addListener(this); } }; + preferencesService.getSearchPreferences().getObservableSearchFlags().addListener(new SetChangeListener() { + @Override + public void onChanged(Change change) { + getSortOrder().removeListener(scoreSortOderPrioritizer); + updateSortOrder(); + getSortOrder().addListener(scoreSortOderPrioritizer); + } + }); + // insert score sort order this.getSortOrder().add(0, this.getColumns().get(0)); mainTablePreferences.getColumnPreferences().getColumnSortOrder().forEach(columnModel -> @@ -208,6 +220,13 @@ public void onChanged(Change> c database.getDatabase().registerListener(this); } + private void updateSortOrder() { + if (preferencesService.getSearchPreferences().isSortByScore()) { + getSortOrder().removeAll(getColumns().get(0)); + getSortOrder().add(0, getColumns().get(0)); + } + } + /** * This is called, if a user starts typing some characters into the keyboard with focus on main table. The {@link MainTable} will scroll to the cell with the same starting column value and typed string * diff --git a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java index 3fc3884e414..646ff598f09 100644 --- a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java +++ b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java @@ -94,6 +94,7 @@ public class GlobalSearchBar extends HBox { private final Button openGlobalSearchButton; private final ToggleButton keepSearchString; private final ToggleButton floatingModeButton; + private final ToggleButton sortByScoreButton; // private final Button searchModeButton; private final Tooltip searchFieldTooltip = new Tooltip(); private final Label currentResults = new Label(""); @@ -145,6 +146,7 @@ public GlobalSearchBar(JabRefFrame frame, StateManager stateManager, Preferences openGlobalSearchButton = IconTheme.JabRefIcons.OPEN_GLOBAL_SEARCH.asButton(); keepSearchString = IconTheme.JabRefIcons.KEEP_SEARCH_STRING.asToggleButton(); floatingModeButton = IconTheme.JabRefIcons.FLOATING_MODE.asToggleButton(); + sortByScoreButton = IconTheme.JabRefIcons.SORT_BY_SCORE.asToggleButton(); initSearchModifierButtons(); @@ -153,6 +155,7 @@ public GlobalSearchBar(JabRefFrame frame, StateManager stateManager, Preferences .or(fulltextButton.focusedProperty()) .or(keepSearchString.focusedProperty()) .or(floatingModeButton.focusedProperty()) + .or(sortByScoreButton.focusedProperty()) .or(searchField.textProperty() .isNotEmpty()); @@ -164,8 +167,10 @@ public GlobalSearchBar(JabRefFrame frame, StateManager stateManager, Preferences keepSearchString.visibleProperty().bind(focusedOrActive); floatingModeButton.visibleProperty().unbind(); floatingModeButton.visibleProperty().bind(focusedOrActive); + sortByScoreButton.visibleProperty().unbind(); + sortByScoreButton.visibleProperty().bind(focusedOrActive); - StackPane modifierButtons = new StackPane(new HBox(regularExpressionButton, fulltextButton, keepSearchString, floatingModeButton)); + StackPane modifierButtons = new StackPane(new HBox(regularExpressionButton, fulltextButton, keepSearchString, floatingModeButton, sortByScoreButton)); modifierButtons.setAlignment(Pos.CENTER); searchField.setRight(new HBox(searchField.getRight(), modifierButtons)); searchField.getStyleClass().add("search-field"); @@ -236,6 +241,14 @@ private void initSearchModifierButtons() { performSearch(); }); + sortByScoreButton.setSelected(searchPreferences.isSortByScore()); + sortByScoreButton.setTooltip(new Tooltip(Localization.lang("Always sort by score"))); + initSearchModifierButton(sortByScoreButton); + sortByScoreButton.setOnAction(event -> { + searchPreferences.setSearchFlag(SearchRules.SearchFlags.SORT_BY_SCORE, sortByScoreButton.isSelected()); + performSearch(); + }); + openGlobalSearchButton.disableProperty().bindBidirectional(globalSearchActive); openGlobalSearchButton.setTooltip(new Tooltip(Localization.lang("Search across libraries in a new window"))); initSearchModifierButton(openGlobalSearchButton); diff --git a/src/main/java/org/jabref/model/search/rules/SearchRules.java b/src/main/java/org/jabref/model/search/rules/SearchRules.java index b0b7b7542af..b386eb09f27 100644 --- a/src/main/java/org/jabref/model/search/rules/SearchRules.java +++ b/src/main/java/org/jabref/model/search/rules/SearchRules.java @@ -44,6 +44,6 @@ static SearchRule getSearchRule(EnumSet searchFlags) { } public enum SearchFlags { - REGULAR_EXPRESSION, FULLTEXT, KEEP_SEARCH_STRING, FLOATING_SEARCH; + REGULAR_EXPRESSION, FULLTEXT, KEEP_SEARCH_STRING, FLOATING_SEARCH, SORT_BY_SCORE; } } diff --git a/src/main/java/org/jabref/preferences/JabRefPreferences.java b/src/main/java/org/jabref/preferences/JabRefPreferences.java index 0a55855b0b2..56f7d8bc195 100644 --- a/src/main/java/org/jabref/preferences/JabRefPreferences.java +++ b/src/main/java/org/jabref/preferences/JabRefPreferences.java @@ -253,6 +253,7 @@ public class JabRefPreferences implements PreferencesService { public static final String SEARCH_KEEP_SEARCH_STRING = "keepSearchString"; public static final String SEARCH_KEEP_GLOBAL_WINDOW_ON_TOP = "keepOnTop"; public static final String SEARCH_FLOATING_MODE = "floatingSearch"; + public static final String SEARCH_SORT_BY_SCORE = "sortByScore"; public static final String GENERATE_KEY_ON_IMPORT = "generateKeyOnImport"; public static final String GROBID_ENABLED = "grobidEnabled"; @@ -478,6 +479,7 @@ private JabRefPreferences() { defaults.put(SEARCH_FULLTEXT, Boolean.TRUE); defaults.put(SEARCH_KEEP_SEARCH_STRING, Boolean.FALSE); defaults.put(SEARCH_KEEP_GLOBAL_WINDOW_ON_TOP, Boolean.TRUE); + defaults.put(SEARCH_SORT_BY_SCORE, Boolean.TRUE); defaults.put(SEARCH_FLOATING_MODE, Boolean.FALSE); defaults.put(GENERATE_KEY_ON_IMPORT, Boolean.TRUE); @@ -2642,6 +2644,7 @@ public SearchPreferences getSearchPreferences() { getBoolean(SEARCH_FULLTEXT), getBoolean(SEARCH_KEEP_SEARCH_STRING), getBoolean(SEARCH_FLOATING_MODE), + getBoolean(SEARCH_SORT_BY_SCORE), getBoolean(SEARCH_KEEP_GLOBAL_WINDOW_ON_TOP)); EasyBind.listen(searchPreferences.searchDisplayModeProperty(), (obs, oldValue, newValue) -> put(SEARCH_DISPLAY_MODE, Objects.requireNonNull(searchPreferences.getSearchDisplayMode()).toString())); @@ -2650,6 +2653,7 @@ public SearchPreferences getSearchPreferences() { putBoolean(SEARCH_FULLTEXT, searchPreferences.getObservableSearchFlags().contains(SearchRules.SearchFlags.FULLTEXT)); putBoolean(SEARCH_KEEP_SEARCH_STRING, searchPreferences.getObservableSearchFlags().contains(SearchRules.SearchFlags.KEEP_SEARCH_STRING)); putBoolean(SEARCH_FLOATING_MODE, searchPreferences.getObservableSearchFlags().contains(SearchRules.SearchFlags.FLOATING_SEARCH)); + putBoolean(SEARCH_SORT_BY_SCORE, searchPreferences.getObservableSearchFlags().contains(SearchRules.SearchFlags.SORT_BY_SCORE)); }); EasyBind.listen(searchPreferences.keepWindowOnTopProperty(), (obs, oldValue, newValue) -> putBoolean(SEARCH_KEEP_GLOBAL_WINDOW_ON_TOP, searchPreferences.shouldKeepWindowOnTop())); diff --git a/src/main/java/org/jabref/preferences/SearchPreferences.java b/src/main/java/org/jabref/preferences/SearchPreferences.java index 40d8d6f16c4..e5d4b0e29f6 100644 --- a/src/main/java/org/jabref/preferences/SearchPreferences.java +++ b/src/main/java/org/jabref/preferences/SearchPreferences.java @@ -18,7 +18,7 @@ public class SearchPreferences { private final ObservableSet searchFlags; private final BooleanProperty keepWindowOnTop; - public SearchPreferences(SearchDisplayMode searchDisplayMode, boolean isRegularExpression, boolean isFulltext, boolean isKeepSearchString, boolean isFloatingMode, boolean keepWindowOnTop) { + public SearchPreferences(SearchDisplayMode searchDisplayMode, boolean isRegularExpression, boolean isFulltext, boolean isKeepSearchString, boolean isFloatingMode, boolean isSortByScore, boolean keepWindowOnTop) { this.searchDisplayMode = new SimpleObjectProperty<>(searchDisplayMode); this.keepWindowOnTop = new SimpleBooleanProperty(keepWindowOnTop); @@ -35,6 +35,9 @@ public SearchPreferences(SearchDisplayMode searchDisplayMode, boolean isRegularE if (isFloatingMode) { searchFlags.add(SearchFlags.FLOATING_SEARCH); } + if (isSortByScore) { + searchFlags.add(SearchFlags.SORT_BY_SCORE); + } } public SearchPreferences(SearchDisplayMode searchDisplayMode, EnumSet searchFlags, boolean keepWindowOnTop) { @@ -52,7 +55,7 @@ public EnumSet getSearchFlags() { return EnumSet.copyOf(searchFlags); } - protected ObservableSet getObservableSearchFlags() { + public ObservableSet getObservableSearchFlags() { return searchFlags; } @@ -92,6 +95,10 @@ public boolean isFloatingMode() { return searchFlags.contains(SearchFlags.FLOATING_SEARCH); } + public boolean isSortByScore() { + return searchFlags.contains(SearchFlags.SORT_BY_SCORE); + } + public boolean shouldKeepWindowOnTop() { return keepWindowOnTop.get(); } diff --git a/src/main/resources/l10n/JabRef_en.properties b/src/main/resources/l10n/JabRef_en.properties index b15b581e974..6194517816f 100644 --- a/src/main/resources/l10n/JabRef_en.properties +++ b/src/main/resources/l10n/JabRef_en.properties @@ -390,6 +390,8 @@ Fulltext\ for=Fulltext for Floating\ search=Floating search +Always\ sort\ by\ score=Always sort by score + Further\ information\ about\ Mr.\ DLib\ for\ JabRef\ users.=Further information about Mr. DLib for JabRef users. General=General From f308f87fa9a15a4f2dc2ce2a190fa582c7ad7823 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Tue, 16 Aug 2022 17:46:12 +0200 Subject: [PATCH 032/256] Only show two decimal places of the search score --- .../java/org/jabref/gui/maintable/MainTableColumnFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/jabref/gui/maintable/MainTableColumnFactory.java b/src/main/java/org/jabref/gui/maintable/MainTableColumnFactory.java index e5331894c36..a0a552032a2 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableColumnFactory.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableColumnFactory.java @@ -141,7 +141,7 @@ private TableColumn createScoreColumn(MainTableC Tooltip.install(header, new Tooltip(MainTableColumnModel.Type.SCORE.getDisplayName())); column.setGraphic(header); column.setStyle("-fx-alignment: CENTER-RIGHT;"); - column.setCellValueFactory(cellData -> cellData.getValue().searchScoreProperty().asString()); + column.setCellValueFactory(cellData -> cellData.getValue().searchScoreProperty().asString("%.2f")); new ValueTableCellFactory() .withText(text -> text) .install(column); From 25341a14a3f5dad569ed17f9267120327f3b0e07 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Tue, 16 Aug 2022 17:47:31 +0200 Subject: [PATCH 033/256] Remove unused icon from theme --- src/main/java/org/jabref/gui/icon/IconTheme.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/jabref/gui/icon/IconTheme.java b/src/main/java/org/jabref/gui/icon/IconTheme.java index 6eedbdd70fe..3ff6fc98714 100644 --- a/src/main/java/org/jabref/gui/icon/IconTheme.java +++ b/src/main/java/org/jabref/gui/icon/IconTheme.java @@ -279,7 +279,6 @@ public enum JabRefIcons implements JabRefIcon { CHECK(MaterialDesignC.CHECK), WARNING(MaterialDesignA.ALERT), ERROR(MaterialDesignA.ALERT_CIRCLE), - CASE_SENSITIVE(MaterialDesignA.ALPHABETICAL), REG_EX(MaterialDesignR.REGEX), FULLTEXT(MaterialDesignF.FILE_EYE), FLOATING_MODE(MaterialDesignF.FILTER), From 87a00c7c4b78d63c9e71da11573f27449428e408 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Tue, 16 Aug 2022 19:43:12 +0200 Subject: [PATCH 034/256] Gray-out non-hits Co-authored-by: Carl Christian Snethlage <50491877+calixtus@users.noreply.github.com> --- .../gui/maintable/BibEntryTableViewModel.java | 2 +- .../org/jabref/gui/maintable/MainTable.css | 6 ++++ .../org/jabref/gui/maintable/MainTable.java | 2 ++ .../gui/util/ViewModelTableRowFactory.java | 28 +++++++++++++++++++ 4 files changed, 37 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jabref/gui/maintable/BibEntryTableViewModel.java b/src/main/java/org/jabref/gui/maintable/BibEntryTableViewModel.java index 29124d5cbe6..53477952fbb 100644 --- a/src/main/java/org/jabref/gui/maintable/BibEntryTableViewModel.java +++ b/src/main/java/org/jabref/gui/maintable/BibEntryTableViewModel.java @@ -50,7 +50,7 @@ public class BibEntryTableViewModel { private final Binding> matchedGroups; private final BibDatabaseContext bibDatabaseContext; - private final FloatProperty searchScore = new SimpleFloatProperty(); + private final FloatProperty searchScore = new SimpleFloatProperty(0); public BibEntryTableViewModel(BibEntry entry, BibDatabaseContext bibDatabaseContext, ObservableValue fieldValueFormatter, StateManager stateManager) { this.entry = entry; diff --git a/src/main/java/org/jabref/gui/maintable/MainTable.css b/src/main/java/org/jabref/gui/maintable/MainTable.css index 51059819604..1b1fb7f66bd 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTable.css +++ b/src/main/java/org/jabref/gui/maintable/MainTable.css @@ -36,6 +36,11 @@ -fx-padding: -2 0 0 0; } +.table-row-cell:entry-not-matching-search { + -fx-background-color: -jr-base; + -fx-opacity: 50%; +} + .rating > .container { -fx-spacing: 2; } @@ -57,3 +62,4 @@ .rating > .container > .button:hover { -fx-effect: dropshadow(three-pass-box, rgba(0, 0, 0, 0.6), 8, 0.0, 0, 0); } + diff --git a/src/main/java/org/jabref/gui/maintable/MainTable.java b/src/main/java/org/jabref/gui/maintable/MainTable.java index 6fc306557db..face9ec6cfb 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTable.java +++ b/src/main/java/org/jabref/gui/maintable/MainTable.java @@ -13,6 +13,7 @@ import javafx.collections.ListChangeListener; import javafx.collections.SetChangeListener; +import javafx.css.PseudoClass; import javafx.scene.control.SelectionMode; import javafx.scene.control.TableColumn; import javafx.scene.control.TableRow; @@ -137,6 +138,7 @@ public MainTable(MainTableDataModel model, Globals.getClipboardManager(), Globals.TASK_EXECUTOR, Globals.entryTypesManager)) + .withPseudoClass(PseudoClass.getPseudoClass("entry-not-matching-search"), entry -> stateManager.activeSearchQueryProperty().isPresent().and(entry.searchScoreProperty().isEqualTo(0))) .setOnDragDetected(this::handleOnDragDetected) .setOnDragDropped(this::handleOnDragDropped) .setOnDragOver(this::handleOnDragOver) diff --git a/src/main/java/org/jabref/gui/util/ViewModelTableRowFactory.java b/src/main/java/org/jabref/gui/util/ViewModelTableRowFactory.java index 87114e3b6a0..bbb25924408 100644 --- a/src/main/java/org/jabref/gui/util/ViewModelTableRowFactory.java +++ b/src/main/java/org/jabref/gui/util/ViewModelTableRowFactory.java @@ -1,8 +1,14 @@ package org.jabref.gui.util; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.function.BiConsumer; import java.util.function.Function; +import javafx.beans.value.ObservableValue; +import javafx.css.PseudoClass; import javafx.geometry.Bounds; import javafx.geometry.Point2D; import javafx.scene.control.ContextMenu; @@ -19,6 +25,7 @@ import org.jabref.model.strings.StringUtil; +import com.tobiasdiez.easybind.Subscription; import org.reactfx.util.TriConsumer; /** @@ -37,6 +44,7 @@ public class ViewModelTableRowFactory implements Callback, Table private TriConsumer, S, ? super DragEvent> toOnDragOver; private TriConsumer, S, ? super MouseDragEvent> toOnMouseDragEntered; private Callback toTooltip; + private final Map>> pseudoClasses = new HashMap<>(); public ViewModelTableRowFactory withOnMouseClickedEvent(BiConsumer onMouseClickedEvent) { this.onMouseClickedEvent = onMouseClickedEvent; @@ -104,6 +112,11 @@ public ViewModelTableRowFactory withTooltip(Callback toTooltip) { return this; } + public ViewModelTableRowFactory withPseudoClass(PseudoClass pseudoClass, Callback> toCondition) { + this.pseudoClasses.putIfAbsent(pseudoClass, toCondition); + return this; + } + @Override public TableRow call(TableView tableView) { TableRow row = new TableRow<>(); @@ -194,6 +207,21 @@ public TableRow call(TableView tableView) { } }); } + + final List subscriptions = new ArrayList<>(); + row.itemProperty().addListener((observable, oldValue, newValue) -> { + subscriptions.forEach(Subscription::unsubscribe); + subscriptions.clear(); + if (row.getItem() != null) { + for (Map.Entry>> pseudoClassWithCondition : pseudoClasses.entrySet()) { + ObservableValue condition = pseudoClassWithCondition.getValue().call(row.getItem()); + subscriptions.add(BindingsHelper.includePseudoClassWhen( + row, + pseudoClassWithCondition.getKey(), + condition)); + } + } + }); return row; } From 4da86d81c3c4ab87d438df95265513acce88c7c3 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Wed, 17 Aug 2022 19:30:39 +0200 Subject: [PATCH 035/256] Better floating mode --- src/main/java/org/jabref/gui/maintable/MainTable.css | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/org/jabref/gui/maintable/MainTable.css b/src/main/java/org/jabref/gui/maintable/MainTable.css index 1b1fb7f66bd..a759c3fce7e 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTable.css +++ b/src/main/java/org/jabref/gui/maintable/MainTable.css @@ -37,8 +37,7 @@ } .table-row-cell:entry-not-matching-search { - -fx-background-color: -jr-base; - -fx-opacity: 50%; + -fx-opacity: 35%; } .rating > .container { From 9e8d5493a7d3d2ed959f3f4c9cdcb8c9a365a813 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Wed, 17 Aug 2022 19:37:38 +0200 Subject: [PATCH 036/256] Fix force-score-ordering bug --- src/main/java/org/jabref/gui/maintable/MainTable.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/org/jabref/gui/maintable/MainTable.java b/src/main/java/org/jabref/gui/maintable/MainTable.java index face9ec6cfb..e67e97cd93b 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTable.java +++ b/src/main/java/org/jabref/gui/maintable/MainTable.java @@ -180,8 +180,6 @@ public void onChanged(Change change) { } }); - // insert score sort order - this.getSortOrder().add(0, this.getColumns().get(0)); mainTablePreferences.getColumnPreferences().getColumnSortOrder().forEach(columnModel -> this.getColumns().stream() .map(column -> (MainTableColumn) column) From a4ccaf0460e5c5ed7145a13afea3bd949a453e07 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Wed, 17 Aug 2022 19:45:06 +0200 Subject: [PATCH 037/256] Remove SearchDisplayMode Everything is in the EnumSet now. --- .../jabref/gui/search/SearchDisplayMode.java | 33 ------------------- .../jabref/preferences/JabRefPreferences.java | 14 +------- .../jabref/preferences/SearchPreferences.java | 22 ++----------- 3 files changed, 3 insertions(+), 66 deletions(-) delete mode 100644 src/main/java/org/jabref/gui/search/SearchDisplayMode.java diff --git a/src/main/java/org/jabref/gui/search/SearchDisplayMode.java b/src/main/java/org/jabref/gui/search/SearchDisplayMode.java deleted file mode 100644 index 18074f57a71..00000000000 --- a/src/main/java/org/jabref/gui/search/SearchDisplayMode.java +++ /dev/null @@ -1,33 +0,0 @@ -package org.jabref.gui.search; - -import java.util.function.Supplier; - -import org.jabref.logic.l10n.Localization; - -/** - * Collects the possible search modes - */ -public enum SearchDisplayMode { - - FLOAT(() -> Localization.lang("Float"), () -> Localization.lang("Gray out non-hits")), - FILTER(() -> Localization.lang("Filter"), () -> Localization.lang("Hide non-hits")); - - private final Supplier displayName; - private final Supplier toolTipText; - - /** - * We have to use supplier for the localized text so that language changes are correctly reflected. - */ - SearchDisplayMode(Supplier displayName, Supplier toolTipText) { - this.displayName = displayName; - this.toolTipText = toolTipText; - } - - public String getDisplayName() { - return displayName.get(); - } - - public String getToolTipText() { - return toolTipText.get(); - } -} diff --git a/src/main/java/org/jabref/preferences/JabRefPreferences.java b/src/main/java/org/jabref/preferences/JabRefPreferences.java index 56f7d8bc195..90a72e19070 100644 --- a/src/main/java/org/jabref/preferences/JabRefPreferences.java +++ b/src/main/java/org/jabref/preferences/JabRefPreferences.java @@ -54,7 +54,6 @@ import org.jabref.gui.maintable.MainTableNameFormatPreferences.DisplayStyle; import org.jabref.gui.maintable.MainTablePreferences; import org.jabref.gui.mergeentries.DiffMode; -import org.jabref.gui.search.SearchDisplayMode; import org.jabref.gui.sidepane.SidePaneType; import org.jabref.gui.specialfields.SpecialFieldsPreferences; import org.jabref.gui.theme.Theme; @@ -471,10 +470,9 @@ private JabRefPreferences() { // Since some of the preference settings themselves use localized strings, we cannot set the language after // the initialization of the preferences in main // Otherwise that language framework will be instantiated and more importantly, statically initialized preferences - // like the SearchDisplayMode will never be translated. + // will never be translated. Localization.setLanguage(getLanguage()); - defaults.put(SEARCH_DISPLAY_MODE, SearchDisplayMode.FILTER.toString()); defaults.put(SEARCH_REG_EXP, Boolean.FALSE); defaults.put(SEARCH_FULLTEXT, Boolean.TRUE); defaults.put(SEARCH_KEEP_SEARCH_STRING, Boolean.FALSE); @@ -2630,16 +2628,7 @@ public SearchPreferences getSearchPreferences() { return searchPreferences; } - SearchDisplayMode searchDisplayMode; - try { - searchDisplayMode = SearchDisplayMode.valueOf(get(SEARCH_DISPLAY_MODE)); - } catch (IllegalArgumentException ex) { - // Should only occur when the searchmode is set directly via preferences.put and the enum was not used - searchDisplayMode = SearchDisplayMode.valueOf((String) defaults.get(SEARCH_DISPLAY_MODE)); - } - searchPreferences = new SearchPreferences( - searchDisplayMode, getBoolean(SEARCH_REG_EXP), getBoolean(SEARCH_FULLTEXT), getBoolean(SEARCH_KEEP_SEARCH_STRING), @@ -2647,7 +2636,6 @@ public SearchPreferences getSearchPreferences() { getBoolean(SEARCH_SORT_BY_SCORE), getBoolean(SEARCH_KEEP_GLOBAL_WINDOW_ON_TOP)); - EasyBind.listen(searchPreferences.searchDisplayModeProperty(), (obs, oldValue, newValue) -> put(SEARCH_DISPLAY_MODE, Objects.requireNonNull(searchPreferences.getSearchDisplayMode()).toString())); searchPreferences.getObservableSearchFlags().addListener((SetChangeListener) c -> { putBoolean(SEARCH_REG_EXP, searchPreferences.getObservableSearchFlags().contains(SearchRules.SearchFlags.REGULAR_EXPRESSION)); putBoolean(SEARCH_FULLTEXT, searchPreferences.getObservableSearchFlags().contains(SearchRules.SearchFlags.FULLTEXT)); diff --git a/src/main/java/org/jabref/preferences/SearchPreferences.java b/src/main/java/org/jabref/preferences/SearchPreferences.java index e5d4b0e29f6..db37674feba 100644 --- a/src/main/java/org/jabref/preferences/SearchPreferences.java +++ b/src/main/java/org/jabref/preferences/SearchPreferences.java @@ -3,23 +3,18 @@ import java.util.EnumSet; import javafx.beans.property.BooleanProperty; -import javafx.beans.property.ObjectProperty; import javafx.beans.property.SimpleBooleanProperty; -import javafx.beans.property.SimpleObjectProperty; import javafx.collections.FXCollections; import javafx.collections.ObservableSet; -import org.jabref.gui.search.SearchDisplayMode; import org.jabref.model.search.rules.SearchRules.SearchFlags; public class SearchPreferences { - private final ObjectProperty searchDisplayMode; private final ObservableSet searchFlags; private final BooleanProperty keepWindowOnTop; - public SearchPreferences(SearchDisplayMode searchDisplayMode, boolean isRegularExpression, boolean isFulltext, boolean isKeepSearchString, boolean isFloatingMode, boolean isSortByScore, boolean keepWindowOnTop) { - this.searchDisplayMode = new SimpleObjectProperty<>(searchDisplayMode); + public SearchPreferences(boolean isRegularExpression, boolean isFulltext, boolean isKeepSearchString, boolean isFloatingMode, boolean isSortByScore, boolean keepWindowOnTop) { this.keepWindowOnTop = new SimpleBooleanProperty(keepWindowOnTop); searchFlags = FXCollections.observableSet(EnumSet.noneOf(SearchFlags.class)); @@ -40,8 +35,7 @@ public SearchPreferences(SearchDisplayMode searchDisplayMode, boolean isRegularE } } - public SearchPreferences(SearchDisplayMode searchDisplayMode, EnumSet searchFlags, boolean keepWindowOnTop) { - this.searchDisplayMode = new SimpleObjectProperty<>(searchDisplayMode); + public SearchPreferences(EnumSet searchFlags, boolean keepWindowOnTop) { this.keepWindowOnTop = new SimpleBooleanProperty(keepWindowOnTop); this.searchFlags = FXCollections.observableSet(searchFlags); @@ -59,18 +53,6 @@ public ObservableSet getObservableSearchFlags() { return searchFlags; } - public SearchDisplayMode getSearchDisplayMode() { - return searchDisplayMode.get(); - } - - public ObjectProperty searchDisplayModeProperty() { - return searchDisplayMode; - } - - public void setSearchDisplayMode(SearchDisplayMode searchDisplayMode) { - this.searchDisplayMode.set(searchDisplayMode); - } - public void setSearchFlag(SearchFlags flag, boolean value) { if (searchFlags.contains(flag) && !value) { searchFlags.remove(flag); From 1d4bef3c7eaa21ffc7b1de721cbae24fe13d96c8 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Sat, 3 Sep 2022 14:41:20 +0200 Subject: [PATCH 038/256] Always pre- and append wildcards for simple queries --- .../org/jabref/logic/search/retrieval/LuceneSearcher.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java b/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java index d967dbc9a9a..5f02a0ecf91 100644 --- a/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java +++ b/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.HashMap; +import java.util.stream.Collectors; import org.jabref.gui.LibraryTab; import org.jabref.logic.search.SearchQuery; @@ -67,7 +68,12 @@ public HashMap search(SearchQuery query) { queryString = "/" + queryString + "/"; } } - Query luceneQuery = new MultiFieldQueryParser(fieldsToSearchArray, new EnglishStemAnalyzer(), boosts).parse(queryString); + MultiFieldQueryParser queryParser = new MultiFieldQueryParser(fieldsToSearchArray, new EnglishStemAnalyzer(), boosts); + queryParser.setAllowLeadingWildcard(true); + if (!queryString.contains("\"") && !queryString.contains(":") && !queryString.contains("*") && !queryString.contains("~")) { + queryString = Arrays.stream(queryString.split(" ")).map(s -> "*" + s + "*").collect(Collectors.joining(" ")); + } + Query luceneQuery = queryParser.parse(queryString); TopDocs docs = searcher.search(luceneQuery, Integer.MAX_VALUE); for (ScoreDoc scoreDoc : docs.scoreDocs) { SearchResult searchResult = new SearchResult(searcher, luceneQuery, scoreDoc); From 44aae6e0616955fdffdd3970098b0e30611368fd Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Sat, 3 Sep 2022 14:50:45 +0200 Subject: [PATCH 039/256] Rename floating search to filtering search --- .../java/org/jabref/gui/icon/IconTheme.java | 2 +- .../gui/maintable/MainTableDataModel.java | 2 +- .../jabref/gui/search/GlobalSearchBar.java | 23 +++++++++---------- .../model/search/rules/SearchRules.java | 2 +- .../jabref/preferences/JabRefPreferences.java | 8 +++---- .../jabref/preferences/SearchPreferences.java | 10 ++++---- src/main/resources/l10n/JabRef_en.properties | 2 +- 7 files changed, 24 insertions(+), 25 deletions(-) diff --git a/src/main/java/org/jabref/gui/icon/IconTheme.java b/src/main/java/org/jabref/gui/icon/IconTheme.java index 3ff6fc98714..9b3449bb011 100644 --- a/src/main/java/org/jabref/gui/icon/IconTheme.java +++ b/src/main/java/org/jabref/gui/icon/IconTheme.java @@ -281,7 +281,7 @@ public enum JabRefIcons implements JabRefIcon { ERROR(MaterialDesignA.ALERT_CIRCLE), REG_EX(MaterialDesignR.REGEX), FULLTEXT(MaterialDesignF.FILE_EYE), - FLOATING_MODE(MaterialDesignF.FILTER), + FILTER(MaterialDesignF.FILTER), SORT_BY_SCORE(MaterialDesignS.SORT_VARIANT), CONSOLE(MaterialDesignC.CONSOLE), FORUM(MaterialDesignF.FORUM), diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index 12013ba7859..fcf945296bc 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -93,7 +93,7 @@ private boolean isMatched(ObservableList groups, Optional query, BibEntryTableViewModel entry) { - if (query.isPresent() && query.get().getSearchFlags().contains(SearchRules.SearchFlags.FLOATING_SEARCH)) { + if (query.isPresent() && query.get().getSearchFlags().contains(SearchRules.SearchFlags.FILTERING_SEARCH)) { return true; } return query.map(matcher -> matcher.isMatch(entry.getEntry())) diff --git a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java index 646ff598f09..f657518d39f 100644 --- a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java +++ b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java @@ -93,9 +93,8 @@ public class GlobalSearchBar extends HBox { private final ToggleButton fulltextButton; private final Button openGlobalSearchButton; private final ToggleButton keepSearchString; - private final ToggleButton floatingModeButton; + private final ToggleButton filterModeButton; private final ToggleButton sortByScoreButton; - // private final Button searchModeButton; private final Tooltip searchFieldTooltip = new Tooltip(); private final Label currentResults = new Label(""); @@ -145,7 +144,7 @@ public GlobalSearchBar(JabRefFrame frame, StateManager stateManager, Preferences fulltextButton = IconTheme.JabRefIcons.FULLTEXT.asToggleButton(); openGlobalSearchButton = IconTheme.JabRefIcons.OPEN_GLOBAL_SEARCH.asButton(); keepSearchString = IconTheme.JabRefIcons.KEEP_SEARCH_STRING.asToggleButton(); - floatingModeButton = IconTheme.JabRefIcons.FLOATING_MODE.asToggleButton(); + filterModeButton = IconTheme.JabRefIcons.FILTER.asToggleButton(); sortByScoreButton = IconTheme.JabRefIcons.SORT_BY_SCORE.asToggleButton(); initSearchModifierButtons(); @@ -154,7 +153,7 @@ public GlobalSearchBar(JabRefFrame frame, StateManager stateManager, Preferences .or(regularExpressionButton.focusedProperty()) .or(fulltextButton.focusedProperty()) .or(keepSearchString.focusedProperty()) - .or(floatingModeButton.focusedProperty()) + .or(filterModeButton.focusedProperty()) .or(sortByScoreButton.focusedProperty()) .or(searchField.textProperty() .isNotEmpty()); @@ -165,12 +164,12 @@ public GlobalSearchBar(JabRefFrame frame, StateManager stateManager, Preferences fulltextButton.visibleProperty().bind(focusedOrActive); keepSearchString.visibleProperty().unbind(); keepSearchString.visibleProperty().bind(focusedOrActive); - floatingModeButton.visibleProperty().unbind(); - floatingModeButton.visibleProperty().bind(focusedOrActive); + filterModeButton.visibleProperty().unbind(); + filterModeButton.visibleProperty().bind(focusedOrActive); sortByScoreButton.visibleProperty().unbind(); sortByScoreButton.visibleProperty().bind(focusedOrActive); - StackPane modifierButtons = new StackPane(new HBox(regularExpressionButton, fulltextButton, keepSearchString, floatingModeButton, sortByScoreButton)); + StackPane modifierButtons = new StackPane(new HBox(regularExpressionButton, fulltextButton, keepSearchString, filterModeButton, sortByScoreButton)); modifierButtons.setAlignment(Pos.CENTER); searchField.setRight(new HBox(searchField.getRight(), modifierButtons)); searchField.getStyleClass().add("search-field"); @@ -233,11 +232,11 @@ private void initSearchModifierButtons() { performSearch(); }); - floatingModeButton.setSelected(!searchPreferences.isFloatingMode()); - floatingModeButton.setTooltip(new Tooltip(Localization.lang("Floating search"))); - initSearchModifierButton(floatingModeButton); - floatingModeButton.setOnAction(event -> { - searchPreferences.setSearchFlag(SearchRules.SearchFlags.FLOATING_SEARCH, !floatingModeButton.isSelected()); + filterModeButton.setSelected(!searchPreferences.isFilteringMode()); + filterModeButton.setTooltip(new Tooltip(Localization.lang("Filter search results"))); + initSearchModifierButton(filterModeButton); + filterModeButton.setOnAction(event -> { + searchPreferences.setSearchFlag(SearchRules.SearchFlags.FILTERING_SEARCH, !filterModeButton.isSelected()); performSearch(); }); diff --git a/src/main/java/org/jabref/model/search/rules/SearchRules.java b/src/main/java/org/jabref/model/search/rules/SearchRules.java index b386eb09f27..48fe4ee022f 100644 --- a/src/main/java/org/jabref/model/search/rules/SearchRules.java +++ b/src/main/java/org/jabref/model/search/rules/SearchRules.java @@ -44,6 +44,6 @@ static SearchRule getSearchRule(EnumSet searchFlags) { } public enum SearchFlags { - REGULAR_EXPRESSION, FULLTEXT, KEEP_SEARCH_STRING, FLOATING_SEARCH, SORT_BY_SCORE; + REGULAR_EXPRESSION, FULLTEXT, KEEP_SEARCH_STRING, FILTERING_SEARCH, SORT_BY_SCORE; } } diff --git a/src/main/java/org/jabref/preferences/JabRefPreferences.java b/src/main/java/org/jabref/preferences/JabRefPreferences.java index 90a72e19070..2333b5e22d2 100644 --- a/src/main/java/org/jabref/preferences/JabRefPreferences.java +++ b/src/main/java/org/jabref/preferences/JabRefPreferences.java @@ -251,7 +251,7 @@ public class JabRefPreferences implements PreferencesService { public static final String SEARCH_FULLTEXT = "fulltextSearch"; public static final String SEARCH_KEEP_SEARCH_STRING = "keepSearchString"; public static final String SEARCH_KEEP_GLOBAL_WINDOW_ON_TOP = "keepOnTop"; - public static final String SEARCH_FLOATING_MODE = "floatingSearch"; + public static final String SEARCH_FILTERING_MODE = "filteringSearch"; public static final String SEARCH_SORT_BY_SCORE = "sortByScore"; public static final String GENERATE_KEY_ON_IMPORT = "generateKeyOnImport"; @@ -478,7 +478,7 @@ private JabRefPreferences() { defaults.put(SEARCH_KEEP_SEARCH_STRING, Boolean.FALSE); defaults.put(SEARCH_KEEP_GLOBAL_WINDOW_ON_TOP, Boolean.TRUE); defaults.put(SEARCH_SORT_BY_SCORE, Boolean.TRUE); - defaults.put(SEARCH_FLOATING_MODE, Boolean.FALSE); + defaults.put(SEARCH_FILTERING_MODE, Boolean.FALSE); defaults.put(GENERATE_KEY_ON_IMPORT, Boolean.TRUE); defaults.put(GROBID_ENABLED, Boolean.FALSE); @@ -2632,7 +2632,7 @@ public SearchPreferences getSearchPreferences() { getBoolean(SEARCH_REG_EXP), getBoolean(SEARCH_FULLTEXT), getBoolean(SEARCH_KEEP_SEARCH_STRING), - getBoolean(SEARCH_FLOATING_MODE), + getBoolean(SEARCH_FILTERING_MODE), getBoolean(SEARCH_SORT_BY_SCORE), getBoolean(SEARCH_KEEP_GLOBAL_WINDOW_ON_TOP)); @@ -2640,7 +2640,7 @@ public SearchPreferences getSearchPreferences() { putBoolean(SEARCH_REG_EXP, searchPreferences.getObservableSearchFlags().contains(SearchRules.SearchFlags.REGULAR_EXPRESSION)); putBoolean(SEARCH_FULLTEXT, searchPreferences.getObservableSearchFlags().contains(SearchRules.SearchFlags.FULLTEXT)); putBoolean(SEARCH_KEEP_SEARCH_STRING, searchPreferences.getObservableSearchFlags().contains(SearchRules.SearchFlags.KEEP_SEARCH_STRING)); - putBoolean(SEARCH_FLOATING_MODE, searchPreferences.getObservableSearchFlags().contains(SearchRules.SearchFlags.FLOATING_SEARCH)); + putBoolean(SEARCH_FILTERING_MODE, searchPreferences.getObservableSearchFlags().contains(SearchRules.SearchFlags.FILTERING_SEARCH)); putBoolean(SEARCH_SORT_BY_SCORE, searchPreferences.getObservableSearchFlags().contains(SearchRules.SearchFlags.SORT_BY_SCORE)); }); EasyBind.listen(searchPreferences.keepWindowOnTopProperty(), (obs, oldValue, newValue) -> putBoolean(SEARCH_KEEP_GLOBAL_WINDOW_ON_TOP, searchPreferences.shouldKeepWindowOnTop())); diff --git a/src/main/java/org/jabref/preferences/SearchPreferences.java b/src/main/java/org/jabref/preferences/SearchPreferences.java index db37674feba..fe6c2deed4b 100644 --- a/src/main/java/org/jabref/preferences/SearchPreferences.java +++ b/src/main/java/org/jabref/preferences/SearchPreferences.java @@ -14,7 +14,7 @@ public class SearchPreferences { private final ObservableSet searchFlags; private final BooleanProperty keepWindowOnTop; - public SearchPreferences(boolean isRegularExpression, boolean isFulltext, boolean isKeepSearchString, boolean isFloatingMode, boolean isSortByScore, boolean keepWindowOnTop) { + public SearchPreferences(boolean isRegularExpression, boolean isFulltext, boolean isKeepSearchString, boolean isFilteringMode, boolean isSortByScore, boolean keepWindowOnTop) { this.keepWindowOnTop = new SimpleBooleanProperty(keepWindowOnTop); searchFlags = FXCollections.observableSet(EnumSet.noneOf(SearchFlags.class)); @@ -27,8 +27,8 @@ public SearchPreferences(boolean isRegularExpression, boolean isFulltext, boolea if (isKeepSearchString) { searchFlags.add(SearchFlags.KEEP_SEARCH_STRING); } - if (isFloatingMode) { - searchFlags.add(SearchFlags.FLOATING_SEARCH); + if (isFilteringMode) { + searchFlags.add(SearchFlags.FILTERING_SEARCH); } if (isSortByScore) { searchFlags.add(SearchFlags.SORT_BY_SCORE); @@ -73,8 +73,8 @@ public boolean shouldKeepSearchString() { return searchFlags.contains(SearchFlags.KEEP_SEARCH_STRING); } - public boolean isFloatingMode() { - return searchFlags.contains(SearchFlags.FLOATING_SEARCH); + public boolean isFilteringMode() { + return searchFlags.contains(SearchFlags.FILTERING_SEARCH); } public boolean isSortByScore() { diff --git a/src/main/resources/l10n/JabRef_en.properties b/src/main/resources/l10n/JabRef_en.properties index 6194517816f..f9e9388d37b 100644 --- a/src/main/resources/l10n/JabRef_en.properties +++ b/src/main/resources/l10n/JabRef_en.properties @@ -388,7 +388,7 @@ Fulltext\ search=Fulltext search Fulltext\ for=Fulltext for -Floating\ search=Floating search +Filter\ search\ results=Filter search results Always\ sort\ by\ score=Always sort by score From 79e1b9dcad6b2abb9d81124db2db406fdfc76888 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Sat, 3 Sep 2022 15:01:34 +0200 Subject: [PATCH 040/256] Index groups individually --- .../org/jabref/logic/search/indexing/LuceneIndexer.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java b/src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java index 45fc3ea86f8..be20dc9bd4e 100644 --- a/src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java @@ -4,6 +4,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; +import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -169,6 +170,11 @@ public void addBibFieldsToIndex(BibEntry bibEntry) { for (Keyword keyword : keywords) { document.add(new StringField(field.getKey().getName(), keyword.toString(), org.apache.lucene.document.Field.Store.YES)); } + } else if (field.getKey() == StandardField.GROUPS) { + List groups = Arrays.stream(field.getValue().split(preferences.getKeywordDelimiter().toString())).map(String::trim).toList(); + for (String group : groups) { + document.add(new StringField(field.getKey().getName(), group, org.apache.lucene.document.Field.Store.YES)); + } } else { document.add(new TextField(field.getKey().getName(), field.getValue(), org.apache.lucene.document.Field.Store.YES)); SearchFieldConstants.searchableBibFields.add(field.getKey().getName()); From 61973a3f4ac464266f0ac012583f98272677ce49 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Sat, 3 Sep 2022 15:14:04 +0200 Subject: [PATCH 041/256] Localize score column header --- .../java/org/jabref/gui/maintable/MainTableColumnFactory.java | 2 +- src/main/resources/l10n/JabRef_en.properties | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jabref/gui/maintable/MainTableColumnFactory.java b/src/main/java/org/jabref/gui/maintable/MainTableColumnFactory.java index a0a552032a2..402a34e3ec5 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableColumnFactory.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableColumnFactory.java @@ -136,7 +136,7 @@ public static void setExactWidth(TableColumn column, double width) { */ private TableColumn createScoreColumn(MainTableColumnModel columnModel) { TableColumn column = new MainTableColumn<>(columnModel); - Node header = new Text("Score"); + Node header = new Text(Localization.lang("Score")); header.getStyleClass().add("mainTable-header"); Tooltip.install(header, new Tooltip(MainTableColumnModel.Type.SCORE.getDisplayName())); column.setGraphic(header); diff --git a/src/main/resources/l10n/JabRef_en.properties b/src/main/resources/l10n/JabRef_en.properties index f9e9388d37b..1bfc64849df 100644 --- a/src/main/resources/l10n/JabRef_en.properties +++ b/src/main/resources/l10n/JabRef_en.properties @@ -378,6 +378,7 @@ Format\:\ Tab\:field;field;...\ (e.g.\ General\:url;pdf;note...)=Format\: Tab\:f Format\ of\ author\ and\ editor\ names=Format of author and editor names Format\ string=Format string +Score=Score Format\ used=Format used Formatter\ name=Formatter name From 356d942645627e07d6dda2eafe00972fdd2a9fbb Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Sat, 3 Sep 2022 15:15:30 +0200 Subject: [PATCH 042/256] Remove score from sort-order when disabling force sort by score --- src/main/java/org/jabref/gui/maintable/MainTable.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/jabref/gui/maintable/MainTable.java b/src/main/java/org/jabref/gui/maintable/MainTable.java index e67e97cd93b..638eeb9ac5f 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTable.java +++ b/src/main/java/org/jabref/gui/maintable/MainTable.java @@ -221,8 +221,8 @@ public void onChanged(Change change) { } private void updateSortOrder() { + getSortOrder().removeAll(getColumns().get(0)); if (preferencesService.getSearchPreferences().isSortByScore()) { - getSortOrder().removeAll(getColumns().get(0)); getSortOrder().add(0, getColumns().get(0)); } } From 41d93fb76b0856449bb2992e9daeb1755da5a139 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Sat, 3 Sep 2022 17:18:02 +0200 Subject: [PATCH 043/256] Fix search filter --- .../java/org/jabref/gui/maintable/MainTableDataModel.java | 7 +++---- src/main/java/org/jabref/gui/search/GlobalSearchBar.java | 4 ++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index fcf945296bc..5af50b3b8b0 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -54,13 +54,13 @@ public MainTableDataModel(BibDatabaseContext context, PreferencesService prefere stateManager.activeSearchQueryProperty(), groupsPreferences.groupViewModeProperty(), (groups, query, groupViewMode) -> { + doSearch(query); return entry -> { updateSearchGroups(); return isMatched(groups, query, entry); }; }) ); - stateManager.activeSearchQueryProperty().addListener((observable, oldValue, newValue) -> doSearch(newValue)); // We need to wrap the list since otherwise sorting in the table does not work entriesSorted = new SortedList<>(entriesFiltered); @@ -93,11 +93,10 @@ private boolean isMatched(ObservableList groups, Optional query, BibEntryTableViewModel entry) { - if (query.isPresent() && query.get().getSearchFlags().contains(SearchRules.SearchFlags.FILTERING_SEARCH)) { + if (!query.isPresent() || !query.get().getSearchFlags().contains(SearchRules.SearchFlags.FILTERING_SEARCH)) { return true; } - return query.map(matcher -> matcher.isMatch(entry.getEntry())) - .orElse(true); + return stateManager.getSearchResults().containsKey(entry.getEntry()); } private boolean isMatchedByGroup(ObservableList groups, BibEntryTableViewModel entry) { diff --git a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java index f657518d39f..0664397b6d0 100644 --- a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java +++ b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java @@ -232,11 +232,11 @@ private void initSearchModifierButtons() { performSearch(); }); - filterModeButton.setSelected(!searchPreferences.isFilteringMode()); + filterModeButton.setSelected(searchPreferences.isFilteringMode()); filterModeButton.setTooltip(new Tooltip(Localization.lang("Filter search results"))); initSearchModifierButton(filterModeButton); filterModeButton.setOnAction(event -> { - searchPreferences.setSearchFlag(SearchRules.SearchFlags.FILTERING_SEARCH, !filterModeButton.isSelected()); + searchPreferences.setSearchFlag(SearchRules.SearchFlags.FILTERING_SEARCH, filterModeButton.isSelected()); performSearch(); }); From c19b53043562d6f0a23969544b171c61c81fa4d0 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Sat, 3 Sep 2022 19:05:37 +0200 Subject: [PATCH 044/256] Make groupviewmode an enumset To prepare to support 'floating' mode --- .../jabref/gui/groups/GroupModeViewModel.java | 19 +++++------ .../org/jabref/gui/groups/GroupViewMode.java | 2 +- .../jabref/gui/groups/GroupsPreferences.java | 32 +++++++++++++------ .../gui/maintable/MainTableDataModel.java | 2 +- .../groups/GroupsTabViewModel.java | 17 ++++------ .../gui/sidepane/GroupsSidePaneComponent.java | 19 +++++------ .../jabref/preferences/JabRefPreferences.java | 13 +++++--- .../gui/groups/GroupNodeViewModelTest.java | 2 +- .../gui/groups/GroupTreeViewModelTest.java | 2 +- 9 files changed, 61 insertions(+), 47 deletions(-) diff --git a/src/main/java/org/jabref/gui/groups/GroupModeViewModel.java b/src/main/java/org/jabref/gui/groups/GroupModeViewModel.java index 022941a97d5..b623055c1e2 100644 --- a/src/main/java/org/jabref/gui/groups/GroupModeViewModel.java +++ b/src/main/java/org/jabref/gui/groups/GroupModeViewModel.java @@ -1,5 +1,6 @@ package org.jabref.gui.groups; +import javafx.collections.ObservableSet; import javafx.scene.Node; import javafx.scene.control.Tooltip; @@ -8,29 +9,25 @@ public class GroupModeViewModel { - private final GroupViewMode mode; + private final ObservableSet mode; - public GroupModeViewModel(GroupViewMode mode) { + public GroupModeViewModel(ObservableSet mode) { this.mode = mode; } public Node getUnionIntersectionGraphic() { - if (mode == GroupViewMode.UNION) { - return JabRefIcons.GROUP_UNION.getGraphicNode(); - } else if (mode == GroupViewMode.INTERSECTION) { + if (mode.contains(GroupViewMode.INTERSECTION)) { return JabRefIcons.GROUP_INTERSECTION.getGraphicNode(); + } else { + return JabRefIcons.GROUP_UNION.getGraphicNode(); } - - // As there is no concept like an empty node/icon, we return simply the other icon - return JabRefIcons.GROUP_INTERSECTION.getGraphicNode(); } public Tooltip getUnionIntersectionTooltip() { - if (mode == GroupViewMode.UNION) { + if (mode.contains(GroupViewMode.INTERSECTION)) { return new Tooltip(Localization.lang("Toggle intersection")); - } else if (mode == GroupViewMode.INTERSECTION) { + } else { return new Tooltip(Localization.lang("Toggle union")); } - return new Tooltip(); } } diff --git a/src/main/java/org/jabref/gui/groups/GroupViewMode.java b/src/main/java/org/jabref/gui/groups/GroupViewMode.java index 564c85496e6..66f06b2646b 100644 --- a/src/main/java/org/jabref/gui/groups/GroupViewMode.java +++ b/src/main/java/org/jabref/gui/groups/GroupViewMode.java @@ -1,4 +1,4 @@ package org.jabref.gui.groups; -public enum GroupViewMode { INTERSECTION, UNION } +public enum GroupViewMode { INTERSECTION } diff --git a/src/main/java/org/jabref/gui/groups/GroupsPreferences.java b/src/main/java/org/jabref/gui/groups/GroupsPreferences.java index a6bf6f67641..93e8a651fa8 100644 --- a/src/main/java/org/jabref/gui/groups/GroupsPreferences.java +++ b/src/main/java/org/jabref/gui/groups/GroupsPreferences.java @@ -1,38 +1,52 @@ package org.jabref.gui.groups; +import java.util.EnumSet; + import javafx.beans.property.BooleanProperty; import javafx.beans.property.ObjectProperty; +import javafx.beans.property.SetProperty; import javafx.beans.property.SimpleBooleanProperty; -import javafx.beans.property.SimpleObjectProperty; +import javafx.beans.property.SimpleSetProperty; +import javafx.collections.FXCollections; public class GroupsPreferences { - private final ObjectProperty groupViewMode; + private final SetProperty groupViewMode; private final BooleanProperty shouldAutoAssignGroup; private final BooleanProperty shouldDisplayGroupCount; private final ObjectProperty keywordSeparator; - public GroupsPreferences(GroupViewMode groupViewMode, + public GroupsPreferences(boolean viewModeIntersection, boolean shouldAutoAssignGroup, boolean shouldDisplayGroupCount, ObjectProperty keywordSeparator) { - this.groupViewMode = new SimpleObjectProperty<>(groupViewMode); + this.groupViewMode = new SimpleSetProperty<>(FXCollections.observableSet()); + if (viewModeIntersection) { + this.groupViewMode.add(GroupViewMode.INTERSECTION); + } this.shouldAutoAssignGroup = new SimpleBooleanProperty(shouldAutoAssignGroup); this.shouldDisplayGroupCount = new SimpleBooleanProperty(shouldDisplayGroupCount); this.keywordSeparator = keywordSeparator; } - public GroupViewMode getGroupViewMode() { - return groupViewMode.getValue(); + public EnumSet getGroupViewMode() { + if (groupViewMode.isEmpty()) { + return EnumSet.noneOf(GroupViewMode.class); + } + return EnumSet.copyOf(groupViewMode); } - public ObjectProperty groupViewModeProperty() { + public SetProperty groupViewModeProperty() { return groupViewMode; } - public void setGroupViewMode(GroupViewMode groupViewMode) { - this.groupViewMode.set(groupViewMode); + public void setGroupViewMode(GroupViewMode mode, boolean value) { + if (groupViewMode.contains(mode) && !value) { + groupViewMode.remove(mode); + } else if (!groupViewMode.contains(mode) && value) { + groupViewMode.add(mode); + } } public boolean shouldAutoAssignGroup() { diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index 5af50b3b8b0..aac88c5bed3 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -112,7 +112,7 @@ private Optional createGroupMatcher(List selectedGrou } final MatcherSet searchRules = MatcherSets.build( - groupsPreferences.getGroupViewMode() == GroupViewMode.INTERSECTION + groupsPreferences.getGroupViewMode().contains(GroupViewMode.INTERSECTION) ? MatcherSets.MatcherType.AND : MatcherSets.MatcherType.OR); diff --git a/src/main/java/org/jabref/gui/preferences/groups/GroupsTabViewModel.java b/src/main/java/org/jabref/gui/preferences/groups/GroupsTabViewModel.java index b3a0a8a4a8b..9b9fab7ce06 100644 --- a/src/main/java/org/jabref/gui/preferences/groups/GroupsTabViewModel.java +++ b/src/main/java/org/jabref/gui/preferences/groups/GroupsTabViewModel.java @@ -28,15 +28,12 @@ public GroupsTabViewModel(DialogService dialogService, GroupsPreferences groupsP @Override public void setValues() { - switch (groupsPreferences.getGroupViewMode()) { - case INTERSECTION -> { - groupViewModeIntersectionProperty.setValue(true); - groupViewModeUnionProperty.setValue(false); - } - case UNION -> { - groupViewModeIntersectionProperty.setValue(false); - groupViewModeUnionProperty.setValue(true); - } + if (groupsPreferences.getGroupViewMode().contains(GroupViewMode.INTERSECTION)) { + groupViewModeIntersectionProperty.setValue(true); + groupViewModeUnionProperty.setValue(false); + } else { + groupViewModeIntersectionProperty.setValue(false); + groupViewModeUnionProperty.setValue(true); } autoAssignGroupProperty.setValue(groupsPreferences.shouldAutoAssignGroup()); displayGroupCountProperty.setValue(groupsPreferences.shouldDisplayGroupCount()); @@ -45,7 +42,7 @@ public void setValues() { @Override public void storeSettings() { - groupsPreferences.setGroupViewMode(groupViewModeIntersectionProperty.getValue() ? GroupViewMode.INTERSECTION : GroupViewMode.UNION); + groupsPreferences.setGroupViewMode(GroupViewMode.INTERSECTION, groupViewModeIntersectionProperty.getValue()); groupsPreferences.setAutoAssignGroup(autoAssignGroupProperty.getValue()); groupsPreferences.setDisplayGroupCount(displayGroupCountProperty.getValue()); groupsPreferences.keywordSeparatorProperty().setValue(keywordSeparatorProperty.getValue().charAt(0)); diff --git a/src/main/java/org/jabref/gui/sidepane/GroupsSidePaneComponent.java b/src/main/java/org/jabref/gui/sidepane/GroupsSidePaneComponent.java index 1ead8207f46..b65bbe6279f 100644 --- a/src/main/java/org/jabref/gui/sidepane/GroupsSidePaneComponent.java +++ b/src/main/java/org/jabref/gui/sidepane/GroupsSidePaneComponent.java @@ -1,5 +1,8 @@ package org.jabref.gui.sidepane; +import java.util.EnumSet; + +import javafx.collections.SetChangeListener; import javafx.scene.control.Button; import org.jabref.gui.DialogService; @@ -10,8 +13,6 @@ import org.jabref.gui.icon.IconTheme; import org.jabref.logic.l10n.Localization; -import com.tobiasdiez.easybind.EasyBind; - public class GroupsSidePaneComponent extends SidePaneComponent { private final GroupsPreferences groupsPreferences; private final DialogService dialogService; @@ -28,8 +29,8 @@ public GroupsSidePaneComponent(SimpleCommand closeCommand, this.dialogService = dialogService; setupIntersectionUnionToggle(); - EasyBind.subscribe(groupsPreferences.groupViewModeProperty(), mode -> { - GroupModeViewModel modeViewModel = new GroupModeViewModel(mode); + groupsPreferences.groupViewModeProperty().addListener((SetChangeListener) change -> { + GroupModeViewModel modeViewModel = new GroupModeViewModel(groupsPreferences.groupViewModeProperty()); intersectionUnionToggle.setGraphic(modeViewModel.getUnionIntersectionGraphic()); intersectionUnionToggle.setTooltip(modeViewModel.getUnionIntersectionTooltip()); }); @@ -44,13 +45,13 @@ private class ToggleUnionIntersectionAction extends SimpleCommand { @Override public void execute() { - GroupViewMode mode = groupsPreferences.getGroupViewMode(); + EnumSet mode = groupsPreferences.getGroupViewMode(); - if (mode == GroupViewMode.UNION) { - groupsPreferences.setGroupViewMode(GroupViewMode.INTERSECTION); + if (mode.contains(GroupViewMode.INTERSECTION)) { + groupsPreferences.setGroupViewMode(GroupViewMode.INTERSECTION, false); dialogService.notify(Localization.lang("Group view mode set to intersection")); - } else if (mode == GroupViewMode.INTERSECTION) { - groupsPreferences.setGroupViewMode(GroupViewMode.UNION); + } else { + groupsPreferences.setGroupViewMode(GroupViewMode.INTERSECTION, true); dialogService.notify(Localization.lang("Group view mode set to union")); } } diff --git a/src/main/java/org/jabref/preferences/JabRefPreferences.java b/src/main/java/org/jabref/preferences/JabRefPreferences.java index 2333b5e22d2..69a15ee98e9 100644 --- a/src/main/java/org/jabref/preferences/JabRefPreferences.java +++ b/src/main/java/org/jabref/preferences/JabRefPreferences.java @@ -374,7 +374,7 @@ public class JabRefPreferences implements PreferencesService { private static final String PROTECTED_TERMS_DISABLED_INTERNAL = "protectedTermsDisabledInternal"; // GroupViewMode - private static final String GROUP_INTERSECT_UNION_VIEW_MODE = "groupIntersectUnionViewModes"; + private static final String GROUP_VIEW_INTERSECTION = "groupIntersection"; // Dialog states private static final String PREFS_EXPORT_PATH = "prefsExportPath"; @@ -598,7 +598,7 @@ private JabRefPreferences() { defaults.put(AUTOCOMPLETER_COMPLETE_FIELDS, "author;editor;title;journal;publisher;keywords;crossref;related;entryset"); defaults.put(AUTO_ASSIGN_GROUP, Boolean.TRUE); defaults.put(DISPLAY_GROUP_COUNT, Boolean.TRUE); - defaults.put(GROUP_INTERSECT_UNION_VIEW_MODE, GroupViewMode.INTERSECTION.name()); + defaults.put(GROUP_VIEW_INTERSECTION, Boolean.TRUE); defaults.put(KEYWORD_SEPARATOR, ", "); defaults.put(DEFAULT_ENCODING, StandardCharsets.UTF_8.name()); defaults.put(DEFAULT_OWNER, System.getProperty("user.name")); @@ -1419,13 +1419,18 @@ public GroupsPreferences getGroupsPreferences() { } groupsPreferences = new GroupsPreferences( - GroupViewMode.valueOf(get(GROUP_INTERSECT_UNION_VIEW_MODE)), + getBoolean(GROUP_VIEW_INTERSECTION), getBoolean(AUTO_ASSIGN_GROUP), getBoolean(DISPLAY_GROUP_COUNT), getInternalPreferences().keywordSeparatorProperty() ); - EasyBind.listen(groupsPreferences.groupViewModeProperty(), (obs, oldValue, newValue) -> put(GROUP_INTERSECT_UNION_VIEW_MODE, newValue.name())); + groupsPreferences.groupViewModeProperty().addListener(new SetChangeListener() { + @Override + public void onChanged(Change change) { + putBoolean(GROUP_VIEW_INTERSECTION, groupsPreferences.groupViewModeProperty().contains(GroupViewMode.INTERSECTION)); + } + }); EasyBind.listen(groupsPreferences.autoAssignGroupProperty(), (obs, oldValue, newValue) -> putBoolean(AUTO_ASSIGN_GROUP, newValue)); EasyBind.listen(groupsPreferences.displayGroupCountProperty(), (obs, oldValue, newValue) -> putBoolean(DISPLAY_GROUP_COUNT, newValue)); // KeywordSeparator is handled by JabRefPreferences::getInternalPreferences diff --git a/src/test/java/org/jabref/gui/groups/GroupNodeViewModelTest.java b/src/test/java/org/jabref/gui/groups/GroupNodeViewModelTest.java index 973868ff0e8..4257ff4790f 100644 --- a/src/test/java/org/jabref/gui/groups/GroupNodeViewModelTest.java +++ b/src/test/java/org/jabref/gui/groups/GroupNodeViewModelTest.java @@ -46,7 +46,7 @@ void setUp() { taskExecutor = new CurrentThreadTaskExecutor(); preferencesService = mock(PreferencesService.class); when(preferencesService.getGroupsPreferences()).thenReturn(new GroupsPreferences( - GroupViewMode.UNION, + false, true, true, new SimpleObjectProperty<>(',') diff --git a/src/test/java/org/jabref/gui/groups/GroupTreeViewModelTest.java b/src/test/java/org/jabref/gui/groups/GroupTreeViewModelTest.java index c96d5262396..f4277506724 100644 --- a/src/test/java/org/jabref/gui/groups/GroupTreeViewModelTest.java +++ b/src/test/java/org/jabref/gui/groups/GroupTreeViewModelTest.java @@ -40,7 +40,7 @@ void setUp() { taskExecutor = new CurrentThreadTaskExecutor(); preferencesService = mock(PreferencesService.class); when(preferencesService.getGroupsPreferences()).thenReturn(new GroupsPreferences( - GroupViewMode.UNION, + false, true, true, new SimpleObjectProperty<>(',') From 79a753b44dc5435fa80126c12b07aec559ecb62e Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Sat, 3 Sep 2022 19:07:03 +0200 Subject: [PATCH 045/256] Checkstyle --- src/main/java/org/jabref/gui/LibraryTab.java | 2 +- src/main/java/org/jabref/gui/maintable/MainTableDataModel.java | 2 +- src/main/java/org/jabref/gui/preview/PreviewPanel.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/jabref/gui/LibraryTab.java b/src/main/java/org/jabref/gui/LibraryTab.java index 71eb0071dc2..ece4d7d4c40 100644 --- a/src/main/java/org/jabref/gui/LibraryTab.java +++ b/src/main/java/org/jabref/gui/LibraryTab.java @@ -48,9 +48,9 @@ import org.jabref.logic.importer.util.FileFieldParser; import org.jabref.logic.l10n.Localization; import org.jabref.logic.pdf.FileAnnotationCache; +import org.jabref.logic.search.SearchQuery; import org.jabref.logic.search.indexing.IndexingTaskManager; import org.jabref.logic.search.indexing.LuceneIndexer; -import org.jabref.logic.search.SearchQuery; import org.jabref.logic.shared.DatabaseLocation; import org.jabref.logic.util.UpdateField; import org.jabref.logic.util.io.FileUtil; diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index aac88c5bed3..f9b7ad98ecf 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -14,8 +14,8 @@ import org.jabref.gui.groups.GroupViewMode; import org.jabref.gui.groups.GroupsPreferences; import org.jabref.gui.util.BindingsHelper; -import org.jabref.logic.search.retrieval.LuceneSearcher; import org.jabref.logic.search.SearchQuery; +import org.jabref.logic.search.retrieval.LuceneSearcher; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.groups.GroupTreeNode; diff --git a/src/main/java/org/jabref/gui/preview/PreviewPanel.java b/src/main/java/org/jabref/gui/preview/PreviewPanel.java index 822725cbc9d..b04ba7f0735 100644 --- a/src/main/java/org/jabref/gui/preview/PreviewPanel.java +++ b/src/main/java/org/jabref/gui/preview/PreviewPanel.java @@ -24,8 +24,8 @@ import org.jabref.gui.keyboard.KeyBindingRepository; import org.jabref.gui.theme.ThemeManager; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.search.indexing.IndexingTaskManager; import org.jabref.logic.preview.PreviewLayout; +import org.jabref.logic.search.indexing.IndexingTaskManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.preferences.PreferencesService; From c63a261bad19fadc50add8d3ea985442ddfc7416 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Sun, 4 Sep 2022 10:57:36 +0200 Subject: [PATCH 046/256] Floating groups --- src/main/java/org/jabref/gui/LibraryTab.java | 2 +- src/main/java/org/jabref/gui/StateManager.java | 2 +- .../jabref/gui/externalfiles/ImportHandler.java | 4 ++-- .../jabref/gui/groups/GroupNodeViewModel.java | 2 +- .../jabref/gui/groups/GroupTreeViewModel.java | 4 ++-- .../org/jabref/gui/groups/GroupViewMode.java | 2 +- .../org/jabref/gui/groups/GroupsPreferences.java | 4 ++++ .../java/org/jabref/gui/maintable/MainTable.css | 4 ++++ .../java/org/jabref/gui/maintable/MainTable.java | 16 ++++++++++++++++ .../jabref/gui/maintable/MainTableDataModel.java | 13 ++++++++----- .../gui/sidepane/GroupsSidePaneComponent.java | 13 ++++++++++++- .../jabref/gui/sidepane/SidePaneComponent.java | 2 +- .../jabref/preferences/JabRefPreferences.java | 4 ++++ src/main/resources/l10n/JabRef_en.properties | 1 + .../gui/groups/GroupNodeViewModelTest.java | 1 + .../gui/groups/GroupTreeViewModelTest.java | 3 ++- 16 files changed, 61 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/jabref/gui/LibraryTab.java b/src/main/java/org/jabref/gui/LibraryTab.java index ece4d7d4c40..07c9b56c817 100644 --- a/src/main/java/org/jabref/gui/LibraryTab.java +++ b/src/main/java/org/jabref/gui/LibraryTab.java @@ -824,7 +824,7 @@ public void listen(EntriesAddedEvent addedEntriesEvent) { // Automatically add new entries to the selected group (or set of groups) if (preferencesService.getGroupsPreferences().shouldAutoAssignGroup()) { - stateManager.getSelectedGroup(bibDatabaseContext).forEach( + stateManager.getSelectedGroups(bibDatabaseContext).forEach( selectedGroup -> selectedGroup.addEntriesToGroup(addedEntriesEvent.getBibEntries())); } } diff --git a/src/main/java/org/jabref/gui/StateManager.java b/src/main/java/org/jabref/gui/StateManager.java index d6bd4571f2c..66d3a3df7fb 100644 --- a/src/main/java/org/jabref/gui/StateManager.java +++ b/src/main/java/org/jabref/gui/StateManager.java @@ -107,7 +107,7 @@ public void setSelectedGroups(BibDatabaseContext database, List n selectedGroups.put(database, FXCollections.observableArrayList(newSelectedGroups)); } - public ObservableList getSelectedGroup(BibDatabaseContext database) { + public ObservableList getSelectedGroups(BibDatabaseContext database) { ObservableList selectedGroupsForDatabase = selectedGroups.get(database); return selectedGroupsForDatabase != null ? selectedGroupsForDatabase : FXCollections.observableArrayList(); } diff --git a/src/main/java/org/jabref/gui/externalfiles/ImportHandler.java b/src/main/java/org/jabref/gui/externalfiles/ImportHandler.java index 0c59b8b7803..dd10144a9d8 100644 --- a/src/main/java/org/jabref/gui/externalfiles/ImportHandler.java +++ b/src/main/java/org/jabref/gui/externalfiles/ImportHandler.java @@ -188,7 +188,7 @@ public void importEntries(List entries) { } // Add to group - addToGroups(entries, stateManager.getSelectedGroup(bibDatabaseContext)); + addToGroups(entries, stateManager.getSelectedGroups(bibDatabaseContext)); } public void importEntryWithDuplicateCheck(BibDatabaseContext bibDatabaseContext, BibEntry entry) { @@ -227,7 +227,7 @@ public void importEntryWithDuplicateCheck(BibDatabaseContext bibDatabaseContext, preferencesService.getOwnerPreferences(), preferencesService.getTimestampPreferences()); - addToGroups(List.of(entry), stateManager.getSelectedGroup(this.bibDatabaseContext)); + addToGroups(List.of(entry), stateManager.getSelectedGroups(this.bibDatabaseContext)); } private void addToGroups(List entries, Collection groups) { diff --git a/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java b/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java index f63eafc2470..71ecf9cbe10 100644 --- a/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java +++ b/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java @@ -254,7 +254,7 @@ private void refreshGroup() { DefaultTaskExecutor.runInJavaFXThread(() -> { updateMatchedEntries(); // Update the entries matched by the group // "Re-add" to the selected groups if it were selected, this refreshes the entries the user views - ObservableList selectedGroups = this.stateManager.getSelectedGroup(this.databaseContext); + ObservableList selectedGroups = this.stateManager.getSelectedGroups(this.databaseContext); if (selectedGroups.remove(this.groupNode)) { selectedGroups.add(this.groupNode); } diff --git a/src/main/java/org/jabref/gui/groups/GroupTreeViewModel.java b/src/main/java/org/jabref/gui/groups/GroupTreeViewModel.java index 670130757ad..ddbf5c9917c 100644 --- a/src/main/java/org/jabref/gui/groups/GroupTreeViewModel.java +++ b/src/main/java/org/jabref/gui/groups/GroupTreeViewModel.java @@ -135,11 +135,11 @@ private void onActiveDatabaseChanged(Optional newDatabase) { .orElse(GroupNodeViewModel.getAllEntriesGroup(newDatabase.get(), stateManager, taskExecutor, localDragboard, preferences)); rootGroup.setValue(newRoot); - if (stateManager.getSelectedGroup(newDatabase.get()).isEmpty()) { + if (stateManager.getSelectedGroups(newDatabase.get()).isEmpty()) { stateManager.setSelectedGroups(newDatabase.get(), Collections.singletonList(newRoot.getGroupNode())); } selectedGroups.setAll( - stateManager.getSelectedGroup(newDatabase.get()).stream() + stateManager.getSelectedGroups(newDatabase.get()).stream() .map(selectedGroup -> new GroupNodeViewModel(newDatabase.get(), stateManager, taskExecutor, selectedGroup, localDragboard, preferences)) .collect(Collectors.toList())); } else { diff --git a/src/main/java/org/jabref/gui/groups/GroupViewMode.java b/src/main/java/org/jabref/gui/groups/GroupViewMode.java index 66f06b2646b..615bc5f41e5 100644 --- a/src/main/java/org/jabref/gui/groups/GroupViewMode.java +++ b/src/main/java/org/jabref/gui/groups/GroupViewMode.java @@ -1,4 +1,4 @@ package org.jabref.gui.groups; -public enum GroupViewMode { INTERSECTION } +public enum GroupViewMode { INTERSECTION, FILTER } diff --git a/src/main/java/org/jabref/gui/groups/GroupsPreferences.java b/src/main/java/org/jabref/gui/groups/GroupsPreferences.java index 93e8a651fa8..44df2be6dd5 100644 --- a/src/main/java/org/jabref/gui/groups/GroupsPreferences.java +++ b/src/main/java/org/jabref/gui/groups/GroupsPreferences.java @@ -17,6 +17,7 @@ public class GroupsPreferences { private final ObjectProperty keywordSeparator; public GroupsPreferences(boolean viewModeIntersection, + boolean viewModeFilter, boolean shouldAutoAssignGroup, boolean shouldDisplayGroupCount, ObjectProperty keywordSeparator) { @@ -25,6 +26,9 @@ public GroupsPreferences(boolean viewModeIntersection, if (viewModeIntersection) { this.groupViewMode.add(GroupViewMode.INTERSECTION); } + if (viewModeFilter) { + this.groupViewMode.add(GroupViewMode.FILTER); + } this.shouldAutoAssignGroup = new SimpleBooleanProperty(shouldAutoAssignGroup); this.shouldDisplayGroupCount = new SimpleBooleanProperty(shouldDisplayGroupCount); this.keywordSeparator = keywordSeparator; diff --git a/src/main/java/org/jabref/gui/maintable/MainTable.css b/src/main/java/org/jabref/gui/maintable/MainTable.css index a759c3fce7e..7734dd76b1c 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTable.css +++ b/src/main/java/org/jabref/gui/maintable/MainTable.css @@ -40,6 +40,10 @@ -fx-opacity: 35%; } +.table-row-cell:entry-not-matching-groups { + -fx-background-color: -jr-gray-2; +} + .rating > .container { -fx-spacing: 2; } diff --git a/src/main/java/org/jabref/gui/maintable/MainTable.java b/src/main/java/org/jabref/gui/maintable/MainTable.java index 638eeb9ac5f..1e80bb89fa6 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTable.java +++ b/src/main/java/org/jabref/gui/maintable/MainTable.java @@ -36,6 +36,7 @@ import org.jabref.gui.actions.StandardActions; import org.jabref.gui.edit.EditAction; import org.jabref.gui.externalfiles.ImportHandler; +import org.jabref.gui.groups.GroupViewMode; import org.jabref.gui.keyboard.KeyBinding; import org.jabref.gui.keyboard.KeyBindingRepository; import org.jabref.gui.maintable.columns.LibraryColumn; @@ -56,6 +57,8 @@ import org.jabref.preferences.PreferencesService; import com.google.common.eventbus.Subscribe; +import com.tobiasdiez.easybind.EasyBind; +import com.tobiasdiez.easybind.EasyBinding; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -139,6 +142,19 @@ public MainTable(MainTableDataModel model, Globals.TASK_EXECUTOR, Globals.entryTypesManager)) .withPseudoClass(PseudoClass.getPseudoClass("entry-not-matching-search"), entry -> stateManager.activeSearchQueryProperty().isPresent().and(entry.searchScoreProperty().isEqualTo(0))) + .withPseudoClass(PseudoClass.getPseudoClass("entry-not-matching-groups"), entry -> { + EasyBinding matchesGroup = EasyBind.combine( + stateManager.activeGroupProperty(), + preferencesService.getGroupsPreferences().groupViewModeProperty(), + (groups, viewMode) -> { + if (viewMode.contains(GroupViewMode.FILTER)) { + return Boolean.valueOf(false); + } + MainTableDataModel.updateSearchGroups(stateManager, database); + return Boolean.valueOf(!MainTableDataModel.createGroupMatcher(stateManager.activeGroupProperty(), preferencesService.getGroupsPreferences()).map(matcher -> matcher.isMatch(entry.getEntry())).orElse(true)); + }); + return matchesGroup; + }) .setOnDragDetected(this::handleOnDragDetected) .setOnDragDropped(this::handleOnDragDropped) .setOnDragOver(this::handleOnDragOver) diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index f9b7ad98ecf..17a869e82d7 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -56,7 +56,7 @@ public MainTableDataModel(BibDatabaseContext context, PreferencesService prefere (groups, query, groupViewMode) -> { doSearch(query); return entry -> { - updateSearchGroups(); + updateSearchGroups(stateManager, bibDatabaseContext); return isMatched(groups, query, entry); }; }) @@ -66,8 +66,8 @@ public MainTableDataModel(BibDatabaseContext context, PreferencesService prefere entriesSorted = new SortedList<>(entriesFiltered); } - private void updateSearchGroups() { - stateManager.getSelectedGroup(bibDatabaseContext).stream().map(GroupTreeNode::getGroup).filter(g -> g instanceof SearchGroup).map(g -> ((SearchGroup) g)).forEach(g -> g.updateMatches(bibDatabaseContext)); + public static void updateSearchGroups(StateManager stateManager, BibDatabaseContext bibDatabaseContext) { + stateManager.getSelectedGroups(bibDatabaseContext).stream().map(GroupTreeNode::getGroup).filter(g -> g instanceof SearchGroup).map(g -> ((SearchGroup) g)).forEach(g -> g.updateMatches(bibDatabaseContext)); } private void doSearch(Optional query) { @@ -100,12 +100,15 @@ private boolean isMatchedBySearch(Optional query, BibEntryTableView } private boolean isMatchedByGroup(ObservableList groups, BibEntryTableViewModel entry) { - return createGroupMatcher(groups) + if (!preferencesService.getGroupsPreferences().groupViewModeProperty().contains(GroupViewMode.FILTER)) { + return true; + } + return createGroupMatcher(groups, groupsPreferences) .map(matcher -> matcher.isMatch(entry.getEntry())) .orElse(true); } - private Optional createGroupMatcher(List selectedGroups) { + public static Optional createGroupMatcher(List selectedGroups, GroupsPreferences groupsPreferences) { if ((selectedGroups == null) || selectedGroups.isEmpty()) { // No selected group, show all entries return Optional.empty(); diff --git a/src/main/java/org/jabref/gui/sidepane/GroupsSidePaneComponent.java b/src/main/java/org/jabref/gui/sidepane/GroupsSidePaneComponent.java index b65bbe6279f..43da9a59c8b 100644 --- a/src/main/java/org/jabref/gui/sidepane/GroupsSidePaneComponent.java +++ b/src/main/java/org/jabref/gui/sidepane/GroupsSidePaneComponent.java @@ -4,6 +4,8 @@ import javafx.collections.SetChangeListener; import javafx.scene.control.Button; +import javafx.scene.control.ToggleButton; +import javafx.scene.control.Tooltip; import org.jabref.gui.DialogService; import org.jabref.gui.actions.SimpleCommand; @@ -17,6 +19,7 @@ public class GroupsSidePaneComponent extends SidePaneComponent { private final GroupsPreferences groupsPreferences; private final DialogService dialogService; private final Button intersectionUnionToggle = IconTheme.JabRefIcons.GROUP_INTERSECTION.asButton(); + private final ToggleButton filterToggle = IconTheme.JabRefIcons.FILTER.asToggleButton(); public GroupsSidePaneComponent(SimpleCommand closeCommand, SimpleCommand moveUpCommand, @@ -28,6 +31,7 @@ public GroupsSidePaneComponent(SimpleCommand closeCommand, this.groupsPreferences = groupsPreferences; this.dialogService = dialogService; setupIntersectionUnionToggle(); + setupFilterToggle(); groupsPreferences.groupViewModeProperty().addListener((SetChangeListener) change -> { GroupModeViewModel modeViewModel = new GroupModeViewModel(groupsPreferences.groupViewModeProperty()); @@ -37,10 +41,17 @@ public GroupsSidePaneComponent(SimpleCommand closeCommand, } private void setupIntersectionUnionToggle() { - addExtraButtonToHeader(intersectionUnionToggle, 0); + addExtraNodeToHeader(intersectionUnionToggle, 0); intersectionUnionToggle.setOnAction(event -> new ToggleUnionIntersectionAction().execute()); } + private void setupFilterToggle() { + addExtraNodeToHeader(filterToggle, 0); + filterToggle.setSelected(groupsPreferences.groupViewModeProperty().contains(GroupViewMode.FILTER)); + filterToggle.selectedProperty().addListener((observable, oldValue, newValue) -> groupsPreferences.setGroupViewMode(GroupViewMode.FILTER, newValue)); + filterToggle.setTooltip(new Tooltip(Localization.lang("Filter by groups"))); + } + private class ToggleUnionIntersectionAction extends SimpleCommand { @Override diff --git a/src/main/java/org/jabref/gui/sidepane/SidePaneComponent.java b/src/main/java/org/jabref/gui/sidepane/SidePaneComponent.java index 104a3c42518..8499beea6f9 100644 --- a/src/main/java/org/jabref/gui/sidepane/SidePaneComponent.java +++ b/src/main/java/org/jabref/gui/sidepane/SidePaneComponent.java @@ -68,7 +68,7 @@ private Node createHeaderView() { return headerView; } - protected void addExtraButtonToHeader(Button button, int position) { + protected void addExtraNodeToHeader(Node button, int position) { this.buttonContainer.getChildren().add(position, button); } } diff --git a/src/main/java/org/jabref/preferences/JabRefPreferences.java b/src/main/java/org/jabref/preferences/JabRefPreferences.java index 69a15ee98e9..8ee6e31310f 100644 --- a/src/main/java/org/jabref/preferences/JabRefPreferences.java +++ b/src/main/java/org/jabref/preferences/JabRefPreferences.java @@ -375,6 +375,7 @@ public class JabRefPreferences implements PreferencesService { // GroupViewMode private static final String GROUP_VIEW_INTERSECTION = "groupIntersection"; + private static final String GROUP_VIEW_FILTER = "groupFilter"; // Dialog states private static final String PREFS_EXPORT_PATH = "prefsExportPath"; @@ -599,6 +600,7 @@ private JabRefPreferences() { defaults.put(AUTO_ASSIGN_GROUP, Boolean.TRUE); defaults.put(DISPLAY_GROUP_COUNT, Boolean.TRUE); defaults.put(GROUP_VIEW_INTERSECTION, Boolean.TRUE); + defaults.put(GROUP_VIEW_FILTER, Boolean.TRUE); defaults.put(KEYWORD_SEPARATOR, ", "); defaults.put(DEFAULT_ENCODING, StandardCharsets.UTF_8.name()); defaults.put(DEFAULT_OWNER, System.getProperty("user.name")); @@ -1420,6 +1422,7 @@ public GroupsPreferences getGroupsPreferences() { groupsPreferences = new GroupsPreferences( getBoolean(GROUP_VIEW_INTERSECTION), + getBoolean(GROUP_VIEW_FILTER), getBoolean(AUTO_ASSIGN_GROUP), getBoolean(DISPLAY_GROUP_COUNT), getInternalPreferences().keywordSeparatorProperty() @@ -1429,6 +1432,7 @@ public GroupsPreferences getGroupsPreferences() { @Override public void onChanged(Change change) { putBoolean(GROUP_VIEW_INTERSECTION, groupsPreferences.groupViewModeProperty().contains(GroupViewMode.INTERSECTION)); + putBoolean(GROUP_VIEW_FILTER, groupsPreferences.groupViewModeProperty().contains(GroupViewMode.FILTER)); } }); EasyBind.listen(groupsPreferences.autoAssignGroupProperty(), (obs, oldValue, newValue) -> putBoolean(AUTO_ASSIGN_GROUP, newValue)); diff --git a/src/main/resources/l10n/JabRef_en.properties b/src/main/resources/l10n/JabRef_en.properties index 1bfc64849df..ac08210d953 100644 --- a/src/main/resources/l10n/JabRef_en.properties +++ b/src/main/resources/l10n/JabRef_en.properties @@ -379,6 +379,7 @@ Format\:\ Tab\:field;field;...\ (e.g.\ General\:url;pdf;note...)=Format\: Tab\:f Format\ of\ author\ and\ editor\ names=Format of author and editor names Format\ string=Format string Score=Score +Filter\ by\ groups=Filter by groups Format\ used=Format used Formatter\ name=Formatter name diff --git a/src/test/java/org/jabref/gui/groups/GroupNodeViewModelTest.java b/src/test/java/org/jabref/gui/groups/GroupNodeViewModelTest.java index 4257ff4790f..f9ab8d6beb9 100644 --- a/src/test/java/org/jabref/gui/groups/GroupNodeViewModelTest.java +++ b/src/test/java/org/jabref/gui/groups/GroupNodeViewModelTest.java @@ -49,6 +49,7 @@ void setUp() { false, true, true, + true, new SimpleObjectProperty<>(',') )); diff --git a/src/test/java/org/jabref/gui/groups/GroupTreeViewModelTest.java b/src/test/java/org/jabref/gui/groups/GroupTreeViewModelTest.java index f4277506724..48d0ed6ab45 100644 --- a/src/test/java/org/jabref/gui/groups/GroupTreeViewModelTest.java +++ b/src/test/java/org/jabref/gui/groups/GroupTreeViewModelTest.java @@ -43,6 +43,7 @@ void setUp() { false, true, true, + true, new SimpleObjectProperty<>(',') )); groupTree = new GroupTreeViewModel(stateManager, mock(DialogService.class), preferencesService, taskExecutor, new CustomLocalDragboard()); @@ -56,7 +57,7 @@ void rootGroupIsAllEntriesByDefault() { @Test void rootGroupIsSelectedByDefault() { - assertEquals(groupTree.rootGroupProperty().get().getGroupNode(), stateManager.getSelectedGroup(databaseContext).get(0)); + assertEquals(groupTree.rootGroupProperty().get().getGroupNode(), stateManager.getSelectedGroups(databaseContext).get(0)); } @Test From 0ac40900e764e183f2527a4ec3bad332dd54c3db Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Sun, 4 Sep 2022 11:11:10 +0200 Subject: [PATCH 047/256] Invert-mode for groups --- .../java/org/jabref/gui/groups/GroupViewMode.java | 2 +- .../java/org/jabref/gui/groups/GroupsPreferences.java | 4 ++++ src/main/java/org/jabref/gui/icon/IconTheme.java | 1 + src/main/java/org/jabref/gui/maintable/MainTable.java | 2 +- .../org/jabref/gui/maintable/MainTableDataModel.java | 2 +- .../jabref/gui/sidepane/GroupsSidePaneComponent.java | 11 ++++++++++- .../org/jabref/preferences/JabRefPreferences.java | 4 ++++ src/main/resources/l10n/JabRef_en.properties | 1 + .../org/jabref/gui/groups/GroupNodeViewModelTest.java | 1 + .../org/jabref/gui/groups/GroupTreeViewModelTest.java | 1 + 10 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/jabref/gui/groups/GroupViewMode.java b/src/main/java/org/jabref/gui/groups/GroupViewMode.java index 615bc5f41e5..b365595ae32 100644 --- a/src/main/java/org/jabref/gui/groups/GroupViewMode.java +++ b/src/main/java/org/jabref/gui/groups/GroupViewMode.java @@ -1,4 +1,4 @@ package org.jabref.gui.groups; -public enum GroupViewMode { INTERSECTION, FILTER } +public enum GroupViewMode { INTERSECTION, FILTER , INVERT } diff --git a/src/main/java/org/jabref/gui/groups/GroupsPreferences.java b/src/main/java/org/jabref/gui/groups/GroupsPreferences.java index 44df2be6dd5..387eae2e68f 100644 --- a/src/main/java/org/jabref/gui/groups/GroupsPreferences.java +++ b/src/main/java/org/jabref/gui/groups/GroupsPreferences.java @@ -18,6 +18,7 @@ public class GroupsPreferences { public GroupsPreferences(boolean viewModeIntersection, boolean viewModeFilter, + boolean viewModeInvert, boolean shouldAutoAssignGroup, boolean shouldDisplayGroupCount, ObjectProperty keywordSeparator) { @@ -29,6 +30,9 @@ public GroupsPreferences(boolean viewModeIntersection, if (viewModeFilter) { this.groupViewMode.add(GroupViewMode.FILTER); } + if (viewModeInvert) { + this.groupViewMode.add(GroupViewMode.INVERT); + } this.shouldAutoAssignGroup = new SimpleBooleanProperty(shouldAutoAssignGroup); this.shouldDisplayGroupCount = new SimpleBooleanProperty(shouldDisplayGroupCount); this.keywordSeparator = keywordSeparator; diff --git a/src/main/java/org/jabref/gui/icon/IconTheme.java b/src/main/java/org/jabref/gui/icon/IconTheme.java index 9b3449bb011..3f583bbd265 100644 --- a/src/main/java/org/jabref/gui/icon/IconTheme.java +++ b/src/main/java/org/jabref/gui/icon/IconTheme.java @@ -282,6 +282,7 @@ public enum JabRefIcons implements JabRefIcon { REG_EX(MaterialDesignR.REGEX), FULLTEXT(MaterialDesignF.FILE_EYE), FILTER(MaterialDesignF.FILTER), + INVERT(MaterialDesignI.INVERT_COLORS), SORT_BY_SCORE(MaterialDesignS.SORT_VARIANT), CONSOLE(MaterialDesignC.CONSOLE), FORUM(MaterialDesignF.FORUM), diff --git a/src/main/java/org/jabref/gui/maintable/MainTable.java b/src/main/java/org/jabref/gui/maintable/MainTable.java index 1e80bb89fa6..d9795a52af8 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTable.java +++ b/src/main/java/org/jabref/gui/maintable/MainTable.java @@ -151,7 +151,7 @@ public MainTable(MainTableDataModel model, return Boolean.valueOf(false); } MainTableDataModel.updateSearchGroups(stateManager, database); - return Boolean.valueOf(!MainTableDataModel.createGroupMatcher(stateManager.activeGroupProperty(), preferencesService.getGroupsPreferences()).map(matcher -> matcher.isMatch(entry.getEntry())).orElse(true)); + return Boolean.valueOf(preferencesService.getGroupsPreferences().getGroupViewMode().contains(GroupViewMode.INVERT) ^ !MainTableDataModel.createGroupMatcher(stateManager.activeGroupProperty(), preferencesService.getGroupsPreferences()).map(matcher -> matcher.isMatch(entry.getEntry())).orElse(true)); }); return matchesGroup; }) diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index 17a869e82d7..056b22e3435 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -104,7 +104,7 @@ private boolean isMatchedByGroup(ObservableList groups, BibEntryT return true; } return createGroupMatcher(groups, groupsPreferences) - .map(matcher -> matcher.isMatch(entry.getEntry())) + .map(matcher -> preferencesService.getGroupsPreferences().getGroupViewMode().contains(GroupViewMode.INVERT) ^ matcher.isMatch(entry.getEntry())) .orElse(true); } diff --git a/src/main/java/org/jabref/gui/sidepane/GroupsSidePaneComponent.java b/src/main/java/org/jabref/gui/sidepane/GroupsSidePaneComponent.java index 43da9a59c8b..bbec37ec5e0 100644 --- a/src/main/java/org/jabref/gui/sidepane/GroupsSidePaneComponent.java +++ b/src/main/java/org/jabref/gui/sidepane/GroupsSidePaneComponent.java @@ -20,6 +20,7 @@ public class GroupsSidePaneComponent extends SidePaneComponent { private final DialogService dialogService; private final Button intersectionUnionToggle = IconTheme.JabRefIcons.GROUP_INTERSECTION.asButton(); private final ToggleButton filterToggle = IconTheme.JabRefIcons.FILTER.asToggleButton(); + private final ToggleButton invertToggle = IconTheme.JabRefIcons.INVERT.asToggleButton(); public GroupsSidePaneComponent(SimpleCommand closeCommand, SimpleCommand moveUpCommand, @@ -30,8 +31,9 @@ public GroupsSidePaneComponent(SimpleCommand closeCommand, super(SidePaneType.GROUPS, closeCommand, moveUpCommand, moveDownCommand, contentFactory); this.groupsPreferences = groupsPreferences; this.dialogService = dialogService; - setupIntersectionUnionToggle(); + setupInvertToggle(); setupFilterToggle(); + setupIntersectionUnionToggle(); groupsPreferences.groupViewModeProperty().addListener((SetChangeListener) change -> { GroupModeViewModel modeViewModel = new GroupModeViewModel(groupsPreferences.groupViewModeProperty()); @@ -52,6 +54,13 @@ private void setupFilterToggle() { filterToggle.setTooltip(new Tooltip(Localization.lang("Filter by groups"))); } + private void setupInvertToggle() { + addExtraNodeToHeader(invertToggle, 0); + invertToggle.setSelected(groupsPreferences.groupViewModeProperty().contains(GroupViewMode.INVERT)); + invertToggle.selectedProperty().addListener((observable, oldValue, newValue) -> groupsPreferences.setGroupViewMode(GroupViewMode.INVERT, newValue)); + invertToggle.setTooltip(new Tooltip(Localization.lang("Invert groups"))); + } + private class ToggleUnionIntersectionAction extends SimpleCommand { @Override diff --git a/src/main/java/org/jabref/preferences/JabRefPreferences.java b/src/main/java/org/jabref/preferences/JabRefPreferences.java index 8ee6e31310f..870facc27a5 100644 --- a/src/main/java/org/jabref/preferences/JabRefPreferences.java +++ b/src/main/java/org/jabref/preferences/JabRefPreferences.java @@ -376,6 +376,7 @@ public class JabRefPreferences implements PreferencesService { // GroupViewMode private static final String GROUP_VIEW_INTERSECTION = "groupIntersection"; private static final String GROUP_VIEW_FILTER = "groupFilter"; + private static final String GROUP_VIEW_INVERT = "groupInvert"; // Dialog states private static final String PREFS_EXPORT_PATH = "prefsExportPath"; @@ -601,6 +602,7 @@ private JabRefPreferences() { defaults.put(DISPLAY_GROUP_COUNT, Boolean.TRUE); defaults.put(GROUP_VIEW_INTERSECTION, Boolean.TRUE); defaults.put(GROUP_VIEW_FILTER, Boolean.TRUE); + defaults.put(GROUP_VIEW_INVERT, Boolean.FALSE); defaults.put(KEYWORD_SEPARATOR, ", "); defaults.put(DEFAULT_ENCODING, StandardCharsets.UTF_8.name()); defaults.put(DEFAULT_OWNER, System.getProperty("user.name")); @@ -1423,6 +1425,7 @@ public GroupsPreferences getGroupsPreferences() { groupsPreferences = new GroupsPreferences( getBoolean(GROUP_VIEW_INTERSECTION), getBoolean(GROUP_VIEW_FILTER), + getBoolean(GROUP_VIEW_INVERT), getBoolean(AUTO_ASSIGN_GROUP), getBoolean(DISPLAY_GROUP_COUNT), getInternalPreferences().keywordSeparatorProperty() @@ -1433,6 +1436,7 @@ public GroupsPreferences getGroupsPreferences() { public void onChanged(Change change) { putBoolean(GROUP_VIEW_INTERSECTION, groupsPreferences.groupViewModeProperty().contains(GroupViewMode.INTERSECTION)); putBoolean(GROUP_VIEW_FILTER, groupsPreferences.groupViewModeProperty().contains(GroupViewMode.FILTER)); + putBoolean(GROUP_VIEW_INVERT, groupsPreferences.groupViewModeProperty().contains(GroupViewMode.INVERT)); } }); EasyBind.listen(groupsPreferences.autoAssignGroupProperty(), (obs, oldValue, newValue) -> putBoolean(AUTO_ASSIGN_GROUP, newValue)); diff --git a/src/main/resources/l10n/JabRef_en.properties b/src/main/resources/l10n/JabRef_en.properties index ac08210d953..25dccc9a0f1 100644 --- a/src/main/resources/l10n/JabRef_en.properties +++ b/src/main/resources/l10n/JabRef_en.properties @@ -380,6 +380,7 @@ Format\ of\ author\ and\ editor\ names=Format of author and editor names Format\ string=Format string Score=Score Filter\ by\ groups=Filter by groups +Invert\ groups=Invert groups Format\ used=Format used Formatter\ name=Formatter name diff --git a/src/test/java/org/jabref/gui/groups/GroupNodeViewModelTest.java b/src/test/java/org/jabref/gui/groups/GroupNodeViewModelTest.java index f9ab8d6beb9..d4be2136b19 100644 --- a/src/test/java/org/jabref/gui/groups/GroupNodeViewModelTest.java +++ b/src/test/java/org/jabref/gui/groups/GroupNodeViewModelTest.java @@ -48,6 +48,7 @@ void setUp() { when(preferencesService.getGroupsPreferences()).thenReturn(new GroupsPreferences( false, true, + false, true, true, new SimpleObjectProperty<>(',') diff --git a/src/test/java/org/jabref/gui/groups/GroupTreeViewModelTest.java b/src/test/java/org/jabref/gui/groups/GroupTreeViewModelTest.java index 48d0ed6ab45..cdd04628cc5 100644 --- a/src/test/java/org/jabref/gui/groups/GroupTreeViewModelTest.java +++ b/src/test/java/org/jabref/gui/groups/GroupTreeViewModelTest.java @@ -42,6 +42,7 @@ void setUp() { when(preferencesService.getGroupsPreferences()).thenReturn(new GroupsPreferences( false, true, + false, true, true, new SimpleObjectProperty<>(',') From 00cefc211f17e0dd15b647b1a43a9afabafad681 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Sun, 4 Sep 2022 17:23:18 +0200 Subject: [PATCH 048/256] Fix indexing Commit d8b48dd1ea8034df583b23dadb69a022566131f1 introduced a bug that prohibited the fulltext-indexer to start. It now uses the OpenDatabaseAction that does not trigger the indexing listener. This suggests that openinga new library never triggered an index-creation as that always used that action. --- src/main/java/org/jabref/gui/LibraryTab.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/java/org/jabref/gui/LibraryTab.java b/src/main/java/org/jabref/gui/LibraryTab.java index 2166157ba3f..fb7a39c8f10 100644 --- a/src/main/java/org/jabref/gui/LibraryTab.java +++ b/src/main/java/org/jabref/gui/LibraryTab.java @@ -213,6 +213,14 @@ public void onDatabaseLoadingSucceed(ParserResult result) { feedData(context); + if (preferencesService.getFilePreferences().shouldFulltextIndexLinkedFiles()) { + try { + indexingTaskManager.updateIndex(PdfIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences()), bibDatabaseContext); + } catch (IOException e) { + LOGGER.error("Cannot access lucene index", e); + } + } + // a temporary workaround to update groups pane stateManager.activeDatabaseProperty().bind( EasyBind.map(frame.getTabbedPane().getSelectionModel().selectedItemProperty(), From 3a4e6651c4326a063e900904408ba41f2ce03d2e Mon Sep 17 00:00:00 2001 From: Siedlerchr Date: Sun, 4 Sep 2022 23:12:15 +0200 Subject: [PATCH 049/256] Prepare CHANGELOG.md # Conflicts: # CHANGELOG.md --- CHANGELOG.md | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7a56567cda..f497b9b0adf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,17 @@ In case, there is no issue present, the pull request implementing the feature is Note that this project **does not** adhere to [Semantic Versioning](http://semver.org/). -## [Unreleased] +## [version6] - Big changes + +The branch `version6` is intended to collect improvements, where multiple pull requests build on each other. +From the user perspective, the combination of the pull request needs to be tested. +(We created the branch to support lowering the amount of open pull requests) + + + + + +## [Unreleased] - version 5.8 - not yet released ### Added @@ -893,7 +903,8 @@ The changelog of JabRef 4.x is available at the [v4.3.1 tag](https://github.com/ The changelog of JabRef 3.x is available at the [v3.8.2 tag](https://github.com/JabRef/jabref/blob/v3.8.2/CHANGELOG.md). The changelog of JabRef 2.11 and all previous versions is available as [text file in the v2.11.1 tag](https://github.com/JabRef/jabref/blob/v2.11.1/CHANGELOG). -[Unreleased]: https://github.com/JabRef/jabref/compare/v5.7...HEAD +[version6]: https://github.com/JabRef/jabref/compare/main...version6 +[Unreleased]: https://github.com/JabRef/jabref/compare/v5.8...HEAD [5.7]: https://github.com/JabRef/jabref/compare/v5.6...v5.7 [5.6]: https://github.com/JabRef/jabref/compare/v5.5...v5.6 [5.5]: https://github.com/JabRef/jabref/compare/v5.4...v5.5 From bc6710e2b219325de531b159b2673144dc081072 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Sat, 10 Sep 2022 16:31:39 +0200 Subject: [PATCH 050/256] Checkstyle --- src/main/java/org/jabref/gui/groups/GroupViewMode.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/jabref/gui/groups/GroupViewMode.java b/src/main/java/org/jabref/gui/groups/GroupViewMode.java index b365595ae32..b9b96cce436 100644 --- a/src/main/java/org/jabref/gui/groups/GroupViewMode.java +++ b/src/main/java/org/jabref/gui/groups/GroupViewMode.java @@ -1,4 +1,4 @@ package org.jabref.gui.groups; -public enum GroupViewMode { INTERSECTION, FILTER , INVERT } +public enum GroupViewMode { INTERSECTION, FILTER, INVERT } From 3280d23ba731e5d0134dc634df1e92e48d84dba1 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Sun, 11 Sep 2022 17:37:38 +0200 Subject: [PATCH 051/256] Temporary fix for linking to bibentries by hash --- .../java/org/jabref/gui/maintable/MainTableDataModel.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index 056b22e3435..a9b82dc1662 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -72,12 +72,13 @@ public static void updateSearchGroups(StateManager stateManager, BibDatabaseCont private void doSearch(Optional query) { if (query.isPresent()) { - String searchString = query.get().getQuery(); try { stateManager.getSearchResults().clear(); + /** These are now all empty. Why? for (BibDatabaseContext context : stateManager.getOpenDatabases()) { stateManager.getSearchResults().putAll(LuceneSearcher.of(context).search(query.get())); - } + }*/ + stateManager.getSearchResults().putAll(LuceneSearcher.of(stateManager.getActiveDatabase().get()).search(query.get())); } catch (IOException e) { e.printStackTrace(); } From 4412f6e3e50d9ee17bab5f4bffa2abd8098cdc8a Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Sun, 11 Sep 2022 17:38:01 +0200 Subject: [PATCH 052/256] Improve DocumentReader --- .../logic/search/indexing/DocumentReader.java | 71 ++++++++++--------- 1 file changed, 37 insertions(+), 34 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/indexing/DocumentReader.java b/src/main/java/org/jabref/logic/search/indexing/DocumentReader.java index e3ce895e2b3..2426f84a26b 100644 --- a/src/main/java/org/jabref/logic/search/indexing/DocumentReader.java +++ b/src/main/java/org/jabref/logic/search/indexing/DocumentReader.java @@ -71,11 +71,7 @@ public DocumentReader(BibEntry bibEntry, FilePreferences filePreferences) { public Optional> readLinkedPdf(BibDatabaseContext databaseContext, LinkedFile pdf) { Optional pdfPath = pdf.findIn(databaseContext, filePreferences); if (pdfPath.isPresent()) { - try { - return Optional.of(readPdfContents(pdf, pdfPath.get())); - } catch (IOException e) { - LOGGER.error("Could not read pdf file {}!", pdf.getLink(), e); - } + return Optional.of(readPdfContents(pdf, pdfPath.get())); } return Optional.empty(); } @@ -94,19 +90,30 @@ public List readLinkedPdfs(BibDatabaseContext databaseContext) { .collect(Collectors.toList()); } - private List readPdfContents(LinkedFile pdf, Path resolvedPdfPath) throws IOException { + private List readPdfContents(LinkedFile pdf, Path resolvedPdfPath) { + List pages = new ArrayList<>(); try (PDDocument pdfDocument = Loader.loadPDF(resolvedPdfPath.toFile())) { - List pages = new ArrayList<>(); - - for (int pageNumber = 0; pageNumber < pdfDocument.getNumberOfPages(); pageNumber++) { - Document newDocument = new Document(); - addIdentifiers(newDocument, pdf.getLink()); - addMetaData(newDocument, resolvedPdfPath, pageNumber); - addContentIfNotEmpty(pdfDocument, newDocument, pageNumber); - pages.add(newDocument); - } - return pages; + for (int pageNumber = 0; pageNumber < pdfDocument.getNumberOfPages(); pageNumber++) { + Document newDocument = new Document(); + addIdentifiers(newDocument, pdf.getLink()); + addMetaData(newDocument, resolvedPdfPath, pageNumber); + try { + addContentIfNotEmpty(pdfDocument, newDocument, pageNumber); + } catch (IOException e) { + LOGGER.warn("Could not read page " + pageNumber + " of " + resolvedPdfPath.toAbsolutePath(), e); + } + pages.add(newDocument); + } + } catch (IOException e) { + LOGGER.warn("Could not read " + resolvedPdfPath.toAbsolutePath(), e); + } + if (pages.isEmpty()) { + Document newDocument = new Document(); + addIdentifiers(newDocument, pdf.getLink()); + addMetaData(newDocument, resolvedPdfPath, 0); + pages.add(newDocument); } + return pages; } private void addMetaData(Document newDocument, Path resolvedPdfPath, int pageNumber) { @@ -135,24 +142,20 @@ public static String mergeLines(String text) { return LINEBREAK_WITHOUT_PERIOD_PATTERN.matcher(mergedHyphenNewlines).replaceAll("$1 "); } - private void addContentIfNotEmpty(PDDocument pdfDocument, Document newDocument, int pageNumber) { - try { - PDFTextStripper pdfTextStripper = new PDFTextStripper(); - pdfTextStripper.setLineSeparator("\n"); - pdfTextStripper.setStartPage(pageNumber); - pdfTextStripper.setEndPage(pageNumber); - - String pdfContent = pdfTextStripper.getText(pdfDocument); - if (StringUtil.isNotBlank(pdfContent)) { - newDocument.add(new TextField(CONTENT, mergeLines(pdfContent), Field.Store.YES)); - } - PDPage page = pdfDocument.getPage(pageNumber); - List annotations = page.getAnnotations().stream().filter((annotation) -> annotation.getContents() != null).map(PDAnnotation::getContents).collect(Collectors.toList()); - if (annotations.size() > 0) { - newDocument.add(new TextField(ANNOTATIONS, annotations.stream().collect(Collectors.joining("\n")), Field.Store.YES)); - } - } catch (IOException e) { - LOGGER.info("Could not read contents of PDF document \"{}\"", pdfDocument.toString(), e); + private void addContentIfNotEmpty(PDDocument pdfDocument, Document newDocument, int pageNumber) throws IOException { + PDFTextStripper pdfTextStripper = new PDFTextStripper(); + pdfTextStripper.setLineSeparator("\n"); + pdfTextStripper.setStartPage(pageNumber); + pdfTextStripper.setEndPage(pageNumber); + + String pdfContent = pdfTextStripper.getText(pdfDocument); + if (StringUtil.isNotBlank(pdfContent)) { + newDocument.add(new TextField(CONTENT, mergeLines(pdfContent), Field.Store.YES)); + } + PDPage page = pdfDocument.getPage(pageNumber); + List annotations = page.getAnnotations().stream().filter((annotation) -> annotation.getContents() != null).map(PDAnnotation::getContents).collect(Collectors.toList()); + if (annotations.size() > 0) { + newDocument.add(new TextField(ANNOTATIONS, annotations.stream().collect(Collectors.joining("\n")), Field.Store.YES)); } } From b7e39b3c73364b88d18bb0c81c5a07aff689519f Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Sun, 11 Sep 2022 17:38:40 +0200 Subject: [PATCH 053/256] Always make all fields searchable --- .../java/org/jabref/logic/search/indexing/LuceneIndexer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java b/src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java index be20dc9bd4e..6522a925d8d 100644 --- a/src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java @@ -165,6 +165,7 @@ public void addBibFieldsToIndex(BibEntry bibEntry) { Document document = new Document(); document.add(new StringField(SearchFieldConstants.BIB_ENTRY_ID_HASH, String.valueOf(bibEntry.getLastIndexHash()), org.apache.lucene.document.Field.Store.YES)); for (Map.Entry field : bibEntry.getFieldMap().entrySet()) { + SearchFieldConstants.searchableBibFields.add(field.getKey().getName()); if (field.getKey() == StandardField.KEYWORDS) { KeywordList keywords = KeywordList.parse(field.getValue(), preferences.getKeywordDelimiter()); for (Keyword keyword : keywords) { @@ -177,7 +178,6 @@ public void addBibFieldsToIndex(BibEntry bibEntry) { } } else { document.add(new TextField(field.getKey().getName(), field.getValue(), org.apache.lucene.document.Field.Store.YES)); - SearchFieldConstants.searchableBibFields.add(field.getKey().getName()); } } indexWriter.addDocument(document); From 5ade8ef855e4a22ec81ba88f0a1095a149de6554 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Sun, 11 Sep 2022 17:39:15 +0200 Subject: [PATCH 054/256] Make index-path output a debug message --- src/main/java/org/jabref/model/database/BibDatabaseContext.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/jabref/model/database/BibDatabaseContext.java b/src/main/java/org/jabref/model/database/BibDatabaseContext.java index 7b5fe5120b1..0548dc09fbc 100644 --- a/src/main/java/org/jabref/model/database/BibDatabaseContext.java +++ b/src/main/java/org/jabref/model/database/BibDatabaseContext.java @@ -247,7 +247,7 @@ public Path getFulltextIndexPath() { if (getDatabasePath().isPresent()) { Path databasePath = appData.resolve(String.valueOf(this.getDatabasePath().get().hashCode())); - LOGGER.info("Index path for {} is {}", getDatabasePath().get(), databasePath); + LOGGER.debug("Index path for {} is {}", getDatabasePath().get(), databasePath); return databasePath; } From dfd3c95489eb640079b512fd22df2cca35f02427 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Sun, 11 Sep 2022 17:52:02 +0200 Subject: [PATCH 055/256] Fix deletition of old entries --- .../search/indexing/IndexingTaskManager.java | 4 +-- .../logic/search/indexing/LuceneIndexer.java | 36 ++++++++++++++----- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/indexing/IndexingTaskManager.java b/src/main/java/org/jabref/logic/search/indexing/IndexingTaskManager.java index 405a18fc4ba..5ce512461de 100644 --- a/src/main/java/org/jabref/logic/search/indexing/IndexingTaskManager.java +++ b/src/main/java/org/jabref/logic/search/indexing/IndexingTaskManager.java @@ -3,7 +3,6 @@ import java.util.List; import java.util.Set; import java.util.concurrent.ConcurrentLinkedDeque; -import java.util.stream.Collectors; import org.jabref.gui.util.BackgroundTask; import org.jabref.gui.util.DefaultTaskExecutor; @@ -106,7 +105,8 @@ public void manageFulltextIndexAccordingToPrefs(LuceneIndexer indexer) { public void updateIndex(LuceneIndexer indexer) { Set pathsToRemove = indexer.getListOfFilePaths(); - Set hashesOfEntriesToRemove = indexer.getDatabaseContext().getEntries().stream().map(BibEntry::updateAndGetIndexHash).collect(Collectors.toSet()); + Set hashesOfEntriesToRemove = indexer.getListOfHashes(); + indexer.getDatabaseContext().getEntries().stream().forEach(BibEntry::updateAndGetIndexHash); for (BibEntry entry : indexer.getDatabaseContext().getEntries()) { enqueueTask(() -> indexer.addBibFieldsToIndex(entry), true); enqueueTask(() -> indexer.addLinkedFilesToIndex(entry), false); diff --git a/src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java b/src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java index 6522a925d8d..33b03cc91c9 100644 --- a/src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java @@ -11,6 +11,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import org.jabref.gui.LibraryTab; import org.jabref.logic.util.StandardFileType; @@ -261,26 +262,45 @@ public void updateLinkedFilesInIndex(BibEntry entry, List removedFil } /** - * Lists the paths of all the files that are stored in the index + * Lists all values of a given field stored in the index * - * @return all file paths + * @param field the field to get the values for + * @return all values for this field */ - public Set getListOfFilePaths() { - Set paths = new HashSet<>(); + private Set getListOfField(String field) { + Set values = new HashSet<>(); try (IndexReader reader = DirectoryReader.open(directoryToIndex)) { IndexSearcher searcher = new IndexSearcher(reader); MatchAllDocsQuery query = new MatchAllDocsQuery(); TopDocs allDocs = searcher.search(query, Integer.MAX_VALUE); for (ScoreDoc scoreDoc : allDocs.scoreDocs) { Document doc = reader.document(scoreDoc.doc); - if (doc.getField(SearchFieldConstants.PATH) != null) { - paths.add(doc.getField(SearchFieldConstants.PATH).stringValue()); + if (doc.getField(field) != null) { + values.add(doc.getField(field).stringValue()); } } } catch (IOException e) { - return paths; + return values; } - return paths; + return values; + } + + /** + * Lists the paths of all the files that are stored in the index + * + * @return all file paths + */ + public Set getListOfFilePaths() { + return getListOfField(SearchFieldConstants.PATH); + } + + /** + * Lists the hashes of all the entries that are stored in the index + * + * @return all entry hashes + */ + public Set getListOfHashes() { + return getListOfField(SearchFieldConstants.BIB_ENTRY_ID_HASH).stream().map(Integer::valueOf).collect(Collectors.toSet()); } public void deleteLinkedFilesIndex() { From 4b17f10e2cb3142e6741d14a8b89f223a88e9533 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Sun, 11 Sep 2022 19:00:15 +0200 Subject: [PATCH 056/256] Fix re-indexing on database loading --- src/main/java/org/jabref/gui/LibraryTab.java | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/main/java/org/jabref/gui/LibraryTab.java b/src/main/java/org/jabref/gui/LibraryTab.java index 5e594fa5b49..a04b8cce6d7 100644 --- a/src/main/java/org/jabref/gui/LibraryTab.java +++ b/src/main/java/org/jabref/gui/LibraryTab.java @@ -885,15 +885,6 @@ public void listen(EntriesRemovedEvent removedEntriesEvent) { private class IndexUpdateListener { - public IndexUpdateListener() { - try { - indexingTaskManager.manageFulltextIndexAccordingToPrefs(LuceneIndexer.of(bibDatabaseContext, preferencesService, preferencesService.getFilePreferences())); - indexingTaskManager.updateIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService, preferencesService.getFilePreferences())); - } catch (IOException e) { - LOGGER.error("Cannot access lucene index", e); - } - } - @Subscribe public void listen(EntriesAddedEvent addedEntryEvent) { try { From 7e098d6014819e9cfc8d9de9b307c047177aa6d4 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Fri, 16 Sep 2022 16:43:39 +0200 Subject: [PATCH 057/256] Fixed empty databases in open database list --- src/main/java/org/jabref/gui/LibraryTab.java | 2 ++ .../java/org/jabref/gui/maintable/MainTableDataModel.java | 4 +--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/jabref/gui/LibraryTab.java b/src/main/java/org/jabref/gui/LibraryTab.java index a04b8cce6d7..88f0e5270db 100644 --- a/src/main/java/org/jabref/gui/LibraryTab.java +++ b/src/main/java/org/jabref/gui/LibraryTab.java @@ -245,7 +245,9 @@ public void onDatabaseLoadingFailed(Exception ex) { public void feedData(BibDatabaseContext bibDatabaseContext) { cleanUp(); + stateManager.getOpenDatabases().remove(this.bibDatabaseContext); this.bibDatabaseContext = Objects.requireNonNull(bibDatabaseContext); + stateManager.getOpenDatabases().add(this.bibDatabaseContext); bibDatabaseContext.getDatabase().registerListener(this); bibDatabaseContext.getMetaData().registerListener(this); diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index a9b82dc1662..d7b3b9f8f92 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -74,11 +74,9 @@ private void doSearch(Optional query) { if (query.isPresent()) { try { stateManager.getSearchResults().clear(); - /** These are now all empty. Why? for (BibDatabaseContext context : stateManager.getOpenDatabases()) { stateManager.getSearchResults().putAll(LuceneSearcher.of(context).search(query.get())); - }*/ - stateManager.getSearchResults().putAll(LuceneSearcher.of(stateManager.getActiveDatabase().get()).search(query.get())); + } } catch (IOException e) { e.printStackTrace(); } From e4552488199b70e6684a2bb90391e2755c2ab82c Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Sat, 17 Sep 2022 14:38:05 +0200 Subject: [PATCH 058/256] Make databasecontext in LibraryTab a property The object may change after data-loading (see feedData), but is bound to stateManager.openDatabases on creation of the tab. --- src/main/java/org/jabref/gui/JabRefFrame.java | 5 +- src/main/java/org/jabref/gui/LibraryTab.java | 89 ++++++++++--------- .../java/org/jabref/gui/StateManager.java | 4 +- .../gui/maintable/MainTableDataModel.java | 4 +- .../gui/openoffice/OpenOfficePanel.java | 5 +- .../search/SearchResultsTableDataModel.java | 6 +- 6 files changed, 60 insertions(+), 53 deletions(-) diff --git a/src/main/java/org/jabref/gui/JabRefFrame.java b/src/main/java/org/jabref/gui/JabRefFrame.java index 037b4f1170e..eba658ef924 100644 --- a/src/main/java/org/jabref/gui/JabRefFrame.java +++ b/src/main/java/org/jabref/gui/JabRefFrame.java @@ -17,6 +17,7 @@ import javafx.beans.Observable; import javafx.beans.binding.Bindings; import javafx.beans.binding.StringBinding; +import javafx.beans.property.ObjectProperty; import javafx.collections.transformation.FilteredList; import javafx.concurrent.Task; import javafx.geometry.Orientation; @@ -166,7 +167,7 @@ public class JabRefFrame extends BorderPane { private final FileHistoryMenu fileHistory; - @SuppressWarnings({"FieldCanBeLocal"}) private EasyObservableList openDatabaseList; + @SuppressWarnings({"FieldCanBeLocal"}) private EasyObservableList> openDatabaseList; private final Stage mainStage; private final StateManager stateManager; @@ -636,7 +637,7 @@ public void init() { filteredTabs.setPredicate(tab -> tab instanceof LibraryTab); // This variable cannot be inlined, since otherwise the list created by EasyBind is being garbage collected - openDatabaseList = EasyBind.map(filteredTabs, tab -> ((LibraryTab) tab).getBibDatabaseContext()); + openDatabaseList = EasyBind.map(filteredTabs, tab -> ((LibraryTab) tab).getBibDatabaseContextProperty()); EasyBind.bindContent(stateManager.getOpenDatabases(), openDatabaseList); // the binding for stateManager.activeDatabaseProperty() is at org.jabref.gui.LibraryTab.onDatabaseLoadingSucceed diff --git a/src/main/java/org/jabref/gui/LibraryTab.java b/src/main/java/org/jabref/gui/LibraryTab.java index 88f0e5270db..1481492a9e2 100644 --- a/src/main/java/org/jabref/gui/LibraryTab.java +++ b/src/main/java/org/jabref/gui/LibraryTab.java @@ -12,7 +12,9 @@ import javafx.animation.PauseTransition; import javafx.application.Platform; import javafx.beans.property.BooleanProperty; +import javafx.beans.property.ObjectProperty; import javafx.beans.property.SimpleBooleanProperty; +import javafx.beans.property.SimpleObjectProperty; import javafx.collections.ListChangeListener; import javafx.geometry.Orientation; import javafx.scene.Node; @@ -90,7 +92,7 @@ public class LibraryTab extends Tab { private final BooleanProperty changedProperty = new SimpleBooleanProperty(false); private final BooleanProperty nonUndoableChangeProperty = new SimpleBooleanProperty(false); - private BibDatabaseContext bibDatabaseContext; + private ObjectProperty bibDatabaseContext; private MainTableDataModel tableModel; private CitationStyleCache citationStyleCache; private FileAnnotationCache annotationCache; @@ -128,7 +130,7 @@ public LibraryTab(JabRefFrame frame, BibDatabaseContext bibDatabaseContext, ImportFormatReader importFormatReader) { this.frame = Objects.requireNonNull(frame); - this.bibDatabaseContext = Objects.requireNonNull(bibDatabaseContext); + this.bibDatabaseContext = new SimpleObjectProperty<>(Objects.requireNonNull(bibDatabaseContext)); this.undoManager = frame.getUndoManager(); this.dialogService = frame.getDialogService(); this.preferencesService = Objects.requireNonNull(preferencesService); @@ -152,9 +154,9 @@ public LibraryTab(JabRefFrame frame, this.getDatabase().registerListener(new EntriesRemovedListener()); // ensure that at each addition of a new entry, the entry is added to the groups interface - this.bibDatabaseContext.getDatabase().registerListener(new GroupTreeListener()); + this.bibDatabaseContext.get().getDatabase().registerListener(new GroupTreeListener()); // ensure that all entry changes mark the panel as changed - this.bibDatabaseContext.getDatabase().registerListener(this); + this.bibDatabaseContext.get().getDatabase().registerListener(this); this.getDatabase().registerListener(new UpdateTimestampListener(preferencesService)); @@ -166,7 +168,7 @@ public LibraryTab(JabRefFrame frame, Platform.runLater(() -> { EasyBind.subscribe(changedProperty, this::updateTabTitle); - stateManager.getOpenDatabases().addListener((ListChangeListener) c -> + stateManager.getOpenDatabases().addListener((ListChangeListener>) c -> updateTabTitle(changedProperty.getValue())); }); } @@ -220,8 +222,8 @@ public void onDatabaseLoadingSucceed(ParserResult result) { feedData(context); try { - indexingTaskManager.manageFulltextIndexAccordingToPrefs(LuceneIndexer.of(bibDatabaseContext, preferencesService, preferencesService.getFilePreferences())); - indexingTaskManager.updateIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService, preferencesService.getFilePreferences())); + indexingTaskManager.manageFulltextIndexAccordingToPrefs(LuceneIndexer.of(bibDatabaseContext.get(), preferencesService, preferencesService.getFilePreferences())); + indexingTaskManager.updateIndex(LuceneIndexer.of(bibDatabaseContext.get(), preferencesService, preferencesService.getFilePreferences())); } catch (IOException e) { LOGGER.error("Cannot access lucene index", e); } @@ -245,9 +247,7 @@ public void onDatabaseLoadingFailed(Exception ex) { public void feedData(BibDatabaseContext bibDatabaseContext) { cleanUp(); - stateManager.getOpenDatabases().remove(this.bibDatabaseContext); - this.bibDatabaseContext = Objects.requireNonNull(bibDatabaseContext); - stateManager.getOpenDatabases().add(this.bibDatabaseContext); + this.bibDatabaseContext.setValue(Objects.requireNonNull(bibDatabaseContext)); bibDatabaseContext.getDatabase().registerListener(this); bibDatabaseContext.getMetaData().registerListener(this); @@ -263,9 +263,9 @@ public void feedData(BibDatabaseContext bibDatabaseContext) { this.getDatabase().registerListener(new EntriesRemovedListener()); // ensure that at each addition of a new entry, the entry is added to the groups interface - this.bibDatabaseContext.getDatabase().registerListener(new GroupTreeListener()); + this.bibDatabaseContext.get().getDatabase().registerListener(new GroupTreeListener()); // ensure that all entry changes mark the panel as changed - this.bibDatabaseContext.getDatabase().registerListener(this); + this.bibDatabaseContext.get().getDatabase().registerListener(this); this.getDatabase().registerListener(new UpdateTimestampListener(preferencesService)); @@ -273,7 +273,7 @@ public void feedData(BibDatabaseContext bibDatabaseContext) { Platform.runLater(() -> { EasyBind.subscribe(changedProperty, this::updateTabTitle); - stateManager.getOpenDatabases().addListener((ListChangeListener) c -> + stateManager.getOpenDatabases().addListener((ListChangeListener>) c -> updateTabTitle(changedProperty.getValue())); }); @@ -281,12 +281,12 @@ public void feedData(BibDatabaseContext bibDatabaseContext) { } public void installAutosaveManagerAndBackupManager() { - if (isDatabaseReadyForAutoSave(bibDatabaseContext)) { - AutosaveManager autosaveManager = AutosaveManager.start(bibDatabaseContext); + if (isDatabaseReadyForAutoSave(bibDatabaseContext.get())) { + AutosaveManager autosaveManager = AutosaveManager.start(bibDatabaseContext.get()); autosaveManager.registerListener(new AutosaveUiManager(this)); } - if (isDatabaseReadyForBackup(bibDatabaseContext)) { - BackupManager.start(bibDatabaseContext, Globals.entryTypesManager, preferencesService); + if (isDatabaseReadyForBackup(bibDatabaseContext.get())) { + BackupManager.start(bibDatabaseContext.get(), Globals.entryTypesManager, preferencesService); } } @@ -311,8 +311,8 @@ private boolean isDatabaseReadyForBackup(BibDatabaseContext context) { public void updateTabTitle(boolean isChanged) { boolean isAutosaveEnabled = preferencesService.getImportExportPreferences().shouldAutoSave(); - DatabaseLocation databaseLocation = bibDatabaseContext.getLocation(); - Optional file = bibDatabaseContext.getDatabasePath(); + DatabaseLocation databaseLocation = bibDatabaseContext.get().getLocation(); + Optional file = bibDatabaseContext.get().getDatabasePath(); StringBuilder tabTitle = new StringBuilder(); StringBuilder toolTipText = new StringBuilder(); @@ -331,13 +331,13 @@ public void updateTabTitle(boolean isChanged) { if (databaseLocation == DatabaseLocation.SHARED) { tabTitle.append(" \u2013 "); - addSharedDbInformation(tabTitle, bibDatabaseContext); + addSharedDbInformation(tabTitle, bibDatabaseContext.get()); toolTipText.append(' '); - addSharedDbInformation(toolTipText, bibDatabaseContext); + addSharedDbInformation(toolTipText, bibDatabaseContext.get()); } // Database mode - addModeInfo(toolTipText, bibDatabaseContext); + addModeInfo(toolTipText, bibDatabaseContext.get()); // Changed information (tooltip) if (isChanged && !isAutosaveEnabled) { @@ -350,18 +350,18 @@ public void updateTabTitle(boolean isChanged) { } else { if (databaseLocation == DatabaseLocation.LOCAL) { tabTitle.append(Localization.lang("untitled")); - if (bibDatabaseContext.getDatabase().hasEntries()) { + if (bibDatabaseContext.get().getDatabase().hasEntries()) { // if the database is not empty and no file is assigned, // the database came from an import and has to be treated somehow // -> mark as changed tabTitle.append('*'); } } else { - addSharedDbInformation(tabTitle, bibDatabaseContext); - addSharedDbInformation(toolTipText, bibDatabaseContext); + addSharedDbInformation(tabTitle, bibDatabaseContext.get()); + addSharedDbInformation(toolTipText, bibDatabaseContext.get()); } - addModeInfo(toolTipText, bibDatabaseContext); - if ((databaseLocation == DatabaseLocation.LOCAL) && bibDatabaseContext.getDatabase().hasEntries()) { + addModeInfo(toolTipText, bibDatabaseContext.get()); + if ((databaseLocation == DatabaseLocation.LOCAL) && bibDatabaseContext.get().getDatabase().hasEntries()) { addChangedInformation(toolTipText, Localization.lang("untitled")); } } @@ -377,6 +377,7 @@ public void updateTabTitle(boolean isChanged) { private List collectAllDatabasePaths() { List list = new ArrayList<>(); stateManager.getOpenDatabases().stream() + .map(ObjectProperty::get) .map(BibDatabaseContext::getDatabasePath) .forEachOrdered(pathOptional -> pathOptional.ifPresentOrElse( path -> list.add(path.toAbsolutePath().toString()), @@ -430,8 +431,8 @@ private void delete(boolean cut, List entries) { return; } - getUndoManager().addEdit(new UndoableRemoveEntries(bibDatabaseContext.getDatabase(), entries, cut)); - bibDatabaseContext.getDatabase().removeEntries(entries); + getUndoManager().addEdit(new UndoableRemoveEntries(bibDatabaseContext.get().getDatabase(), entries, cut)); + bibDatabaseContext.get().getDatabase().removeEntries(entries); ensureNotShowingBottomPanel(entries); this.changedProperty.setValue(true); @@ -470,7 +471,7 @@ public void insertEntry(final BibEntry bibEntry) { public void insertEntries(final List entries) { if (!entries.isEmpty()) { - bibDatabaseContext.getDatabase().insertEntries(entries); + bibDatabaseContext.get().getDatabase().insertEntries(entries); // Set owner and timestamp for (BibEntry entry : entries) { @@ -481,7 +482,7 @@ public void insertEntries(final List entries) { preferencesService.getTimestampPreferences()); } // Create an UndoableInsertEntries object. - getUndoManager().addEdit(new UndoableInsertEntries(bibDatabaseContext.getDatabase(), entries)); + getUndoManager().addEdit(new UndoableInsertEntries(bibDatabaseContext.get().getDatabase(), entries)); this.changedProperty.setValue(true); // The database just changed. if (preferencesService.getEntryEditorPreferences().shouldOpenOnNewEntry()) { @@ -503,7 +504,7 @@ public void editEntryAndFocusField(BibEntry entry, Field field) { private void createMainTable() { mainTable = new MainTable(tableModel, this, - bibDatabaseContext, + bibDatabaseContext.get(), preferencesService, dialogService, stateManager, @@ -538,11 +539,11 @@ public void setupMainPanel() { .subscribeToValues(this::saveDividerLocation); // Add changePane in case a file is present - otherwise just add the splitPane to the panel - Optional file = bibDatabaseContext.getDatabasePath(); + Optional file = bibDatabaseContext.get().getDatabasePath(); if (file.isPresent()) { resetChangeMonitor(); } else { - if (bibDatabaseContext.getDatabase().hasEntries()) { + if (bibDatabaseContext.get().getDatabase().hasEntries()) { // if the database is not empty and no file is assigned, // the database came from an import and has to be treated somehow // -> mark as changed @@ -666,7 +667,7 @@ public synchronized void markChangedOrUnChanged() { } public BibDatabase getDatabase() { - return bibDatabaseContext.getDatabase(); + return bibDatabaseContext.get().getDatabase(); } private boolean showDeleteConfirmationDialog(int numberOfEntries) { @@ -707,8 +708,8 @@ private void saveDividerLocation(Number position) { */ public void cleanUp() { changeMonitor.ifPresent(DatabaseChangeMonitor::unregister); - AutosaveManager.shutdown(bibDatabaseContext); - BackupManager.shutdown(bibDatabaseContext); + AutosaveManager.shutdown(bibDatabaseContext.get()); + BackupManager.shutdown(bibDatabaseContext.get()); } /** @@ -721,6 +722,10 @@ public List getSelectedEntries() { } public BibDatabaseContext getBibDatabaseContext() { + return this.bibDatabaseContext.get(); + } + + public ObjectProperty getBibDatabaseContextProperty() { return this.bibDatabaseContext; } @@ -769,7 +774,7 @@ public FileAnnotationCache getAnnotationCache() { public void resetChangeMonitor() { changeMonitor.ifPresent(DatabaseChangeMonitor::unregister); - changeMonitor = Optional.of(new DatabaseChangeMonitor(bibDatabaseContext, + changeMonitor = Optional.of(new DatabaseChangeMonitor(bibDatabaseContext.get(), Globals.getFileUpdateMonitor(), Globals.TASK_EXECUTOR, dialogService, @@ -849,7 +854,7 @@ public void listen(EntriesAddedEvent addedEntriesEvent) { // Automatically add new entries to the selected group (or set of groups) if (preferencesService.getGroupsPreferences().shouldAutoAssignGroup()) { - stateManager.getSelectedGroups(bibDatabaseContext).forEach( + stateManager.getSelectedGroups(bibDatabaseContext.get()).forEach( selectedGroup -> selectedGroup.addEntriesToGroup(addedEntriesEvent.getBibEntries())); } } @@ -890,7 +895,7 @@ private class IndexUpdateListener { @Subscribe public void listen(EntriesAddedEvent addedEntryEvent) { try { - LuceneIndexer luceneIndexer = LuceneIndexer.of(bibDatabaseContext, preferencesService, preferencesService.getFilePreferences()); + LuceneIndexer luceneIndexer = LuceneIndexer.of(bibDatabaseContext.get(), preferencesService, preferencesService.getFilePreferences()); for (BibEntry addedEntry : addedEntryEvent.getBibEntries()) { indexingTaskManager.addToIndex(luceneIndexer, addedEntry); } @@ -902,7 +907,7 @@ public void listen(EntriesAddedEvent addedEntryEvent) { @Subscribe public void listen(EntriesRemovedEvent removedEntriesEvent) { try { - LuceneIndexer luceneIndexer = LuceneIndexer.of(bibDatabaseContext, preferencesService, preferencesService.getFilePreferences()); + LuceneIndexer luceneIndexer = LuceneIndexer.of(bibDatabaseContext.get(), preferencesService, preferencesService.getFilePreferences()); for (BibEntry removedEntry : removedEntriesEvent.getBibEntries()) { indexingTaskManager.removeFromIndex(luceneIndexer, removedEntry); } @@ -921,7 +926,7 @@ public void listen(FieldChangedEvent fieldChangedEvent) { List newFileList = FileFieldParser.parse(fieldChangedEvent.getNewValue()); removedFiles.remove(newFileList); } - indexingTaskManager.updateIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService, preferencesService.getFilePreferences()), bibEntry, removedFiles); + indexingTaskManager.updateIndex(LuceneIndexer.of(bibDatabaseContext.get(), preferencesService, preferencesService.getFilePreferences()), bibEntry, removedFiles); } catch (IOException e) { LOGGER.warn("I/O error when writing lucene index", e); } diff --git a/src/main/java/org/jabref/gui/StateManager.java b/src/main/java/org/jabref/gui/StateManager.java index eb4dd664b42..540d90eca5f 100644 --- a/src/main/java/org/jabref/gui/StateManager.java +++ b/src/main/java/org/jabref/gui/StateManager.java @@ -49,7 +49,7 @@ public class StateManager { private final CustomLocalDragboard localDragboard = new CustomLocalDragboard(); - private final ObservableList openDatabases = FXCollections.observableArrayList(); + private final ObservableList> openDatabases = FXCollections.observableArrayList(); private final OptionalObjectProperty activeDatabase = OptionalObjectProperty.empty(); private final ReadOnlyListWrapper activeGroups = new ReadOnlyListWrapper<>(FXCollections.observableArrayList()); private final ObservableList selectedEntries = FXCollections.observableArrayList(); @@ -78,7 +78,7 @@ public CustomLocalDragboard getLocalDragboard() { return localDragboard; } - public ObservableList getOpenDatabases() { + public ObservableList> getOpenDatabases() { return openDatabases; } diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index d7b3b9f8f92..51003e477bf 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -74,8 +74,8 @@ private void doSearch(Optional query) { if (query.isPresent()) { try { stateManager.getSearchResults().clear(); - for (BibDatabaseContext context : stateManager.getOpenDatabases()) { - stateManager.getSearchResults().putAll(LuceneSearcher.of(context).search(query.get())); + for (ObjectProperty context : stateManager.getOpenDatabases()) { + stateManager.getSearchResults().putAll(LuceneSearcher.of(context.get()).search(query.get())); } } catch (IOException e) { e.printStackTrace(); diff --git a/src/main/java/org/jabref/gui/openoffice/OpenOfficePanel.java b/src/main/java/org/jabref/gui/openoffice/OpenOfficePanel.java index 298c0149309..88111890826 100644 --- a/src/main/java/org/jabref/gui/openoffice/OpenOfficePanel.java +++ b/src/main/java/org/jabref/gui/openoffice/OpenOfficePanel.java @@ -8,6 +8,7 @@ import javax.swing.undo.UndoManager; +import javafx.beans.property.ObjectProperty; import javafx.concurrent.Task; import javafx.geometry.Insets; import javafx.geometry.Side; @@ -266,8 +267,8 @@ private void exportEntries() { private List getBaseList() { List databases = new ArrayList<>(); if (openOfficePreferences.getUseAllDatabases()) { - for (BibDatabaseContext database : stateManager.getOpenDatabases()) { - databases.add(database.getDatabase()); + for (ObjectProperty database : stateManager.getOpenDatabases()) { + databases.add(database.get().getDatabase()); } } else { databases.add(stateManager.getActiveDatabase() diff --git a/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java b/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java index e95a8ff03fe..5b225ad8bf4 100644 --- a/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java +++ b/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java @@ -34,9 +34,9 @@ public SearchResultsTableDataModel(BibDatabaseContext bibDatabaseContext, Prefer this.fieldValueFormatter = new SimpleObjectProperty<>(new MainTableFieldValueFormatter(preferencesService, bibDatabaseContext)); ObservableList entriesViewModel = FXCollections.observableArrayList(); - for (BibDatabaseContext context : stateManager.getOpenDatabases()) { - ObservableList entriesForDb = context.getDatabase().getEntries(); - List viewModelForDb = EasyBind.mapBacked(entriesForDb, entry -> new BibEntryTableViewModel(entry, context, fieldValueFormatter, stateManager)); + for (ObjectProperty context : stateManager.getOpenDatabases()) { + ObservableList entriesForDb = context.get().getDatabase().getEntries(); + List viewModelForDb = EasyBind.mapBacked(entriesForDb, entry -> new BibEntryTableViewModel(entry, context.get(), fieldValueFormatter, stateManager)); entriesViewModel.addAll(viewModelForDb); } From 0adcad8ee8f2cd791ebbbe6144946fc7f8893056 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Sun, 16 Oct 2022 09:29:27 +0200 Subject: [PATCH 059/256] Cleanup (remove search rules) Breaks some functionality, marked with TODO. Needs to be fixed before merging. --- .../org/jabref/benchmarks/Benchmarks.java | 1 - .../org/jabref/gui/entryeditor/SourceTab.java | 3 +- .../FulltextSearchResultsTab.java | 6 +- .../jabref/gui/groups/GroupDialogView.java | 5 +- .../org/jabref/gui/maintable/MainTable.java | 6 +- .../org/jabref/gui/preview/PreviewViewer.java | 2 +- .../jabref/gui/search/GlobalSearchBar.java | 25 +- .../search/SearchResultsTableDataModel.java | 7 - ...tainsAndRegexBasedSearchRuleDescriber.java | 47 ---- .../GrammarBasedSearchRuleDescriber.java | 126 --------- .../rules/describer/SearchDescriber.java | 9 - .../rules/describer/SearchDescribers.java | 30 -- .../logic/exporter/GroupSerializer.java | 4 +- .../jabref/logic/search/DatabaseSearcher.java | 5 +- .../org/jabref/logic/search/SearchQuery.java | 181 +++--------- .../search/retrieval/LuceneSearcher.java | 34 +-- .../org/jabref/model/groups/SearchGroup.java | 3 +- .../jabref/model/search/GroupSearchQuery.java | 6 - .../search/rules/ContainsBasedSearchRule.java | 48 ---- .../search/rules/FullTextSearchRule.java | 41 --- .../search/rules/GrammarBasedSearchRule.java | 261 ------------------ .../search/rules/RegexBasedSearchRule.java | 60 ---- .../jabref/model/search/rules/SearchRule.java | 10 - .../model/search/rules/SearchRules.java | 38 --- .../model/search/rules/SentenceAnalyzer.java | 57 ---- .../jabref/preferences/JabRefPreferences.java | 14 +- .../jabref/preferences/SearchPreferences.java | 36 +-- ...sAndRegexBasedSearchRuleDescriberTest.java | 3 - .../GrammarBasedSearchRuleDescriberTest.java | 4 - .../logic/exporter/GroupSerializerTest.java | 1 - .../logic/importer/util/GroupsParserTest.java | 1 - .../logic/search/DatabaseSearcherTest.java | 1 - .../jabref/logic/search/SearchQueryTest.java | 2 - .../model/groups/GroupTreeNodeTest.java | 2 - .../jabref/model/groups/SearchGroupTest.java | 1 - 35 files changed, 92 insertions(+), 988 deletions(-) delete mode 100644 src/main/java/org/jabref/gui/search/rules/describer/ContainsAndRegexBasedSearchRuleDescriber.java delete mode 100644 src/main/java/org/jabref/gui/search/rules/describer/GrammarBasedSearchRuleDescriber.java delete mode 100644 src/main/java/org/jabref/gui/search/rules/describer/SearchDescriber.java delete mode 100644 src/main/java/org/jabref/gui/search/rules/describer/SearchDescribers.java delete mode 100644 src/main/java/org/jabref/model/search/rules/ContainsBasedSearchRule.java delete mode 100644 src/main/java/org/jabref/model/search/rules/FullTextSearchRule.java delete mode 100644 src/main/java/org/jabref/model/search/rules/GrammarBasedSearchRule.java delete mode 100644 src/main/java/org/jabref/model/search/rules/RegexBasedSearchRule.java delete mode 100644 src/main/java/org/jabref/model/search/rules/SearchRule.java delete mode 100644 src/main/java/org/jabref/model/search/rules/SentenceAnalyzer.java diff --git a/src/jmh/java/org/jabref/benchmarks/Benchmarks.java b/src/jmh/java/org/jabref/benchmarks/Benchmarks.java index b0073114ac5..49e8d32fe56 100644 --- a/src/jmh/java/org/jabref/benchmarks/Benchmarks.java +++ b/src/jmh/java/org/jabref/benchmarks/Benchmarks.java @@ -31,7 +31,6 @@ import org.jabref.model.groups.KeywordGroup; import org.jabref.model.groups.WordKeywordGroup; import org.jabref.model.metadata.MetaData; -import org.jabref.model.search.rules.SearchRules.SearchFlags; import org.jabref.model.util.DummyFileUpdateMonitor; import org.jabref.preferences.GeneralPreferences; import org.jabref.preferences.JabRefPreferences; diff --git a/src/main/java/org/jabref/gui/entryeditor/SourceTab.java b/src/main/java/org/jabref/gui/entryeditor/SourceTab.java index 1e3de019011..e2281c08194 100644 --- a/src/main/java/org/jabref/gui/entryeditor/SourceTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/SourceTab.java @@ -43,7 +43,6 @@ import org.jabref.logic.importer.ParserResult; import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.search.SearchQuery; import org.jabref.logic.util.OS; import org.jabref.model.database.BibDatabase; import org.jabref.model.database.BibDatabaseContext; @@ -112,7 +111,7 @@ public SourceTab(BibDatabaseContext bibDatabaseContext, CountingUndoManager undo this.keyBindingRepository = keyBindingRepository; stateManager.activeSearchQueryProperty().addListener((observable, oldValue, newValue) -> { - searchHighlightPattern = newValue.flatMap(SearchQuery::getPatternForWords); + // searchHighlightPattern = newValue.flatMap(SearchQuery::getPatternForWords); TODO btut: Pattern-Highlighting with lucene highlightSearchPattern(); }); } diff --git a/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java b/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java index acadf39a1dd..23ab2de139e 100644 --- a/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java @@ -30,7 +30,7 @@ import org.jabref.model.entry.LinkedFile; import org.jabref.model.pdf.search.LuceneSearchResults; import org.jabref.model.pdf.search.SearchResult; -import org.jabref.model.search.rules.SearchRules; +import org.jabref.model.search.rules.SearchRules.SearchFlags; import org.jabref.preferences.PreferencesService; import org.slf4j.Logger; @@ -70,8 +70,8 @@ public FulltextSearchResultsTab(StateManager stateManager, PreferencesService pr public boolean shouldShow(BibEntry entry) { return this.stateManager.activeSearchQueryProperty().isPresent().get() && this.stateManager.activeSearchQueryProperty().get().isPresent() && - this.stateManager.activeSearchQueryProperty().get().get().getSearchFlags().contains(SearchRules.SearchFlags.FULLTEXT) && - this.stateManager.activeSearchQueryProperty().get().get().getQuery().length() > 0; + this.stateManager.activeSearchQueryProperty().get().get().getSearchFlags().contains(SearchFlags.FULLTEXT) && + this.stateManager.activeSearchQueryProperty().get().get().toString().length() > 0; } @Override diff --git a/src/main/java/org/jabref/gui/groups/GroupDialogView.java b/src/main/java/org/jabref/gui/groups/GroupDialogView.java index 9ea122616af..138559dd899 100644 --- a/src/main/java/org/jabref/gui/groups/GroupDialogView.java +++ b/src/main/java/org/jabref/gui/groups/GroupDialogView.java @@ -36,7 +36,6 @@ import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.groups.AbstractGroup; import org.jabref.model.groups.GroupHierarchyType; -import org.jabref.model.search.rules.SearchRules; import org.jabref.model.search.rules.SearchRules.SearchFlags; import org.jabref.preferences.PreferencesService; @@ -156,9 +155,9 @@ public void initialize() { searchGroupRegex.selectedProperty().addListener((observable, oldValue, newValue) -> { EnumSet searchFlags = viewModel.searchFlagsProperty().get(); if (newValue) { - searchFlags.add(SearchRules.SearchFlags.REGULAR_EXPRESSION); + searchFlags.add(SearchFlags.REGULAR_EXPRESSION); } else { - searchFlags.remove(SearchRules.SearchFlags.REGULAR_EXPRESSION); + searchFlags.remove(SearchFlags.REGULAR_EXPRESSION); } viewModel.searchFlagsProperty().set(searchFlags); }); diff --git a/src/main/java/org/jabref/gui/maintable/MainTable.java b/src/main/java/org/jabref/gui/maintable/MainTable.java index d9795a52af8..4e32d698eb2 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTable.java +++ b/src/main/java/org/jabref/gui/maintable/MainTable.java @@ -53,7 +53,7 @@ import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.database.event.EntriesAddedEvent; import org.jabref.model.entry.BibEntry; -import org.jabref.model.search.rules.SearchRules; +import org.jabref.model.search.rules.SearchRules.SearchFlags; import org.jabref.preferences.PreferencesService; import com.google.common.eventbus.Subscribe; @@ -187,9 +187,9 @@ public void onChanged(Change> c } }; - preferencesService.getSearchPreferences().getObservableSearchFlags().addListener(new SetChangeListener() { + preferencesService.getSearchPreferences().getObservableSearchFlags().addListener(new SetChangeListener() { @Override - public void onChanged(Change change) { + public void onChanged(Change change) { getSortOrder().removeListener(scoreSortOderPrioritizer); updateSortOrder(); getSortOrder().addListener(scoreSortOderPrioritizer); diff --git a/src/main/java/org/jabref/gui/preview/PreviewViewer.java b/src/main/java/org/jabref/gui/preview/PreviewViewer.java index 78a7d47d2ee..65ad6bc4e08 100644 --- a/src/main/java/org/jabref/gui/preview/PreviewViewer.java +++ b/src/main/java/org/jabref/gui/preview/PreviewViewer.java @@ -134,7 +134,7 @@ function getSelectionHtml() { private boolean registered; private final ChangeListener> listener = (queryObservable, queryOldValue, queryNewValue) -> { - searchHighlightPattern = queryNewValue.flatMap(SearchQuery::getJavaScriptPatternForWords); + // searchHighlightPattern = queryNewValue.flatMap(SearchQuery::getJavaScriptPatternForWords); TODO btut: Pattern-Highlighting with lucene highlightSearchPattern(); }; diff --git a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java index d415090194f..57939608e6e 100644 --- a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java +++ b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java @@ -1,6 +1,7 @@ package org.jabref.gui.search; import java.lang.reflect.Field; +import java.util.EnumSet; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -52,7 +53,6 @@ import org.jabref.gui.icon.IconTheme; import org.jabref.gui.keyboard.KeyBinding; import org.jabref.gui.keyboard.KeyBindingRepository; -import org.jabref.gui.search.rules.describer.SearchDescribers; import org.jabref.gui.undo.CountingUndoManager; import org.jabref.gui.util.BindingsHelper; import org.jabref.gui.util.DefaultTaskExecutor; @@ -72,6 +72,7 @@ import de.saxsys.mvvmfx.utils.validation.Validator; import de.saxsys.mvvmfx.utils.validation.visualization.ControlsFxVisualizer; import impl.org.controlsfx.skin.AutoCompletePopup; +import org.apache.lucene.search.Query; import org.controlsfx.control.textfield.AutoCompletionBinding; import org.controlsfx.control.textfield.CustomTextField; import org.reactfx.util.FxTimer; @@ -199,15 +200,14 @@ public GlobalSearchBar(JabRefFrame frame, StateManager stateManager, Preferences // Async update searchTask.restart(); }, - query -> setSearchTerm(query.map(SearchQuery::getQuery).orElse(""))); + query -> setSearchTerm(query.map(SearchQuery::getQuery).orElse(new SearchQuery("", EnumSet.noneOf(SearchRules.SearchFlags.class)).getQuery()))); this.stateManager.activeSearchQueryProperty().addListener((obs, oldValue, newValue) -> newValue.ifPresent(this::updateSearchResultsForQuery)); this.stateManager.getSearchResults().addListener((MapChangeListener) map -> stateManager.activeSearchQueryProperty().get().ifPresent(this::updateSearchResultsForQuery)); } private void updateSearchResultsForQuery(SearchQuery query) { - updateResults(this.stateManager.getSearchResults().size(), SearchDescribers.getSearchDescriberFor(query).getDescription(), - query.isGrammarBasedSearch()); + updateResults(this.stateManager.getSearchResults().size()); } private void initSearchModifierButtons() { @@ -356,7 +356,7 @@ private AutoCompletePopup getPopup(AutoCompletionBinding autoCompletio } } - private void updateResults(int matched, TextFlow description, boolean grammarBasedSearch) { + private void updateResults(int matched) { if (matched == 0) { currentResults.setText(Localization.lang("No results found.")); searchField.pseudoClassStateChanged(CLASS_NO_RESULTS, true); @@ -364,15 +364,8 @@ private void updateResults(int matched, TextFlow description, boolean grammarBas currentResults.setText(Localization.lang("Found %0 results.", String.valueOf(matched))); searchField.pseudoClassStateChanged(CLASS_RESULTS_FOUND, true); } - if (grammarBasedSearch) { - // TODO: switch Icon color - // searchIcon.setIcon(IconTheme.JabRefIcon.ADVANCED_SEARCH.getIcon()); - } else { - // TODO: switch Icon color - // searchIcon.setIcon(IconTheme.JabRefIcon.SEARCH.getIcon()); - } - setSearchFieldHintTooltip(description); + // setSearchFieldHintTooltip(description); TODO btut: Search-tooltip for lucene } private void setSearchFieldHintTooltip(TextFlow description) { @@ -396,12 +389,12 @@ public void updateHintVisibility() { setSearchFieldHintTooltip(null); } - public void setSearchTerm(String searchTerm) { - if (searchTerm.equals(searchField.getText())) { + public void setSearchTerm(Query searchQuery) { + if (searchQuery.toString().equals(searchField.getText())) { return; } - DefaultTaskExecutor.runInJavaFXThread(() -> searchField.setText(searchTerm)); + DefaultTaskExecutor.runInJavaFXThread(() -> searchField.setText(searchQuery.toString())); } private static class SearchPopupSkin implements Skin> { diff --git a/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java b/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java index 5b225ad8bf4..3eae8674b4b 100644 --- a/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java +++ b/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java @@ -1,7 +1,6 @@ package org.jabref.gui.search; import java.util.List; -import java.util.Optional; import javafx.beans.property.ObjectProperty; import javafx.beans.property.SimpleObjectProperty; @@ -13,7 +12,6 @@ import org.jabref.gui.StateManager; import org.jabref.gui.maintable.BibEntryTableViewModel; import org.jabref.gui.maintable.MainTableFieldValueFormatter; -import org.jabref.logic.search.SearchQuery; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.preferences.PreferencesService; @@ -47,11 +45,6 @@ public SearchResultsTableDataModel(BibDatabaseContext bibDatabaseContext, Prefer entriesSorted = new SortedList<>(entriesFiltered); } - private boolean isMatchedBySearch(Optional query, BibEntryTableViewModel entry) { - return query.map(matcher -> matcher.isMatch(entry.getEntry())) - .orElse(true); - } - public SortedList getEntriesFilteredAndSorted() { return entriesSorted; } diff --git a/src/main/java/org/jabref/gui/search/rules/describer/ContainsAndRegexBasedSearchRuleDescriber.java b/src/main/java/org/jabref/gui/search/rules/describer/ContainsAndRegexBasedSearchRuleDescriber.java deleted file mode 100644 index 00ff67b8387..00000000000 --- a/src/main/java/org/jabref/gui/search/rules/describer/ContainsAndRegexBasedSearchRuleDescriber.java +++ /dev/null @@ -1,47 +0,0 @@ -package org.jabref.gui.search.rules.describer; - -import java.util.EnumSet; -import java.util.List; - -import javafx.scene.text.Text; -import javafx.scene.text.TextFlow; - -import org.jabref.gui.util.TooltipTextUtil; -import org.jabref.logic.l10n.Localization; -import org.jabref.model.search.rules.SearchRules; -import org.jabref.model.search.rules.SearchRules.SearchFlags; -import org.jabref.model.search.rules.SentenceAnalyzer; - -public class ContainsAndRegexBasedSearchRuleDescriber implements SearchDescriber { - - private final EnumSet searchFlags; - private final String query; - - public ContainsAndRegexBasedSearchRuleDescriber(EnumSet searchFlags, String query) { - this.searchFlags = searchFlags; - this.query = query; - } - - @Override - public TextFlow getDescription() { - List words = new SentenceAnalyzer(query).getWords(); - String firstWord = words.isEmpty() ? "" : words.get(0); - - String temp = searchFlags.contains(SearchRules.SearchFlags.REGULAR_EXPRESSION) ? Localization.lang( - "This search contains entries in which any field contains the regular expression %0") - : Localization.lang("This search contains entries in which any field contains the term %0"); - List textList = TooltipTextUtil.formatToTexts(temp, new TooltipTextUtil.TextReplacement("%0", firstWord, TooltipTextUtil.TextType.BOLD)); - - if (words.size() > 1) { - List unprocessedWords = words.subList(1, words.size()); - for (String word : unprocessedWords) { - textList.add(TooltipTextUtil.createText(String.format(" %s ", Localization.lang("and")), TooltipTextUtil.TextType.NORMAL)); - textList.add(TooltipTextUtil.createText(word, TooltipTextUtil.TextType.BOLD)); - } - } - - TextFlow searchDescription = new TextFlow(); - searchDescription.getChildren().setAll(textList); - return searchDescription; - } -} diff --git a/src/main/java/org/jabref/gui/search/rules/describer/GrammarBasedSearchRuleDescriber.java b/src/main/java/org/jabref/gui/search/rules/describer/GrammarBasedSearchRuleDescriber.java deleted file mode 100644 index e49c5702101..00000000000 --- a/src/main/java/org/jabref/gui/search/rules/describer/GrammarBasedSearchRuleDescriber.java +++ /dev/null @@ -1,126 +0,0 @@ -package org.jabref.gui.search.rules.describer; - -import java.util.ArrayList; -import java.util.EnumSet; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.regex.Pattern; - -import javafx.scene.text.Text; -import javafx.scene.text.TextFlow; - -import org.jabref.gui.util.TooltipTextUtil; -import org.jabref.logic.l10n.Localization; -import org.jabref.model.search.rules.GrammarBasedSearchRule; -import org.jabref.model.search.rules.SearchRules; -import org.jabref.model.search.rules.SearchRules.SearchFlags; -import org.jabref.model.strings.StringUtil; -import org.jabref.search.SearchBaseVisitor; -import org.jabref.search.SearchParser; - -import org.antlr.v4.runtime.tree.ParseTree; - -public class GrammarBasedSearchRuleDescriber implements SearchDescriber { - - private final EnumSet searchFlags; - private final ParseTree parseTree; - - public GrammarBasedSearchRuleDescriber(EnumSet searchFlags, ParseTree parseTree) { - this.searchFlags = searchFlags; - this.parseTree = Objects.requireNonNull(parseTree); - } - - @Override - public TextFlow getDescription() { - TextFlow textFlow = new TextFlow(); - DescriptionSearchBaseVisitor descriptionSearchBaseVisitor = new DescriptionSearchBaseVisitor(); - - // describe advanced search expression - textFlow.getChildren().add(TooltipTextUtil.createText(String.format("%s ", Localization.lang("This search contains entries in which")), TooltipTextUtil.TextType.NORMAL)); - textFlow.getChildren().addAll(descriptionSearchBaseVisitor.visit(parseTree)); - textFlow.getChildren().add(TooltipTextUtil.createText(". ", TooltipTextUtil.TextType.NORMAL)); - return textFlow; - } - - private class DescriptionSearchBaseVisitor extends SearchBaseVisitor> { - - @Override - public List visitStart(SearchParser.StartContext context) { - return visit(context.expression()); - } - - @Override - public List visitUnaryExpression(SearchParser.UnaryExpressionContext context) { - List textList = visit(context.expression()); - textList.add(0, TooltipTextUtil.createText(Localization.lang("not").concat(" "), TooltipTextUtil.TextType.NORMAL)); - return textList; - } - - @Override - public List visitParenExpression(SearchParser.ParenExpressionContext context) { - ArrayList textList = new ArrayList<>(); - textList.add(TooltipTextUtil.createText(String.format("%s", context.expression()), TooltipTextUtil.TextType.NORMAL)); - return textList; - } - - @Override - public List visitBinaryExpression(SearchParser.BinaryExpressionContext context) { - List textList = visit(context.left); - if ("AND".equalsIgnoreCase(context.operator.getText())) { - textList.add(TooltipTextUtil.createText(String.format(" %s ", Localization.lang("and")), TooltipTextUtil.TextType.NORMAL)); - } else { - textList.add(TooltipTextUtil.createText(String.format(" %s ", Localization.lang("or")), TooltipTextUtil.TextType.NORMAL)); - } - textList.addAll(visit(context.right)); - return textList; - } - - @Override - public List visitComparison(SearchParser.ComparisonContext context) { - final List textList = new ArrayList<>(); - final Optional fieldDescriptor = Optional.ofNullable(context.left); - final String value = StringUtil.unquote(context.right.getText(), '"'); - if (!fieldDescriptor.isPresent()) { - TextFlow description = new ContainsAndRegexBasedSearchRuleDescriber(searchFlags, value).getDescription(); - description.getChildren().forEach(it -> textList.add((Text) it)); - return textList; - } - - final String field = StringUtil.unquote(fieldDescriptor.get().getText(), '"'); - final GrammarBasedSearchRule.ComparisonOperator operator = GrammarBasedSearchRule.ComparisonOperator.build(context.operator.getText()); - - final boolean regExpFieldSpec = !Pattern.matches("\\w+", field); - String temp = regExpFieldSpec ? Localization.lang( - "any field that matches the regular expression %0") : Localization.lang("the field %0"); - - if (operator == GrammarBasedSearchRule.ComparisonOperator.CONTAINS) { - if (searchFlags.contains(SearchRules.SearchFlags.REGULAR_EXPRESSION)) { - temp = Localization.lang("%0 contains the regular expression %1", temp); - } else { - temp = Localization.lang("%0 contains the term %1", temp); - } - } else if (operator == GrammarBasedSearchRule.ComparisonOperator.EXACT) { - if (searchFlags.contains(SearchRules.SearchFlags.REGULAR_EXPRESSION)) { - temp = Localization.lang("%0 matches the regular expression %1", temp); - } else { - temp = Localization.lang("%0 matches the term %1", temp); - } - } else if (operator == GrammarBasedSearchRule.ComparisonOperator.DOES_NOT_CONTAIN) { - if (searchFlags.contains(SearchRules.SearchFlags.REGULAR_EXPRESSION)) { - temp = Localization.lang("%0 doesn't contain the regular expression %1", temp); - } else { - temp = Localization.lang("%0 doesn't contain the term %1", temp); - } - } else { - throw new IllegalStateException("CANNOT HAPPEN!"); - } - - List formattedTexts = TooltipTextUtil.formatToTexts(temp, - new TooltipTextUtil.TextReplacement("%0", field, TooltipTextUtil.TextType.BOLD), - new TooltipTextUtil.TextReplacement("%1", value, TooltipTextUtil.TextType.BOLD)); - textList.addAll(formattedTexts); - return textList; - } - } -} diff --git a/src/main/java/org/jabref/gui/search/rules/describer/SearchDescriber.java b/src/main/java/org/jabref/gui/search/rules/describer/SearchDescriber.java deleted file mode 100644 index d3a18ecfd60..00000000000 --- a/src/main/java/org/jabref/gui/search/rules/describer/SearchDescriber.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.jabref.gui.search.rules.describer; - -import javafx.scene.text.TextFlow; - -@FunctionalInterface -public interface SearchDescriber { - - TextFlow getDescription(); -} diff --git a/src/main/java/org/jabref/gui/search/rules/describer/SearchDescribers.java b/src/main/java/org/jabref/gui/search/rules/describer/SearchDescribers.java deleted file mode 100644 index de696f03632..00000000000 --- a/src/main/java/org/jabref/gui/search/rules/describer/SearchDescribers.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.jabref.gui.search.rules.describer; - -import org.jabref.logic.search.SearchQuery; -import org.jabref.model.search.rules.ContainsBasedSearchRule; -import org.jabref.model.search.rules.GrammarBasedSearchRule; -import org.jabref.model.search.rules.RegexBasedSearchRule; - -public class SearchDescribers { - - private SearchDescribers() { - } - - /** - * Get the search describer for a given search query. - * - * @param searchQuery the search query - * @return the search describer to turn the search into something human understandable - */ - public static SearchDescriber getSearchDescriberFor(SearchQuery searchQuery) { - if (searchQuery.getRule() instanceof GrammarBasedSearchRule grammarBasedSearchRule) { - return new GrammarBasedSearchRuleDescriber(grammarBasedSearchRule.getSearchFlags(), grammarBasedSearchRule.getTree()); - } else if (searchQuery.getRule() instanceof ContainsBasedSearchRule containBasedSearchRule) { - return new ContainsAndRegexBasedSearchRuleDescriber(containBasedSearchRule.getSearchFlags(), searchQuery.getQuery()); - } else if (searchQuery.getRule() instanceof RegexBasedSearchRule regexBasedSearchRule) { - return new ContainsAndRegexBasedSearchRuleDescriber(regexBasedSearchRule.getSearchFlags(), searchQuery.getQuery()); - } else { - throw new IllegalStateException("Cannot find a describer for searchRule " + searchQuery.getRule() + " and query " + searchQuery.getQuery()); - } - } -} diff --git a/src/main/java/org/jabref/logic/exporter/GroupSerializer.java b/src/main/java/org/jabref/logic/exporter/GroupSerializer.java index 05dd1ba04d9..01b915eafec 100644 --- a/src/main/java/org/jabref/logic/exporter/GroupSerializer.java +++ b/src/main/java/org/jabref/logic/exporter/GroupSerializer.java @@ -18,7 +18,7 @@ import org.jabref.model.groups.RegexKeywordGroup; import org.jabref.model.groups.SearchGroup; import org.jabref.model.groups.TexGroup; -import org.jabref.model.search.rules.SearchRules; +import org.jabref.model.search.rules.SearchRules.SearchFlags; import org.jabref.model.strings.StringUtil; public class GroupSerializer { @@ -72,7 +72,7 @@ private String serializeSearchGroup(SearchGroup group) { sb.append(MetadataSerializationConfiguration.GROUP_UNIT_SEPARATOR); sb.append(StringUtil.booleanToBinaryString(false)); sb.append(MetadataSerializationConfiguration.GROUP_UNIT_SEPARATOR); - sb.append(StringUtil.booleanToBinaryString(group.getSearchFlags().contains(SearchRules.SearchFlags.REGULAR_EXPRESSION))); + sb.append(StringUtil.booleanToBinaryString(group.getSearchFlags().contains(SearchFlags.REGULAR_EXPRESSION))); sb.append(MetadataSerializationConfiguration.GROUP_UNIT_SEPARATOR); appendGroupDetails(sb, group); diff --git a/src/main/java/org/jabref/logic/search/DatabaseSearcher.java b/src/main/java/org/jabref/logic/search/DatabaseSearcher.java index 422d8d4d35c..2cd968e89f9 100644 --- a/src/main/java/org/jabref/logic/search/DatabaseSearcher.java +++ b/src/main/java/org/jabref/logic/search/DatabaseSearcher.java @@ -3,7 +3,6 @@ import java.util.Collections; import java.util.List; import java.util.Objects; -import java.util.stream.Collectors; import org.jabref.model.database.BibDatabase; import org.jabref.model.database.BibDatabases; @@ -32,7 +31,9 @@ public List getMatches() { return Collections.emptyList(); } - List matchEntries = database.getEntries().stream().filter(query::isMatch).collect(Collectors.toList()); + List matchEntries = List.of(); + // List matchEntries = database.getEntries().stream().filter(query::isMatch).collect(Collectors.toList()); + // TODO btut: is this for CLI? We need the databasecontext to access the index return BibDatabases.purgeEmptyEntries(matchEntries); } } diff --git a/src/main/java/org/jabref/logic/search/SearchQuery.java b/src/main/java/org/jabref/logic/search/SearchQuery.java index d8112f15688..3c53dae4671 100644 --- a/src/main/java/org/jabref/logic/search/SearchQuery.java +++ b/src/main/java/org/jabref/logic/search/SearchQuery.java @@ -1,176 +1,73 @@ package org.jabref.logic.search; -import java.util.Collections; +import java.util.Arrays; import java.util.EnumSet; -import java.util.List; +import java.util.HashMap; import java.util.Objects; -import java.util.Optional; -import java.util.regex.Pattern; import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.jabref.logic.l10n.Localization; -import org.jabref.model.entry.BibEntry; -import org.jabref.model.search.SearchMatcher; -import org.jabref.model.search.rules.ContainsBasedSearchRule; -import org.jabref.model.search.rules.GrammarBasedSearchRule; -import org.jabref.model.search.rules.SearchRule; +import org.jabref.model.pdf.search.EnglishStemAnalyzer; +import org.jabref.model.pdf.search.SearchFieldConstants; import org.jabref.model.search.rules.SearchRules; -import org.jabref.model.search.rules.SentenceAnalyzer; -public class SearchQuery implements SearchMatcher { +import org.apache.lucene.queryparser.classic.MultiFieldQueryParser; +import org.apache.lucene.queryparser.classic.ParseException; +import org.apache.lucene.search.Query; - /** - * The mode of escaping special characters in regular expressions - */ - private enum EscapeMode { - /** - * using \Q and \E marks - */ - JAVA { - @Override - String format(String regex) { - return Pattern.quote(regex); - } - }, - /** - * escaping all javascript regex special characters separately - */ - JAVASCRIPT { - @Override - String format(String regex) { - return JAVASCRIPT_ESCAPED_CHARS_PATTERN.matcher(regex).replaceAll("\\\\$0"); - } - }; - - /** - * Regex pattern for escaping special characters in javascript regular expressions - */ - private static final Pattern JAVASCRIPT_ESCAPED_CHARS_PATTERN = Pattern.compile("[.*+?^${}()|\\[\\]\\\\/]"); - - /** - * Attempt to escape all regex special characters. - * - * @param regex a string containing a regex expression - * @return a regex with all special characters escaped - */ - abstract String format(String regex); - } +public class SearchQuery { protected final String query; + protected Query parsedQuery; + protected String parseError; protected EnumSet searchFlags; - protected final SearchRule rule; public SearchQuery(String query, EnumSet searchFlags) { this.query = Objects.requireNonNull(query); this.searchFlags = searchFlags; - this.rule = SearchRules.getSearchRuleByQuery(query, searchFlags); - } - @Override - public String toString() { - return String.format("\"%s\" (%s)", getQuery(), getRegularExpressionDescription()); - } - - @Override - public boolean isMatch(BibEntry entry) { - return rule.applyRule(getQuery(), entry); - } + HashMap boosts = new HashMap<>(); + SearchFieldConstants.searchableBibFields.forEach(field -> boosts.put(field, Float.valueOf(4))); - public boolean isValid() { - return rule.validateSearchStrings(getQuery()); - } - - public boolean isContainsBasedSearch() { - return rule instanceof ContainsBasedSearchRule; - } - - private String getRegularExpressionDescription() { - if (searchFlags.contains(SearchRules.SearchFlags.REGULAR_EXPRESSION)) { - return "regular expression"; - } else { - return "plain text"; + if (searchFlags.contains(SearchRules.SearchFlags.FULLTEXT)) { + Arrays.stream(SearchFieldConstants.PDF_FIELDS).forEach(field -> boosts.put(field, Float.valueOf(1))); } - } - - public String localize() { - return String.format("\"%s\" (%s)", - getQuery(), - getLocalizedRegularExpressionDescription()); - } + String[] fieldsToSearchArray = new String[boosts.size()]; + boosts.keySet().toArray(fieldsToSearchArray); - private String getLocalizedRegularExpressionDescription() { if (searchFlags.contains(SearchRules.SearchFlags.REGULAR_EXPRESSION)) { - return Localization.lang("regular expression"); - } else { - return Localization.lang("plain text"); + if (query.length() > 0 && !(query.startsWith("/") && query.endsWith("/"))) { + query = "/" + query + "/"; + } } - } - /** - * Tests if the query is an advanced search query described as described in the help - * - * @return true if the query is an advanced search query - */ - public boolean isGrammarBasedSearch() { - return rule instanceof GrammarBasedSearchRule; - } - - public String getQuery() { - return query; - } - - public EnumSet getSearchFlags() { - return searchFlags; - } - - /** - * Returns a list of words this query searches for. The returned strings can be a regular expression. - */ - public List getSearchWords() { - if (searchFlags.contains(SearchRules.SearchFlags.REGULAR_EXPRESSION)) { - return Collections.singletonList(getQuery()); - } else { - // Parses the search query for valid words and returns a list these words. - // For example, "The great Vikinger" will give ["The","great","Vikinger"] - return (new SentenceAnalyzer(getQuery())).getWords(); + MultiFieldQueryParser queryParser = new MultiFieldQueryParser(fieldsToSearchArray, new EnglishStemAnalyzer(), boosts); + queryParser.setAllowLeadingWildcard(true); + if (!query.contains("\"") && !query.contains(":") && !query.contains("*") && !query.contains("~")) { + query = Arrays.stream(query.split(" ")).map(s -> "*" + s + "*").collect(Collectors.joining(" ")); + } + try { + parsedQuery = queryParser.parse(query); + parseError = null; + } catch (ParseException e) { + parsedQuery = null; + parseError = e.getMessage(); } } - // Returns a regular expression pattern in the form (w1)|(w2)| ... wi are escaped if no regular expression search is enabled - public Optional getPatternForWords() { - return joinWordsToPattern(EscapeMode.JAVA); + @Override + public String toString() { + return query; } - // Returns a regular expression pattern in the form (w1)|(w2)| ... wi are escaped for javascript if no regular expression search is enabled - public Optional getJavaScriptPatternForWords() { - return joinWordsToPattern(EscapeMode.JAVASCRIPT); + public boolean isValid() { + return parseError == null; } - /** - * Returns a regular expression pattern in the form (w1)|(w2)| ... wi are escaped if no regular expression search is enabled - * - * @param escapeMode the mode of escaping special characters in wi - */ - private Optional joinWordsToPattern(EscapeMode escapeMode) { - List words = getSearchWords(); - - if ((words == null) || words.isEmpty() || words.get(0).isEmpty()) { - return Optional.empty(); - } - - // compile the words to a regular expression in the form (w1)|(w2)|(w3) - Stream joiner = words.stream(); - if (!searchFlags.contains(SearchRules.SearchFlags.REGULAR_EXPRESSION)) { - // Reformat string when we are looking for a literal match - joiner = joiner.map(escapeMode::format); - } - String searchPattern = joiner.collect(Collectors.joining(")|(", "(", ")")); - - return Optional.of(Pattern.compile(searchPattern, Pattern.CASE_INSENSITIVE)); + public Query getQuery() { + return parsedQuery; } - public SearchRule getRule() { - return rule; + public EnumSet getSearchFlags() { + return searchFlags; } } diff --git a/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java b/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java index 5f02a0ecf91..2fdb660b9de 100644 --- a/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java +++ b/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java @@ -1,26 +1,18 @@ package org.jabref.logic.search.retrieval; import java.io.IOException; -import java.util.Arrays; import java.util.HashMap; -import java.util.stream.Collectors; import org.jabref.gui.LibraryTab; import org.jabref.logic.search.SearchQuery; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; -import org.jabref.model.pdf.search.EnglishStemAnalyzer; import org.jabref.model.pdf.search.LuceneSearchResults; -import org.jabref.model.pdf.search.SearchFieldConstants; import org.jabref.model.pdf.search.SearchResult; -import org.jabref.model.search.rules.SearchRules; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.queryparser.classic.MultiFieldQueryParser; -import org.apache.lucene.queryparser.classic.ParseException; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; @@ -52,31 +44,11 @@ public static LuceneSearcher of(BibDatabaseContext databaseContext) throws IOExc */ public HashMap search(SearchQuery query) { HashMap results = new HashMap<>(); - HashMap boosts = new HashMap<>(); - SearchFieldConstants.searchableBibFields.forEach(field -> boosts.put(field, Float.valueOf(4))); - - if (query.getSearchFlags().contains(SearchRules.SearchFlags.FULLTEXT)) { - Arrays.stream(SearchFieldConstants.PDF_FIELDS).forEach(field -> boosts.put(field, Float.valueOf(1))); - } try (IndexReader reader = DirectoryReader.open(indexDirectory)) { IndexSearcher searcher = new IndexSearcher(reader); - String[] fieldsToSearchArray = new String[boosts.size()]; - boosts.keySet().toArray(fieldsToSearchArray); - String queryString = query.getQuery(); - if (query.getSearchFlags().contains(SearchRules.SearchFlags.REGULAR_EXPRESSION)) { - if (queryString.length() > 0 && !(queryString.startsWith("/") && queryString.endsWith("/"))) { - queryString = "/" + queryString + "/"; - } - } - MultiFieldQueryParser queryParser = new MultiFieldQueryParser(fieldsToSearchArray, new EnglishStemAnalyzer(), boosts); - queryParser.setAllowLeadingWildcard(true); - if (!queryString.contains("\"") && !queryString.contains(":") && !queryString.contains("*") && !queryString.contains("~")) { - queryString = Arrays.stream(queryString.split(" ")).map(s -> "*" + s + "*").collect(Collectors.joining(" ")); - } - Query luceneQuery = queryParser.parse(queryString); - TopDocs docs = searcher.search(luceneQuery, Integer.MAX_VALUE); + TopDocs docs = searcher.search(query.getQuery(), Integer.MAX_VALUE); for (ScoreDoc scoreDoc : docs.scoreDocs) { - SearchResult searchResult = new SearchResult(searcher, luceneQuery, scoreDoc); + SearchResult searchResult = new SearchResult(searcher, query.getQuery(), scoreDoc); for (BibEntry match : searchResult.getMatchingEntries(databaseContext)) { if (searchResult.getLuceneScore() > 0) { if (!results.containsKey(match)) { @@ -86,8 +58,6 @@ public HashMap search(SearchQuery query) { } } } - } catch (ParseException e) { - LOGGER.warn("Could not parse query: '{}'!\n{}", query.getQuery(), e.getMessage()); } catch (IOException e) { LOGGER.warn("Could not open Index at: '{}'!\n{}", indexDirectory, e.getMessage()); } diff --git a/src/main/java/org/jabref/model/groups/SearchGroup.java b/src/main/java/org/jabref/model/groups/SearchGroup.java index 2ba8c902b68..f0fa5bdb102 100644 --- a/src/main/java/org/jabref/model/groups/SearchGroup.java +++ b/src/main/java/org/jabref/model/groups/SearchGroup.java @@ -10,6 +10,7 @@ import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.search.GroupSearchQuery; +import org.jabref.model.search.rules.SearchRules; import org.jabref.model.search.rules.SearchRules.SearchFlags; import org.slf4j.Logger; @@ -54,7 +55,7 @@ public boolean contains(BibEntry entry) { return matches.contains(entry); } - public EnumSet getSearchFlags() { + public EnumSet getSearchFlags() { return query.getSearchFlags(); } diff --git a/src/main/java/org/jabref/model/search/GroupSearchQuery.java b/src/main/java/org/jabref/model/search/GroupSearchQuery.java index 0c0becf589f..cbc999204a0 100644 --- a/src/main/java/org/jabref/model/search/GroupSearchQuery.java +++ b/src/main/java/org/jabref/model/search/GroupSearchQuery.java @@ -3,7 +3,6 @@ import java.util.EnumSet; import org.jabref.logic.search.SearchQuery; -import org.jabref.model.entry.BibEntry; import org.jabref.model.search.rules.SearchRules.SearchFlags; public class GroupSearchQuery extends SearchQuery { @@ -12,11 +11,6 @@ public GroupSearchQuery(String query, EnumSet searchFlags) { super(query, searchFlags); } - @Override - public boolean isMatch(BibEntry entry) { - return this.getRule().applyRule(query, entry); - } - public String getSearchExpression() { return query; } diff --git a/src/main/java/org/jabref/model/search/rules/ContainsBasedSearchRule.java b/src/main/java/org/jabref/model/search/rules/ContainsBasedSearchRule.java deleted file mode 100644 index 9d0232eca1e..00000000000 --- a/src/main/java/org/jabref/model/search/rules/ContainsBasedSearchRule.java +++ /dev/null @@ -1,48 +0,0 @@ -package org.jabref.model.search.rules; - -import java.util.EnumSet; -import java.util.Iterator; -import java.util.List; - -import org.jabref.model.entry.BibEntry; -import org.jabref.model.entry.field.Field; -import org.jabref.model.search.rules.SearchRules.SearchFlags; -import org.jabref.model.strings.StringUtil; - -/** - * Search rule for a search based on String.contains() - */ -public class ContainsBasedSearchRule extends FullTextSearchRule { - - public ContainsBasedSearchRule(EnumSet searchFlags) { - super(searchFlags); - } - - @Override - public boolean validateSearchStrings(String query) { - return true; - } - - @Override - public boolean applyRule(String query, BibEntry bibEntry) { - String searchString = query; - List unmatchedWords = new SentenceAnalyzer(searchString).getWords(); - - for (Field fieldKey : bibEntry.getFields()) { - String formattedFieldContent = StringUtil.stripAccents(bibEntry.getLatexFreeField(fieldKey).get()); - - Iterator unmatchedWordsIterator = unmatchedWords.iterator(); - while (unmatchedWordsIterator.hasNext()) { - String word = StringUtil.stripAccents(unmatchedWordsIterator.next()); - if (formattedFieldContent.contains(word)) { - unmatchedWordsIterator.remove(); - } - } - - if (unmatchedWords.isEmpty()) { - return true; - } - } - return false; - } -} diff --git a/src/main/java/org/jabref/model/search/rules/FullTextSearchRule.java b/src/main/java/org/jabref/model/search/rules/FullTextSearchRule.java deleted file mode 100644 index 2b34872fe66..00000000000 --- a/src/main/java/org/jabref/model/search/rules/FullTextSearchRule.java +++ /dev/null @@ -1,41 +0,0 @@ -package org.jabref.model.search.rules; - -import java.util.Collections; -import java.util.EnumSet; -import java.util.List; - -import org.jabref.gui.Globals; -import org.jabref.model.database.BibDatabaseContext; -import org.jabref.model.pdf.search.SearchResult; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * All classes providing full text search results inherit from this class. - *

- * Some kind of caching of the full text search results is implemented. - */ -public abstract class FullTextSearchRule implements SearchRule { - - private static final Logger LOGGER = LoggerFactory.getLogger(FullTextSearchRule.class); - - protected final EnumSet searchFlags; - - protected String lastQuery; - protected List lastSearchResults; - - private final BibDatabaseContext databaseContext; - - public FullTextSearchRule(EnumSet searchFlags) { - this.searchFlags = searchFlags; - this.lastQuery = ""; - lastSearchResults = Collections.emptyList(); - - databaseContext = Globals.stateManager.getActiveDatabase().orElse(null); - } - - public EnumSet getSearchFlags() { - return searchFlags; - } -} diff --git a/src/main/java/org/jabref/model/search/rules/GrammarBasedSearchRule.java b/src/main/java/org/jabref/model/search/rules/GrammarBasedSearchRule.java deleted file mode 100644 index 207801e0a3a..00000000000 --- a/src/main/java/org/jabref/model/search/rules/GrammarBasedSearchRule.java +++ /dev/null @@ -1,261 +0,0 @@ -package org.jabref.model.search.rules; - -import java.util.ArrayList; -import java.util.EnumSet; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.function.Predicate; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -import org.jabref.gui.Globals; -import org.jabref.model.database.BibDatabaseContext; -import org.jabref.model.entry.BibEntry; -import org.jabref.model.entry.Keyword; -import org.jabref.model.entry.field.Field; -import org.jabref.model.entry.field.InternalField; -import org.jabref.model.pdf.search.SearchResult; -import org.jabref.model.search.rules.SearchRules.SearchFlags; -import org.jabref.model.strings.StringUtil; -import org.jabref.search.SearchBaseVisitor; -import org.jabref.search.SearchLexer; -import org.jabref.search.SearchParser; - -import org.antlr.v4.runtime.ANTLRInputStream; -import org.antlr.v4.runtime.BailErrorStrategy; -import org.antlr.v4.runtime.BaseErrorListener; -import org.antlr.v4.runtime.CommonTokenStream; -import org.antlr.v4.runtime.RecognitionException; -import org.antlr.v4.runtime.Recognizer; -import org.antlr.v4.runtime.misc.ParseCancellationException; -import org.antlr.v4.runtime.tree.ParseTree; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * The search query must be specified in an expression that is acceptable by the Search.g4 grammar. - *

- * This class implements the "Advanced Search Mode" described in the help - */ -public class GrammarBasedSearchRule implements SearchRule { - - private static final Logger LOGGER = LoggerFactory.getLogger(GrammarBasedSearchRule.class); - - private final EnumSet searchFlags; - - private ParseTree tree; - private String query; - private List searchResults = new ArrayList<>(); - - private final BibDatabaseContext databaseContext; - - public static class ThrowingErrorListener extends BaseErrorListener { - - public static final ThrowingErrorListener INSTANCE = new ThrowingErrorListener(); - - @Override - public void syntaxError(Recognizer recognizer, Object offendingSymbol, - int line, int charPositionInLine, String msg, RecognitionException e) - throws ParseCancellationException { - throw new ParseCancellationException("line " + line + ":" + charPositionInLine + " " + msg); - } - } - - public GrammarBasedSearchRule(EnumSet searchFlags) throws RecognitionException { - this.searchFlags = searchFlags; - databaseContext = Globals.stateManager.getActiveDatabase().orElse(null); - } - - public static boolean isValid(EnumSet searchFlags, String query) { - return new GrammarBasedSearchRule(searchFlags).validateSearchStrings(query); - } - - public ParseTree getTree() { - return this.tree; - } - - public String getQuery() { - return this.query; - } - - private void init(String query) throws ParseCancellationException { - if (Objects.equals(this.query, query)) { - return; - } - - SearchLexer lexer = new SearchLexer(new ANTLRInputStream(query)); - lexer.removeErrorListeners(); // no infos on file system - lexer.addErrorListener(ThrowingErrorListener.INSTANCE); - SearchParser parser = new SearchParser(new CommonTokenStream(lexer)); - parser.removeErrorListeners(); // no infos on file system - parser.addErrorListener(ThrowingErrorListener.INSTANCE); - parser.setErrorHandler(new BailErrorStrategy()); // ParseCancelationException on parse errors - tree = parser.start(); - this.query = query; - } - - @Override - public boolean applyRule(String query, BibEntry bibEntry) { - try { - return new BibtexSearchVisitor(searchFlags, bibEntry).visit(tree); - } catch (Exception e) { - LOGGER.debug("Search failed", e); - return false; - } - } - - @Override - public boolean validateSearchStrings(String query) { - try { - init(query); - return true; - } catch (ParseCancellationException e) { - LOGGER.debug("Search query invalid", e); - return false; - } - } - - public EnumSet getSearchFlags() { - return searchFlags; - } - - public enum ComparisonOperator { - EXACT, CONTAINS, DOES_NOT_CONTAIN; - - public static ComparisonOperator build(String value) { - if ("CONTAINS".equalsIgnoreCase(value) || "=".equals(value)) { - return CONTAINS; - } else if ("MATCHES".equalsIgnoreCase(value) || "==".equals(value)) { - return EXACT; - } else { - return DOES_NOT_CONTAIN; - } - } - } - - public static class Comparator { - - private final ComparisonOperator operator; - private final Pattern fieldPattern; - private final Pattern valuePattern; - - public Comparator(String field, String value, ComparisonOperator operator, EnumSet searchFlags) { - this.operator = operator; - - this.fieldPattern = Pattern.compile(searchFlags.contains(SearchRules.SearchFlags.REGULAR_EXPRESSION) ? StringUtil.stripAccents(field) : "\\Q" + StringUtil.stripAccents(field) + "\\E", Pattern.CASE_INSENSITIVE); - this.valuePattern = Pattern.compile(searchFlags.contains(SearchRules.SearchFlags.REGULAR_EXPRESSION) ? StringUtil.stripAccents(value) : "\\Q" + StringUtil.stripAccents(value) + "\\E", Pattern.CASE_INSENSITIVE); - } - - public boolean compare(BibEntry entry) { - // special case for searching for entrytype=phdthesis - if (fieldPattern.matcher(InternalField.TYPE_HEADER.getName()).matches()) { - return matchFieldValue(entry.getType().getName()); - } - - // special case for searching a single keyword - if (fieldPattern.matcher("anykeyword").matches()) { - return entry.getKeywords(',').stream().map(Keyword::toString).anyMatch(this::matchFieldValue); - } - - // specification of fieldsKeys to search is done in the search expression itself - Set fieldsKeys = entry.getFields(); - - // special case for searching allfields=cat and title=dog - if (!fieldPattern.matcher("anyfield").matches()) { - // Filter out the requested fields - fieldsKeys = fieldsKeys.stream().filter(matchFieldKey()).collect(Collectors.toSet()); - } - - for (Field field : fieldsKeys) { - Optional fieldValue = entry.getLatexFreeField(field); - if (fieldValue.isPresent()) { - if (matchFieldValue(StringUtil.stripAccents(fieldValue.get()))) { - return true; - } - } - } - - // special case of asdf!=whatever and entry does not contain asdf - return fieldsKeys.isEmpty() && (operator == ComparisonOperator.DOES_NOT_CONTAIN); - } - - private Predicate matchFieldKey() { - return field -> fieldPattern.matcher(field.getName()).matches(); - } - - public boolean matchFieldValue(String content) { - Matcher matcher = valuePattern.matcher(content); - if (operator == ComparisonOperator.CONTAINS) { - return matcher.find(); - } else if (operator == ComparisonOperator.EXACT) { - return matcher.matches(); - } else if (operator == ComparisonOperator.DOES_NOT_CONTAIN) { - return !matcher.find(); - } else { - throw new IllegalStateException("MUST NOT HAPPEN"); - } - } - } - - /** - * Search results in boolean. It may be later on converted to an int. - */ - static class BibtexSearchVisitor extends SearchBaseVisitor { - - private final EnumSet searchFlags; - - private final BibEntry entry; - - public BibtexSearchVisitor(EnumSet searchFlags, BibEntry bibEntry) { - this.searchFlags = searchFlags; - this.entry = bibEntry; - } - - public boolean comparison(String field, ComparisonOperator operator, String value) { - return new Comparator(field, value, operator, searchFlags).compare(entry); - } - - @Override - public Boolean visitStart(SearchParser.StartContext ctx) { - return visit(ctx.expression()); - } - - @Override - public Boolean visitComparison(SearchParser.ComparisonContext context) { - // remove possible enclosing " symbols - String right = context.right.getText(); - if (right.startsWith("\"") && right.endsWith("\"")) { - right = right.substring(1, right.length() - 1); - } - - Optional fieldDescriptor = Optional.ofNullable(context.left); - if (fieldDescriptor.isPresent()) { - return comparison(fieldDescriptor.get().getText(), ComparisonOperator.build(context.operator.getText()), right); - } else { - return SearchRules.getSearchRule(searchFlags).applyRule(right, entry); - } - } - - @Override - public Boolean visitUnaryExpression(SearchParser.UnaryExpressionContext ctx) { - return !visit(ctx.expression()); // negate - } - - @Override - public Boolean visitParenExpression(SearchParser.ParenExpressionContext ctx) { - return visit(ctx.expression()); // ignore parenthesis - } - - @Override - public Boolean visitBinaryExpression(SearchParser.BinaryExpressionContext ctx) { - if ("AND".equalsIgnoreCase(ctx.operator.getText())) { - return visit(ctx.left) && visit(ctx.right); // and - } else { - return visit(ctx.left) || visit(ctx.right); // or - } - } - } -} diff --git a/src/main/java/org/jabref/model/search/rules/RegexBasedSearchRule.java b/src/main/java/org/jabref/model/search/rules/RegexBasedSearchRule.java deleted file mode 100644 index f90d89842ac..00000000000 --- a/src/main/java/org/jabref/model/search/rules/RegexBasedSearchRule.java +++ /dev/null @@ -1,60 +0,0 @@ -package org.jabref.model.search.rules; - -import java.util.EnumSet; -import java.util.Optional; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.regex.PatternSyntaxException; - -import org.jabref.model.entry.BibEntry; -import org.jabref.model.entry.field.Field; -import org.jabref.model.search.rules.SearchRules.SearchFlags; -import org.jabref.model.strings.StringUtil; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Search rule for regex-based search. - */ -public class RegexBasedSearchRule extends FullTextSearchRule { - - private static final Logger LOGGER = LoggerFactory.getLogger(RegexBasedSearchRule.class); - - public RegexBasedSearchRule(EnumSet searchFlags) { - super(searchFlags); - } - - @Override - public boolean validateSearchStrings(String query) { - try { - Pattern.compile(query, Pattern.CASE_INSENSITIVE); - } catch (PatternSyntaxException ex) { - return false; - } - return true; - } - - @Override - public boolean applyRule(String query, BibEntry bibEntry) { - Pattern pattern; - try { - pattern = Pattern.compile(StringUtil.stripAccents(query), Pattern.CASE_INSENSITIVE); - } catch (PatternSyntaxException ex) { - LOGGER.debug("Could not compile regex {}", query, ex); - return false; - } - - for (Field field : bibEntry.getFields()) { - Optional fieldOptional = bibEntry.getField(field); - if (fieldOptional.isPresent()) { - String fieldContentNoBrackets = StringUtil.stripAccents(bibEntry.getLatexFreeField(field).get()); - Matcher m = pattern.matcher(fieldContentNoBrackets); - if (m.find()) { - return true; - } - } - } - return false; - } -} diff --git a/src/main/java/org/jabref/model/search/rules/SearchRule.java b/src/main/java/org/jabref/model/search/rules/SearchRule.java deleted file mode 100644 index ffc4dced699..00000000000 --- a/src/main/java/org/jabref/model/search/rules/SearchRule.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.jabref.model.search.rules; - -import org.jabref.model.entry.BibEntry; - -public interface SearchRule { - - boolean applyRule(String query, BibEntry bibEntry); - - boolean validateSearchStrings(String query); -} diff --git a/src/main/java/org/jabref/model/search/rules/SearchRules.java b/src/main/java/org/jabref/model/search/rules/SearchRules.java index 48fe4ee022f..708ca1df316 100644 --- a/src/main/java/org/jabref/model/search/rules/SearchRules.java +++ b/src/main/java/org/jabref/model/search/rules/SearchRules.java @@ -1,48 +1,10 @@ package org.jabref.model.search.rules; -import java.util.EnumSet; -import java.util.regex.Pattern; - /** * This is a factory to instantiate the matching SearchRule implementation matching a given query */ public class SearchRules { - private static final Pattern SIMPLE_EXPRESSION = Pattern.compile("[^\\p{Punct}]*"); - - private SearchRules() { - } - - /** - * Returns the appropriate search rule that fits best to the given parameter. - */ - public static SearchRule getSearchRuleByQuery(String query, EnumSet searchFlags) { - if (isSimpleQuery(query)) { - return new ContainsBasedSearchRule(searchFlags); - } - - // this searches specified fields if specified, - // and all fields otherwise - SearchRule searchExpression = new GrammarBasedSearchRule(searchFlags); - if (searchExpression.validateSearchStrings(query)) { - return searchExpression; - } else { - return getSearchRule(searchFlags); - } - } - - private static boolean isSimpleQuery(String query) { - return SIMPLE_EXPRESSION.matcher(query).matches(); - } - - static SearchRule getSearchRule(EnumSet searchFlags) { - if (searchFlags.contains(SearchFlags.REGULAR_EXPRESSION)) { - return new RegexBasedSearchRule(searchFlags); - } else { - return new ContainsBasedSearchRule(searchFlags); - } - } - public enum SearchFlags { REGULAR_EXPRESSION, FULLTEXT, KEEP_SEARCH_STRING, FILTERING_SEARCH, SORT_BY_SCORE; } diff --git a/src/main/java/org/jabref/model/search/rules/SentenceAnalyzer.java b/src/main/java/org/jabref/model/search/rules/SentenceAnalyzer.java deleted file mode 100644 index 89987d9d5ac..00000000000 --- a/src/main/java/org/jabref/model/search/rules/SentenceAnalyzer.java +++ /dev/null @@ -1,57 +0,0 @@ -package org.jabref.model.search.rules; - -import java.util.ArrayList; -import java.util.List; - -public class SentenceAnalyzer { - - public static final char ESCAPE_CHAR = '\\'; - public static final char QUOTE_CHAR = '"'; - - private final String query; - - public SentenceAnalyzer(String query) { - this.query = query; - } - - public List getWords() { - List result = new ArrayList<>(); - - StringBuilder stringBuilder = new StringBuilder(); - boolean escaped = false; - boolean quoted = false; - for (char c : query.toCharArray()) { - // Check if we are entering an escape sequence: - if (!escaped && c == ESCAPE_CHAR) { - escaped = true; - } else { - // See if we have reached the end of a word: - if (!escaped && !quoted && Character.isWhitespace(c)) { - if (stringBuilder.length() > 0) { - result.add(stringBuilder.toString()); - stringBuilder = new StringBuilder(); - } - } else if (c == QUOTE_CHAR) { - // Whether it is a start or end quote, store the current - // word if any: - if (stringBuilder.length() > 0) { - result.add(stringBuilder.toString()); - stringBuilder = new StringBuilder(); - } - quoted = !quoted; - } else { - // All other possibilities exhausted, we add the char to - // the current word: - stringBuilder.append(c); - } - escaped = false; - } - } - // Finished with the loop. If we have a current word, add it: - if (stringBuilder.length() > 0) { - result.add(stringBuilder.toString()); - } - - return result; - } -} diff --git a/src/main/java/org/jabref/preferences/JabRefPreferences.java b/src/main/java/org/jabref/preferences/JabRefPreferences.java index dc852b2df8e..d0ff4950f2b 100644 --- a/src/main/java/org/jabref/preferences/JabRefPreferences.java +++ b/src/main/java/org/jabref/preferences/JabRefPreferences.java @@ -111,7 +111,7 @@ import org.jabref.model.entry.types.EntryType; import org.jabref.model.entry.types.EntryTypeFactory; import org.jabref.model.metadata.SaveOrderConfig; -import org.jabref.model.search.rules.SearchRules; +import org.jabref.model.search.rules.SearchRules.SearchFlags; import org.jabref.model.strings.StringUtil; import com.tobiasdiez.easybind.EasyBind; @@ -2628,12 +2628,12 @@ public SearchPreferences getSearchPreferences() { getBoolean(SEARCH_SORT_BY_SCORE), getBoolean(SEARCH_KEEP_GLOBAL_WINDOW_ON_TOP)); - searchPreferences.getObservableSearchFlags().addListener((SetChangeListener) c -> { - putBoolean(SEARCH_REG_EXP, searchPreferences.getObservableSearchFlags().contains(SearchRules.SearchFlags.REGULAR_EXPRESSION)); - putBoolean(SEARCH_FULLTEXT, searchPreferences.getObservableSearchFlags().contains(SearchRules.SearchFlags.FULLTEXT)); - putBoolean(SEARCH_KEEP_SEARCH_STRING, searchPreferences.getObservableSearchFlags().contains(SearchRules.SearchFlags.KEEP_SEARCH_STRING)); - putBoolean(SEARCH_FILTERING_MODE, searchPreferences.getObservableSearchFlags().contains(SearchRules.SearchFlags.FILTERING_SEARCH)); - putBoolean(SEARCH_SORT_BY_SCORE, searchPreferences.getObservableSearchFlags().contains(SearchRules.SearchFlags.SORT_BY_SCORE)); + searchPreferences.getObservableSearchFlags().addListener((SetChangeListener) c -> { + putBoolean(SEARCH_REG_EXP, searchPreferences.getObservableSearchFlags().contains(SearchFlags.REGULAR_EXPRESSION)); + putBoolean(SEARCH_FULLTEXT, searchPreferences.getObservableSearchFlags().contains(SearchFlags.FULLTEXT)); + putBoolean(SEARCH_KEEP_SEARCH_STRING, searchPreferences.getObservableSearchFlags().contains(SearchFlags.KEEP_SEARCH_STRING)); + putBoolean(SEARCH_FILTERING_MODE, searchPreferences.getObservableSearchFlags().contains(SearchFlags.FILTERING_SEARCH)); + putBoolean(SEARCH_SORT_BY_SCORE, searchPreferences.getObservableSearchFlags().contains(SearchFlags.SORT_BY_SCORE)); }); EasyBind.listen(searchPreferences.keepWindowOnTopProperty(), (obs, oldValue, newValue) -> putBoolean(SEARCH_KEEP_GLOBAL_WINDOW_ON_TOP, searchPreferences.shouldKeepWindowOnTop())); diff --git a/src/main/java/org/jabref/preferences/SearchPreferences.java b/src/main/java/org/jabref/preferences/SearchPreferences.java index fe6c2deed4b..64df152e79d 100644 --- a/src/main/java/org/jabref/preferences/SearchPreferences.java +++ b/src/main/java/org/jabref/preferences/SearchPreferences.java @@ -7,53 +7,53 @@ import javafx.collections.FXCollections; import javafx.collections.ObservableSet; -import org.jabref.model.search.rules.SearchRules.SearchFlags; +import org.jabref.model.search.rules.SearchRules; public class SearchPreferences { - private final ObservableSet searchFlags; + private final ObservableSet searchFlags; private final BooleanProperty keepWindowOnTop; public SearchPreferences(boolean isRegularExpression, boolean isFulltext, boolean isKeepSearchString, boolean isFilteringMode, boolean isSortByScore, boolean keepWindowOnTop) { this.keepWindowOnTop = new SimpleBooleanProperty(keepWindowOnTop); - searchFlags = FXCollections.observableSet(EnumSet.noneOf(SearchFlags.class)); + searchFlags = FXCollections.observableSet(EnumSet.noneOf(SearchRules.SearchFlags.class)); if (isRegularExpression) { - searchFlags.add(SearchFlags.REGULAR_EXPRESSION); + searchFlags.add(SearchRules.SearchFlags.REGULAR_EXPRESSION); } if (isFulltext) { - searchFlags.add(SearchFlags.FULLTEXT); + searchFlags.add(SearchRules.SearchFlags.FULLTEXT); } if (isKeepSearchString) { - searchFlags.add(SearchFlags.KEEP_SEARCH_STRING); + searchFlags.add(SearchRules.SearchFlags.KEEP_SEARCH_STRING); } if (isFilteringMode) { - searchFlags.add(SearchFlags.FILTERING_SEARCH); + searchFlags.add(SearchRules.SearchFlags.FILTERING_SEARCH); } if (isSortByScore) { - searchFlags.add(SearchFlags.SORT_BY_SCORE); + searchFlags.add(SearchRules.SearchFlags.SORT_BY_SCORE); } } - public SearchPreferences(EnumSet searchFlags, boolean keepWindowOnTop) { + public SearchPreferences(EnumSet searchFlags, boolean keepWindowOnTop) { this.keepWindowOnTop = new SimpleBooleanProperty(keepWindowOnTop); this.searchFlags = FXCollections.observableSet(searchFlags); } - public EnumSet getSearchFlags() { + public EnumSet getSearchFlags() { // copy of returns an exception when the EnumSet is empty if (searchFlags.isEmpty()) { - return EnumSet.noneOf(SearchFlags.class); + return EnumSet.noneOf(SearchRules.SearchFlags.class); } return EnumSet.copyOf(searchFlags); } - public ObservableSet getObservableSearchFlags() { + public ObservableSet getObservableSearchFlags() { return searchFlags; } - public void setSearchFlag(SearchFlags flag, boolean value) { + public void setSearchFlag(SearchRules.SearchFlags flag, boolean value) { if (searchFlags.contains(flag) && !value) { searchFlags.remove(flag); } else if (!searchFlags.contains(flag) && value) { @@ -62,23 +62,23 @@ public void setSearchFlag(SearchFlags flag, boolean value) { } public boolean isRegularExpression() { - return searchFlags.contains(SearchFlags.REGULAR_EXPRESSION); + return searchFlags.contains(SearchRules.SearchFlags.REGULAR_EXPRESSION); } public boolean isFulltext() { - return searchFlags.contains(SearchFlags.FULLTEXT); + return searchFlags.contains(SearchRules.SearchFlags.FULLTEXT); } public boolean shouldKeepSearchString() { - return searchFlags.contains(SearchFlags.KEEP_SEARCH_STRING); + return searchFlags.contains(SearchRules.SearchFlags.KEEP_SEARCH_STRING); } public boolean isFilteringMode() { - return searchFlags.contains(SearchFlags.FILTERING_SEARCH); + return searchFlags.contains(SearchRules.SearchFlags.FILTERING_SEARCH); } public boolean isSortByScore() { - return searchFlags.contains(SearchFlags.SORT_BY_SCORE); + return searchFlags.contains(SearchRules.SearchFlags.SORT_BY_SCORE); } public boolean shouldKeepWindowOnTop() { diff --git a/src/test/java/org/jabref/gui/search/ContainsAndRegexBasedSearchRuleDescriberTest.java b/src/test/java/org/jabref/gui/search/ContainsAndRegexBasedSearchRuleDescriberTest.java index bc3406ec43c..2a179fc46e8 100644 --- a/src/test/java/org/jabref/gui/search/ContainsAndRegexBasedSearchRuleDescriberTest.java +++ b/src/test/java/org/jabref/gui/search/ContainsAndRegexBasedSearchRuleDescriberTest.java @@ -7,10 +7,7 @@ import javafx.scene.text.TextFlow; import javafx.stage.Stage; -import org.jabref.gui.search.rules.describer.ContainsAndRegexBasedSearchRuleDescriber; import org.jabref.gui.util.TooltipTextUtil; -import org.jabref.model.search.rules.SearchRules; -import org.jabref.model.search.rules.SearchRules.SearchFlags; import org.jabref.testutils.category.GUITest; import org.junit.jupiter.api.Test; diff --git a/src/test/java/org/jabref/gui/search/GrammarBasedSearchRuleDescriberTest.java b/src/test/java/org/jabref/gui/search/GrammarBasedSearchRuleDescriberTest.java index ce2c4cc3a49..20b2f8f64fe 100644 --- a/src/test/java/org/jabref/gui/search/GrammarBasedSearchRuleDescriberTest.java +++ b/src/test/java/org/jabref/gui/search/GrammarBasedSearchRuleDescriberTest.java @@ -8,11 +8,7 @@ import javafx.scene.text.TextFlow; import javafx.stage.Stage; -import org.jabref.gui.search.rules.describer.GrammarBasedSearchRuleDescriber; import org.jabref.gui.util.TooltipTextUtil; -import org.jabref.model.search.rules.GrammarBasedSearchRule; -import org.jabref.model.search.rules.SearchRules; -import org.jabref.model.search.rules.SearchRules.SearchFlags; import org.jabref.testutils.category.GUITest; import org.junit.jupiter.api.Test; diff --git a/src/test/java/org/jabref/logic/exporter/GroupSerializerTest.java b/src/test/java/org/jabref/logic/exporter/GroupSerializerTest.java index 1753495506a..664ebbcae45 100644 --- a/src/test/java/org/jabref/logic/exporter/GroupSerializerTest.java +++ b/src/test/java/org/jabref/logic/exporter/GroupSerializerTest.java @@ -25,7 +25,6 @@ import org.jabref.model.groups.TexGroup; import org.jabref.model.groups.WordKeywordGroup; import org.jabref.model.metadata.MetaData; -import org.jabref.model.search.rules.SearchRules; import org.jabref.model.util.DummyFileUpdateMonitor; import org.junit.jupiter.api.BeforeEach; diff --git a/src/test/java/org/jabref/logic/importer/util/GroupsParserTest.java b/src/test/java/org/jabref/logic/importer/util/GroupsParserTest.java index e600056b879..10c45f1a7f4 100644 --- a/src/test/java/org/jabref/logic/importer/util/GroupsParserTest.java +++ b/src/test/java/org/jabref/logic/importer/util/GroupsParserTest.java @@ -21,7 +21,6 @@ import org.jabref.model.groups.SearchGroup; import org.jabref.model.groups.TexGroup; import org.jabref.model.metadata.MetaData; -import org.jabref.model.search.rules.SearchRules; import org.jabref.model.util.DummyFileUpdateMonitor; import org.jabref.model.util.FileUpdateMonitor; diff --git a/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java b/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java index f276f9ddfaf..0ffa1636f4d 100644 --- a/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java +++ b/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java @@ -8,7 +8,6 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.types.StandardEntryType; -import org.jabref.model.search.rules.SearchRules; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/src/test/java/org/jabref/logic/search/SearchQueryTest.java b/src/test/java/org/jabref/logic/search/SearchQueryTest.java index 4bf01fadb73..d924ee762c1 100644 --- a/src/test/java/org/jabref/logic/search/SearchQueryTest.java +++ b/src/test/java/org/jabref/logic/search/SearchQueryTest.java @@ -7,8 +7,6 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.types.StandardEntryType; -import org.jabref.model.search.rules.SearchRules; -import org.jabref.model.search.rules.SearchRules.SearchFlags; import org.junit.jupiter.api.Test; diff --git a/src/test/java/org/jabref/model/groups/GroupTreeNodeTest.java b/src/test/java/org/jabref/model/groups/GroupTreeNodeTest.java index 8878219ecda..4c8689ce0d2 100644 --- a/src/test/java/org/jabref/model/groups/GroupTreeNodeTest.java +++ b/src/test/java/org/jabref/model/groups/GroupTreeNodeTest.java @@ -12,8 +12,6 @@ import org.jabref.model.entry.field.StandardField; import org.jabref.model.search.matchers.AndMatcher; import org.jabref.model.search.matchers.OrMatcher; -import org.jabref.model.search.rules.SearchRules; -import org.jabref.model.search.rules.SearchRules.SearchFlags; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/src/test/java/org/jabref/model/groups/SearchGroupTest.java b/src/test/java/org/jabref/model/groups/SearchGroupTest.java index 5bdec41d87e..40185f8bfc8 100644 --- a/src/test/java/org/jabref/model/groups/SearchGroupTest.java +++ b/src/test/java/org/jabref/model/groups/SearchGroupTest.java @@ -4,7 +4,6 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; -import org.jabref.model.search.rules.SearchRules; import org.junit.jupiter.api.Test; From 3d042a80f0f0ef6fdece8a6879502c2a25cc31b2 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Sat, 5 Nov 2022 19:33:54 +0100 Subject: [PATCH 060/256] Store search-results per context --- src/main/java/org/jabref/gui/StateManager.java | 5 +++-- .../org/jabref/gui/entryeditor/EntryEditor.java | 2 +- .../fileannotationtab/FulltextSearchResultsTab.java | 12 ++++++++++-- .../gui/maintable/BibEntryTableViewModel.java | 6 +++--- .../jabref/gui/maintable/MainTableDataModel.java | 5 ++--- .../jabref/gui/maintable/columns/FileColumn.java | 2 +- .../java/org/jabref/gui/search/GlobalSearchBar.java | 13 ++++++++----- .../gui/search/SearchResultsTableDataModel.java | 2 +- 8 files changed, 29 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/jabref/gui/StateManager.java b/src/main/java/org/jabref/gui/StateManager.java index 540d90eca5f..9abb10f9de0 100644 --- a/src/main/java/org/jabref/gui/StateManager.java +++ b/src/main/java/org/jabref/gui/StateManager.java @@ -1,6 +1,7 @@ package org.jabref.gui; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; @@ -55,7 +56,7 @@ public class StateManager { private final ObservableList selectedEntries = FXCollections.observableArrayList(); private final ObservableMap> selectedGroups = FXCollections.observableHashMap(); private final OptionalObjectProperty activeSearchQuery = OptionalObjectProperty.empty(); - private final ObservableMap searchResults = FXCollections.observableHashMap(); + private final ObservableMap> searchResults = FXCollections.observableHashMap(); private final OptionalObjectProperty focusOwner = OptionalObjectProperty.empty(); private final ObservableList>> backgroundTasks = FXCollections.observableArrayList(task -> new Observable[] {task.getValue().progressProperty(), task.getValue().runningProperty()}); private final EasyBinding anyTaskRunning = EasyBind.reduce(backgroundTasks, tasks -> tasks.map(Pair::getValue).anyMatch(Task::isRunning)); @@ -181,7 +182,7 @@ public void setLastAutomaticFieldEditorEdit(LastAutomaticFieldEditorEdit automat lastAutomaticFieldEditorEditProperty().set(automaticFieldEditorEdit); } - public ObservableMap getSearchResults() { + public ObservableMap> getSearchResults() { return searchResults; } } diff --git a/src/main/java/org/jabref/gui/entryeditor/EntryEditor.java b/src/main/java/org/jabref/gui/entryeditor/EntryEditor.java index f136ccbe15f..97f92db1a36 100644 --- a/src/main/java/org/jabref/gui/entryeditor/EntryEditor.java +++ b/src/main/java/org/jabref/gui/entryeditor/EntryEditor.java @@ -277,7 +277,7 @@ private List createTabs() { // LaTeX citations tab entryEditorTabs.add(new LatexCitationsTab(databaseContext, preferencesService, taskExecutor, dialogService)); - entryEditorTabs.add(new FulltextSearchResultsTab(stateManager, preferencesService, dialogService)); + entryEditorTabs.add(new FulltextSearchResultsTab(stateManager, preferencesService, dialogService, databaseContext)); return entryEditorTabs; } diff --git a/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java b/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java index 23ab2de139e..5de53a24589 100644 --- a/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java @@ -26,6 +26,7 @@ import org.jabref.gui.maintable.OpenFolderAction; import org.jabref.gui.util.TooltipTextUtil; import org.jabref.logic.l10n.Localization; +import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.LinkedFile; import org.jabref.model.pdf.search.LuceneSearchResults; @@ -44,6 +45,7 @@ public class FulltextSearchResultsTab extends EntryEditorTab { private final PreferencesService preferencesService; private final DialogService dialogService; private final ActionFactory actionFactory; + private final BibDatabaseContext context; private final TextFlow content; @@ -51,10 +53,11 @@ public class FulltextSearchResultsTab extends EntryEditorTab { private DocumentViewerView documentViewerView; - public FulltextSearchResultsTab(StateManager stateManager, PreferencesService preferencesService, DialogService dialogService) { + public FulltextSearchResultsTab(StateManager stateManager, PreferencesService preferencesService, DialogService dialogService, BibDatabaseContext context) { this.stateManager = stateManager; this.preferencesService = preferencesService; this.dialogService = dialogService; + this.context = context; this.actionFactory = new ActionFactory(preferencesService.getKeyBindingRepository()); content = new TextFlow(); @@ -83,10 +86,15 @@ protected void bindToEntry(BibEntry entry) { documentViewerView = new DocumentViewerView(); } this.entry = entry; - LuceneSearchResults searchResults = stateManager.getSearchResults().get(entry); content.getChildren().clear(); + if (!stateManager.getSearchResults().containsKey(context)) { + return; + } + + LuceneSearchResults searchResults = stateManager.getSearchResults().get(context).get(entry); + if (searchResults.numSearchResults() == 0) { content.getChildren().add(new Text(Localization.lang("No search matches."))); } diff --git a/src/main/java/org/jabref/gui/maintable/BibEntryTableViewModel.java b/src/main/java/org/jabref/gui/maintable/BibEntryTableViewModel.java index bece85e5df8..6ceecebf071 100644 --- a/src/main/java/org/jabref/gui/maintable/BibEntryTableViewModel.java +++ b/src/main/java/org/jabref/gui/maintable/BibEntryTableViewModel.java @@ -61,9 +61,9 @@ public BibEntryTableViewModel(BibEntry entry, BibDatabaseContext bibDatabaseCont this.matchedGroups = createMatchedGroupsBinding(bibDatabaseContext, entry); this.bibDatabaseContext = bibDatabaseContext; - stateManager.getSearchResults().addListener((MapChangeListener) change -> { - if (stateManager.getSearchResults().containsKey(entry)) { - searchScore.set(stateManager.getSearchResults().get(entry).getSearchScore()); + stateManager.getSearchResults().addListener((MapChangeListener>) change -> { + if (stateManager.getSearchResults().containsKey(bibDatabaseContext) && stateManager.getSearchResults().get(bibDatabaseContext).containsKey(entry)) { + searchScore.set(stateManager.getSearchResults().get(bibDatabaseContext).get(entry).getSearchScore()); } else { searchScore.set(0); } diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index 51003e477bf..e4ab9a8c8dc 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -73,9 +73,8 @@ public static void updateSearchGroups(StateManager stateManager, BibDatabaseCont private void doSearch(Optional query) { if (query.isPresent()) { try { - stateManager.getSearchResults().clear(); for (ObjectProperty context : stateManager.getOpenDatabases()) { - stateManager.getSearchResults().putAll(LuceneSearcher.of(context.get()).search(query.get())); + stateManager.getSearchResults().put(bibDatabaseContext, LuceneSearcher.of(context.get()).search(query.get())); } } catch (IOException e) { e.printStackTrace(); @@ -95,7 +94,7 @@ private boolean isMatchedBySearch(Optional query, BibEntryTableView if (!query.isPresent() || !query.get().getSearchFlags().contains(SearchRules.SearchFlags.FILTERING_SEARCH)) { return true; } - return stateManager.getSearchResults().containsKey(entry.getEntry()); + return entry.getSearchScore() > 0; } private boolean isMatchedByGroup(ObservableList groups, BibEntryTableViewModel entry) { diff --git a/src/main/java/org/jabref/gui/maintable/columns/FileColumn.java b/src/main/java/org/jabref/gui/maintable/columns/FileColumn.java index a5de2e4e9b7..d2b55f7d70d 100644 --- a/src/main/java/org/jabref/gui/maintable/columns/FileColumn.java +++ b/src/main/java/org/jabref/gui/maintable/columns/FileColumn.java @@ -142,7 +142,7 @@ private ContextMenu createFileMenu(BibEntryTableViewModel entry, List linkedFiles) { - if (linkedFiles.size() > 0 && stateManager.getSearchResults().containsKey(entry.getEntry()) && stateManager.getSearchResults().get(entry.getEntry()).hasFulltextResults()) { + if (linkedFiles.size() > 0 && stateManager.getSearchResults().containsKey(database) && stateManager.getSearchResults().get(database).containsKey(entry.getEntry()) && stateManager.getSearchResults().get(database).get(entry.getEntry()).hasFulltextResults()) { return IconTheme.JabRefIcons.FILE_SEARCH.getGraphicNode(); } if (linkedFiles.size() > 1) { diff --git a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java index 57939608e6e..e6226b48fe1 100644 --- a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java +++ b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java @@ -3,6 +3,7 @@ import java.lang.reflect.Field; import java.util.EnumSet; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.regex.Pattern; @@ -60,6 +61,7 @@ import org.jabref.gui.util.TooltipTextUtil; import org.jabref.logic.l10n.Localization; import org.jabref.logic.search.SearchQuery; +import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.Author; import org.jabref.model.entry.BibEntry; import org.jabref.model.pdf.search.LuceneSearchResults; @@ -72,7 +74,6 @@ import de.saxsys.mvvmfx.utils.validation.Validator; import de.saxsys.mvvmfx.utils.validation.visualization.ControlsFxVisualizer; import impl.org.controlsfx.skin.AutoCompletePopup; -import org.apache.lucene.search.Query; import org.controlsfx.control.textfield.AutoCompletionBinding; import org.controlsfx.control.textfield.CustomTextField; import org.reactfx.util.FxTimer; @@ -200,14 +201,16 @@ public GlobalSearchBar(JabRefFrame frame, StateManager stateManager, Preferences // Async update searchTask.restart(); }, - query -> setSearchTerm(query.map(SearchQuery::getQuery).orElse(new SearchQuery("", EnumSet.noneOf(SearchRules.SearchFlags.class)).getQuery()))); + query -> setSearchTerm(query.orElse(new SearchQuery("", EnumSet.noneOf(SearchRules.SearchFlags.class))))); this.stateManager.activeSearchQueryProperty().addListener((obs, oldValue, newValue) -> newValue.ifPresent(this::updateSearchResultsForQuery)); - this.stateManager.getSearchResults().addListener((MapChangeListener) map -> stateManager.activeSearchQueryProperty().get().ifPresent(this::updateSearchResultsForQuery)); + this.stateManager.getSearchResults().addListener((MapChangeListener>) change -> { + stateManager.activeSearchQueryProperty().get().ifPresent(this::updateSearchResultsForQuery); + }); } private void updateSearchResultsForQuery(SearchQuery query) { - updateResults(this.stateManager.getSearchResults().size()); + updateResults(this.stateManager.getSearchResults().values().stream().map(Map::size).reduce(0, Integer::sum)); } private void initSearchModifierButtons() { @@ -389,7 +392,7 @@ public void updateHintVisibility() { setSearchFieldHintTooltip(null); } - public void setSearchTerm(Query searchQuery) { + public void setSearchTerm(SearchQuery searchQuery) { if (searchQuery.toString().equals(searchField.getText())) { return; } diff --git a/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java b/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java index 3eae8674b4b..cfbe12973af 100644 --- a/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java +++ b/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java @@ -39,7 +39,7 @@ public SearchResultsTableDataModel(BibDatabaseContext bibDatabaseContext, Prefer } entriesFiltered = new FilteredList<>(entriesViewModel); - entriesFiltered.predicateProperty().bind(EasyBind.map(stateManager.activeSearchQueryProperty(), (query) -> entry -> stateManager.getSearchResults().containsKey(entry.getEntry()))); + entriesFiltered.setPredicate(entry -> entry.getSearchScore() > 0); // We need to wrap the list since otherwise sorting in the table does not work entriesSorted = new SortedList<>(entriesFiltered); From c3eb8bf71ae45a5ce3fc0273fedca9633890b2cd Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Sun, 6 Nov 2022 18:29:34 +0100 Subject: [PATCH 061/256] Fixes and performance improvements - performance: remove bindings from inactive MainTableDataModels - global search: use search score instead of maps to check for matches - do not repeat searches if the query is the same --- src/main/java/org/jabref/gui/LibraryTab.java | 3 +++ .../gui/maintable/BibEntryTableViewModel.java | 17 ++++++++++++----- .../gui/maintable/MainTableDataModel.java | 17 +++++++++++++---- .../org/jabref/gui/search/GlobalSearchBar.java | 1 - .../org/jabref/logic/search/SearchQuery.java | 15 ++++++++++++++- 5 files changed, 42 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/jabref/gui/LibraryTab.java b/src/main/java/org/jabref/gui/LibraryTab.java index 1481492a9e2..63c88cd9039 100644 --- a/src/main/java/org/jabref/gui/LibraryTab.java +++ b/src/main/java/org/jabref/gui/LibraryTab.java @@ -252,6 +252,9 @@ public void feedData(BibDatabaseContext bibDatabaseContext) { bibDatabaseContext.getDatabase().registerListener(this); bibDatabaseContext.getMetaData().registerListener(this); + if (this.tableModel != null) { + this.tableModel.removeBindings(); + } this.tableModel = new MainTableDataModel(getBibDatabaseContext(), preferencesService, stateManager); citationStyleCache = new CitationStyleCache(bibDatabaseContext); annotationCache = new FileAnnotationCache(bibDatabaseContext, preferencesService.getFilePreferences()); diff --git a/src/main/java/org/jabref/gui/maintable/BibEntryTableViewModel.java b/src/main/java/org/jabref/gui/maintable/BibEntryTableViewModel.java index 6ceecebf071..2971a706291 100644 --- a/src/main/java/org/jabref/gui/maintable/BibEntryTableViewModel.java +++ b/src/main/java/org/jabref/gui/maintable/BibEntryTableViewModel.java @@ -49,6 +49,7 @@ public class BibEntryTableViewModel { private final EasyBinding> linkedIdentifiers; private final Binding> matchedGroups; private final BibDatabaseContext bibDatabaseContext; + private final StateManager stateManager; private final FloatProperty searchScore = new SimpleFloatProperty(0); @@ -60,13 +61,11 @@ public BibEntryTableViewModel(BibEntry entry, BibDatabaseContext bibDatabaseCont this.linkedIdentifiers = createLinkedIdentifiersBinding(entry); this.matchedGroups = createMatchedGroupsBinding(bibDatabaseContext, entry); this.bibDatabaseContext = bibDatabaseContext; + this.stateManager = stateManager; + updateSearchScore(); stateManager.getSearchResults().addListener((MapChangeListener>) change -> { - if (stateManager.getSearchResults().containsKey(bibDatabaseContext) && stateManager.getSearchResults().get(bibDatabaseContext).containsKey(entry)) { - searchScore.set(stateManager.getSearchResults().get(bibDatabaseContext).get(entry).getSearchScore()); - } else { - searchScore.set(0); - } + updateSearchScore(); }); } @@ -155,4 +154,12 @@ public float getSearchScore() { public FloatProperty searchScoreProperty() { return searchScore; } + + public void updateSearchScore() { + if (stateManager.getSearchResults().containsKey(bibDatabaseContext) && stateManager.getSearchResults().get(bibDatabaseContext).containsKey(entry)) { + searchScore.set(stateManager.getSearchResults().get(bibDatabaseContext).get(entry).getSearchScore()); + } else { + searchScore.set(0); + } + } } diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index e4ab9a8c8dc..f9cb80edd8d 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -35,6 +35,7 @@ public class MainTableDataModel { private final GroupsPreferences groupsPreferences; private final BibDatabaseContext bibDatabaseContext; private final StateManager stateManager; + private Optional lastSearchQuery = Optional.empty(); public MainTableDataModel(BibDatabaseContext context, PreferencesService preferencesService, StateManager stateManager) { this.preferencesService = preferencesService; @@ -66,16 +67,24 @@ public MainTableDataModel(BibDatabaseContext context, PreferencesService prefere entriesSorted = new SortedList<>(entriesFiltered); } + public void removeBindings() { + entriesFiltered.predicateProperty().unbind(); + } + public static void updateSearchGroups(StateManager stateManager, BibDatabaseContext bibDatabaseContext) { stateManager.getSelectedGroups(bibDatabaseContext).stream().map(GroupTreeNode::getGroup).filter(g -> g instanceof SearchGroup).map(g -> ((SearchGroup) g)).forEach(g -> g.updateMatches(bibDatabaseContext)); } private void doSearch(Optional query) { - if (query.isPresent()) { + if (lastSearchQuery != null && lastSearchQuery.equals(query)) { + return; + } + lastSearchQuery = query; + stateManager.getSearchResults().remove(bibDatabaseContext); + if (query.isPresent() && query.get().toString().length() > 0) { try { - for (ObjectProperty context : stateManager.getOpenDatabases()) { - stateManager.getSearchResults().put(bibDatabaseContext, LuceneSearcher.of(context.get()).search(query.get())); - } + // TODO btut: maybe do in background? + stateManager.getSearchResults().put(bibDatabaseContext, LuceneSearcher.of(bibDatabaseContext).search(query.get())); } catch (IOException e) { e.printStackTrace(); } diff --git a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java index e6226b48fe1..daf590c9036 100644 --- a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java +++ b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java @@ -203,7 +203,6 @@ public GlobalSearchBar(JabRefFrame frame, StateManager stateManager, Preferences }, query -> setSearchTerm(query.orElse(new SearchQuery("", EnumSet.noneOf(SearchRules.SearchFlags.class))))); - this.stateManager.activeSearchQueryProperty().addListener((obs, oldValue, newValue) -> newValue.ifPresent(this::updateSearchResultsForQuery)); this.stateManager.getSearchResults().addListener((MapChangeListener>) change -> { stateManager.activeSearchQueryProperty().get().ifPresent(this::updateSearchResultsForQuery); }); diff --git a/src/main/java/org/jabref/logic/search/SearchQuery.java b/src/main/java/org/jabref/logic/search/SearchQuery.java index 3c53dae4671..9d68999bcab 100644 --- a/src/main/java/org/jabref/logic/search/SearchQuery.java +++ b/src/main/java/org/jabref/logic/search/SearchQuery.java @@ -42,7 +42,7 @@ public SearchQuery(String query, EnumSet searchFlags) { MultiFieldQueryParser queryParser = new MultiFieldQueryParser(fieldsToSearchArray, new EnglishStemAnalyzer(), boosts); queryParser.setAllowLeadingWildcard(true); - if (!query.contains("\"") && !query.contains(":") && !query.contains("*") && !query.contains("~")) { + if (!query.contains("\"") && !query.contains(":") && !query.contains("*") && !query.contains("~") & query.length() > 0) { query = Arrays.stream(query.split(" ")).map(s -> "*" + s + "*").collect(Collectors.joining(" ")); } try { @@ -59,6 +59,19 @@ public String toString() { return query; } + @Override + public boolean equals(Object other) { + if (other instanceof SearchQuery) { + return ((SearchQuery) other).query.equals(this.query) && ((SearchQuery) other).searchFlags.equals(this.searchFlags); + } + return false; + } + + @Override + public int hashCode() { + return this.query.hashCode(); + } + public boolean isValid() { return parseError == null; } From 60de8b8048709dac23251ed4c55fb4726ff7759a Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Sun, 6 Nov 2022 18:36:01 +0100 Subject: [PATCH 062/256] Update search when adding entries --- .../org/jabref/gui/maintable/MainTableDataModel.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index f9cb80edd8d..40819d35530 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -6,6 +6,7 @@ import javafx.beans.property.ObjectProperty; import javafx.beans.property.SimpleObjectProperty; +import javafx.collections.ListChangeListener; import javafx.collections.ObservableList; import javafx.collections.transformation.FilteredList; import javafx.collections.transformation.SortedList; @@ -62,6 +63,16 @@ public MainTableDataModel(BibDatabaseContext context, PreferencesService prefere }; }) ); + entriesViewModel.addListener((ListChangeListener) c -> { + if (stateManager.activeSearchQueryProperty().isPresent().get()) { + while (c.next()) { + if (c.wasAdded()) { + doSearch(stateManager.activeSearchQueryProperty().get()); + return; + } + } + } + }); // We need to wrap the list since otherwise sorting in the table does not work entriesSorted = new SortedList<>(entriesFiltered); From 0f32e91c807235ff3dafde9314ee1abb08c86265 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Sun, 6 Nov 2022 19:28:35 +0100 Subject: [PATCH 063/256] Use pattern matching for cast Co-authored-by: Christoph --- src/main/java/org/jabref/logic/search/SearchQuery.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jabref/logic/search/SearchQuery.java b/src/main/java/org/jabref/logic/search/SearchQuery.java index 9d68999bcab..8a652fb4bb1 100644 --- a/src/main/java/org/jabref/logic/search/SearchQuery.java +++ b/src/main/java/org/jabref/logic/search/SearchQuery.java @@ -61,7 +61,8 @@ public String toString() { @Override public boolean equals(Object other) { - if (other instanceof SearchQuery) { + if (other instanceof SearchQuery searchQuery) { + return other.query... return ((SearchQuery) other).query.equals(this.query) && ((SearchQuery) other).searchFlags.equals(this.searchFlags); } return false; From 231f20005be721e55d8235b7dadb7cb504d1f045 Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Mon, 7 Nov 2022 18:31:17 +0100 Subject: [PATCH 064/256] Fix pattern matching --- src/main/java/org/jabref/logic/search/SearchQuery.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/SearchQuery.java b/src/main/java/org/jabref/logic/search/SearchQuery.java index 8a652fb4bb1..4b265e008c4 100644 --- a/src/main/java/org/jabref/logic/search/SearchQuery.java +++ b/src/main/java/org/jabref/logic/search/SearchQuery.java @@ -62,8 +62,7 @@ public String toString() { @Override public boolean equals(Object other) { if (other instanceof SearchQuery searchQuery) { - return other.query... - return ((SearchQuery) other).query.equals(this.query) && ((SearchQuery) other).searchFlags.equals(this.searchFlags); + return searchQuery.query.equals(this.query) && searchQuery.searchFlags.equals(this.searchFlags); } return false; } From 675d75c533f3a7cbb8bcb424895aa9122118913f Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Mon, 7 Nov 2022 20:55:34 +0100 Subject: [PATCH 065/256] Fix merge --- src/main/java/org/jabref/gui/LibraryTab.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/jabref/gui/LibraryTab.java b/src/main/java/org/jabref/gui/LibraryTab.java index c3365bf536c..fdd85dec035 100644 --- a/src/main/java/org/jabref/gui/LibraryTab.java +++ b/src/main/java/org/jabref/gui/LibraryTab.java @@ -239,7 +239,7 @@ public void onDatabaseLoadingFailed(Exception ex) { public void feedData(BibDatabaseContext bibDatabaseContext) { cleanUp(); - this.bibDatabaseContext = Objects.requireNonNull(bibDatabaseContext); + this.bibDatabaseContext = new SimpleObjectProperty<>(Objects.requireNonNull(bibDatabaseContext)); // When you open an existing library, a library tab with a loading animation is added immediately. // At that point, the library tab is given a temporary bibDatabaseContext with no entries. // This line is necessary because, while there is already a binding that updates the active database when a new tab is added, From 49eeb1ddc87a07ff15940c5222bdb17e594b2ffa Mon Sep 17 00:00:00 2001 From: Benedikt Tutzer Date: Mon, 7 Nov 2022 21:16:28 +0100 Subject: [PATCH 066/256] Speed up switches between sorting/filtering modes --- .../gui/maintable/MainTableDataModel.java | 1 + .../org/jabref/logic/search/SearchQuery.java | 20 ++++++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index 40819d35530..75bd2907990 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -56,6 +56,7 @@ public MainTableDataModel(BibDatabaseContext context, PreferencesService prefere stateManager.activeSearchQueryProperty(), groupsPreferences.groupViewModeProperty(), (groups, query, groupViewMode) -> { + // TODO btut: do not repeat search if display mode changes. Check if the same can be done for groups doSearch(query); return entry -> { updateSearchGroups(stateManager, bibDatabaseContext); diff --git a/src/main/java/org/jabref/logic/search/SearchQuery.java b/src/main/java/org/jabref/logic/search/SearchQuery.java index 4b265e008c4..6fe8b1a5dcf 100644 --- a/src/main/java/org/jabref/logic/search/SearchQuery.java +++ b/src/main/java/org/jabref/logic/search/SearchQuery.java @@ -4,6 +4,7 @@ import java.util.EnumSet; import java.util.HashMap; import java.util.Objects; +import java.util.Set; import java.util.stream.Collectors; import org.jabref.model.pdf.search.EnglishStemAnalyzer; @@ -14,6 +15,10 @@ import org.apache.lucene.queryparser.classic.ParseException; import org.apache.lucene.search.Query; +import static org.jabref.model.search.rules.SearchRules.SearchFlags.FILTERING_SEARCH; +import static org.jabref.model.search.rules.SearchRules.SearchFlags.KEEP_SEARCH_STRING; +import static org.jabref.model.search.rules.SearchRules.SearchFlags.SORT_BY_SCORE; + public class SearchQuery { protected final String query; @@ -59,10 +64,23 @@ public String toString() { return query; } + /** + * Equals, but only partially compares SearchFlags + * + * @return true if the search query is the same except for the filtering/sorting flags + */ @Override public boolean equals(Object other) { if (other instanceof SearchQuery searchQuery) { - return searchQuery.query.equals(this.query) && searchQuery.searchFlags.equals(this.searchFlags); + if (!searchQuery.query.equals(this.query)) { + return false; + } + Set thisSearchRulesWithoutFilterAndSort = this.searchFlags.clone(); + Set otherSearchRulesWithoutFilterAndSort = searchQuery.searchFlags.clone(); + Set filterAndSortFlags = EnumSet.of(SORT_BY_SCORE, FILTERING_SEARCH, KEEP_SEARCH_STRING); + thisSearchRulesWithoutFilterAndSort.removeAll(filterAndSortFlags); + otherSearchRulesWithoutFilterAndSort.removeAll(filterAndSortFlags); + return thisSearchRulesWithoutFilterAndSort.equals(otherSearchRulesWithoutFilterAndSort); } return false; } From 08aca2501643df4d5d650051225a2cd21d2255aa Mon Sep 17 00:00:00 2001 From: Carl Christian Snethlage <50491877+calixtus@users.noreply.github.com> Date: Tue, 27 Dec 2022 19:06:51 +0100 Subject: [PATCH 067/256] Fixed merge errors --- src/main/java/org/jabref/gui/JabRefFrame.java | 7 ++-- src/main/java/org/jabref/gui/LibraryTab.java | 37 +++++-------------- .../java/org/jabref/gui/StateManager.java | 4 +- .../search/SearchResultsTableDataModel.java | 6 +-- .../logic/search/indexing/LuceneIndexer.java | 4 +- 5 files changed, 19 insertions(+), 39 deletions(-) diff --git a/src/main/java/org/jabref/gui/JabRefFrame.java b/src/main/java/org/jabref/gui/JabRefFrame.java index 738a44e956a..f11328aa50d 100644 --- a/src/main/java/org/jabref/gui/JabRefFrame.java +++ b/src/main/java/org/jabref/gui/JabRefFrame.java @@ -16,7 +16,6 @@ import javafx.beans.Observable; import javafx.beans.binding.Bindings; import javafx.beans.binding.StringBinding; -import javafx.beans.property.ObjectProperty; import javafx.collections.transformation.FilteredList; import javafx.concurrent.Task; import javafx.geometry.Orientation; @@ -168,7 +167,7 @@ public class JabRefFrame extends BorderPane { private final FileHistoryMenu fileHistory; - @SuppressWarnings({"FieldCanBeLocal"}) private EasyObservableList> openDatabaseList; + @SuppressWarnings({"FieldCanBeLocal"}) private EasyObservableList openDatabaseList; private final Stage mainStage; private final StateManager stateManager; @@ -663,7 +662,7 @@ public void init() { filteredTabs.setPredicate(tab -> tab instanceof LibraryTab); // This variable cannot be inlined, since otherwise the list created by EasyBind is being garbage collected - openDatabaseList = EasyBind.map(filteredTabs, tab -> ((LibraryTab) tab).getBibDatabaseContextProperty()); + openDatabaseList = EasyBind.map(filteredTabs, tab -> ((LibraryTab) tab).getBibDatabaseContext()); EasyBind.bindContent(stateManager.getOpenDatabases(), openDatabaseList); // the binding for stateManager.activeDatabaseProperty() is at org.jabref.gui.LibraryTab.onDatabaseLoadingSucceed @@ -1158,7 +1157,7 @@ public void addTab(LibraryTab libraryTab, boolean raisePanel) { /** * Opens a new tab with existing data. - * Asynchronous loading is done at {@link #createLibraryTab(BackgroundTask, Path, PreferencesService, StateManager, JabRefFrame, ThemeManager)}. + * Asynchronous loading is done at {@link LibraryTab#createLibraryTab(BackgroundTask, Path, PreferencesService, StateManager, JabRefFrame, ThemeManager)}. */ public LibraryTab addTab(BibDatabaseContext databaseContext, boolean raisePanel) { Objects.requireNonNull(databaseContext); diff --git a/src/main/java/org/jabref/gui/LibraryTab.java b/src/main/java/org/jabref/gui/LibraryTab.java index b103a51d050..db11fa8a539 100644 --- a/src/main/java/org/jabref/gui/LibraryTab.java +++ b/src/main/java/org/jabref/gui/LibraryTab.java @@ -12,9 +12,7 @@ import javafx.animation.PauseTransition; import javafx.application.Platform; import javafx.beans.property.BooleanProperty; -import javafx.beans.property.ObjectProperty; import javafx.beans.property.SimpleBooleanProperty; -import javafx.beans.property.SimpleObjectProperty; import javafx.collections.ListChangeListener; import javafx.geometry.Orientation; import javafx.scene.Node; @@ -220,8 +218,8 @@ public void onDatabaseLoadingSucceed(ParserResult result) { feedData(context); try { - indexingTaskManager.manageFulltextIndexAccordingToPrefs(LuceneIndexer.of(bibDatabaseContext.get(), preferencesService, preferencesService.getFilePreferences())); - indexingTaskManager.updateIndex(LuceneIndexer.of(bibDatabaseContext.get(), preferencesService, preferencesService.getFilePreferences())); + indexingTaskManager.manageFulltextIndexAccordingToPrefs(LuceneIndexer.of(bibDatabaseContext, preferencesService, preferencesService.getFilePreferences())); + indexingTaskManager.updateIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService, preferencesService.getFilePreferences())); } catch (IOException e) { LOGGER.error("Cannot access lucene index", e); } @@ -822,7 +820,7 @@ public void resetChangedProperties() { /** * Creates a new library tab. Contents are loaded by the {@code dataLoadingTask}. Most of the other parameters are required by {@code resetChangeMonitor()}. * - * @param dataLoadingTask The task to execute to load the data. It is executed using {@link Globals.TASK_EXECUTOR}. + * @param dataLoadingTask The task to execute to load the data. It is executed using {@link Globals#TASK_EXECUTOR}. * @param file the path to the file (loaded by the dataLoadingTask) */ public static LibraryTab createLibraryTab(BackgroundTask dataLoadingTask, Path file, PreferencesService preferencesService, StateManager stateManager, JabRefFrame frame, ThemeManager themeManager) { @@ -892,7 +890,7 @@ private class IndexUpdateListener { @Subscribe public void listen(EntriesAddedEvent addedEntryEvent) { try { - LuceneIndexer luceneIndexer = LuceneIndexer.of(bibDatabaseContext.get(), preferencesService, preferencesService.getFilePreferences()); + LuceneIndexer luceneIndexer = LuceneIndexer.of(bibDatabaseContext, preferencesService, preferencesService.getFilePreferences()); for (BibEntry addedEntry : addedEntryEvent.getBibEntries()) { indexingTaskManager.addToIndex(luceneIndexer, addedEntry); } @@ -904,7 +902,7 @@ public void listen(EntriesAddedEvent addedEntryEvent) { @Subscribe public void listen(EntriesRemovedEvent removedEntriesEvent) { try { - LuceneIndexer luceneIndexer = LuceneIndexer.of(bibDatabaseContext.get(), preferencesService, preferencesService.getFilePreferences()); + LuceneIndexer luceneIndexer = LuceneIndexer.of(bibDatabaseContext, preferencesService, preferencesService.getFilePreferences()); for (BibEntry removedEntry : removedEntriesEvent.getBibEntries()) { indexingTaskManager.removeFromIndex(luceneIndexer, removedEntry); } @@ -917,33 +915,16 @@ public void listen(EntriesRemovedEvent removedEntriesEvent) { public void listen(FieldChangedEvent fieldChangedEvent) { for (BibEntry bibEntry : fieldChangedEvent.getBibEntries()) { try { - List removedFiles = new ArrayList<>(); + List toRemoveList = new ArrayList<>(); if (fieldChangedEvent.getField().equals(StandardField.FILE)) { - List oldFileList = FileFieldParser.parse(fieldChangedEvent.getOldValue()); - List newFileList = FileFieldParser.parse(fieldChangedEvent.getNewValue()); - removedFiles.remove(newFileList); + toRemoveList.addAll(FileFieldParser.parse(fieldChangedEvent.getOldValue())); + toRemoveList.removeAll(FileFieldParser.parse(fieldChangedEvent.getNewValue())); } - indexingTaskManager.updateIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService, preferencesService.getFilePreferences()), bibEntry, removedFiles); + indexingTaskManager.updateIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService, preferencesService.getFilePreferences()), bibEntry, toRemoveList); } catch (IOException e) { LOGGER.warn("I/O error when writing lucene index", e); } } -// if (fieldChangedEvent.getField().equals(StandardField.FILE)) { -// List oldFileList = FileFieldParser.parse(fieldChangedEvent.getOldValue()); -// List newFileList = FileFieldParser.parse(fieldChangedEvent.getNewValue()); -// -// List addedFiles = new ArrayList<>(newFileList); -// addedFiles.remove(oldFileList); -// List removedFiles = new ArrayList<>(oldFileList); -// removedFiles.remove(newFileList); -// -// try { -// indexingTaskManager.addToIndex(PdfIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences()), fieldChangedEvent.getBibEntry(), addedFiles, bibDatabaseContext); -// indexingTaskManager.removeFromIndex(PdfIndexer.of(bibDatabaseContext, preferencesService.getFilePreferences()), fieldChangedEvent.getBibEntry(), removedFiles); -// } catch (IOException e) { -// LOGGER.warn("I/O error when writing lucene index", e); -// } -// } } } diff --git a/src/main/java/org/jabref/gui/StateManager.java b/src/main/java/org/jabref/gui/StateManager.java index c2422011a3a..6845c5dba22 100644 --- a/src/main/java/org/jabref/gui/StateManager.java +++ b/src/main/java/org/jabref/gui/StateManager.java @@ -54,7 +54,7 @@ public class StateManager { private static final Logger LOGGER = LoggerFactory.getLogger(StateManager.class); private final CustomLocalDragboard localDragboard = new CustomLocalDragboard(); - private final ObservableList> openDatabases = FXCollections.observableArrayList(); + private final ObservableList openDatabases = FXCollections.observableArrayList(); private final OptionalObjectProperty activeDatabase = OptionalObjectProperty.empty(); private final ReadOnlyListWrapper activeGroups = new ReadOnlyListWrapper<>(FXCollections.observableArrayList()); private final ObservableList selectedEntries = FXCollections.observableArrayList(); @@ -83,7 +83,7 @@ public CustomLocalDragboard getLocalDragboard() { return localDragboard; } - public ObservableList> getOpenDatabases() { + public ObservableList getOpenDatabases() { return openDatabases; } diff --git a/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java b/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java index cfbe12973af..7cc11d47932 100644 --- a/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java +++ b/src/main/java/org/jabref/gui/search/SearchResultsTableDataModel.java @@ -32,9 +32,9 @@ public SearchResultsTableDataModel(BibDatabaseContext bibDatabaseContext, Prefer this.fieldValueFormatter = new SimpleObjectProperty<>(new MainTableFieldValueFormatter(preferencesService, bibDatabaseContext)); ObservableList entriesViewModel = FXCollections.observableArrayList(); - for (ObjectProperty context : stateManager.getOpenDatabases()) { - ObservableList entriesForDb = context.get().getDatabase().getEntries(); - List viewModelForDb = EasyBind.mapBacked(entriesForDb, entry -> new BibEntryTableViewModel(entry, context.get(), fieldValueFormatter, stateManager)); + for (BibDatabaseContext context : stateManager.getOpenDatabases()) { + ObservableList entriesForDb = context.getDatabase().getEntries(); + List viewModelForDb = EasyBind.mapBacked(entriesForDb, entry -> new BibEntryTableViewModel(entry, context, fieldValueFormatter, stateManager)); entriesViewModel.addAll(viewModelForDb); } diff --git a/src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java b/src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java index 33b03cc91c9..874d661e128 100644 --- a/src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java @@ -168,12 +168,12 @@ public void addBibFieldsToIndex(BibEntry bibEntry) { for (Map.Entry field : bibEntry.getFieldMap().entrySet()) { SearchFieldConstants.searchableBibFields.add(field.getKey().getName()); if (field.getKey() == StandardField.KEYWORDS) { - KeywordList keywords = KeywordList.parse(field.getValue(), preferences.getKeywordDelimiter()); + KeywordList keywords = KeywordList.parse(field.getValue(), preferences.getBibEntryPreferences().getKeywordSeparator()); for (Keyword keyword : keywords) { document.add(new StringField(field.getKey().getName(), keyword.toString(), org.apache.lucene.document.Field.Store.YES)); } } else if (field.getKey() == StandardField.GROUPS) { - List groups = Arrays.stream(field.getValue().split(preferences.getKeywordDelimiter().toString())).map(String::trim).toList(); + List groups = Arrays.stream(field.getValue().split(preferences.getBibEntryPreferences().getKeywordSeparator().toString())).map(String::trim).toList(); for (String group : groups) { document.add(new StringField(field.getKey().getName(), group, org.apache.lucene.document.Field.Store.YES)); } From 15b2152b81fbac2ed291bac58c56155ef558f6a5 Mon Sep 17 00:00:00 2001 From: Carl Christian Snethlage <50491877+calixtus@users.noreply.github.com> Date: Sun, 1 Jan 2023 21:51:24 +0100 Subject: [PATCH 068/256] Fixed small issues --- .../org/jabref/gui/maintable/MainTableDataModel.java | 10 +++++++--- .../org/jabref/gui/openoffice/OpenOfficePanel.java | 1 - .../jabref/logic/search/retrieval/LuceneSearcher.java | 4 +--- .../java/org/jabref/preferences/JabRefPreferences.java | 1 - 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index 75bd2907990..937d100b527 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -27,8 +27,12 @@ import org.jabref.preferences.PreferencesService; import com.tobiasdiez.easybind.EasyBind; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class MainTableDataModel { + private static final Logger LOGGER = LoggerFactory.getLogger(MainTableDataModel.class); + private final FilteredList entriesFiltered; private final SortedList entriesSorted; private final ObjectProperty fieldValueFormatter; @@ -88,7 +92,7 @@ public static void updateSearchGroups(StateManager stateManager, BibDatabaseCont } private void doSearch(Optional query) { - if (lastSearchQuery != null && lastSearchQuery.equals(query)) { + if (lastSearchQuery.isPresent() && lastSearchQuery.equals(query)) { return; } lastSearchQuery = query; @@ -98,7 +102,7 @@ private void doSearch(Optional query) { // TODO btut: maybe do in background? stateManager.getSearchResults().put(bibDatabaseContext, LuceneSearcher.of(bibDatabaseContext).search(query.get())); } catch (IOException e) { - e.printStackTrace(); + LOGGER.debug("Failed to run database search '{}'", query.get(), e); } } } @@ -112,7 +116,7 @@ private boolean isMatched(ObservableList groups, Optional query, BibEntryTableViewModel entry) { - if (!query.isPresent() || !query.get().getSearchFlags().contains(SearchRules.SearchFlags.FILTERING_SEARCH)) { + if (query.isEmpty() || !query.get().getSearchFlags().contains(SearchRules.SearchFlags.FILTERING_SEARCH)) { return true; } return entry.getSearchScore() > 0; diff --git a/src/main/java/org/jabref/gui/openoffice/OpenOfficePanel.java b/src/main/java/org/jabref/gui/openoffice/OpenOfficePanel.java index 28f35606f90..4ddf2ac7a42 100644 --- a/src/main/java/org/jabref/gui/openoffice/OpenOfficePanel.java +++ b/src/main/java/org/jabref/gui/openoffice/OpenOfficePanel.java @@ -8,7 +8,6 @@ import javax.swing.undo.UndoManager; -import javafx.beans.property.ObjectProperty; import javafx.concurrent.Task; import javafx.geometry.Insets; import javafx.geometry.Side; diff --git a/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java b/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java index 2fdb660b9de..3dfc1bb7e18 100644 --- a/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java +++ b/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java @@ -3,7 +3,6 @@ import java.io.IOException; import java.util.HashMap; -import org.jabref.gui.LibraryTab; import org.jabref.logic.search.SearchQuery; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; @@ -21,8 +20,7 @@ import org.slf4j.LoggerFactory; public final class LuceneSearcher { - - private static final Logger LOGGER = LoggerFactory.getLogger(LibraryTab.class); + private static final Logger LOGGER = LoggerFactory.getLogger(LuceneSearcher.class); private final BibDatabaseContext databaseContext; private final Directory indexDirectory; diff --git a/src/main/java/org/jabref/preferences/JabRefPreferences.java b/src/main/java/org/jabref/preferences/JabRefPreferences.java index 42553d67349..d440e83d430 100644 --- a/src/main/java/org/jabref/preferences/JabRefPreferences.java +++ b/src/main/java/org/jabref/preferences/JabRefPreferences.java @@ -244,7 +244,6 @@ public class JabRefPreferences implements PreferencesService { public static final String FILE_BROWSER_COMMAND = "fileBrowserCommand"; public static final String MAIN_FILE_DIRECTORY = "fileDirectory"; - public static final String SEARCH_DISPLAY_MODE = "searchDisplayMode"; public static final String SEARCH_REG_EXP = "regExpSearch"; public static final String SEARCH_FULLTEXT = "fulltextSearch"; public static final String SEARCH_KEEP_SEARCH_STRING = "keepSearchString"; From f8b643be2454a8ab9afc46d1663abebad45ca109 Mon Sep 17 00:00:00 2001 From: Carl Christian Snethlage <50491877+calixtus@users.noreply.github.com> Date: Wed, 4 Jan 2023 00:01:00 +0100 Subject: [PATCH 069/256] Removed obsolete tests, fixed some tests --- src/main/java/org/jabref/gui/LibraryTab.java | 10 +- .../ExternalFilesEntryLinker.java | 4 +- .../jabref/gui/groups/GroupsPreferences.java | 19 +-- .../RebuildFulltextSearchIndexAction.java | 4 +- .../search/indexing/IndexingTaskManager.java | 10 +- .../logic/search/indexing/LuceneIndexer.java | 16 +-- .../gui/groups/GroupTreeViewModelTest.java | 7 +- ...sAndRegexBasedSearchRuleDescriberTest.java | 95 ------------- .../GrammarBasedSearchRuleDescriberTest.java | 127 ------------------ .../logic/exporter/GroupSerializerTest.java | 5 +- .../logic/importer/util/GroupsParserTest.java | 1 + .../search/indexing/LuceneIndexerTest.java | 5 +- .../search/retrieval/LuceneSearcherTest.java | 5 +- .../logic/search/DatabaseSearcherTest.java | 21 +-- .../jabref/logic/search/SearchQueryTest.java | 73 ++++------ .../model/groups/GroupTreeNodeTest.java | 9 +- .../jabref/model/groups/SearchGroupTest.java | 5 +- .../rules/ContainsBasedSearchRuleTest.java | 58 -------- .../rules/GrammarBasedSearchRuleTest.java | 101 -------------- .../search/rules/SentenceAnalyzerTest.java | 34 ----- 20 files changed, 96 insertions(+), 513 deletions(-) delete mode 100644 src/test/java/org/jabref/gui/search/ContainsAndRegexBasedSearchRuleDescriberTest.java delete mode 100644 src/test/java/org/jabref/gui/search/GrammarBasedSearchRuleDescriberTest.java delete mode 100644 src/test/java/org/jabref/model/search/rules/ContainsBasedSearchRuleTest.java delete mode 100644 src/test/java/org/jabref/model/search/rules/GrammarBasedSearchRuleTest.java delete mode 100644 src/test/java/org/jabref/model/search/rules/SentenceAnalyzerTest.java diff --git a/src/main/java/org/jabref/gui/LibraryTab.java b/src/main/java/org/jabref/gui/LibraryTab.java index db11fa8a539..1672b792d69 100644 --- a/src/main/java/org/jabref/gui/LibraryTab.java +++ b/src/main/java/org/jabref/gui/LibraryTab.java @@ -218,8 +218,8 @@ public void onDatabaseLoadingSucceed(ParserResult result) { feedData(context); try { - indexingTaskManager.manageFulltextIndexAccordingToPrefs(LuceneIndexer.of(bibDatabaseContext, preferencesService, preferencesService.getFilePreferences())); - indexingTaskManager.updateIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService, preferencesService.getFilePreferences())); + indexingTaskManager.manageFulltextIndexAccordingToPrefs(LuceneIndexer.of(bibDatabaseContext, preferencesService)); + indexingTaskManager.updateIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService)); } catch (IOException e) { LOGGER.error("Cannot access lucene index", e); } @@ -890,7 +890,7 @@ private class IndexUpdateListener { @Subscribe public void listen(EntriesAddedEvent addedEntryEvent) { try { - LuceneIndexer luceneIndexer = LuceneIndexer.of(bibDatabaseContext, preferencesService, preferencesService.getFilePreferences()); + LuceneIndexer luceneIndexer = LuceneIndexer.of(bibDatabaseContext, preferencesService); for (BibEntry addedEntry : addedEntryEvent.getBibEntries()) { indexingTaskManager.addToIndex(luceneIndexer, addedEntry); } @@ -902,7 +902,7 @@ public void listen(EntriesAddedEvent addedEntryEvent) { @Subscribe public void listen(EntriesRemovedEvent removedEntriesEvent) { try { - LuceneIndexer luceneIndexer = LuceneIndexer.of(bibDatabaseContext, preferencesService, preferencesService.getFilePreferences()); + LuceneIndexer luceneIndexer = LuceneIndexer.of(bibDatabaseContext, preferencesService); for (BibEntry removedEntry : removedEntriesEvent.getBibEntries()) { indexingTaskManager.removeFromIndex(luceneIndexer, removedEntry); } @@ -920,7 +920,7 @@ public void listen(FieldChangedEvent fieldChangedEvent) { toRemoveList.addAll(FileFieldParser.parse(fieldChangedEvent.getOldValue())); toRemoveList.removeAll(FileFieldParser.parse(fieldChangedEvent.getNewValue())); } - indexingTaskManager.updateIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService, preferencesService.getFilePreferences()), bibEntry, toRemoveList); + indexingTaskManager.updateIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService), bibEntry, toRemoveList); } catch (IOException e) { LOGGER.warn("I/O error when writing lucene index", e); } diff --git a/src/main/java/org/jabref/gui/externalfiles/ExternalFilesEntryLinker.java b/src/main/java/org/jabref/gui/externalfiles/ExternalFilesEntryLinker.java index 34e62fa949b..64ceafc59a3 100644 --- a/src/main/java/org/jabref/gui/externalfiles/ExternalFilesEntryLinker.java +++ b/src/main/java/org/jabref/gui/externalfiles/ExternalFilesEntryLinker.java @@ -82,7 +82,7 @@ public void moveFilesToFileDirAndAddToEntry(BibEntry entry, List files, In } try { - indexingTaskManager.addToIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService, filePreferences), entry); + indexingTaskManager.addToIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService), entry); } catch (IOException e) { LOGGER.error("Could not access Fulltext-Index", e); } @@ -100,7 +100,7 @@ public void copyFilesToFileDirAndAddToEntry(BibEntry entry, List files, In } try { - indexingTaskManager.addToIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService, filePreferences), entry); + indexingTaskManager.addToIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService), entry); } catch (IOException e) { LOGGER.error("Could not access Fulltext-Index", e); } diff --git a/src/main/java/org/jabref/gui/groups/GroupsPreferences.java b/src/main/java/org/jabref/gui/groups/GroupsPreferences.java index 8bd5102e496..f3bd7e9734a 100644 --- a/src/main/java/org/jabref/gui/groups/GroupsPreferences.java +++ b/src/main/java/org/jabref/gui/groups/GroupsPreferences.java @@ -3,7 +3,6 @@ import java.util.EnumSet; import javafx.beans.property.BooleanProperty; -import javafx.beans.property.ObjectProperty; import javafx.beans.property.SetProperty; import javafx.beans.property.SimpleBooleanProperty; import javafx.beans.property.SimpleSetProperty; @@ -11,17 +10,15 @@ public class GroupsPreferences { - private final SetProperty groupViewMode; - private final BooleanProperty shouldAutoAssignGroup; - private final BooleanProperty shouldDisplayGroupCount; + private final SetProperty groupViewMode = new SimpleSetProperty<>(FXCollections.observableSet());; + private final BooleanProperty shouldAutoAssignGroup = new SimpleBooleanProperty(); + private final BooleanProperty shouldDisplayGroupCount = new SimpleBooleanProperty(); public GroupsPreferences(boolean viewModeIntersection, boolean viewModeFilter, boolean viewModeInvert, boolean shouldAutoAssignGroup, boolean shouldDisplayGroupCount) { - - this.groupViewMode = new SimpleSetProperty<>(FXCollections.observableSet()); if (viewModeIntersection) { this.groupViewMode.add(GroupViewMode.INTERSECTION); } @@ -31,8 +28,14 @@ public GroupsPreferences(boolean viewModeIntersection, if (viewModeInvert) { this.groupViewMode.add(GroupViewMode.INVERT); } - this.shouldAutoAssignGroup = new SimpleBooleanProperty(shouldAutoAssignGroup); - this.shouldDisplayGroupCount = new SimpleBooleanProperty(shouldDisplayGroupCount); + this.shouldAutoAssignGroup.set(shouldAutoAssignGroup); + this.shouldDisplayGroupCount.set(shouldDisplayGroupCount); + } + + public GroupsPreferences(EnumSet groupViewModes, boolean shouldAutoAssignGroup, boolean shouldDisplayGroupCount) { + this.groupViewMode.addAll(groupViewModes); + this.shouldAutoAssignGroup.set(shouldAutoAssignGroup); + this.shouldDisplayGroupCount.set(shouldDisplayGroupCount); } public EnumSet getGroupViewMode() { diff --git a/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java b/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java index 85b3f3a2bc0..7a9f42dacc9 100644 --- a/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java +++ b/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java @@ -71,8 +71,8 @@ private void rebuildIndex() { return; } try { - currentLibraryTab.get().getIndexingTaskManager().createIndex(LuceneIndexer.of(databaseContext, preferencesService, filePreferences)); - currentLibraryTab.get().getIndexingTaskManager().updateIndex(LuceneIndexer.of(databaseContext, preferencesService, filePreferences)); + currentLibraryTab.get().getIndexingTaskManager().createIndex(LuceneIndexer.of(databaseContext, preferencesService)); + currentLibraryTab.get().getIndexingTaskManager().updateIndex(LuceneIndexer.of(databaseContext, preferencesService)); } catch (IOException e) { dialogService.notify(Localization.lang("Failed to access fulltext search index")); LOGGER.error("Failed to access fulltext search index", e); diff --git a/src/main/java/org/jabref/logic/search/indexing/IndexingTaskManager.java b/src/main/java/org/jabref/logic/search/indexing/IndexingTaskManager.java index 5ce512461de..5fd8a41075f 100644 --- a/src/main/java/org/jabref/logic/search/indexing/IndexingTaskManager.java +++ b/src/main/java/org/jabref/logic/search/indexing/IndexingTaskManager.java @@ -17,7 +17,7 @@ public class IndexingTaskManager extends BackgroundTask { private final ConcurrentLinkedDeque taskQueue = new ConcurrentLinkedDeque<>(); - private TaskExecutor taskExecutor; + private final TaskExecutor taskExecutor; private int numOfIndexedFiles = 0; private final Object lock = new Object(); @@ -88,17 +88,17 @@ public AutoCloseable blockNewTasks() { } public void createIndex(LuceneIndexer indexer) { - enqueueTask(() -> indexer.createIndex(), true); + enqueueTask(indexer::createIndex, true); } public void manageFulltextIndexAccordingToPrefs(LuceneIndexer indexer) { indexer.getFilePreferences().fulltextIndexLinkedFilesProperty().addListener((observable, oldValue, newValue) -> { - if (newValue.booleanValue()) { + if (newValue) { for (BibEntry bibEntry : indexer.getDatabaseContext().getEntries()) { enqueueTask(() -> indexer.updateLinkedFilesInIndex(bibEntry, List.of()), false); } } else { - enqueueTask(() -> indexer.deleteLinkedFilesIndex(), true); + enqueueTask(indexer::deleteLinkedFilesIndex, true); } }); } @@ -106,7 +106,7 @@ public void manageFulltextIndexAccordingToPrefs(LuceneIndexer indexer) { public void updateIndex(LuceneIndexer indexer) { Set pathsToRemove = indexer.getListOfFilePaths(); Set hashesOfEntriesToRemove = indexer.getListOfHashes(); - indexer.getDatabaseContext().getEntries().stream().forEach(BibEntry::updateAndGetIndexHash); + indexer.getDatabaseContext().getEntries().forEach(BibEntry::updateAndGetIndexHash); for (BibEntry entry : indexer.getDatabaseContext().getEntries()) { enqueueTask(() -> indexer.addBibFieldsToIndex(entry), true); enqueueTask(() -> indexer.addLinkedFilesToIndex(entry), false); diff --git a/src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java b/src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java index 874d661e128..9503ec9445c 100644 --- a/src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java @@ -59,17 +59,15 @@ public class LuceneIndexer { private final BibDatabaseContext databaseContext; private final PreferencesService preferences; - private final FilePreferences filePreferences; - public LuceneIndexer(BibDatabaseContext databaseContext, PreferencesService preferences, FilePreferences filePreferences) throws IOException { + public LuceneIndexer(BibDatabaseContext databaseContext, PreferencesService preferences) throws IOException { this.databaseContext = databaseContext; this.directoryToIndex = new NIOFSDirectory(databaseContext.getFulltextIndexPath()); this.preferences = preferences; - this.filePreferences = filePreferences; } - public static LuceneIndexer of(BibDatabaseContext databaseContext, PreferencesService preferences, FilePreferences filePreferences) throws IOException { - return new LuceneIndexer(databaseContext, preferences, filePreferences); + public static LuceneIndexer of(BibDatabaseContext databaseContext, PreferencesService preferences) throws IOException { + return new LuceneIndexer(databaseContext, preferences); } public BibDatabaseContext getDatabaseContext() { @@ -197,13 +195,13 @@ public void addBibFieldsToIndex(BibEntry bibEntry) { * @param linkedFile the file to write to the index */ private void writeFileToIndex(BibEntry entry, LinkedFile linkedFile) { - if (!filePreferences.shouldFulltextIndexLinkedFiles()) { + if (!preferences.getFilePreferences().shouldFulltextIndexLinkedFiles()) { return; } if (linkedFile.isOnlineLink() || !StandardFileType.PDF.getName().equals(linkedFile.getFileType())) { return; } - Optional resolvedPath = linkedFile.findIn(databaseContext, filePreferences); + Optional resolvedPath = linkedFile.findIn(databaseContext, preferences.getFilePreferences()); if (resolvedPath.isEmpty()) { LOGGER.warn("Could not find {}", linkedFile.getLink()); return; @@ -229,7 +227,7 @@ private void writeFileToIndex(BibEntry entry, LinkedFile linkedFile) { // if there is no index yet, don't need to check anything! } // If no document was found, add the new one - Optional> pages = new DocumentReader(entry, filePreferences).readLinkedPdf(this.databaseContext, linkedFile); + Optional> pages = new DocumentReader(entry, preferences.getFilePreferences()).readLinkedPdf(this.databaseContext, linkedFile); if (pages.isPresent()) { try (IndexWriter indexWriter = new IndexWriter(directoryToIndex, new IndexWriterConfig( @@ -320,6 +318,6 @@ public void deleteLinkedFilesIndex() { } public FilePreferences getFilePreferences() { - return filePreferences; + return preferences.getFilePreferences(); } } diff --git a/src/test/java/org/jabref/gui/groups/GroupTreeViewModelTest.java b/src/test/java/org/jabref/gui/groups/GroupTreeViewModelTest.java index c8ddd435fb2..7a4d88639c6 100644 --- a/src/test/java/org/jabref/gui/groups/GroupTreeViewModelTest.java +++ b/src/test/java/org/jabref/gui/groups/GroupTreeViewModelTest.java @@ -1,5 +1,6 @@ package org.jabref.gui.groups; +import java.util.EnumSet; import java.util.Optional; import org.jabref.gui.DialogService; @@ -46,9 +47,9 @@ void setUp() { dialogService = mock(DialogService.class, Answers.RETURNS_DEEP_STUBS); when(preferencesService.getGroupsPreferences()).thenReturn(new GroupsPreferences( - GroupViewMode.UNION, - true, - true)); + EnumSet.of(GroupViewMode.FILTER), + true, + true)); groupTree = new GroupTreeViewModel(stateManager, mock(DialogService.class), preferencesService, taskExecutor, new CustomLocalDragboard()); } diff --git a/src/test/java/org/jabref/gui/search/ContainsAndRegexBasedSearchRuleDescriberTest.java b/src/test/java/org/jabref/gui/search/ContainsAndRegexBasedSearchRuleDescriberTest.java deleted file mode 100644 index 2a179fc46e8..00000000000 --- a/src/test/java/org/jabref/gui/search/ContainsAndRegexBasedSearchRuleDescriberTest.java +++ /dev/null @@ -1,95 +0,0 @@ -package org.jabref.gui.search; - -import java.util.EnumSet; -import java.util.List; - -import javafx.scene.text.Text; -import javafx.scene.text.TextFlow; -import javafx.stage.Stage; - -import org.jabref.gui.util.TooltipTextUtil; -import org.jabref.testutils.category.GUITest; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.testfx.framework.junit5.ApplicationExtension; -import org.testfx.framework.junit5.Start; - -@GUITest -@ExtendWith(ApplicationExtension.class) -class ContainsAndRegexBasedSearchRuleDescriberTest { - - @Start - void onStart(Stage stage) { - // Needed to init JavaFX thread - stage.show(); - } - - @Test - void testSimpleTerm() { - String query = "test"; - List expectedTexts = List.of( - TooltipTextUtil.createText("This search contains entries in which any field contains the term "), - TooltipTextUtil.createText("test", TooltipTextUtil.TextType.BOLD), - TooltipTextUtil.createText(" (case insensitive). ")); - TextFlow description = new ContainsAndRegexBasedSearchRuleDescriber(EnumSet.noneOf(SearchFlags.class), query).getDescription(); - - TextFlowEqualityHelper.assertEquals(expectedTexts, description); - } - - @Test - void testNoAst() { - String query = "a b"; - List expectedTexts = List.of( - TooltipTextUtil.createText("This search contains entries in which any field contains the term "), - TooltipTextUtil.createText("a", TooltipTextUtil.TextType.BOLD), - TooltipTextUtil.createText(" and "), - TooltipTextUtil.createText("b", TooltipTextUtil.TextType.BOLD), - TooltipTextUtil.createText(" (case insensitive). ")); - TextFlow description = new ContainsAndRegexBasedSearchRuleDescriber(EnumSet.noneOf(SearchFlags.class), query).getDescription(); - - TextFlowEqualityHelper.assertEquals(expectedTexts, description); - } - - @Test - void testNoAstRegex() { - String query = "a b"; - List expectedTexts = List.of( - TooltipTextUtil.createText("This search contains entries in which any field contains the regular expression "), - TooltipTextUtil.createText("a", TooltipTextUtil.TextType.BOLD), - TooltipTextUtil.createText(" and "), - TooltipTextUtil.createText("b", TooltipTextUtil.TextType.BOLD), - TooltipTextUtil.createText(" (case insensitive). ")); - TextFlow description = new ContainsAndRegexBasedSearchRuleDescriber(EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION), query).getDescription(); - - TextFlowEqualityHelper.assertEquals(expectedTexts, description); - } - - @Test - void testNoAstRegexCaseSensitive() { - String query = "a b"; - List expectedTexts = List.of( - TooltipTextUtil.createText("This search contains entries in which any field contains the regular expression "), - TooltipTextUtil.createText("a", TooltipTextUtil.TextType.BOLD), - TooltipTextUtil.createText(" and "), - TooltipTextUtil.createText("b", TooltipTextUtil.TextType.BOLD), - TooltipTextUtil.createText(" (case sensitive). ")); - TextFlow description = new ContainsAndRegexBasedSearchRuleDescriber(EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION), query).getDescription(); - - TextFlowEqualityHelper.assertEquals(expectedTexts, description); - } - - @Test - void testNoAstCaseSensitive() { - String query = "a b"; - List expectedTexts = List.of( - TooltipTextUtil.createText("This search contains entries in which any field contains the term "), - TooltipTextUtil.createText("a", TooltipTextUtil.TextType.BOLD), - TooltipTextUtil.createText(" and "), - TooltipTextUtil.createText("b", TooltipTextUtil.TextType.BOLD), - TooltipTextUtil.createText(" (case sensitive). ")); - TextFlow description = new ContainsAndRegexBasedSearchRuleDescriber(EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE), query).getDescription(); - - TextFlowEqualityHelper.assertEquals(expectedTexts, description); - } -} diff --git a/src/test/java/org/jabref/gui/search/GrammarBasedSearchRuleDescriberTest.java b/src/test/java/org/jabref/gui/search/GrammarBasedSearchRuleDescriberTest.java deleted file mode 100644 index 20b2f8f64fe..00000000000 --- a/src/test/java/org/jabref/gui/search/GrammarBasedSearchRuleDescriberTest.java +++ /dev/null @@ -1,127 +0,0 @@ -package org.jabref.gui.search; - -import java.util.Arrays; -import java.util.EnumSet; -import java.util.List; - -import javafx.scene.text.Text; -import javafx.scene.text.TextFlow; -import javafx.stage.Stage; - -import org.jabref.gui.util.TooltipTextUtil; -import org.jabref.testutils.category.GUITest; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.testfx.framework.junit5.ApplicationExtension; -import org.testfx.framework.junit5.Start; - -import static org.junit.jupiter.api.Assertions.assertTrue; - -@GUITest -@ExtendWith(ApplicationExtension.class) -class GrammarBasedSearchRuleDescriberTest { - - @Start - void onStart(Stage stage) { - // Needed to init JavaFX thread - stage.show(); - } - - private TextFlow createDescription(String query, EnumSet searchFlags) { - GrammarBasedSearchRule grammarBasedSearchRule = new GrammarBasedSearchRule(searchFlags); - assertTrue(grammarBasedSearchRule.validateSearchStrings(query)); - GrammarBasedSearchRuleDescriber describer = new GrammarBasedSearchRuleDescriber(searchFlags, grammarBasedSearchRule.getTree()); - return describer.getDescription(); - } - - @Test - void testSimpleQueryCaseSensitiveRegex() { - String query = "a=b"; - List expectedTexts = Arrays.asList(TooltipTextUtil.createText("This search contains entries in which "), TooltipTextUtil.createText("the field "), TooltipTextUtil.createText("a", TooltipTextUtil.TextType.BOLD), - TooltipTextUtil.createText(" contains the regular expression "), TooltipTextUtil.createText("b", TooltipTextUtil.TextType.BOLD), TooltipTextUtil.createText(". "), - TooltipTextUtil.createText("The search is case-sensitive.")); - TextFlow description = createDescription(query, EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)); - - TextFlowEqualityHelper.assertEquals(expectedTexts, description); - } - - @Test - void testSimpleQueryCaseSensitive() { - String query = "a=b"; - List expectedTexts = Arrays.asList(TooltipTextUtil.createText("This search contains entries in which "), TooltipTextUtil.createText("the field "), TooltipTextUtil.createText("a", TooltipTextUtil.TextType.BOLD), - TooltipTextUtil.createText(" contains the term "), TooltipTextUtil.createText("b", TooltipTextUtil.TextType.BOLD), TooltipTextUtil.createText(". "), - TooltipTextUtil.createText("The search is case-sensitive.")); - TextFlow description = createDescription(query, EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE)); - - TextFlowEqualityHelper.assertEquals(expectedTexts, description); - } - - @Test - void testSimpleQuery() { - String query = "a=b"; - List expectedTexts = Arrays.asList(TooltipTextUtil.createText("This search contains entries in which "), TooltipTextUtil.createText("the field "), TooltipTextUtil.createText("a", TooltipTextUtil.TextType.BOLD), - TooltipTextUtil.createText(" contains the term "), TooltipTextUtil.createText("b", TooltipTextUtil.TextType.BOLD), TooltipTextUtil.createText(". "), - TooltipTextUtil.createText("The search is case-insensitive.")); - TextFlow description = createDescription(query, EnumSet.noneOf(SearchFlags.class)); - - TextFlowEqualityHelper.assertEquals(expectedTexts, description); - } - - @Test - void testSimpleQueryRegex() { - String query = "a=b"; - List expectedTexts = Arrays.asList(TooltipTextUtil.createText("This search contains entries in which "), TooltipTextUtil.createText("the field "), TooltipTextUtil.createText("a", TooltipTextUtil.TextType.BOLD), - TooltipTextUtil.createText(" contains the regular expression "), TooltipTextUtil.createText("b", TooltipTextUtil.TextType.BOLD), TooltipTextUtil.createText(". "), - TooltipTextUtil.createText("The search is case-insensitive.")); - TextFlow description = createDescription(query, EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)); - - TextFlowEqualityHelper.assertEquals(expectedTexts, description); - } - - @Test - void testComplexQueryCaseSensitiveRegex() { - String query = "not a=b and c=e or e=\"x\""; - List expectedTexts = Arrays.asList(TooltipTextUtil.createText("This search contains entries in which "), TooltipTextUtil.createText("not "), TooltipTextUtil.createText("the field "), TooltipTextUtil.createText("a", TooltipTextUtil.TextType.BOLD), - TooltipTextUtil.createText(" contains the regular expression "), TooltipTextUtil.createText("b", TooltipTextUtil.TextType.BOLD), TooltipTextUtil.createText(" and "), TooltipTextUtil.createText("the field "), TooltipTextUtil.createText("c", TooltipTextUtil.TextType.BOLD), TooltipTextUtil.createText(" contains the regular expression "), - TooltipTextUtil.createText("e", TooltipTextUtil.TextType.BOLD), TooltipTextUtil.createText(" or "), TooltipTextUtil.createText("the field "), TooltipTextUtil.createText("e", TooltipTextUtil.TextType.BOLD), TooltipTextUtil.createText(" contains the regular expression "), - TooltipTextUtil.createText("x", TooltipTextUtil.TextType.BOLD), TooltipTextUtil.createText(". "), TooltipTextUtil.createText("The search is case-sensitive.")); - TextFlow description = createDescription(query, EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)); - - TextFlowEqualityHelper.assertEquals(expectedTexts, description); - } - - @Test - void testComplexQueryRegex() { - String query = "not a=b and c=e or e=\"x\""; - List expectedTexts = Arrays.asList(TooltipTextUtil.createText("This search contains entries in which "), TooltipTextUtil.createText("not "), TooltipTextUtil.createText("the field "), TooltipTextUtil.createText("a", TooltipTextUtil.TextType.BOLD), - TooltipTextUtil.createText(" contains the regular expression "), TooltipTextUtil.createText("b", TooltipTextUtil.TextType.BOLD), TooltipTextUtil.createText(" and "), TooltipTextUtil.createText("the field "), TooltipTextUtil.createText("c", TooltipTextUtil.TextType.BOLD), TooltipTextUtil.createText(" contains the regular expression "), - TooltipTextUtil.createText("e", TooltipTextUtil.TextType.BOLD), TooltipTextUtil.createText(" or "), TooltipTextUtil.createText("the field "), TooltipTextUtil.createText("e", TooltipTextUtil.TextType.BOLD), TooltipTextUtil.createText(" contains the regular expression "), - TooltipTextUtil.createText("x", TooltipTextUtil.TextType.BOLD), TooltipTextUtil.createText(". "), TooltipTextUtil.createText("The search is case-insensitive.")); - TextFlow description = createDescription(query, EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)); - - TextFlowEqualityHelper.assertEquals(expectedTexts, description); - } - - @Test - void testComplexQueryCaseSensitive() { - String query = "not a=b and c=e or e=\"x\""; - List expectedTexts = Arrays.asList(TooltipTextUtil.createText("This search contains entries in which "), TooltipTextUtil.createText("not "), TooltipTextUtil.createText("the field "), TooltipTextUtil.createText("a", TooltipTextUtil.TextType.BOLD), - TooltipTextUtil.createText(" contains the term "), TooltipTextUtil.createText("b", TooltipTextUtil.TextType.BOLD), TooltipTextUtil.createText(" and "), TooltipTextUtil.createText("the field "), TooltipTextUtil.createText("c", TooltipTextUtil.TextType.BOLD), TooltipTextUtil.createText(" contains the term "), TooltipTextUtil.createText("e", TooltipTextUtil.TextType.BOLD), - TooltipTextUtil.createText(" or "), TooltipTextUtil.createText("the field "), TooltipTextUtil.createText("e", TooltipTextUtil.TextType.BOLD), TooltipTextUtil.createText(" contains the term "), TooltipTextUtil.createText("x", TooltipTextUtil.TextType.BOLD), TooltipTextUtil.createText(". "), TooltipTextUtil.createText("The search is case-sensitive.")); - TextFlow description = createDescription(query, EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE)); - - TextFlowEqualityHelper.assertEquals(expectedTexts, description); - } - - @Test - void testComplexQuery() { - String query = "not a=b and c=e or e=\"x\""; - List expectedTexts = Arrays.asList(TooltipTextUtil.createText("This search contains entries in which "), TooltipTextUtil.createText("not "), TooltipTextUtil.createText("the field "), TooltipTextUtil.createText("a", TooltipTextUtil.TextType.BOLD), - TooltipTextUtil.createText(" contains the term "), TooltipTextUtil.createText("b", TooltipTextUtil.TextType.BOLD), TooltipTextUtil.createText(" and "), TooltipTextUtil.createText("the field "), TooltipTextUtil.createText("c", TooltipTextUtil.TextType.BOLD), TooltipTextUtil.createText(" contains the term "), TooltipTextUtil.createText("e", TooltipTextUtil.TextType.BOLD), - TooltipTextUtil.createText(" or "), TooltipTextUtil.createText("the field "), TooltipTextUtil.createText("e", TooltipTextUtil.TextType.BOLD), TooltipTextUtil.createText(" contains the term "), TooltipTextUtil.createText("x", TooltipTextUtil.TextType.BOLD), TooltipTextUtil.createText(". "), TooltipTextUtil.createText("The search is case-insensitive.")); - TextFlow description = createDescription(query, EnumSet.noneOf(SearchFlags.class)); - - TextFlowEqualityHelper.assertEquals(expectedTexts, description); - } -} diff --git a/src/test/java/org/jabref/logic/exporter/GroupSerializerTest.java b/src/test/java/org/jabref/logic/exporter/GroupSerializerTest.java index 664ebbcae45..4e75315a344 100644 --- a/src/test/java/org/jabref/logic/exporter/GroupSerializerTest.java +++ b/src/test/java/org/jabref/logic/exporter/GroupSerializerTest.java @@ -25,6 +25,7 @@ import org.jabref.model.groups.TexGroup; import org.jabref.model.groups.WordKeywordGroup; import org.jabref.model.metadata.MetaData; +import org.jabref.model.search.rules.SearchRules; import org.jabref.model.util.DummyFileUpdateMonitor; import org.junit.jupiter.api.BeforeEach; @@ -90,14 +91,14 @@ void serializeSingleRegexKeywordGroup() { @Test void serializeSingleSearchGroup() { - SearchGroup group = new SearchGroup("myExplicitGroup", GroupHierarchyType.INDEPENDENT, "author=harrer", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)); + SearchGroup group = new SearchGroup("myExplicitGroup", GroupHierarchyType.INDEPENDENT, "author=harrer", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)); List serialization = groupSerializer.serializeTree(GroupTreeNode.fromGroup(group)); assertEquals(Collections.singletonList("0 SearchGroup:myExplicitGroup;0;author=harrer;1;1;1;;;;"), serialization); } @Test void serializeSingleSearchGroupWithRegex() { - SearchGroup group = new SearchGroup("myExplicitGroup", GroupHierarchyType.INCLUDING, "author=\"harrer\"", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE)); + SearchGroup group = new SearchGroup("myExplicitGroup", GroupHierarchyType.INCLUDING, "author=\"harrer\"", EnumSet.noneOf(SearchRules.SearchFlags.class)); List serialization = groupSerializer.serializeTree(GroupTreeNode.fromGroup(group)); assertEquals(Collections.singletonList("0 SearchGroup:myExplicitGroup;2;author=\"harrer\";1;0;1;;;;"), serialization); } diff --git a/src/test/java/org/jabref/logic/importer/util/GroupsParserTest.java b/src/test/java/org/jabref/logic/importer/util/GroupsParserTest.java index 10c45f1a7f4..e600056b879 100644 --- a/src/test/java/org/jabref/logic/importer/util/GroupsParserTest.java +++ b/src/test/java/org/jabref/logic/importer/util/GroupsParserTest.java @@ -21,6 +21,7 @@ import org.jabref.model.groups.SearchGroup; import org.jabref.model.groups.TexGroup; import org.jabref.model.metadata.MetaData; +import org.jabref.model.search.rules.SearchRules; import org.jabref.model.util.DummyFileUpdateMonitor; import org.jabref.model.util.FileUpdateMonitor; diff --git a/src/test/java/org/jabref/logic/pdf/search/indexing/LuceneIndexerTest.java b/src/test/java/org/jabref/logic/pdf/search/indexing/LuceneIndexerTest.java index 15edef099aa..5cd9156e715 100644 --- a/src/test/java/org/jabref/logic/pdf/search/indexing/LuceneIndexerTest.java +++ b/src/test/java/org/jabref/logic/pdf/search/indexing/LuceneIndexerTest.java @@ -13,6 +13,7 @@ import org.jabref.model.entry.LinkedFile; import org.jabref.model.entry.types.StandardEntryType; import org.jabref.preferences.FilePreferences; +import org.jabref.preferences.PreferencesService; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; @@ -36,6 +37,8 @@ public class LuceneIndexerTest { public void setUp(@TempDir Path indexDir) throws IOException { FilePreferences filePreferences = mock(FilePreferences.class); when(filePreferences.shouldFulltextIndexLinkedFiles()).thenReturn(true); + PreferencesService preferencesService = mock(PreferencesService.class); + when(preferencesService.getFilePreferences()).thenReturn(filePreferences); this.database = new BibDatabase(); this.context = mock(BibDatabaseContext.class); @@ -44,7 +47,7 @@ public void setUp(@TempDir Path indexDir) throws IOException { when(context.getFulltextIndexPath()).thenReturn(indexDir); when(context.getDatabase()).thenReturn(database); when(context.getEntries()).thenReturn(database.getEntries()); - this.indexer = LuceneIndexer.of(context, filePreferences); + this.indexer = LuceneIndexer.of(context, preferencesService); } @Test diff --git a/src/test/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcherTest.java b/src/test/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcherTest.java index 20a9700a694..4822452f6df 100644 --- a/src/test/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcherTest.java +++ b/src/test/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcherTest.java @@ -14,6 +14,7 @@ import org.jabref.model.entry.types.StandardEntryType; import org.jabref.model.pdf.search.LuceneSearchResults; import org.jabref.preferences.FilePreferences; +import org.jabref.preferences.PreferencesService; import org.apache.lucene.queryparser.classic.ParseException; import org.junit.jupiter.api.BeforeEach; @@ -34,6 +35,8 @@ public class LuceneSearcherTest { public void setUp(@TempDir Path indexDir) throws IOException { FilePreferences filePreferences = mock(FilePreferences.class); when(filePreferences.shouldFulltextIndexLinkedFiles()).thenReturn(true); + PreferencesService preferencesService = mock(PreferencesService.class); + when(preferencesService.getFilePreferences()).thenReturn(filePreferences); // given BibDatabase database = new BibDatabase(); BibDatabaseContext context = mock(BibDatabaseContext.class); @@ -55,7 +58,7 @@ public void setUp(@TempDir Path indexDir) throws IOException { exampleThesis.setCitationKey("ExampleThesis"); database.insertEntry(exampleThesis); - LuceneIndexer indexer = LuceneIndexer.of(context, filePreferences); + LuceneIndexer indexer = LuceneIndexer.of(context, preferencesService); search = LuceneSearcher.of(context); indexer.createIndex(); diff --git a/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java b/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java index 0ffa1636f4d..2b86997d101 100644 --- a/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java +++ b/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java @@ -8,6 +8,7 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.types.StandardEntryType; +import org.jabref.model.search.rules.SearchRules; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -16,7 +17,7 @@ public class DatabaseSearcherTest { - public static final SearchQuery INVALID_SEARCH_QUERY = new SearchQuery("\\asd123{}asdf", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)); + public static final SearchQuery INVALID_SEARCH_QUERY = new SearchQuery("\\asd123{}asdf", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)); private BibDatabase database; @@ -27,7 +28,7 @@ public void setUp() { @Test public void testNoMatchesFromEmptyDatabase() { - List matches = new DatabaseSearcher(new SearchQuery("whatever", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)), database).getMatches(); + List matches = new DatabaseSearcher(new SearchQuery("whatever", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)), database).getMatches(); assertEquals(Collections.emptyList(), matches); } @@ -40,7 +41,7 @@ public void testNoMatchesFromEmptyDatabaseWithInvalidSearchExpression() { @Test public void testGetDatabaseFromMatchesDatabaseWithEmptyEntries() { database.insertEntry(new BibEntry()); - List matches = new DatabaseSearcher(new SearchQuery("whatever", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)), database).getMatches(); + List matches = new DatabaseSearcher(new SearchQuery("whatever", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)), database).getMatches(); assertEquals(Collections.emptyList(), matches); } @@ -49,7 +50,7 @@ public void testNoMatchesFromDatabaseWithArticleTypeEntry() { BibEntry entry = new BibEntry(StandardEntryType.Article); entry.setField(StandardField.AUTHOR, "harrer"); database.insertEntry(entry); - List matches = new DatabaseSearcher(new SearchQuery("whatever", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)), database).getMatches(); + List matches = new DatabaseSearcher(new SearchQuery("whatever", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)), database).getMatches(); assertEquals(Collections.emptyList(), matches); } @@ -58,13 +59,13 @@ public void testCorrectMatchFromDatabaseWithArticleTypeEntry() { BibEntry entry = new BibEntry(StandardEntryType.Article); entry.setField(StandardField.AUTHOR, "harrer"); database.insertEntry(entry); - List matches = new DatabaseSearcher(new SearchQuery("harrer", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)), database).getMatches(); + List matches = new DatabaseSearcher(new SearchQuery("harrer", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)), database).getMatches(); assertEquals(Collections.singletonList(entry), matches); } @Test public void testNoMatchesFromEmptyDatabaseWithInvalidQuery() { - SearchQuery query = new SearchQuery("asdf[", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)); + SearchQuery query = new SearchQuery("asdf[", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)); DatabaseSearcher databaseSearcher = new DatabaseSearcher(query, database); @@ -77,7 +78,7 @@ public void testCorrectMatchFromDatabaseWithIncollectionTypeEntry() { entry.setField(StandardField.AUTHOR, "tonho"); database.insertEntry(entry); - SearchQuery query = new SearchQuery("tonho", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)); + SearchQuery query = new SearchQuery("tonho", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)); List matches = new DatabaseSearcher(query, database).getMatches(); assertEquals(Collections.singletonList(entry), matches); @@ -92,7 +93,7 @@ public void testNoMatchesFromDatabaseWithTwoEntries() { entry.setField(StandardField.AUTHOR, "tonho"); database.insertEntry(entry); - SearchQuery query = new SearchQuery("tonho", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)); + SearchQuery query = new SearchQuery("tonho", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)); DatabaseSearcher databaseSearcher = new DatabaseSearcher(query, database); assertEquals(Collections.singletonList(entry), databaseSearcher.getMatches()); @@ -104,7 +105,7 @@ public void testNoMatchesFromDabaseWithIncollectionTypeEntry() { entry.setField(StandardField.AUTHOR, "tonho"); database.insertEntry(entry); - SearchQuery query = new SearchQuery("asdf", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)); + SearchQuery query = new SearchQuery("asdf", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)); DatabaseSearcher databaseSearcher = new DatabaseSearcher(query, database); assertEquals(Collections.emptyList(), databaseSearcher.getMatches()); @@ -115,7 +116,7 @@ public void testNoMatchFromDatabaseWithEmptyEntry() { BibEntry entry = new BibEntry(); database.insertEntry(entry); - SearchQuery query = new SearchQuery("tonho", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)); + SearchQuery query = new SearchQuery("tonho", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)); DatabaseSearcher databaseSearcher = new DatabaseSearcher(query, database); assertEquals(Collections.emptyList(), databaseSearcher.getMatches()); diff --git a/src/test/java/org/jabref/logic/search/SearchQueryTest.java b/src/test/java/org/jabref/logic/search/SearchQueryTest.java index d924ee762c1..abca53033f6 100644 --- a/src/test/java/org/jabref/logic/search/SearchQueryTest.java +++ b/src/test/java/org/jabref/logic/search/SearchQueryTest.java @@ -7,40 +7,26 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.types.StandardEntryType; +import org.jabref.model.search.rules.SearchRules; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; public class SearchQueryTest { @Test public void testToString() { - assertEquals("\"asdf\" (case sensitive, regular expression)", new SearchQuery("asdf", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)).toString()); - assertEquals("\"asdf\" (case insensitive, plain text)", new SearchQuery("asdf", EnumSet.noneOf(SearchFlags.class)).toString()); - } - - @Test - public void testIsContainsBasedSearch() { - assertTrue(new SearchQuery("asdf", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE)).isContainsBasedSearch()); - assertTrue(new SearchQuery("asdf", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)).isContainsBasedSearch()); - assertFalse(new SearchQuery("author=asdf", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE)).isContainsBasedSearch()); - } - - @Test - public void testIsGrammarBasedSearch() { - assertFalse(new SearchQuery("asdf", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE)).isGrammarBasedSearch()); - assertFalse(new SearchQuery("asdf", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)).isGrammarBasedSearch()); - assertTrue(new SearchQuery("author=asdf", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE)).isGrammarBasedSearch()); + assertEquals("\"asdf\" (case sensitive, regular expression)", new SearchQuery("asdf", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)).toString()); + assertEquals("\"asdf\" (case insensitive, plain text)", new SearchQuery("asdf", EnumSet.noneOf(SearchRules.SearchFlags.class)).toString()); } @Test public void testGrammarSearch() { BibEntry entry = new BibEntry(); entry.addKeyword("one two", ','); - SearchQuery searchQuery = new SearchQuery("keywords=\"one two\"", EnumSet.noneOf(SearchFlags.class)); + SearchQuery searchQuery = new SearchQuery("keywords=\"one two\"", EnumSet.noneOf(SearchRules.SearchFlags.class)); assertTrue(searchQuery.isMatch(entry)); } @@ -48,7 +34,7 @@ public void testGrammarSearch() { public void testGrammarSearchFullEntryLastCharMissing() { BibEntry entry = new BibEntry(); entry.setField(StandardField.TITLE, "systematic revie"); - SearchQuery searchQuery = new SearchQuery("title=\"systematic review\"", EnumSet.noneOf(SearchFlags.class)); + SearchQuery searchQuery = new SearchQuery("title=\"systematic review\"", EnumSet.noneOf(SearchRules.SearchFlags.class)); assertFalse(searchQuery.isMatch(entry)); } @@ -56,7 +42,7 @@ public void testGrammarSearchFullEntryLastCharMissing() { public void testGrammarSearchFullEntry() { BibEntry entry = new BibEntry(); entry.setField(StandardField.TITLE, "systematic review"); - SearchQuery searchQuery = new SearchQuery("title=\"systematic review\"", EnumSet.noneOf(SearchFlags.class)); + SearchQuery searchQuery = new SearchQuery("title=\"systematic review\"", EnumSet.noneOf(SearchRules.SearchFlags.class)); assertTrue(searchQuery.isMatch(entry)); } @@ -65,7 +51,7 @@ public void testSearchingForOpenBraketInBooktitle() { BibEntry e = new BibEntry(StandardEntryType.InProceedings); e.setField(StandardField.BOOKTITLE, "Super Conference (SC)"); - SearchQuery searchQuery = new SearchQuery("booktitle=\"(\"", EnumSet.noneOf(SearchFlags.class)); + SearchQuery searchQuery = new SearchQuery("booktitle=\"(\"", EnumSet.noneOf(SearchRules.SearchFlags.class)); assertTrue(searchQuery.isMatch(e)); } @@ -74,7 +60,7 @@ public void testSearchMatchesSingleKeywordNotPart() { BibEntry e = new BibEntry(StandardEntryType.InProceedings); e.setField(StandardField.KEYWORDS, "banana, pineapple, orange"); - SearchQuery searchQuery = new SearchQuery("anykeyword==apple", EnumSet.noneOf(SearchFlags.class)); + SearchQuery searchQuery = new SearchQuery("anykeyword==apple", EnumSet.noneOf(SearchRules.SearchFlags.class)); assertFalse(searchQuery.isMatch(e)); } @@ -83,7 +69,7 @@ public void testSearchMatchesSingleKeyword() { BibEntry e = new BibEntry(StandardEntryType.InProceedings); e.setField(StandardField.KEYWORDS, "banana, pineapple, orange"); - SearchQuery searchQuery = new SearchQuery("anykeyword==pineapple", EnumSet.noneOf(SearchFlags.class)); + SearchQuery searchQuery = new SearchQuery("anykeyword==pineapple", EnumSet.noneOf(SearchRules.SearchFlags.class)); assertTrue(searchQuery.isMatch(e)); } @@ -93,7 +79,7 @@ public void testSearchAllFields() { e.setField(StandardField.TITLE, "Fruity features"); e.setField(StandardField.KEYWORDS, "banana, pineapple, orange"); - SearchQuery searchQuery = new SearchQuery("anyfield==\"fruity features\"", EnumSet.noneOf(SearchFlags.class)); + SearchQuery searchQuery = new SearchQuery("anyfield==\"fruity features\"", EnumSet.noneOf(SearchRules.SearchFlags.class)); assertTrue(searchQuery.isMatch(e)); } @@ -103,7 +89,7 @@ public void testSearchAllFieldsNotForSpecificField() { e.setField(StandardField.TITLE, "Fruity features"); e.setField(StandardField.KEYWORDS, "banana, pineapple, orange"); - SearchQuery searchQuery = new SearchQuery("anyfield=fruit and keywords!=banana", EnumSet.noneOf(SearchFlags.class)); + SearchQuery searchQuery = new SearchQuery("anyfield=fruit and keywords!=banana", EnumSet.noneOf(SearchRules.SearchFlags.class)); assertFalse(searchQuery.isMatch(e)); } @@ -113,7 +99,7 @@ public void testSearchAllFieldsAndSpecificField() { e.setField(StandardField.TITLE, "Fruity features"); e.setField(StandardField.KEYWORDS, "banana, pineapple, orange"); - SearchQuery searchQuery = new SearchQuery("anyfield=fruit and keywords=apple", EnumSet.noneOf(SearchFlags.class)); + SearchQuery searchQuery = new SearchQuery("anyfield=fruit and keywords=apple", EnumSet.noneOf(SearchRules.SearchFlags.class)); assertTrue(searchQuery.isMatch(e)); } @@ -123,59 +109,59 @@ public void testIsMatch() { entry.setType(StandardEntryType.Article); entry.setField(StandardField.AUTHOR, "asdf"); - assertFalse(new SearchQuery("BiblatexEntryType", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)).isMatch(entry)); - assertTrue(new SearchQuery("asdf", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)).isMatch(entry)); - assertTrue(new SearchQuery("author=asdf", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)).isMatch(entry)); + assertFalse(new SearchQuery("BiblatexEntryType", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)).isMatch(entry)); + assertTrue(new SearchQuery("asdf", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)).isMatch(entry)); + assertTrue(new SearchQuery("author=asdf", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)).isMatch(entry)); } @Test public void testIsValidQueryNotAsRegEx() { - assertTrue(new SearchQuery("asdf", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE)).isValid()); + assertTrue(new SearchQuery("asdf", EnumSet.noneOf(SearchRules.SearchFlags.class)).isValid()); } @Test public void testIsValidQueryContainsBracketNotAsRegEx() { - assertTrue(new SearchQuery("asdf[", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE)).isValid()); + assertTrue(new SearchQuery("asdf[", EnumSet.noneOf(SearchRules.SearchFlags.class)).isValid()); } @Test public void testIsNotValidQueryContainsBracketNotAsRegEx() { - assertTrue(new SearchQuery("asdf[", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)).isValid()); + assertTrue(new SearchQuery("asdf[", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)).isValid()); } @Test public void testIsValidQueryAsRegEx() { - assertTrue(new SearchQuery("asdf", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)).isValid()); + assertTrue(new SearchQuery("asdf", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)).isValid()); } @Test public void testIsValidQueryWithNumbersAsRegEx() { - assertTrue(new SearchQuery("123", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)).isValid()); + assertTrue(new SearchQuery("123", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)).isValid()); } @Test public void testIsValidQueryContainsBracketAsRegEx() { - assertTrue(new SearchQuery("asdf[", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)).isValid()); + assertTrue(new SearchQuery("asdf[", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)).isValid()); } @Test public void testIsValidQueryWithEqualSignAsRegEx() { - assertTrue(new SearchQuery("author=asdf", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)).isValid()); + assertTrue(new SearchQuery("author=asdf", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)).isValid()); } @Test public void testIsValidQueryWithNumbersAndEqualSignAsRegEx() { - assertTrue(new SearchQuery("author=123", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)).isValid()); + assertTrue(new SearchQuery("author=123", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)).isValid()); } @Test public void testIsValidQueryWithEqualSignNotAsRegEx() { - assertTrue(new SearchQuery("author=asdf", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE)).isValid()); + assertTrue(new SearchQuery("author=asdf", EnumSet.noneOf(SearchRules.SearchFlags.class)).isValid()); } @Test public void testIsValidQueryWithNumbersAndEqualSignNotAsRegEx() { - assertTrue(new SearchQuery("author=123", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE)).isValid()); + assertTrue(new SearchQuery("author=123", EnumSet.noneOf(SearchRules.SearchFlags.class)).isValid()); } @Test @@ -185,15 +171,14 @@ public void isMatchedForNormalAndFieldBasedSearchMixed() { entry.setField(StandardField.AUTHOR, "asdf"); entry.setField(StandardField.ABSTRACT, "text"); - assertTrue(new SearchQuery("text AND author=asdf", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)).isMatch(entry)); + assertTrue(new SearchQuery("text AND author=asdf", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)).isMatch(entry)); } @Test public void testSimpleTerm() { String query = "progress"; - SearchQuery result = new SearchQuery(query, EnumSet.noneOf(SearchFlags.class)); - assertFalse(result.isGrammarBasedSearch()); + SearchQuery result = new SearchQuery(query, EnumSet.noneOf(SearchRules.SearchFlags.class)); } @Test @@ -225,7 +210,7 @@ public void testGetRegexpJavascriptPattern() { public void testEscapingInPattern() { // first word contain all java special regex characters String queryText = "<([{\\\\^-=$!|]})?*+.> word1 word2."; - SearchQuery textQueryWithSpecialChars = new SearchQuery(queryText, EnumSet.noneOf(SearchFlags.class)); + SearchQuery textQueryWithSpecialChars = new SearchQuery(queryText, EnumSet.noneOf(SearchRules.SearchFlags.class)); String pattern = "(\\Q<([{\\^-=$!|]})?*+.>\\E)|(\\Qword1\\E)|(\\Qword2.\\E)"; assertEquals(Optional.of(pattern), textQueryWithSpecialChars.getPatternForWords().map(Pattern::toString)); } @@ -234,7 +219,7 @@ public void testEscapingInPattern() { public void testEscapingInJavascriptPattern() { // first word contain all javascript special regex characters that should be escaped individually in text based search String queryText = "([{\\\\^$|]})?*+./ word1 word2."; - SearchQuery textQueryWithSpecialChars = new SearchQuery(queryText, EnumSet.noneOf(SearchFlags.class)); + SearchQuery textQueryWithSpecialChars = new SearchQuery(queryText, EnumSet.noneOf(SearchRules.SearchFlags.class)); String pattern = "(\\(\\[\\{\\\\\\^\\$\\|\\]\\}\\)\\?\\*\\+\\.\\/)|(word1)|(word2\\.)"; assertEquals(Optional.of(pattern), textQueryWithSpecialChars.getJavaScriptPatternForWords().map(Pattern::toString)); } diff --git a/src/test/java/org/jabref/model/groups/GroupTreeNodeTest.java b/src/test/java/org/jabref/model/groups/GroupTreeNodeTest.java index 4c8689ce0d2..142c2a0fedf 100644 --- a/src/test/java/org/jabref/model/groups/GroupTreeNodeTest.java +++ b/src/test/java/org/jabref/model/groups/GroupTreeNodeTest.java @@ -12,6 +12,7 @@ import org.jabref.model.entry.field.StandardField; import org.jabref.model.search.matchers.AndMatcher; import org.jabref.model.search.matchers.OrMatcher; +import org.jabref.model.search.rules.SearchRules; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -82,7 +83,7 @@ private static AbstractGroup getKeywordGroup(String name) { } private static AbstractGroup getSearchGroup(String name) { - return new SearchGroup(name, GroupHierarchyType.INCLUDING, "searchExpression", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE)); + return new SearchGroup(name, GroupHierarchyType.INCLUDING, "searchExpression", EnumSet.noneOf(SearchRules.SearchFlags.class)); } private static AbstractGroup getExplict(String name) { @@ -254,7 +255,7 @@ void setGroupExplicitToSearchDoesNotKeepPreviousAssignments() { ExplicitGroup oldGroup = new ExplicitGroup("OldGroup", GroupHierarchyType.INDEPENDENT, ','); oldGroup.add(entry); GroupTreeNode node = GroupTreeNode.fromGroup(oldGroup); - AbstractGroup newGroup = new SearchGroup("NewGroup", GroupHierarchyType.INDEPENDENT, "test", EnumSet.noneOf(SearchFlags.class)); + AbstractGroup newGroup = new SearchGroup("NewGroup", GroupHierarchyType.INDEPENDENT, "test", EnumSet.noneOf(SearchRules.SearchFlags.class)); node.setGroup(newGroup, true, true, entries); @@ -332,7 +333,7 @@ void onlySubgroupsContainAllEntries() { @Test void addEntriesToGroupWorksNotForGroupsNotSupportingExplicitAddingOfEntries() { - GroupTreeNode searchGroup = new GroupTreeNode(new SearchGroup("Search A", GroupHierarchyType.INCLUDING, "searchExpression", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE))); + GroupTreeNode searchGroup = new GroupTreeNode(new SearchGroup("Search A", GroupHierarchyType.INCLUDING, "searchExpression", EnumSet.noneOf(SearchRules.SearchFlags.class))); List fieldChanges = searchGroup.addEntriesToGroup(entries); assertEquals(Collections.emptyList(), fieldChanges); @@ -340,7 +341,7 @@ void addEntriesToGroupWorksNotForGroupsNotSupportingExplicitAddingOfEntries() { @Test void removeEntriesFromGroupWorksNotForGroupsNotSupportingExplicitRemovalOfEntries() { - GroupTreeNode searchGroup = new GroupTreeNode(new SearchGroup("Search A", GroupHierarchyType.INCLUDING, "searchExpression", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE))); + GroupTreeNode searchGroup = new GroupTreeNode(new SearchGroup("Search A", GroupHierarchyType.INCLUDING, "searchExpression", EnumSet.noneOf(SearchRules.SearchFlags.class))); List fieldChanges = searchGroup.removeEntriesFromGroup(entries); assertEquals(Collections.emptyList(), fieldChanges); diff --git a/src/test/java/org/jabref/model/groups/SearchGroupTest.java b/src/test/java/org/jabref/model/groups/SearchGroupTest.java index 40185f8bfc8..1e7ecbdb4de 100644 --- a/src/test/java/org/jabref/model/groups/SearchGroupTest.java +++ b/src/test/java/org/jabref/model/groups/SearchGroupTest.java @@ -4,6 +4,7 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; +import org.jabref.model.search.rules.SearchRules; import org.junit.jupiter.api.Test; @@ -14,7 +15,7 @@ public class SearchGroupTest { @Test public void containsFindsWordWithRegularExpression() { - SearchGroup group = new SearchGroup("myExplicitGroup", GroupHierarchyType.INDEPENDENT, "anyfield=rev*", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)); + SearchGroup group = new SearchGroup("myExplicitGroup", GroupHierarchyType.INDEPENDENT, "anyfield=rev*", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)); BibEntry entry = new BibEntry(); entry.addKeyword("review", ','); @@ -23,7 +24,7 @@ public void containsFindsWordWithRegularExpression() { @Test public void containsDoesNotFindsWordWithInvalidRegularExpression() { - SearchGroup group = new SearchGroup("myExplicitGroup", GroupHierarchyType.INDEPENDENT, "anyfield=*rev*", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)); + SearchGroup group = new SearchGroup("myExplicitGroup", GroupHierarchyType.INDEPENDENT, "anyfield=*rev*", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)); BibEntry entry = new BibEntry(); entry.addKeyword("review", ','); diff --git a/src/test/java/org/jabref/model/search/rules/ContainsBasedSearchRuleTest.java b/src/test/java/org/jabref/model/search/rules/ContainsBasedSearchRuleTest.java deleted file mode 100644 index ed77afd40be..00000000000 --- a/src/test/java/org/jabref/model/search/rules/ContainsBasedSearchRuleTest.java +++ /dev/null @@ -1,58 +0,0 @@ -package org.jabref.model.search.rules; - -import java.util.EnumSet; - -import org.jabref.model.entry.BibEntry; -import org.jabref.model.entry.field.StandardField; -import org.jabref.model.entry.types.StandardEntryType; - -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -/** - * Test case for ContainBasedSearchRule. - */ -public class ContainsBasedSearchRuleTest { - - private final BibEntry be = new BibEntry(StandardEntryType.InCollection) - .withCitationKey("shields01") - .withField(StandardField.TITLE, "Marine finfish larviculture in Europe") - .withField(StandardField.YEAR, "2001") - .withField(StandardField.AUTHOR, "Kevin Shields"); - private final ContainsBasedSearchRule bsCaseSensitive = new ContainsBasedSearchRule(EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)); - private final ContainsBasedSearchRule bsCaseInsensitive = new ContainsBasedSearchRule(EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)); - private final RegexBasedSearchRule bsCaseSensitiveRegexp = new RegexBasedSearchRule(EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)); - private final RegexBasedSearchRule bsCaseInsensitiveRegexp = new RegexBasedSearchRule(EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)); - - @Test - public void testContentOfSingleField() { - String query = "\"marine larviculture\""; - - assertFalse(bsCaseSensitive.applyRule(query, be)); - assertFalse(bsCaseInsensitive.applyRule(query, be)); - assertFalse(bsCaseSensitiveRegexp.applyRule(query, be)); - assertFalse(bsCaseInsensitiveRegexp.applyRule(query, be)); - } - - @Test - public void testContentDistributedOnMultipleFields() { - String query = "marine 2001 shields"; - - assertFalse(bsCaseSensitive.applyRule(query, be)); - assertTrue(bsCaseInsensitive.applyRule(query, be)); - assertFalse(bsCaseSensitiveRegexp.applyRule(query, be)); - assertFalse(bsCaseInsensitiveRegexp.applyRule(query, be)); - } - - @Test - public void testRegularExpressionMatch() { - String query = "marine [A-Za-z]* larviculture"; - - assertFalse(bsCaseSensitive.applyRule(query, be)); - assertFalse(bsCaseInsensitive.applyRule(query, be)); - assertFalse(bsCaseSensitiveRegexp.applyRule(query, be)); - assertTrue(bsCaseInsensitiveRegexp.applyRule(query, be)); - } -} diff --git a/src/test/java/org/jabref/model/search/rules/GrammarBasedSearchRuleTest.java b/src/test/java/org/jabref/model/search/rules/GrammarBasedSearchRuleTest.java deleted file mode 100644 index 6b410b5a088..00000000000 --- a/src/test/java/org/jabref/model/search/rules/GrammarBasedSearchRuleTest.java +++ /dev/null @@ -1,101 +0,0 @@ -package org.jabref.model.search.rules; - -import java.util.EnumSet; - -import org.jabref.model.entry.BibEntry; -import org.jabref.model.entry.field.StandardField; -import org.jabref.model.entry.types.StandardEntryType; - -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -/** - * Test case for GrammarBasedSearchRuleTest. - */ -public class GrammarBasedSearchRuleTest { - - @Test - void applyRuleMatchesSingleTermWithRegex() { - GrammarBasedSearchRule searchRule = new GrammarBasedSearchRule(EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)); - - String query = "M[a-z]+e"; - assertTrue(searchRule.validateSearchStrings(query)); - assertTrue(searchRule.applyRule(query, makeBibtexEntry())); - } - - @Test - void applyRuleDoesNotMatchSingleTermWithRegex() { - GrammarBasedSearchRule searchRule = new GrammarBasedSearchRule(EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)); - - String query = "M[0-9]+e"; - assertTrue(searchRule.validateSearchStrings(query)); - assertFalse(searchRule.applyRule(query, makeBibtexEntry())); - } - - @Test - void searchRuleOfDocumentationMatches() { - GrammarBasedSearchRule searchRule = new GrammarBasedSearchRule(EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)); - - String query = "(author = miller or title|keywords = \"image processing\") and not author = brown"; - assertTrue(searchRule.validateSearchStrings(query)); - assertTrue(searchRule.applyRule(query, new BibEntry() - .withCitationKey("key") - .withField(StandardField.KEYWORDS, "image processing"))); - assertFalse(searchRule.applyRule(query, new BibEntry() - .withCitationKey("key") - .withField(StandardField.AUTHOR, "Sam Brown") - .withField(StandardField.KEYWORDS, "image processing"))); - } - - @Disabled - @Test - void searchForAnyFieldWorks() { - GrammarBasedSearchRule searchRule = new GrammarBasedSearchRule(EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)); - - String query = "anyfield:fruit"; - assertTrue(searchRule.validateSearchStrings(query)); - assertTrue(searchRule.applyRule(query, new BibEntry() - .withField(StandardField.KEYWORDS, "fruit"))); - } - - @Disabled - @Test - void searchForAnyKeywordWorks() { - GrammarBasedSearchRule searchRule = new GrammarBasedSearchRule(EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)); - - String query = "anykeyword:apple"; - assertTrue(searchRule.validateSearchStrings(query)); - assertTrue(searchRule.applyRule(query, new BibEntry() - .withField(StandardField.KEYWORDS, "apple"))); - assertFalse(searchRule.applyRule(query, new BibEntry() - .withField(StandardField.KEYWORDS, "pineapple"))); - } - - @Test - void searchForCitationKeyWorks() { - GrammarBasedSearchRule searchRule = new GrammarBasedSearchRule(EnumSet.noneOf(SearchRules.SearchFlags.class)); - String query = "citationkey==miller2005"; - assertTrue(searchRule.validateSearchStrings(query)); - assertTrue(searchRule.applyRule(query, new BibEntry() - .withCitationKey("miller2005"))); - } - - @Test - void searchForThesisEntryTypeWorks() { - GrammarBasedSearchRule searchRule = new GrammarBasedSearchRule(EnumSet.noneOf(SearchRules.SearchFlags.class)); - String query = "entrytype=thesis"; - assertTrue(searchRule.validateSearchStrings(query)); - assertTrue(searchRule.applyRule(query, new BibEntry(StandardEntryType.PhdThesis))); - } - - public BibEntry makeBibtexEntry() { - return new BibEntry(StandardEntryType.InCollection) - .withCitationKey("shields01") - .withField(StandardField.TITLE, "Marine finfish larviculture in Europe") - .withField(StandardField.YEAR, "2001") - .withField(StandardField.AUTHOR, "Kevin Shields"); - } -} diff --git a/src/test/java/org/jabref/model/search/rules/SentenceAnalyzerTest.java b/src/test/java/org/jabref/model/search/rules/SentenceAnalyzerTest.java deleted file mode 100644 index 61e05e3def4..00000000000 --- a/src/test/java/org/jabref/model/search/rules/SentenceAnalyzerTest.java +++ /dev/null @@ -1,34 +0,0 @@ -package org.jabref.model.search.rules; - -import java.util.List; -import java.util.stream.Stream; - -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -public class SentenceAnalyzerTest { - - static Stream getParameters() { - return Stream.of( - Arguments.of(List.of("a", "b"), "a b"), - - // Leading and trailing spaces - Arguments.of(List.of("a", "b"), " a b "), - - // Escaped characters and trailing spaces - Arguments.of(List.of("b "), "\"b \" "), - - // Escaped characters and leading spaces. - Arguments.of(List.of(" a"), " \\ a") - ); - } - - @ParameterizedTest - @MethodSource("getParameters") - public void testGetWords(List expected, String input) { - assertEquals(expected, new SentenceAnalyzer(input).getWords()); - } -} From 536ecfaaedf9b14b93e3d50b1a63166f4440661e Mon Sep 17 00:00:00 2001 From: Carl Christian Snethlage <50491877+calixtus@users.noreply.github.com> Date: Wed, 4 Jan 2023 09:03:40 +0100 Subject: [PATCH 070/256] Fixed merge error in CHANGELOG.md --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9f66f34079..95b23e60fa1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,12 +11,15 @@ Note that this project **does not** adhere to [Semantic Versioning](http://semve ### Added +- We added a dropdown menu to let users change the library they want to import into during import.[#6177](https://github.com/JabRef/jabref/issues/6177) +- We added the possibility to add/remove a preview style from the selected list using a double click [#9490](https://github.com/JabRef/jabref/issues/9490) - We reintroduced the floating search in the main table. [#8963](https://github.com/JabRef/jabref/pull/8963) - We added a dropdown menu to let users change the library they want to import into during import. [#6177](https://github.com/JabRef/jabref/issues/6177) - We added the possibility to add/remove a preview style from the selected list using a double click. [#9490](https://github.com/JabRef/jabref/issues/9490) - We added the option to define fields as "multine" directly in the custom entry types dialog. [#6448](https://github.com/JabRef/jabref/issues/6448) + ### Changed - The search in the library now displays probable search hits instead of exact matches. Sorting by hit score can be done by the new score table column. [#8963](https://github.com/JabRef/jabref/pull/8963) From 1117e17d3318b321f2702d881f8165fcd22abe59 Mon Sep 17 00:00:00 2001 From: Carl Christian Snethlage <50491877+calixtus@users.noreply.github.com> Date: Wed, 4 Jan 2023 09:04:43 +0100 Subject: [PATCH 071/256] Fixed checkstyle --- src/main/java/org/jabref/gui/groups/GroupsPreferences.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/jabref/gui/groups/GroupsPreferences.java b/src/main/java/org/jabref/gui/groups/GroupsPreferences.java index f3bd7e9734a..71aab020eb1 100644 --- a/src/main/java/org/jabref/gui/groups/GroupsPreferences.java +++ b/src/main/java/org/jabref/gui/groups/GroupsPreferences.java @@ -10,7 +10,7 @@ public class GroupsPreferences { - private final SetProperty groupViewMode = new SimpleSetProperty<>(FXCollections.observableSet());; + private final SetProperty groupViewMode = new SimpleSetProperty<>(FXCollections.observableSet()); private final BooleanProperty shouldAutoAssignGroup = new SimpleBooleanProperty(); private final BooleanProperty shouldDisplayGroupCount = new SimpleBooleanProperty(); From ff4ad1c5a051135d45b708ea7a832dcf8fc1176e Mon Sep 17 00:00:00 2001 From: Carl Christian Snethlage <50491877+calixtus@users.noreply.github.com> Date: Wed, 4 Jan 2023 10:44:22 +0100 Subject: [PATCH 072/256] Fixed more tests --- .../search/retrieval/LuceneSearcher.java | 3 + .../search/retrieval/LuceneSearcherTest.java | 56 ++++++++++--------- 2 files changed, 33 insertions(+), 26 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java b/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java index 3dfc1bb7e18..a224b5524f5 100644 --- a/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java +++ b/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java @@ -2,6 +2,7 @@ import java.io.IOException; import java.util.HashMap; +import java.util.Objects; import org.jabref.logic.search.SearchQuery; import org.jabref.model.database.BibDatabaseContext; @@ -41,6 +42,8 @@ public static LuceneSearcher of(BibDatabaseContext databaseContext) throws IOExc * @return a result map of all entries that have matches in any fields */ public HashMap search(SearchQuery query) { + Objects.requireNonNull(query); + HashMap results = new HashMap<>(); try (IndexReader reader = DirectoryReader.open(indexDirectory)) { IndexSearcher searcher = new IndexSearcher(reader); diff --git a/src/test/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcherTest.java b/src/test/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcherTest.java index 4822452f6df..f7d2743ed6d 100644 --- a/src/test/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcherTest.java +++ b/src/test/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcherTest.java @@ -3,7 +3,10 @@ import java.io.IOException; import java.nio.file.Path; import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import org.jabref.logic.search.SearchQuery; import org.jabref.logic.search.indexing.LuceneIndexer; import org.jabref.logic.search.retrieval.LuceneSearcher; import org.jabref.logic.util.StandardFileType; @@ -13,10 +16,10 @@ import org.jabref.model.entry.LinkedFile; import org.jabref.model.entry.types.StandardEntryType; import org.jabref.model.pdf.search.LuceneSearchResults; +import org.jabref.model.search.rules.SearchRules; import org.jabref.preferences.FilePreferences; import org.jabref.preferences.PreferencesService; -import org.apache.lucene.queryparser.classic.ParseException; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -69,48 +72,49 @@ public void setUp(@TempDir Path indexDir) throws IOException { } @Test - public void searchForTest() throws IOException, ParseException { - LuceneSearchResults result = search.search("test", 10); - assertEquals(8, result.numSearchResults()); + public void searchForTest() { + HashMap searchResults = search.search(new SearchQuery("", EnumSet.noneOf(SearchRules.SearchFlags.class))); + int hits = searchResults.keySet().stream().mapToInt((key) -> searchResults.get(key).numSearchResults()).sum(); + assertEquals(8, hits); } @Test - public void searchForUniversity() throws IOException, ParseException { - LuceneSearchResults result = search.search("University", 10); - assertEquals(1, result.numSearchResults()); + public void searchForUniversity() { + HashMap searchResults = search.search(new SearchQuery("University", EnumSet.noneOf(SearchRules.SearchFlags.class))); + int hits = searchResults.keySet().stream().mapToInt((key) -> searchResults.get(key).numSearchResults()).sum(); + assertEquals(1, hits); } @Test - public void searchForStopWord() throws IOException, ParseException { - LuceneSearchResults result = search.search("and", 10); - assertEquals(0, result.numSearchResults()); + public void searchForStopWord() { + HashMap searchResults = search.search(new SearchQuery("and", EnumSet.noneOf(SearchRules.SearchFlags.class))); + int hits = searchResults.keySet().stream().mapToInt((key) -> searchResults.get(key).numSearchResults()).sum(); + assertEquals(0, hits); } @Test - public void searchForSecond() throws IOException, ParseException { - LuceneSearchResults result = search.search("second", 10); - assertEquals(4, result.numSearchResults()); + public void searchForSecond() { + HashMap searchResults = search.search(new SearchQuery("second", EnumSet.noneOf(SearchRules.SearchFlags.class))); + int hits = searchResults.keySet().stream().mapToInt((key) -> searchResults.get(key).numSearchResults()).sum(); + assertEquals(4, hits); } @Test - public void searchForAnnotation() throws IOException, ParseException { - LuceneSearchResults result = search.search("annotation", 10); - assertEquals(2, result.numSearchResults()); + public void searchForAnnotation() { + HashMap searchResults = search.search(new SearchQuery("annotation", EnumSet.noneOf(SearchRules.SearchFlags.class))); + int hits = searchResults.keySet().stream().mapToInt((key) -> searchResults.get(key).numSearchResults()).sum(); + assertEquals(2, hits); } @Test - public void searchForEmptyString() throws IOException { - LuceneSearchResults result = search.search("", 10); - assertEquals(0, result.numSearchResults()); + public void searchForEmptyString() { + HashMap searchResults = search.search(new SearchQuery("", EnumSet.noneOf(SearchRules.SearchFlags.class))); + int hits = searchResults.keySet().stream().mapToInt((key) -> searchResults.get(key).numSearchResults()).sum(); + assertEquals(0, hits); } @Test - public void searchWithNullString() throws IOException { - assertThrows(NullPointerException.class, () -> search.search(null, 10)); - } - - @Test - public void searchForZeroResults() throws IOException { - assertThrows(IllegalArgumentException.class, () -> search.search("test", 0)); + public void searchWithNullString() { + assertThrows(NullPointerException.class, () -> search.search(null)); } } From 7bdccf05ad23ec254e131e75baf1f398b82b76bd Mon Sep 17 00:00:00 2001 From: Carl Christian Snethlage <50491877+calixtus@users.noreply.github.com> Date: Wed, 4 Jan 2023 10:46:21 +0100 Subject: [PATCH 073/256] Removed obsolete tests --- .../jabref/logic/search/SearchQueryTest.java | 52 ------------------- 1 file changed, 52 deletions(-) diff --git a/src/test/java/org/jabref/logic/search/SearchQueryTest.java b/src/test/java/org/jabref/logic/search/SearchQueryTest.java index abca53033f6..9ea8a0eafeb 100644 --- a/src/test/java/org/jabref/logic/search/SearchQueryTest.java +++ b/src/test/java/org/jabref/logic/search/SearchQueryTest.java @@ -1,8 +1,6 @@ package org.jabref.logic.search; import java.util.EnumSet; -import java.util.Optional; -import java.util.regex.Pattern; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; @@ -173,54 +171,4 @@ public void isMatchedForNormalAndFieldBasedSearchMixed() { assertTrue(new SearchQuery("text AND author=asdf", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)).isMatch(entry)); } - - @Test - public void testSimpleTerm() { - String query = "progress"; - - SearchQuery result = new SearchQuery(query, EnumSet.noneOf(SearchRules.SearchFlags.class)); - } - - @Test - public void testGetPattern() { - String query = "progress"; - SearchQuery result = new SearchQuery(query, EnumSet.noneOf(SearchFlags.class)); - Pattern pattern = Pattern.compile("(\\Qprogress\\E)"); - // We can't directly compare the pattern objects - assertEquals(Optional.of(pattern.toString()), result.getPatternForWords().map(Pattern::toString)); - } - - @Test - public void testGetRegexpPattern() { - String queryText = "[a-c]\\d* \\d*"; - SearchQuery regexQuery = new SearchQuery(queryText, EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)); - Pattern pattern = Pattern.compile("([a-c]\\d* \\d*)"); - assertEquals(Optional.of(pattern.toString()), regexQuery.getPatternForWords().map(Pattern::toString)); - } - - @Test - public void testGetRegexpJavascriptPattern() { - String queryText = "[a-c]\\d* \\d*"; - SearchQuery regexQuery = new SearchQuery(queryText, EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)); - Pattern pattern = Pattern.compile("([a-c]\\d* \\d*)"); - assertEquals(Optional.of(pattern.toString()), regexQuery.getJavaScriptPatternForWords().map(Pattern::toString)); - } - - @Test - public void testEscapingInPattern() { - // first word contain all java special regex characters - String queryText = "<([{\\\\^-=$!|]})?*+.> word1 word2."; - SearchQuery textQueryWithSpecialChars = new SearchQuery(queryText, EnumSet.noneOf(SearchRules.SearchFlags.class)); - String pattern = "(\\Q<([{\\^-=$!|]})?*+.>\\E)|(\\Qword1\\E)|(\\Qword2.\\E)"; - assertEquals(Optional.of(pattern), textQueryWithSpecialChars.getPatternForWords().map(Pattern::toString)); - } - - @Test - public void testEscapingInJavascriptPattern() { - // first word contain all javascript special regex characters that should be escaped individually in text based search - String queryText = "([{\\\\^$|]})?*+./ word1 word2."; - SearchQuery textQueryWithSpecialChars = new SearchQuery(queryText, EnumSet.noneOf(SearchRules.SearchFlags.class)); - String pattern = "(\\(\\[\\{\\\\\\^\\$\\|\\]\\}\\)\\?\\*\\+\\.\\/)|(word1)|(word2\\.)"; - assertEquals(Optional.of(pattern), textQueryWithSpecialChars.getJavaScriptPatternForWords().map(Pattern::toString)); - } } From 40de9517e6df6436c1d3fb0f3283da04b8ba09d0 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Wed, 4 Jan 2023 13:34:26 +0100 Subject: [PATCH 074/256] Fixes "Fixed merge error in CHANGELOG.md" by removing duplicate entries This reverts commit 536ecfaaedf9b14b93e3d50b1a63166f4440661e. --- CHANGELOG.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95b23e60fa1..a9f66f34079 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,15 +11,12 @@ Note that this project **does not** adhere to [Semantic Versioning](http://semve ### Added -- We added a dropdown menu to let users change the library they want to import into during import.[#6177](https://github.com/JabRef/jabref/issues/6177) -- We added the possibility to add/remove a preview style from the selected list using a double click [#9490](https://github.com/JabRef/jabref/issues/9490) - We reintroduced the floating search in the main table. [#8963](https://github.com/JabRef/jabref/pull/8963) - We added a dropdown menu to let users change the library they want to import into during import. [#6177](https://github.com/JabRef/jabref/issues/6177) - We added the possibility to add/remove a preview style from the selected list using a double click. [#9490](https://github.com/JabRef/jabref/issues/9490) - We added the option to define fields as "multine" directly in the custom entry types dialog. [#6448](https://github.com/JabRef/jabref/issues/6448) - ### Changed - The search in the library now displays probable search hits instead of exact matches. Sorting by hit score can be done by the new score table column. [#8963](https://github.com/JabRef/jabref/pull/8963) From 1f38a68dfb889cdf04d2c3ee2584f32c3ce49e15 Mon Sep 17 00:00:00 2001 From: Carl Christian Snethlage <50491877+calixtus@users.noreply.github.com> Date: Mon, 23 Jan 2023 17:01:28 +0100 Subject: [PATCH 075/256] WiP on tests --- .../search/retrieval/LuceneSearcherTest.java | 109 ++++-- .../jabref/logic/search/SearchQueryTest.java | 363 ++++++++++-------- 2 files changed, 268 insertions(+), 204 deletions(-) diff --git a/src/test/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcherTest.java b/src/test/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcherTest.java index f7d2743ed6d..7743ecdfea8 100644 --- a/src/test/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcherTest.java +++ b/src/test/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcherTest.java @@ -32,89 +32,114 @@ public class LuceneSearcherTest { - private LuceneSearcher search; + private LuceneSearcher searcher; + private PreferencesService preferencesService; + private BibDatabase bibDatabase; + private BibDatabaseContext bibDatabaseContext; @BeforeEach public void setUp(@TempDir Path indexDir) throws IOException { - FilePreferences filePreferences = mock(FilePreferences.class); - when(filePreferences.shouldFulltextIndexLinkedFiles()).thenReturn(true); - PreferencesService preferencesService = mock(PreferencesService.class); - when(preferencesService.getFilePreferences()).thenReturn(filePreferences); - // given - BibDatabase database = new BibDatabase(); - BibDatabaseContext context = mock(BibDatabaseContext.class); - when(context.getFileDirectories(Mockito.any())).thenReturn(Collections.singletonList(Path.of("src/test/resources/pdfs"))); - when(context.getFulltextIndexPath()).thenReturn(indexDir); - when(context.getDatabase()).thenReturn(database); - when(context.getEntries()).thenReturn(database.getEntries()); - BibEntry examplePdf = new BibEntry(StandardEntryType.Article); - examplePdf.setFiles(Collections.singletonList(new LinkedFile("Example Entry", "example.pdf", StandardFileType.PDF.getName()))); - database.insertEntry(examplePdf); - - BibEntry metaDataEntry = new BibEntry(StandardEntryType.Article); - metaDataEntry.setFiles(Collections.singletonList(new LinkedFile("Metadata Entry", "metaData.pdf", StandardFileType.PDF.getName()))); - metaDataEntry.setCitationKey("MetaData2017"); - database.insertEntry(metaDataEntry); - - BibEntry exampleThesis = new BibEntry(StandardEntryType.PhdThesis); - exampleThesis.setFiles(Collections.singletonList(new LinkedFile("Example Thesis", "thesis-example.pdf", StandardFileType.PDF.getName()))); - exampleThesis.setCitationKey("ExampleThesis"); - database.insertEntry(exampleThesis); + preferencesService = mock(PreferencesService.class); + when(preferencesService.getFilePreferences()).thenReturn(mock(FilePreferences.class)); + + bibDatabase = new BibDatabase(); + bibDatabaseContext = mock(BibDatabaseContext.class); + when(bibDatabaseContext.getFileDirectories(Mockito.any())).thenReturn(Collections.singletonList(Path.of("src/test/resources/pdfs"))); + when(bibDatabaseContext.getFulltextIndexPath()).thenReturn(indexDir); + when(bibDatabaseContext.getDatabase()).thenReturn(bibDatabase); + when(bibDatabaseContext.getEntries()).thenReturn(bibDatabase.getEntries()); + } - LuceneIndexer indexer = LuceneIndexer.of(context, preferencesService); - search = LuceneSearcher.of(context); + private void initIndexer() throws IOException { + LuceneIndexer indexer = LuceneIndexer.of(bibDatabaseContext, preferencesService); + searcher = LuceneSearcher.of(bibDatabaseContext); - indexer.createIndex(); - for (BibEntry bibEntry : context.getEntries()) { + for (BibEntry bibEntry : bibDatabaseContext.getEntries()) { indexer.addBibFieldsToIndex(bibEntry); indexer.addLinkedFilesToIndex(bibEntry); } } @Test - public void searchForTest() { - HashMap searchResults = search.search(new SearchQuery("", EnumSet.noneOf(SearchRules.SearchFlags.class))); + public void searchForTest() throws IOException { + insertPdfsForSearch(); + initIndexer(); + + HashMap searchResults = searcher.search(new SearchQuery("", EnumSet.noneOf(SearchRules.SearchFlags.class))); int hits = searchResults.keySet().stream().mapToInt((key) -> searchResults.get(key).numSearchResults()).sum(); assertEquals(8, hits); } @Test - public void searchForUniversity() { - HashMap searchResults = search.search(new SearchQuery("University", EnumSet.noneOf(SearchRules.SearchFlags.class))); + public void searchForUniversity() throws IOException { + insertPdfsForSearch(); + initIndexer(); + + HashMap searchResults = searcher.search(new SearchQuery("University", EnumSet.noneOf(SearchRules.SearchFlags.class))); int hits = searchResults.keySet().stream().mapToInt((key) -> searchResults.get(key).numSearchResults()).sum(); assertEquals(1, hits); } @Test - public void searchForStopWord() { - HashMap searchResults = search.search(new SearchQuery("and", EnumSet.noneOf(SearchRules.SearchFlags.class))); + public void searchForStopWord() throws IOException { + insertPdfsForSearch(); + initIndexer(); + + HashMap searchResults = searcher.search(new SearchQuery("and", EnumSet.noneOf(SearchRules.SearchFlags.class))); int hits = searchResults.keySet().stream().mapToInt((key) -> searchResults.get(key).numSearchResults()).sum(); assertEquals(0, hits); } @Test - public void searchForSecond() { - HashMap searchResults = search.search(new SearchQuery("second", EnumSet.noneOf(SearchRules.SearchFlags.class))); + public void searchForSecond() throws IOException { + insertPdfsForSearch(); + initIndexer(); + + HashMap searchResults = searcher.search(new SearchQuery("second", EnumSet.noneOf(SearchRules.SearchFlags.class))); int hits = searchResults.keySet().stream().mapToInt((key) -> searchResults.get(key).numSearchResults()).sum(); assertEquals(4, hits); } @Test - public void searchForAnnotation() { - HashMap searchResults = search.search(new SearchQuery("annotation", EnumSet.noneOf(SearchRules.SearchFlags.class))); + public void searchForAnnotation() throws IOException { + insertPdfsForSearch(); + initIndexer(); + + HashMap searchResults = searcher.search(new SearchQuery("annotation", EnumSet.noneOf(SearchRules.SearchFlags.class))); int hits = searchResults.keySet().stream().mapToInt((key) -> searchResults.get(key).numSearchResults()).sum(); assertEquals(2, hits); } @Test - public void searchForEmptyString() { - HashMap searchResults = search.search(new SearchQuery("", EnumSet.noneOf(SearchRules.SearchFlags.class))); + public void searchForEmptyString() throws IOException { + insertPdfsForSearch(); + initIndexer(); + + HashMap searchResults = searcher.search(new SearchQuery("", EnumSet.noneOf(SearchRules.SearchFlags.class))); int hits = searchResults.keySet().stream().mapToInt((key) -> searchResults.get(key).numSearchResults()).sum(); assertEquals(0, hits); } @Test public void searchWithNullString() { - assertThrows(NullPointerException.class, () -> search.search(null)); + assertThrows(NullPointerException.class, () -> searcher.search(null)); + } + + private void insertPdfsForSearch() { + when(preferencesService.getFilePreferences().shouldFulltextIndexLinkedFiles()).thenReturn(true); + + BibEntry examplePdf = new BibEntry(StandardEntryType.Article); + examplePdf.setFiles(Collections.singletonList(new LinkedFile("Example Entry", "example.pdf", StandardFileType.PDF.getName()))); + bibDatabase.insertEntry(examplePdf); + + BibEntry metaDataEntry = new BibEntry(StandardEntryType.Article); + metaDataEntry.setFiles(Collections.singletonList(new LinkedFile("Metadata Entry", "metaData.pdf", StandardFileType.PDF.getName()))); + metaDataEntry.setCitationKey("MetaData2017"); + bibDatabase.insertEntry(metaDataEntry); + + BibEntry exampleThesis = new BibEntry(StandardEntryType.PhdThesis); + exampleThesis.setFiles(Collections.singletonList(new LinkedFile("Example Thesis", "thesis-example.pdf", StandardFileType.PDF.getName()))); + exampleThesis.setCitationKey("ExampleThesis"); + bibDatabase.insertEntry(exampleThesis); } } diff --git a/src/test/java/org/jabref/logic/search/SearchQueryTest.java b/src/test/java/org/jabref/logic/search/SearchQueryTest.java index 9ea8a0eafeb..c6ce0728860 100644 --- a/src/test/java/org/jabref/logic/search/SearchQueryTest.java +++ b/src/test/java/org/jabref/logic/search/SearchQueryTest.java @@ -1,174 +1,213 @@ package org.jabref.logic.search; +import java.io.IOException; +import java.nio.file.Path; +import java.util.Collections; import java.util.EnumSet; -import org.jabref.model.entry.BibEntry; -import org.jabref.model.entry.field.StandardField; -import org.jabref.model.entry.types.StandardEntryType; +import org.jabref.logic.search.indexing.LuceneIndexer; +import org.jabref.logic.search.retrieval.LuceneSearcher; +import org.jabref.model.database.BibDatabase; +import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.search.rules.SearchRules; +import org.jabref.preferences.FilePreferences; +import org.jabref.preferences.PreferencesService; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mockito; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class SearchQueryTest { - @Test - public void testToString() { - assertEquals("\"asdf\" (case sensitive, regular expression)", new SearchQuery("asdf", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)).toString()); - assertEquals("\"asdf\" (case insensitive, plain text)", new SearchQuery("asdf", EnumSet.noneOf(SearchRules.SearchFlags.class)).toString()); - } - - @Test - public void testGrammarSearch() { - BibEntry entry = new BibEntry(); - entry.addKeyword("one two", ','); - SearchQuery searchQuery = new SearchQuery("keywords=\"one two\"", EnumSet.noneOf(SearchRules.SearchFlags.class)); - assertTrue(searchQuery.isMatch(entry)); - } - - @Test - public void testGrammarSearchFullEntryLastCharMissing() { - BibEntry entry = new BibEntry(); - entry.setField(StandardField.TITLE, "systematic revie"); - SearchQuery searchQuery = new SearchQuery("title=\"systematic review\"", EnumSet.noneOf(SearchRules.SearchFlags.class)); - assertFalse(searchQuery.isMatch(entry)); - } - - @Test - public void testGrammarSearchFullEntry() { - BibEntry entry = new BibEntry(); - entry.setField(StandardField.TITLE, "systematic review"); - SearchQuery searchQuery = new SearchQuery("title=\"systematic review\"", EnumSet.noneOf(SearchRules.SearchFlags.class)); - assertTrue(searchQuery.isMatch(entry)); - } - - @Test - public void testSearchingForOpenBraketInBooktitle() { - BibEntry e = new BibEntry(StandardEntryType.InProceedings); - e.setField(StandardField.BOOKTITLE, "Super Conference (SC)"); - - SearchQuery searchQuery = new SearchQuery("booktitle=\"(\"", EnumSet.noneOf(SearchRules.SearchFlags.class)); - assertTrue(searchQuery.isMatch(e)); - } - - @Test - public void testSearchMatchesSingleKeywordNotPart() { - BibEntry e = new BibEntry(StandardEntryType.InProceedings); - e.setField(StandardField.KEYWORDS, "banana, pineapple, orange"); - - SearchQuery searchQuery = new SearchQuery("anykeyword==apple", EnumSet.noneOf(SearchRules.SearchFlags.class)); - assertFalse(searchQuery.isMatch(e)); - } - - @Test - public void testSearchMatchesSingleKeyword() { - BibEntry e = new BibEntry(StandardEntryType.InProceedings); - e.setField(StandardField.KEYWORDS, "banana, pineapple, orange"); - - SearchQuery searchQuery = new SearchQuery("anykeyword==pineapple", EnumSet.noneOf(SearchRules.SearchFlags.class)); - assertTrue(searchQuery.isMatch(e)); - } - - @Test - public void testSearchAllFields() { - BibEntry e = new BibEntry(StandardEntryType.InProceedings); - e.setField(StandardField.TITLE, "Fruity features"); - e.setField(StandardField.KEYWORDS, "banana, pineapple, orange"); - - SearchQuery searchQuery = new SearchQuery("anyfield==\"fruity features\"", EnumSet.noneOf(SearchRules.SearchFlags.class)); - assertTrue(searchQuery.isMatch(e)); - } - - @Test - public void testSearchAllFieldsNotForSpecificField() { - BibEntry e = new BibEntry(StandardEntryType.InProceedings); - e.setField(StandardField.TITLE, "Fruity features"); - e.setField(StandardField.KEYWORDS, "banana, pineapple, orange"); - - SearchQuery searchQuery = new SearchQuery("anyfield=fruit and keywords!=banana", EnumSet.noneOf(SearchRules.SearchFlags.class)); - assertFalse(searchQuery.isMatch(e)); - } - - @Test - public void testSearchAllFieldsAndSpecificField() { - BibEntry e = new BibEntry(StandardEntryType.InProceedings); - e.setField(StandardField.TITLE, "Fruity features"); - e.setField(StandardField.KEYWORDS, "banana, pineapple, orange"); - - SearchQuery searchQuery = new SearchQuery("anyfield=fruit and keywords=apple", EnumSet.noneOf(SearchRules.SearchFlags.class)); - assertTrue(searchQuery.isMatch(e)); - } - - @Test - public void testIsMatch() { - BibEntry entry = new BibEntry(); - entry.setType(StandardEntryType.Article); - entry.setField(StandardField.AUTHOR, "asdf"); - - assertFalse(new SearchQuery("BiblatexEntryType", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)).isMatch(entry)); - assertTrue(new SearchQuery("asdf", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)).isMatch(entry)); - assertTrue(new SearchQuery("author=asdf", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)).isMatch(entry)); - } - - @Test - public void testIsValidQueryNotAsRegEx() { - assertTrue(new SearchQuery("asdf", EnumSet.noneOf(SearchRules.SearchFlags.class)).isValid()); - } - - @Test - public void testIsValidQueryContainsBracketNotAsRegEx() { - assertTrue(new SearchQuery("asdf[", EnumSet.noneOf(SearchRules.SearchFlags.class)).isValid()); - } - - @Test - public void testIsNotValidQueryContainsBracketNotAsRegEx() { - assertTrue(new SearchQuery("asdf[", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)).isValid()); - } - - @Test - public void testIsValidQueryAsRegEx() { - assertTrue(new SearchQuery("asdf", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)).isValid()); - } - - @Test - public void testIsValidQueryWithNumbersAsRegEx() { - assertTrue(new SearchQuery("123", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)).isValid()); - } - - @Test - public void testIsValidQueryContainsBracketAsRegEx() { - assertTrue(new SearchQuery("asdf[", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)).isValid()); - } - - @Test - public void testIsValidQueryWithEqualSignAsRegEx() { - assertTrue(new SearchQuery("author=asdf", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)).isValid()); - } - - @Test - public void testIsValidQueryWithNumbersAndEqualSignAsRegEx() { - assertTrue(new SearchQuery("author=123", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)).isValid()); - } - - @Test - public void testIsValidQueryWithEqualSignNotAsRegEx() { - assertTrue(new SearchQuery("author=asdf", EnumSet.noneOf(SearchRules.SearchFlags.class)).isValid()); - } - - @Test - public void testIsValidQueryWithNumbersAndEqualSignNotAsRegEx() { - assertTrue(new SearchQuery("author=123", EnumSet.noneOf(SearchRules.SearchFlags.class)).isValid()); - } - - @Test - public void isMatchedForNormalAndFieldBasedSearchMixed() { - BibEntry entry = new BibEntry(); - entry.setType(StandardEntryType.Article); - entry.setField(StandardField.AUTHOR, "asdf"); - entry.setField(StandardField.ABSTRACT, "text"); - - assertTrue(new SearchQuery("text AND author=asdf", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)).isMatch(entry)); - } + private LuceneSearcher searcher; + private PreferencesService preferencesService; + private BibDatabase bibDatabase; + private BibDatabaseContext bibDatabaseContext; + + @BeforeEach + public void setUp(@TempDir Path indexDir) throws IOException { + preferencesService = mock(PreferencesService.class); + when(preferencesService.getFilePreferences()).thenReturn(mock(FilePreferences.class)); + + bibDatabase = new BibDatabase(); + bibDatabaseContext = mock(BibDatabaseContext.class); + when(bibDatabaseContext.getFileDirectories(Mockito.any())).thenReturn(Collections.singletonList(Path.of("src/test/resources/pdfs"))); + when(bibDatabaseContext.getFulltextIndexPath()).thenReturn(indexDir); + when(bibDatabaseContext.getDatabase()).thenReturn(bibDatabase); + when(bibDatabaseContext.getEntries()).thenReturn(bibDatabase.getEntries()); + + LuceneIndexer indexer = LuceneIndexer.of(bibDatabaseContext, preferencesService); + indexer.createIndex(); + + searcher = LuceneSearcher.of(bibDatabaseContext); + } + + @Test + public void nullWhenQueryBlank() { + assertNull(new SearchQuery("", EnumSet.noneOf(SearchRules.SearchFlags.class)).getQuery()); + } + +// +// @Test +// public void testToString() { +// assertEquals("\"asdf\" (case sensitive, regular expression)", new SearchQuery("asdf", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)).toString()); +// assertEquals("\"asdf\" (case insensitive, plain text)", new SearchQuery("asdf", EnumSet.noneOf(SearchRules.SearchFlags.class)).toString()); +// } +// +// @Test +// public void testGrammarSearch() { +// BibEntry entry = new BibEntry(); +// entry.addKeyword("one two", ','); +// SearchQuery searchQuery = new SearchQuery("keywords=\"one two\"", EnumSet.noneOf(SearchRules.SearchFlags.class)); +// assertTrue(searchQuery.isMatch(entry)); +// } +// +// @Test +// public void testGrammarSearchFullEntryLastCharMissing() { +// BibEntry entry = new BibEntry(); +// entry.setField(StandardField.TITLE, "systematic revie"); +// SearchQuery searchQuery = new SearchQuery("title=\"systematic review\"", EnumSet.noneOf(SearchRules.SearchFlags.class)); +// assertFalse(searchQuery.isMatch(entry)); +// } +// +// @Test +// public void testGrammarSearchFullEntry() { +// BibEntry entry = new BibEntry(); +// entry.setField(StandardField.TITLE, "systematic review"); +// SearchQuery searchQuery = new SearchQuery("title=\"systematic review\"", EnumSet.noneOf(SearchRules.SearchFlags.class)); +// assertTrue(searchQuery.isMatch(entry)); +// } +// +// @Test +// public void testSearchingForOpenBraketInBooktitle() { +// BibEntry e = new BibEntry(StandardEntryType.InProceedings); +// e.setField(StandardField.BOOKTITLE, "Super Conference (SC)"); +// +// SearchQuery searchQuery = new SearchQuery("booktitle=\"(\"", EnumSet.noneOf(SearchRules.SearchFlags.class)); +// assertTrue(searchQuery.isMatch(e)); +// } +// +// @Test +// public void testSearchMatchesSingleKeywordNotPart() { +// BibEntry e = new BibEntry(StandardEntryType.InProceedings); +// e.setField(StandardField.KEYWORDS, "banana, pineapple, orange"); +// +// SearchQuery searchQuery = new SearchQuery("anykeyword==apple", EnumSet.noneOf(SearchRules.SearchFlags.class)); +// assertFalse(searchQuery.isMatch(e)); +// } +// +// @Test +// public void testSearchMatchesSingleKeyword() { +// BibEntry e = new BibEntry(StandardEntryType.InProceedings); +// e.setField(StandardField.KEYWORDS, "banana, pineapple, orange"); +// +// SearchQuery searchQuery = new SearchQuery("anykeyword==pineapple", EnumSet.noneOf(SearchRules.SearchFlags.class)); +// assertTrue(searchQuery.isMatch(e)); +// } +// +// @Test +// public void testSearchAllFields() { +// BibEntry e = new BibEntry(StandardEntryType.InProceedings); +// e.setField(StandardField.TITLE, "Fruity features"); +// e.setField(StandardField.KEYWORDS, "banana, pineapple, orange"); +// +// SearchQuery searchQuery = new SearchQuery("anyfield==\"fruity features\"", EnumSet.noneOf(SearchRules.SearchFlags.class)); +// assertTrue(searchQuery.isMatch(e)); +// } +// +// @Test +// public void testSearchAllFieldsNotForSpecificField() { +// BibEntry e = new BibEntry(StandardEntryType.InProceedings); +// e.setField(StandardField.TITLE, "Fruity features"); +// e.setField(StandardField.KEYWORDS, "banana, pineapple, orange"); +// +// SearchQuery searchQuery = new SearchQuery("anyfield=fruit and keywords!=banana", EnumSet.noneOf(SearchRules.SearchFlags.class)); +// assertFalse(searchQuery.isMatch(e)); +// } +// +// @Test +// public void testSearchAllFieldsAndSpecificField() { +// BibEntry e = new BibEntry(StandardEntryType.InProceedings); +// e.setField(StandardField.TITLE, "Fruity features"); +// e.setField(StandardField.KEYWORDS, "banana, pineapple, orange"); +// +// SearchQuery searchQuery = new SearchQuery("anyfield=fruit and keywords=apple", EnumSet.noneOf(SearchRules.SearchFlags.class)); +// assertTrue(searchQuery.isMatch(e)); +// } +// +// @Test +// public void testIsMatch() { +// BibEntry entry = new BibEntry(); +// entry.setType(StandardEntryType.Article); +// entry.setField(StandardField.AUTHOR, "asdf"); +// +// assertFalse(new SearchQuery("BiblatexEntryType", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)).isMatch(entry)); +// assertTrue(new SearchQuery("asdf", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)).isMatch(entry)); +// assertTrue(new SearchQuery("author=asdf", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)).isMatch(entry)); +// } +// +// @Test +// public void testIsValidQueryNotAsRegEx() { +// assertTrue(new SearchQuery("asdf", EnumSet.noneOf(SearchRules.SearchFlags.class)).isValid()); +// } +// +// @Test +// public void testIsValidQueryContainsBracketNotAsRegEx() { +// assertTrue(new SearchQuery("asdf[", EnumSet.noneOf(SearchRules.SearchFlags.class)).isValid()); +// } +// +// @Test +// public void testIsNotValidQueryContainsBracketNotAsRegEx() { +// assertTrue(new SearchQuery("asdf[", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)).isValid()); +// } +// +// @Test +// public void testIsValidQueryAsRegEx() { +// assertTrue(new SearchQuery("asdf", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)).isValid()); +// } +// +// @Test +// public void testIsValidQueryWithNumbersAsRegEx() { +// assertTrue(new SearchQuery("123", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)).isValid()); +// } +// +// @Test +// public void testIsValidQueryContainsBracketAsRegEx() { +// assertTrue(new SearchQuery("asdf[", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)).isValid()); +// } +// +// @Test +// public void testIsValidQueryWithEqualSignAsRegEx() { +// assertTrue(new SearchQuery("author=asdf", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)).isValid()); +// } +// +// @Test +// public void testIsValidQueryWithNumbersAndEqualSignAsRegEx() { +// assertTrue(new SearchQuery("author=123", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)).isValid()); +// } +// +// @Test +// public void testIsValidQueryWithEqualSignNotAsRegEx() { +// assertTrue(new SearchQuery("author=asdf", EnumSet.noneOf(SearchRules.SearchFlags.class)).isValid()); +// } +// +// @Test +// public void testIsValidQueryWithNumbersAndEqualSignNotAsRegEx() { +// assertTrue(new SearchQuery("author=123", EnumSet.noneOf(SearchRules.SearchFlags.class)).isValid()); +// } +// +// @Test +// public void isMatchedForNormalAndFieldBasedSearchMixed() { +// BibEntry entry = new BibEntry(); +// entry.setType(StandardEntryType.Article); +// entry.setField(StandardField.AUTHOR, "asdf"); +// entry.setField(StandardField.ABSTRACT, "text"); +// +// assertTrue(new SearchQuery("text AND author=asdf", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)).isMatch(entry)); +// } } From 00cecf854747eb9e2214cd47b1e201de4cd7f9a1 Mon Sep 17 00:00:00 2001 From: Carl Christian Snethlage <50491877+calixtus@users.noreply.github.com> Date: Wed, 13 Sep 2023 00:36:49 +0200 Subject: [PATCH 076/256] Checkstyle --- src/jmh/java/org/jabref/benchmarks/Benchmarks.java | 2 -- src/main/java/org/jabref/gui/maintable/MainTable.java | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/jmh/java/org/jabref/benchmarks/Benchmarks.java b/src/jmh/java/org/jabref/benchmarks/Benchmarks.java index dc699a11494..68917ce8ee3 100644 --- a/src/jmh/java/org/jabref/benchmarks/Benchmarks.java +++ b/src/jmh/java/org/jabref/benchmarks/Benchmarks.java @@ -33,8 +33,6 @@ import org.jabref.model.groups.KeywordGroup; import org.jabref.model.groups.WordKeywordGroup; import org.jabref.model.metadata.MetaData; -import org.jabref.model.util.DummyFileUpdateMonitor; -import org.jabref.preferences.GeneralPreferences; import org.jabref.preferences.JabRefPreferences; import org.openjdk.jmh.Main; diff --git a/src/main/java/org/jabref/gui/maintable/MainTable.java b/src/main/java/org/jabref/gui/maintable/MainTable.java index 89a6733cbe7..3011de7f78b 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTable.java +++ b/src/main/java/org/jabref/gui/maintable/MainTable.java @@ -53,8 +53,8 @@ import org.jabref.model.database.event.EntriesAddedEvent; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.BibEntryTypesManager; -import org.jabref.model.util.FileUpdateMonitor; import org.jabref.model.search.rules.SearchRules.SearchFlags; +import org.jabref.model.util.FileUpdateMonitor; import org.jabref.preferences.PreferencesService; import com.google.common.eventbus.Subscribe; From ad85c7704f30c4ad6675b1250054a405189cb232 Mon Sep 17 00:00:00 2001 From: Carl Christian Snethlage <50491877+calixtus@users.noreply.github.com> Date: Wed, 13 Sep 2023 00:51:39 +0200 Subject: [PATCH 077/256] Checkstyle --- .../java/org/jabref/benchmarks/Benchmarks.java | 15 +++++---------- .../org/jabref/gui/search/GlobalSearchBar.java | 2 +- .../org/jabref/preferences/SearchPreferences.java | 2 -- .../org/jabref/cli/ArgumentProcessorTest.java | 2 +- 4 files changed, 7 insertions(+), 14 deletions(-) diff --git a/src/jmh/java/org/jabref/benchmarks/Benchmarks.java b/src/jmh/java/org/jabref/benchmarks/Benchmarks.java index 68917ce8ee3..e39b1dce706 100644 --- a/src/jmh/java/org/jabref/benchmarks/Benchmarks.java +++ b/src/jmh/java/org/jabref/benchmarks/Benchmarks.java @@ -3,10 +3,8 @@ import java.io.IOException; import java.io.StringReader; import java.io.StringWriter; -import java.util.EnumSet; import java.util.List; import java.util.Random; -import java.util.stream.Collectors; import org.jabref.gui.Globals; import org.jabref.logic.bibtex.FieldPreferences; @@ -19,7 +17,6 @@ import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.logic.layout.format.HTMLChars; import org.jabref.logic.layout.format.LatexToUnicodeFormatter; -import org.jabref.logic.search.SearchQuery; import org.jabref.logic.util.OS; import org.jabref.model.database.BibDatabase; import org.jabref.model.database.BibDatabaseContext; @@ -102,16 +99,14 @@ public String write() throws Exception { @Benchmark public List search() { - // FIXME: Reuse SearchWorker here - SearchQuery searchQuery = new SearchQuery("Journal Title 500", EnumSet.noneOf(SearchFlags.class)); - return database.getEntries().stream().filter(searchQuery::isMatch).collect(Collectors.toList()); + // FixMe: Create Benchmark for LuceneSearch + return null; } @Benchmark - public List parallelSearch() { - // FIXME: Reuse SearchWorker here - SearchQuery searchQuery = new SearchQuery("Journal Title 500", EnumSet.noneOf(SearchFlags.class)); - return database.getEntries().parallelStream().filter(searchQuery::isMatch).collect(Collectors.toList()); + public List index() { + // FixMe: Create Benchmark for LuceneIndexer + return null; } @Benchmark diff --git a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java index cc003f00b7e..8d820a714f8 100644 --- a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java +++ b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java @@ -17,8 +17,8 @@ import javafx.beans.binding.BooleanBinding; import javafx.beans.property.BooleanProperty; import javafx.beans.property.SimpleBooleanProperty; -import javafx.collections.MapChangeListener; import javafx.collections.ListChangeListener; +import javafx.collections.MapChangeListener; import javafx.collections.ObservableList; import javafx.css.PseudoClass; import javafx.event.Event; diff --git a/src/main/java/org/jabref/preferences/SearchPreferences.java b/src/main/java/org/jabref/preferences/SearchPreferences.java index 3b5ad8f4b0c..dd4b23b7b97 100644 --- a/src/main/java/org/jabref/preferences/SearchPreferences.java +++ b/src/main/java/org/jabref/preferences/SearchPreferences.java @@ -4,10 +4,8 @@ import javafx.beans.property.BooleanProperty; import javafx.beans.property.DoubleProperty; -import javafx.beans.property.ObjectProperty; import javafx.beans.property.SimpleBooleanProperty; import javafx.beans.property.SimpleDoubleProperty; -import javafx.beans.property.SimpleObjectProperty; import javafx.collections.FXCollections; import javafx.collections.ObservableSet; diff --git a/src/test/java/org/jabref/cli/ArgumentProcessorTest.java b/src/test/java/org/jabref/cli/ArgumentProcessorTest.java index 78dcb77f755..30aafcee635 100644 --- a/src/test/java/org/jabref/cli/ArgumentProcessorTest.java +++ b/src/test/java/org/jabref/cli/ArgumentProcessorTest.java @@ -43,7 +43,7 @@ class ArgumentProcessorTest { void setup() { when(importerPreferences.getCustomImporters()).thenReturn(FXCollections.emptyObservableSet()); when(preferencesService.getSearchPreferences()).thenReturn( - new SearchPreferences(null, EnumSet.noneOf(SearchRules.SearchFlags.class), false) + new SearchPreferences(EnumSet.noneOf(SearchRules.SearchFlags.class), false) ); bibtexImporter = new BibtexImporter(importFormatPreferences, new DummyFileUpdateMonitor()); From ed5cebf902bae384a35ba9f9029be687f14ac538 Mon Sep 17 00:00:00 2001 From: Carl Christian Snethlage <50491877+calixtus@users.noreply.github.com> Date: Mon, 25 Sep 2023 21:32:09 +0200 Subject: [PATCH 078/256] Update Java version --- .devcontainer/devcontainer.json | 2 +- build.gradle | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 11163bde525..6d62ca17488 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -31,7 +31,7 @@ // Install java. // See https://github.com/devcontainers/features/tree/main/src/java#options for details. "ghcr.io/devcontainers/features/java:1": { - "version": "20.0.2-tem", + "version": "21", "installGradle": false, "jdkDistro": "tem" } diff --git a/build.gradle b/build.gradle index 49a6162d17f..0154d3a096b 100644 --- a/build.gradle +++ b/build.gradle @@ -41,15 +41,15 @@ group = "org.jabref" version = project.findProperty('projVersion') ?: '100.0.0' java { - sourceCompatibility = JavaVersion.VERSION_19 - targetCompatibility = JavaVersion.VERSION_19 + sourceCompatibility = JavaVersion.VERSION_21 + targetCompatibility = JavaVersion.VERSION_21 // Workaround needed for Eclipse, probably because of https://github.com/gradle/gradle/issues/16922 // Should be removed as soon as Gradle 7.0.1 is released ( https://github.com/gradle/gradle/issues/16922#issuecomment-828217060 ) modularity.inferModulePath.set(false) toolchain { // If this is updated, also update .devcontainer/devcontainer.json#L34 - languageVersion = JavaLanguageVersion.of(20) + languageVersion = JavaLanguageVersion.of(21) } } From afc4d67226b4ae59ecbb4966ec4eee2405761488 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Mon, 25 Sep 2023 22:52:33 +0200 Subject: [PATCH 079/256] Refine logging --- .../java/org/jabref/logic/search/indexing/LuceneIndexer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java b/src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java index 24ff86a42cd..83d174c46a1 100644 --- a/src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java @@ -311,9 +311,9 @@ public void deleteLinkedFilesIndex() { indexWriter.deleteDocuments(queryParser.parse("*")); indexWriter.commit(); } catch (IOException e) { - LOGGER.warn("Could not initialize the IndexWriter!", e); + LOGGER.warn("Could not initialize the IndexWriter", e); } catch (ParseException e) { - e.printStackTrace(); + LOGGER.error("Could not parse", e); } } From 2f3203f904ae70a131efda8b32cf6c70ad015ba1 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Mon, 25 Sep 2023 23:21:43 +0200 Subject: [PATCH 080/256] Fix compile error --- .../java/org/jabref/gui/maintable/MainTableDataModelTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/jabref/gui/maintable/MainTableDataModelTest.java b/src/test/java/org/jabref/gui/maintable/MainTableDataModelTest.java index 3e9ddc469cb..d29ba9614ce 100644 --- a/src/test/java/org/jabref/gui/maintable/MainTableDataModelTest.java +++ b/src/test/java/org/jabref/gui/maintable/MainTableDataModelTest.java @@ -12,6 +12,7 @@ import javafx.collections.transformation.FilteredList; import javafx.collections.transformation.SortedList; +import org.jabref.gui.StateManager; import org.jabref.logic.bibtex.comparator.EntryComparator; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; @@ -33,7 +34,7 @@ void additionToObservableMapTriggersUpdate() { NameDisplayPreferences nameDisplayPreferences = new NameDisplayPreferences(NameDisplayPreferences.DisplayStyle.AS_IS, NameDisplayPreferences.AbbreviationStyle.FULL); SimpleObjectProperty fieldValueFormatter = new SimpleObjectProperty<>(new MainTableFieldValueFormatter(nameDisplayPreferences, bibDatabaseContext)); ObservableList entriesViewModel = EasyBind.mapBacked(allEntries, entry -> - new BibEntryTableViewModel(entry, bibDatabaseContext, fieldValueFormatter)); + new BibEntryTableViewModel(entry, bibDatabaseContext, fieldValueFormatter, new StateManager())); FilteredList entriesFiltered = new FilteredList<>(entriesViewModel); IntegerProperty resultSize = new SimpleIntegerProperty(); resultSize.bind(Bindings.size(entriesFiltered)); From c7714580d55706e0588a287be23768ee5ab9022f Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Mon, 25 Sep 2023 23:21:51 +0200 Subject: [PATCH 081/256] Add LuceneTest --- .../org/jabref/logic/search/LuceneTest.java | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 src/test/java/org/jabref/logic/search/LuceneTest.java diff --git a/src/test/java/org/jabref/logic/search/LuceneTest.java b/src/test/java/org/jabref/logic/search/LuceneTest.java new file mode 100644 index 00000000000..1acfa23a1c4 --- /dev/null +++ b/src/test/java/org/jabref/logic/search/LuceneTest.java @@ -0,0 +1,53 @@ +package org.jabref.logic.search; + +import org.jabref.model.pdf.search.EnglishStemAnalyzer; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.queryparser.classic.QueryParser; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.store.ByteBuffersDirectory; +import org.apache.lucene.store.Directory; + +public class LuceneTest { + public static void main(String[] args) throws Exception { + // Setup the analyzer + Analyzer analyzer = new EnglishStemAnalyzer(); + + // Store the index in memory + Directory directory = new ByteBuffersDirectory(); + + // Configure index writer + IndexWriterConfig config = new IndexWriterConfig(analyzer); + IndexWriter indexWriter = new IndexWriter(directory, config); + + // Index sample data + String[] texts = {"running", "runner", "ran", "trial", "trials"}; + for (String text : texts) { + Document document = new Document(); + document.add(new TextField("content", text, Field.Store.YES)); + indexWriter.addDocument(document); + } + indexWriter.close(); + + search("trials", directory, analyzer); + } + + public static void search(String queryString, Directory directory, Analyzer analyzer) throws Exception { + Query query = new QueryParser("content", analyzer).parse(queryString); + IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(directory)); + ScoreDoc[] hits = searcher.search(query, 10).scoreDocs; + + for (ScoreDoc scoreDoc : hits) { + Document doc = searcher.doc(scoreDoc.doc); + System.out.println(doc.get("content")); + } + } +} From b795fa60c7c8951df115370fbb5b29fc100c475c Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Thu, 11 Jan 2024 23:42:59 +0100 Subject: [PATCH 082/256] Update CHANGELOG.md --- CHANGELOG.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07dfc4333c9..bac31c8769a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,12 +17,10 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv - We added support for customizing the citation command (e.g., `[@key1,@key2]`) when [pushing to external applications](https://docs.jabref.org/cite/pushtoapplications). [#10133](https://github.com/JabRef/jabref/issues/10133) - We added an integrity check for more special characters. [#8712](https://github.com/JabRef/jabref/issues/8712) - We added protected terms described as "Computer science". [#10222](https://github.com/JabRef/jabref/pull/10222) -- - We reintroduced the floating search in the main table. [#8963](https://github.com/JabRef/jabref/pull/8963) - We added a link "Get more themes..." in the preferences to that points to [themes.jabref.org](https://themes.jabref.org) allowing the user to download new themes. [#10243](https://github.com/JabRef/jabref/issues/10243) - We added a fetcher for [LOBID](https://lobid.org/resources/api) resources. [koppor#386](https://github.com/koppor/jabref/issues/386) - When in `biblatex` mode, the [integrity check](https://docs.jabref.org/finding-sorting-and-cleaning-entries/checkintegrity) for journal titles now also checks the field `journal`. - ### Changed - The export formats `listrefs`, `tablerefs`, `tablerefsabsbib`, now use the ISO date format in the footer [#10383](https://github.com/JabRef/jabref/pull/10383). From 29759bb9a1740b9b8a154d394e7c9e53415d6154 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Fri, 24 May 2024 00:40:39 +0300 Subject: [PATCH 083/256] Move search classes to pdf package --- src/main/java/org/jabref/gui/LibraryTab.java | 4 ++-- src/main/java/org/jabref/gui/entryeditor/CommentsTab.java | 2 +- .../java/org/jabref/gui/entryeditor/DeprecatedFieldsTab.java | 2 +- src/main/java/org/jabref/gui/entryeditor/FieldsEditorTab.java | 2 +- .../java/org/jabref/gui/entryeditor/OptionalFields2Tab.java | 2 +- .../java/org/jabref/gui/entryeditor/OptionalFieldsTab.java | 2 +- .../org/jabref/gui/entryeditor/OptionalFieldsTabBase.java | 2 +- src/main/java/org/jabref/gui/entryeditor/OtherFieldsTab.java | 2 +- src/main/java/org/jabref/gui/entryeditor/PreviewTab.java | 2 +- .../java/org/jabref/gui/entryeditor/RequiredFieldsTab.java | 2 +- .../java/org/jabref/gui/entryeditor/UserDefinedFieldsTab.java | 2 +- .../jabref/gui/externalfiles/ExternalFilesEntryLinker.java | 4 ++-- .../java/org/jabref/gui/maintable/MainTableDataModel.java | 2 +- src/main/java/org/jabref/gui/preview/PreviewPanel.java | 2 +- .../jabref/gui/search/RebuildFulltextSearchIndexAction.java | 2 +- .../logic/{search/indexing => pdf/search}/DocumentReader.java | 2 +- .../{search/indexing => pdf/search}/IndexingTaskManager.java | 2 +- .../logic/{search/indexing => pdf/search}/LuceneIndexer.java | 2 +- .../{search/retrieval => pdf/search}/LuceneSearcher.java | 2 +- .../jabref/logic/pdf/search/indexing/IndexingTaskManager.java | 0 src/main/java/org/jabref/model/groups/SearchGroup.java | 2 +- src/test/java/org/jabref/gui/entryeditor/CommentsTabTest.java | 2 +- .../jabref/logic/pdf/search/indexing/DocumentReaderTest.java | 2 +- .../jabref/logic/pdf/search/indexing/LuceneIndexerTest.java | 2 +- .../jabref/logic/pdf/search/retrieval/LuceneSearcherTest.java | 4 ++-- src/test/java/org/jabref/logic/search/SearchQueryTest.java | 4 ++-- 26 files changed, 29 insertions(+), 29 deletions(-) rename src/main/java/org/jabref/logic/{search/indexing => pdf/search}/DocumentReader.java (99%) rename src/main/java/org/jabref/logic/{search/indexing => pdf/search}/IndexingTaskManager.java (99%) rename src/main/java/org/jabref/logic/{search/indexing => pdf/search}/LuceneIndexer.java (99%) rename src/main/java/org/jabref/logic/{search/retrieval => pdf/search}/LuceneSearcher.java (98%) delete mode 100644 src/main/java/org/jabref/logic/pdf/search/indexing/IndexingTaskManager.java diff --git a/src/main/java/org/jabref/gui/LibraryTab.java b/src/main/java/org/jabref/gui/LibraryTab.java index 29c7927ffda..7916efab189 100644 --- a/src/main/java/org/jabref/gui/LibraryTab.java +++ b/src/main/java/org/jabref/gui/LibraryTab.java @@ -51,8 +51,8 @@ import org.jabref.logic.l10n.Localization; import org.jabref.logic.pdf.FileAnnotationCache; import org.jabref.logic.search.SearchQuery; -import org.jabref.logic.search.indexing.IndexingTaskManager; -import org.jabref.logic.search.indexing.LuceneIndexer; +import org.jabref.logic.pdf.search.IndexingTaskManager; +import org.jabref.logic.pdf.search.LuceneIndexer; import org.jabref.logic.shared.DatabaseLocation; import org.jabref.logic.util.UpdateField; import org.jabref.logic.util.io.FileUtil; diff --git a/src/main/java/org/jabref/gui/entryeditor/CommentsTab.java b/src/main/java/org/jabref/gui/entryeditor/CommentsTab.java index 04e53d76b7f..c64dbc58df8 100644 --- a/src/main/java/org/jabref/gui/entryeditor/CommentsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/CommentsTab.java @@ -16,7 +16,7 @@ import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.journals.JournalAbbreviationRepository; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.search.indexing.IndexingTaskManager; +import org.jabref.logic.pdf.search.IndexingTaskManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.Field; diff --git a/src/main/java/org/jabref/gui/entryeditor/DeprecatedFieldsTab.java b/src/main/java/org/jabref/gui/entryeditor/DeprecatedFieldsTab.java index db6a4c35d4a..f5666a54748 100644 --- a/src/main/java/org/jabref/gui/entryeditor/DeprecatedFieldsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/DeprecatedFieldsTab.java @@ -17,7 +17,7 @@ import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.journals.JournalAbbreviationRepository; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.search.indexing.IndexingTaskManager; +import org.jabref.logic.pdf.search.IndexingTaskManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.database.BibDatabaseMode; import org.jabref.model.entry.BibEntry; diff --git a/src/main/java/org/jabref/gui/entryeditor/FieldsEditorTab.java b/src/main/java/org/jabref/gui/entryeditor/FieldsEditorTab.java index ee50a980cd3..97a99bea8a4 100644 --- a/src/main/java/org/jabref/gui/entryeditor/FieldsEditorTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/FieldsEditorTab.java @@ -33,7 +33,7 @@ import org.jabref.gui.theme.ThemeManager; import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.journals.JournalAbbreviationRepository; -import org.jabref.logic.search.indexing.IndexingTaskManager; +import org.jabref.logic.pdf.search.IndexingTaskManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.Field; diff --git a/src/main/java/org/jabref/gui/entryeditor/OptionalFields2Tab.java b/src/main/java/org/jabref/gui/entryeditor/OptionalFields2Tab.java index 80c437628f5..c63c145cfba 100644 --- a/src/main/java/org/jabref/gui/entryeditor/OptionalFields2Tab.java +++ b/src/main/java/org/jabref/gui/entryeditor/OptionalFields2Tab.java @@ -9,7 +9,7 @@ import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.journals.JournalAbbreviationRepository; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.search.indexing.IndexingTaskManager; +import org.jabref.logic.pdf.search.IndexingTaskManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntryTypesManager; import org.jabref.preferences.PreferencesService; diff --git a/src/main/java/org/jabref/gui/entryeditor/OptionalFieldsTab.java b/src/main/java/org/jabref/gui/entryeditor/OptionalFieldsTab.java index a2bb91712da..ad522d2411f 100644 --- a/src/main/java/org/jabref/gui/entryeditor/OptionalFieldsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/OptionalFieldsTab.java @@ -9,7 +9,7 @@ import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.journals.JournalAbbreviationRepository; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.search.indexing.IndexingTaskManager; +import org.jabref.logic.pdf.search.IndexingTaskManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntryTypesManager; import org.jabref.preferences.PreferencesService; diff --git a/src/main/java/org/jabref/gui/entryeditor/OptionalFieldsTabBase.java b/src/main/java/org/jabref/gui/entryeditor/OptionalFieldsTabBase.java index 2320e2c5da2..df55624701c 100644 --- a/src/main/java/org/jabref/gui/entryeditor/OptionalFieldsTabBase.java +++ b/src/main/java/org/jabref/gui/entryeditor/OptionalFieldsTabBase.java @@ -16,7 +16,7 @@ import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.journals.JournalAbbreviationRepository; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.search.indexing.IndexingTaskManager; +import org.jabref.logic.pdf.search.IndexingTaskManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.database.BibDatabaseMode; import org.jabref.model.entry.BibEntry; diff --git a/src/main/java/org/jabref/gui/entryeditor/OtherFieldsTab.java b/src/main/java/org/jabref/gui/entryeditor/OtherFieldsTab.java index 708566858c9..91ae4980fea 100644 --- a/src/main/java/org/jabref/gui/entryeditor/OtherFieldsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/OtherFieldsTab.java @@ -20,7 +20,7 @@ import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.journals.JournalAbbreviationRepository; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.search.indexing.IndexingTaskManager; +import org.jabref.logic.pdf.search.IndexingTaskManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.database.BibDatabaseMode; import org.jabref.model.entry.BibEntry; diff --git a/src/main/java/org/jabref/gui/entryeditor/PreviewTab.java b/src/main/java/org/jabref/gui/entryeditor/PreviewTab.java index 80f3f73ade1..648cbe003a8 100644 --- a/src/main/java/org/jabref/gui/entryeditor/PreviewTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/PreviewTab.java @@ -7,7 +7,7 @@ import org.jabref.gui.theme.ThemeManager; import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.search.indexing.IndexingTaskManager; +import org.jabref.logic.pdf.search.IndexingTaskManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.preferences.PreferencesService; diff --git a/src/main/java/org/jabref/gui/entryeditor/RequiredFieldsTab.java b/src/main/java/org/jabref/gui/entryeditor/RequiredFieldsTab.java index 9e6dc24d5a8..5c227d86d8f 100644 --- a/src/main/java/org/jabref/gui/entryeditor/RequiredFieldsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/RequiredFieldsTab.java @@ -16,7 +16,7 @@ import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.journals.JournalAbbreviationRepository; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.search.indexing.IndexingTaskManager; +import org.jabref.logic.pdf.search.IndexingTaskManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.BibEntryType; diff --git a/src/main/java/org/jabref/gui/entryeditor/UserDefinedFieldsTab.java b/src/main/java/org/jabref/gui/entryeditor/UserDefinedFieldsTab.java index d467a5855f3..6b70c46c8bd 100644 --- a/src/main/java/org/jabref/gui/entryeditor/UserDefinedFieldsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/UserDefinedFieldsTab.java @@ -12,7 +12,7 @@ import org.jabref.gui.theme.ThemeManager; import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.journals.JournalAbbreviationRepository; -import org.jabref.logic.search.indexing.IndexingTaskManager; +import org.jabref.logic.pdf.search.IndexingTaskManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.Field; diff --git a/src/main/java/org/jabref/gui/externalfiles/ExternalFilesEntryLinker.java b/src/main/java/org/jabref/gui/externalfiles/ExternalFilesEntryLinker.java index 81733ee3d19..8f257fd9904 100644 --- a/src/main/java/org/jabref/gui/externalfiles/ExternalFilesEntryLinker.java +++ b/src/main/java/org/jabref/gui/externalfiles/ExternalFilesEntryLinker.java @@ -15,8 +15,8 @@ import org.jabref.logic.cleanup.MoveFilesCleanup; import org.jabref.logic.cleanup.RenamePdfCleanup; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.search.indexing.IndexingTaskManager; -import org.jabref.logic.search.indexing.LuceneIndexer; +import org.jabref.logic.pdf.search.IndexingTaskManager; +import org.jabref.logic.pdf.search.LuceneIndexer; import org.jabref.logic.util.io.FileNameCleaner; import org.jabref.logic.util.io.FileUtil; import org.jabref.model.database.BibDatabaseContext; diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index da44ba88919..bfb2b76f530 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -16,7 +16,7 @@ import org.jabref.gui.groups.GroupsPreferences; import org.jabref.gui.util.BindingsHelper; import org.jabref.logic.search.SearchQuery; -import org.jabref.logic.search.retrieval.LuceneSearcher; +import org.jabref.logic.pdf.search.LuceneSearcher; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.groups.GroupTreeNode; diff --git a/src/main/java/org/jabref/gui/preview/PreviewPanel.java b/src/main/java/org/jabref/gui/preview/PreviewPanel.java index e98371875c6..080aced2ec3 100644 --- a/src/main/java/org/jabref/gui/preview/PreviewPanel.java +++ b/src/main/java/org/jabref/gui/preview/PreviewPanel.java @@ -26,7 +26,7 @@ import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.l10n.Localization; import org.jabref.logic.preview.PreviewLayout; -import org.jabref.logic.search.indexing.IndexingTaskManager; +import org.jabref.logic.pdf.search.IndexingTaskManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.preferences.PreferencesService; diff --git a/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java b/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java index b96580e0565..92d0cedcb20 100644 --- a/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java +++ b/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java @@ -9,7 +9,7 @@ import org.jabref.gui.util.BackgroundTask; import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.search.indexing.LuceneIndexer; +import org.jabref.logic.pdf.search.LuceneIndexer; import org.jabref.model.database.BibDatabaseContext; import org.jabref.preferences.FilePreferences; import org.jabref.preferences.PreferencesService; diff --git a/src/main/java/org/jabref/logic/search/indexing/DocumentReader.java b/src/main/java/org/jabref/logic/pdf/search/DocumentReader.java similarity index 99% rename from src/main/java/org/jabref/logic/search/indexing/DocumentReader.java rename to src/main/java/org/jabref/logic/pdf/search/DocumentReader.java index 030b3fc43c9..bfb2aa9739b 100644 --- a/src/main/java/org/jabref/logic/search/indexing/DocumentReader.java +++ b/src/main/java/org/jabref/logic/pdf/search/DocumentReader.java @@ -1,4 +1,4 @@ -package org.jabref.logic.search.indexing; +package org.jabref.logic.pdf.search; import java.io.IOException; import java.nio.file.Files; diff --git a/src/main/java/org/jabref/logic/search/indexing/IndexingTaskManager.java b/src/main/java/org/jabref/logic/pdf/search/IndexingTaskManager.java similarity index 99% rename from src/main/java/org/jabref/logic/search/indexing/IndexingTaskManager.java rename to src/main/java/org/jabref/logic/pdf/search/IndexingTaskManager.java index 5fd8a41075f..6e4f39dd935 100644 --- a/src/main/java/org/jabref/logic/search/indexing/IndexingTaskManager.java +++ b/src/main/java/org/jabref/logic/pdf/search/IndexingTaskManager.java @@ -1,4 +1,4 @@ -package org.jabref.logic.search.indexing; +package org.jabref.logic.pdf.search; import java.util.List; import java.util.Set; diff --git a/src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java b/src/main/java/org/jabref/logic/pdf/search/LuceneIndexer.java similarity index 99% rename from src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java rename to src/main/java/org/jabref/logic/pdf/search/LuceneIndexer.java index 83d174c46a1..149fbaafe28 100644 --- a/src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java +++ b/src/main/java/org/jabref/logic/pdf/search/LuceneIndexer.java @@ -1,4 +1,4 @@ -package org.jabref.logic.search.indexing; +package org.jabref.logic.pdf.search; import java.io.IOException; import java.nio.file.Files; diff --git a/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java b/src/main/java/org/jabref/logic/pdf/search/LuceneSearcher.java similarity index 98% rename from src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java rename to src/main/java/org/jabref/logic/pdf/search/LuceneSearcher.java index a224b5524f5..a51b0bbd3b2 100644 --- a/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java +++ b/src/main/java/org/jabref/logic/pdf/search/LuceneSearcher.java @@ -1,4 +1,4 @@ -package org.jabref.logic.search.retrieval; +package org.jabref.logic.pdf.search; import java.io.IOException; import java.util.HashMap; diff --git a/src/main/java/org/jabref/logic/pdf/search/indexing/IndexingTaskManager.java b/src/main/java/org/jabref/logic/pdf/search/indexing/IndexingTaskManager.java deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/src/main/java/org/jabref/model/groups/SearchGroup.java b/src/main/java/org/jabref/model/groups/SearchGroup.java index f0fa5bdb102..855d4079310 100644 --- a/src/main/java/org/jabref/model/groups/SearchGroup.java +++ b/src/main/java/org/jabref/model/groups/SearchGroup.java @@ -6,7 +6,7 @@ import java.util.Objects; import java.util.Set; -import org.jabref.logic.search.retrieval.LuceneSearcher; +import org.jabref.logic.pdf.search.LuceneSearcher; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.search.GroupSearchQuery; diff --git a/src/test/java/org/jabref/gui/entryeditor/CommentsTabTest.java b/src/test/java/org/jabref/gui/entryeditor/CommentsTabTest.java index 7e5a80fe675..b7c527917d6 100644 --- a/src/test/java/org/jabref/gui/entryeditor/CommentsTabTest.java +++ b/src/test/java/org/jabref/gui/entryeditor/CommentsTabTest.java @@ -12,7 +12,7 @@ import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.journals.JournalAbbreviationRepository; import org.jabref.logic.preferences.OwnerPreferences; -import org.jabref.logic.search.indexing.IndexingTaskManager; +import org.jabref.logic.pdf.search.IndexingTaskManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.database.BibDatabaseMode; import org.jabref.model.entry.BibEntry; diff --git a/src/test/java/org/jabref/logic/pdf/search/indexing/DocumentReaderTest.java b/src/test/java/org/jabref/logic/pdf/search/indexing/DocumentReaderTest.java index 2eb2517ceb8..dfd305d335a 100644 --- a/src/test/java/org/jabref/logic/pdf/search/indexing/DocumentReaderTest.java +++ b/src/test/java/org/jabref/logic/pdf/search/indexing/DocumentReaderTest.java @@ -6,7 +6,7 @@ import java.util.Optional; import java.util.stream.Stream; -import org.jabref.logic.search.indexing.DocumentReader; +import org.jabref.logic.pdf.search.DocumentReader; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.LinkedFile; diff --git a/src/test/java/org/jabref/logic/pdf/search/indexing/LuceneIndexerTest.java b/src/test/java/org/jabref/logic/pdf/search/indexing/LuceneIndexerTest.java index 5cd9156e715..a748c0f185b 100644 --- a/src/test/java/org/jabref/logic/pdf/search/indexing/LuceneIndexerTest.java +++ b/src/test/java/org/jabref/logic/pdf/search/indexing/LuceneIndexerTest.java @@ -5,7 +5,7 @@ import java.util.Collections; import java.util.Optional; -import org.jabref.logic.search.indexing.LuceneIndexer; +import org.jabref.logic.pdf.search.LuceneIndexer; import org.jabref.logic.util.StandardFileType; import org.jabref.model.database.BibDatabase; import org.jabref.model.database.BibDatabaseContext; diff --git a/src/test/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcherTest.java b/src/test/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcherTest.java index 7743ecdfea8..5716cccc976 100644 --- a/src/test/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcherTest.java +++ b/src/test/java/org/jabref/logic/pdf/search/retrieval/LuceneSearcherTest.java @@ -7,8 +7,8 @@ import java.util.HashMap; import org.jabref.logic.search.SearchQuery; -import org.jabref.logic.search.indexing.LuceneIndexer; -import org.jabref.logic.search.retrieval.LuceneSearcher; +import org.jabref.logic.pdf.search.LuceneIndexer; +import org.jabref.logic.pdf.search.LuceneSearcher; import org.jabref.logic.util.StandardFileType; import org.jabref.model.database.BibDatabase; import org.jabref.model.database.BibDatabaseContext; diff --git a/src/test/java/org/jabref/logic/search/SearchQueryTest.java b/src/test/java/org/jabref/logic/search/SearchQueryTest.java index c6ce0728860..4216352e90d 100644 --- a/src/test/java/org/jabref/logic/search/SearchQueryTest.java +++ b/src/test/java/org/jabref/logic/search/SearchQueryTest.java @@ -5,8 +5,8 @@ import java.util.Collections; import java.util.EnumSet; -import org.jabref.logic.search.indexing.LuceneIndexer; -import org.jabref.logic.search.retrieval.LuceneSearcher; +import org.jabref.logic.pdf.search.LuceneIndexer; +import org.jabref.logic.pdf.search.LuceneSearcher; import org.jabref.model.database.BibDatabase; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.search.rules.SearchRules; From 0dc1c2d874056beb5b671a2e62d55e5e680b01a9 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Fri, 24 May 2024 04:07:33 +0300 Subject: [PATCH 084/256] Move search classes to search package --- CHANGELOG.md | 2 +- .../org/jabref/gui/JabRefExecutorService.java | 4 +- src/main/java/org/jabref/gui/LibraryTab.java | 4 +- .../java/org/jabref/gui/StateManager.java | 2 +- .../jabref/gui/desktop/os/NativeDesktop.java | 2 +- .../jabref/gui/entryeditor/CommentsTab.java | 9 +- .../gui/entryeditor/DeprecatedFieldsTab.java | 2 +- .../entryeditor/DetailOptionalFieldsTab.java | 2 +- .../jabref/gui/entryeditor/EntryEditor.java | 2 +- .../gui/entryeditor/FieldsEditorTab.java | 2 +- .../ImportantOptionalFieldsTab.java | 2 +- .../entryeditor/OptionalFieldsTabBase.java | 2 +- .../gui/entryeditor/OtherFieldsTab.java | 2 +- .../jabref/gui/entryeditor/PreviewTab.java | 2 +- .../gui/entryeditor/RequiredFieldsTab.java | 2 +- .../org/jabref/gui/entryeditor/SourceTab.java | 5 + .../gui/entryeditor/UserDefinedFieldsTab.java | 2 +- .../FulltextSearchResultsTab.java | 6 +- .../gui/exporter/SaveDatabaseAction.java | 3 +- .../ExternalFilesEntryLinker.java | 4 +- .../jabref/gui/groups/GroupDialogView.java | 2 +- .../gui/groups/GroupDialogViewModel.java | 2 +- .../org/jabref/gui/groups/GroupViewMode.java | 4 +- .../gui/maintable/BibEntryTableViewModel.java | 3 +- .../org/jabref/gui/maintable/MainTable.css | 1 - .../org/jabref/gui/maintable/MainTable.java | 2 +- .../gui/maintable/MainTableDataModel.java | 6 +- .../org/jabref/gui/preview/PreviewPanel.java | 2 +- .../jabref/gui/search/GlobalSearchBar.java | 16 +- .../RebuildFulltextSearchIndexAction.java | 2 +- .../logic/exporter/GroupSerializer.java | 2 +- .../logic/importer/util/GroupsParser.java | 5 +- .../jabref/logic/pdf/search/PdfIndexer.java | 351 ------------------ .../logic/pdf/search/PdfIndexerManager.java | 83 ----- .../org/jabref/logic/search/SearchQuery.java | 26 +- .../indexing}/DocumentReader.java | 12 +- .../indexing}/IndexingTaskManager.java | 2 +- .../indexing}/LuceneIndexer.java | 4 +- .../retrieval}/LuceneSearcher.java | 6 +- .../org/jabref/model/groups/SearchGroup.java | 7 +- .../jabref/model/search/GroupSearchQuery.java | 1 - .../{pdf => }/search/LuceneSearchResults.java | 2 +- .../search/SearchFieldConstants.java | 2 +- .../org/jabref/model/search/SearchFlags.java | 10 + .../model/{pdf => }/search/SearchResult.java | 25 +- .../model/search/rules/SearchRules.java | 11 - .../jabref/preferences/JabRefPreferences.java | 2 +- .../jabref/preferences/SearchPreferences.java | 36 +- .../org/jabref/cli/ArgumentProcessorTest.java | 4 +- .../gui/entryeditor/CommentsTabTest.java | 7 +- .../gui/groups/GroupTreeViewModelTest.java | 6 +- .../gui/search/GlobalSearchBarTest.java | 4 +- .../logic/exporter/GroupSerializerTest.java | 6 +- .../logic/importer/util/GroupsParserTest.java | 4 +- .../logic/search/DatabaseSearcherTest.java | 22 +- .../DatabaseSearcherWithBibFilesTest.java | 291 +++++++-------- .../org/jabref/logic/search/LuceneTest.java | 7 +- .../jabref/logic/search/SearchQueryTest.java | 8 +- .../indexing}/DocumentReaderTest.java | 3 +- .../indexing}/LuceneIndexerTest.java | 4 +- .../search/retrieval/LuceneSearcherTest.java | 144 +++++++ .../model/groups/GroupTreeNodeTest.java | 10 +- .../jabref/model/groups/SearchGroupTest.java | 14 +- 63 files changed, 442 insertions(+), 780 deletions(-) delete mode 100644 src/main/java/org/jabref/logic/pdf/search/PdfIndexer.java delete mode 100644 src/main/java/org/jabref/logic/pdf/search/PdfIndexerManager.java rename src/main/java/org/jabref/logic/{pdf/search => search/indexing}/DocumentReader.java (94%) rename src/main/java/org/jabref/logic/{pdf/search => search/indexing}/IndexingTaskManager.java (99%) rename src/main/java/org/jabref/logic/{pdf/search => search/indexing}/LuceneIndexer.java (99%) rename src/main/java/org/jabref/logic/{pdf/search => search/retrieval}/LuceneSearcher.java (94%) rename src/main/java/org/jabref/model/{pdf => }/search/LuceneSearchResults.java (97%) rename src/main/java/org/jabref/model/{pdf => }/search/SearchFieldConstants.java (95%) create mode 100644 src/main/java/org/jabref/model/search/SearchFlags.java rename src/main/java/org/jabref/model/{pdf => }/search/SearchResult.java (86%) delete mode 100644 src/main/java/org/jabref/model/search/rules/SearchRules.java rename src/test/java/org/jabref/logic/{pdf/search => search/indexing}/DocumentReaderTest.java (96%) rename src/test/java/org/jabref/logic/{pdf/search => search/indexing}/LuceneIndexerTest.java (98%) create mode 100644 src/test/java/org/jabref/logic/search/retrieval/LuceneSearcherTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 86591d9c5da..29ac624e389 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv ### Changed +- The search in the library now displays probable search hits instead of exact matches. Sorting by hit score can be done by the new score table column. [#8963](https://github.com/JabRef/jabref/pull/8963) - We replaced the word "Key bindings" with "Keyboard shortcuts" in the Preferences tab. [#11153](https://github.com/JabRef/jabref/pull/11153) - We slightly improved the duplicate check if ISBNs are present. [#8885](https://github.com/JabRef/jabref/issues/8885) - JabRef no longer downloads HTML files of websites when a PDF was not found. [#10149](https://github.com/JabRef/jabref/issues/10149) @@ -209,7 +210,6 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv ### Fixed -- The search in the library now displays probable search hits instead of exact matches. Sorting by hit score can be done by the new score table column. [#8963](https://github.com/JabRef/jabref/pull/8963) - We fixed an issue where "Move URL in note field to url field" in the cleanup dialog caused an exception if no note field was present [forum#3999](https://discourse.jabref.org/t/cleanup-entries-cant-get-it-to-work/3999) - It is possible again to use "current table sort order" for the order of entries when saving. [#9869](https://github.com/JabRef/jabref/issues/9869) - Passwords can be stored in GNOME key ring. [#10274](https://github.com/JabRef/jabref/issues/10274) diff --git a/src/main/java/org/jabref/gui/JabRefExecutorService.java b/src/main/java/org/jabref/gui/JabRefExecutorService.java index 90c3b31bdae..6212f38bedc 100644 --- a/src/main/java/org/jabref/gui/JabRefExecutorService.java +++ b/src/main/java/org/jabref/gui/JabRefExecutorService.java @@ -13,8 +13,6 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import org.jabref.logic.pdf.search.PdfIndexerManager; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -148,7 +146,7 @@ public void shutdownEverything() { gracefullyShutdown(this.executorService); gracefullyShutdown(this.lowPriorityExecutorService); - PdfIndexerManager.shutdownAllIndexers(); + // PdfIndexerManager.shutdownAllIndexers(); timer.cancel(); } diff --git a/src/main/java/org/jabref/gui/LibraryTab.java b/src/main/java/org/jabref/gui/LibraryTab.java index 7202c215d61..61bdb2d3dfe 100644 --- a/src/main/java/org/jabref/gui/LibraryTab.java +++ b/src/main/java/org/jabref/gui/LibraryTab.java @@ -62,9 +62,9 @@ import org.jabref.logic.importer.util.FileFieldParser; import org.jabref.logic.l10n.Localization; import org.jabref.logic.pdf.FileAnnotationCache; -import org.jabref.logic.pdf.search.IndexingTaskManager; -import org.jabref.logic.pdf.search.LuceneIndexer; import org.jabref.logic.search.SearchQuery; +import org.jabref.logic.search.indexing.IndexingTaskManager; +import org.jabref.logic.search.indexing.LuceneIndexer; import org.jabref.logic.shared.DatabaseLocation; import org.jabref.logic.util.UpdateField; import org.jabref.logic.util.io.FileUtil; diff --git a/src/main/java/org/jabref/gui/StateManager.java b/src/main/java/org/jabref/gui/StateManager.java index 808f374fbc6..865927a1c59 100644 --- a/src/main/java/org/jabref/gui/StateManager.java +++ b/src/main/java/org/jabref/gui/StateManager.java @@ -31,7 +31,7 @@ import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.groups.GroupTreeNode; -import org.jabref.model.pdf.search.LuceneSearchResults; +import org.jabref.model.search.LuceneSearchResults; import com.tobiasdiez.easybind.EasyBind; import com.tobiasdiez.easybind.EasyBinding; diff --git a/src/main/java/org/jabref/gui/desktop/os/NativeDesktop.java b/src/main/java/org/jabref/gui/desktop/os/NativeDesktop.java index 630382396b0..5ea61b542b4 100644 --- a/src/main/java/org/jabref/gui/desktop/os/NativeDesktop.java +++ b/src/main/java/org/jabref/gui/desktop/os/NativeDesktop.java @@ -13,7 +13,7 @@ import org.jabref.gui.DialogService; import org.jabref.logic.util.BuildInfo; import org.jabref.logic.util.OS; -import org.jabref.model.pdf.search.SearchFieldConstants; +import org.jabref.model.search.SearchFieldConstants; import org.jabref.model.strings.StringUtil; import org.jabref.preferences.FilePreferences; diff --git a/src/main/java/org/jabref/gui/entryeditor/CommentsTab.java b/src/main/java/org/jabref/gui/entryeditor/CommentsTab.java index 7758249a8c2..51374f7a1cd 100644 --- a/src/main/java/org/jabref/gui/entryeditor/CommentsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/CommentsTab.java @@ -26,7 +26,7 @@ import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.journals.JournalAbbreviationRepository; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.pdf.search.IndexingTaskManager; +import org.jabref.logic.search.indexing.IndexingTaskManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.Field; @@ -49,9 +49,9 @@ public CommentsTab(PreferencesService preferences, DialogService dialogService, StateManager stateManager, ThemeManager themeManager, + IndexingTaskManager indexingTaskManager, TaskExecutor taskExecutor, - JournalAbbreviationRepository journalAbbreviationRepository, - IndexingTaskManager indexingTaskManager) { + JournalAbbreviationRepository journalAbbreviationRepository) { super( false, databaseContext, @@ -63,7 +63,8 @@ public CommentsTab(PreferencesService preferences, themeManager, taskExecutor, journalAbbreviationRepository, - indexingTaskManager); + indexingTaskManager + ); this.defaultOwner = preferences.getOwnerPreferences().getDefaultOwner().toLowerCase(Locale.ROOT).replaceAll("[^a-z0-9]", "-"); setText(Localization.lang("Comments")); setGraphic(IconTheme.JabRefIcons.COMMENT.getGraphicNode()); diff --git a/src/main/java/org/jabref/gui/entryeditor/DeprecatedFieldsTab.java b/src/main/java/org/jabref/gui/entryeditor/DeprecatedFieldsTab.java index a66339ea7b6..3a98ae79fc2 100644 --- a/src/main/java/org/jabref/gui/entryeditor/DeprecatedFieldsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/DeprecatedFieldsTab.java @@ -17,7 +17,7 @@ import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.journals.JournalAbbreviationRepository; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.pdf.search.IndexingTaskManager; +import org.jabref.logic.search.indexing.IndexingTaskManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.database.BibDatabaseMode; import org.jabref.model.entry.BibEntry; diff --git a/src/main/java/org/jabref/gui/entryeditor/DetailOptionalFieldsTab.java b/src/main/java/org/jabref/gui/entryeditor/DetailOptionalFieldsTab.java index e884d5b297d..dcee79c13c4 100644 --- a/src/main/java/org/jabref/gui/entryeditor/DetailOptionalFieldsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/DetailOptionalFieldsTab.java @@ -9,7 +9,7 @@ import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.journals.JournalAbbreviationRepository; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.pdf.search.IndexingTaskManager; +import org.jabref.logic.search.indexing.IndexingTaskManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntryTypesManager; import org.jabref.preferences.PreferencesService; diff --git a/src/main/java/org/jabref/gui/entryeditor/EntryEditor.java b/src/main/java/org/jabref/gui/entryeditor/EntryEditor.java index c616e058a3f..516b1014f35 100644 --- a/src/main/java/org/jabref/gui/entryeditor/EntryEditor.java +++ b/src/main/java/org/jabref/gui/entryeditor/EntryEditor.java @@ -270,7 +270,7 @@ private List createTabs() { entryEditorTabs.add(new OtherFieldsTab(databaseContext, libraryTab.getSuggestionProviders(), undoManager, dialogService, preferencesService, stateManager, themeManager, libraryTab.getIndexingTaskManager(), bibEntryTypesManager, taskExecutor, journalAbbreviationRepository)); // Comment Tab: Tab for general and user-specific comments - entryEditorTabs.add(new CommentsTab(preferencesService, databaseContext, libraryTab.getSuggestionProviders(), undoManager, dialogService, stateManager, themeManager, taskExecutor, journalAbbreviationRepository, libraryTab.getIndexingTaskManager())); + entryEditorTabs.add(new CommentsTab(preferencesService, databaseContext, libraryTab.getSuggestionProviders(), undoManager, dialogService, stateManager, themeManager, libraryTab.getIndexingTaskManager(), taskExecutor, journalAbbreviationRepository)); // The preferences allow to configure tabs to show (e.g.,"General", "Abstract") // These should be shown. Already hard-coded ones should be removed. diff --git a/src/main/java/org/jabref/gui/entryeditor/FieldsEditorTab.java b/src/main/java/org/jabref/gui/entryeditor/FieldsEditorTab.java index 97348f5838a..e0f98d538e3 100644 --- a/src/main/java/org/jabref/gui/entryeditor/FieldsEditorTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/FieldsEditorTab.java @@ -34,7 +34,7 @@ import org.jabref.gui.theme.ThemeManager; import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.journals.JournalAbbreviationRepository; -import org.jabref.logic.pdf.search.IndexingTaskManager; +import org.jabref.logic.search.indexing.IndexingTaskManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.Field; diff --git a/src/main/java/org/jabref/gui/entryeditor/ImportantOptionalFieldsTab.java b/src/main/java/org/jabref/gui/entryeditor/ImportantOptionalFieldsTab.java index 5258477d3f4..9fd2cf5e78b 100644 --- a/src/main/java/org/jabref/gui/entryeditor/ImportantOptionalFieldsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/ImportantOptionalFieldsTab.java @@ -9,7 +9,7 @@ import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.journals.JournalAbbreviationRepository; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.pdf.search.IndexingTaskManager; +import org.jabref.logic.search.indexing.IndexingTaskManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntryTypesManager; import org.jabref.preferences.PreferencesService; diff --git a/src/main/java/org/jabref/gui/entryeditor/OptionalFieldsTabBase.java b/src/main/java/org/jabref/gui/entryeditor/OptionalFieldsTabBase.java index 5a95c3ddb8b..734fde141db 100644 --- a/src/main/java/org/jabref/gui/entryeditor/OptionalFieldsTabBase.java +++ b/src/main/java/org/jabref/gui/entryeditor/OptionalFieldsTabBase.java @@ -16,7 +16,7 @@ import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.journals.JournalAbbreviationRepository; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.pdf.search.IndexingTaskManager; +import org.jabref.logic.search.indexing.IndexingTaskManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.database.BibDatabaseMode; import org.jabref.model.entry.BibEntry; diff --git a/src/main/java/org/jabref/gui/entryeditor/OtherFieldsTab.java b/src/main/java/org/jabref/gui/entryeditor/OtherFieldsTab.java index 52882b29f5a..4ea87a219b9 100644 --- a/src/main/java/org/jabref/gui/entryeditor/OtherFieldsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/OtherFieldsTab.java @@ -20,7 +20,7 @@ import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.journals.JournalAbbreviationRepository; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.pdf.search.IndexingTaskManager; +import org.jabref.logic.search.indexing.IndexingTaskManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.database.BibDatabaseMode; import org.jabref.model.entry.BibEntry; diff --git a/src/main/java/org/jabref/gui/entryeditor/PreviewTab.java b/src/main/java/org/jabref/gui/entryeditor/PreviewTab.java index 648cbe003a8..80f3f73ade1 100644 --- a/src/main/java/org/jabref/gui/entryeditor/PreviewTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/PreviewTab.java @@ -7,7 +7,7 @@ import org.jabref.gui.theme.ThemeManager; import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.pdf.search.IndexingTaskManager; +import org.jabref.logic.search.indexing.IndexingTaskManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.preferences.PreferencesService; diff --git a/src/main/java/org/jabref/gui/entryeditor/RequiredFieldsTab.java b/src/main/java/org/jabref/gui/entryeditor/RequiredFieldsTab.java index a4939e66d79..7129802bfdc 100644 --- a/src/main/java/org/jabref/gui/entryeditor/RequiredFieldsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/RequiredFieldsTab.java @@ -16,7 +16,7 @@ import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.journals.JournalAbbreviationRepository; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.pdf.search.IndexingTaskManager; +import org.jabref.logic.search.indexing.IndexingTaskManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.BibEntryType; diff --git a/src/main/java/org/jabref/gui/entryeditor/SourceTab.java b/src/main/java/org/jabref/gui/entryeditor/SourceTab.java index a690a16547a..8184416a0ec 100644 --- a/src/main/java/org/jabref/gui/entryeditor/SourceTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/SourceTab.java @@ -124,6 +124,11 @@ public SourceTab(BibDatabaseContext bibDatabaseContext, // searchHighlightPattern = newValue.flatMap(SearchQuery::getPatternForWords); TODO btut: Pattern-Highlighting with lucene highlightSearchPattern(); }); + + stateManager.activeGlobalSearchQueryProperty().addListener((observable, oldValue, newValue) -> { + // searchHighlightPattern = newValue.flatMap(SearchQuery::getPatternForWords); + highlightSearchPattern(); + }); } private void highlightSearchPattern() { diff --git a/src/main/java/org/jabref/gui/entryeditor/UserDefinedFieldsTab.java b/src/main/java/org/jabref/gui/entryeditor/UserDefinedFieldsTab.java index 064855d593d..f0f40df1af3 100644 --- a/src/main/java/org/jabref/gui/entryeditor/UserDefinedFieldsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/UserDefinedFieldsTab.java @@ -13,7 +13,7 @@ import org.jabref.gui.theme.ThemeManager; import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.journals.JournalAbbreviationRepository; -import org.jabref.logic.pdf.search.IndexingTaskManager; +import org.jabref.logic.search.indexing.IndexingTaskManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.Field; diff --git a/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java b/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java index 26ab054e42e..3dd3b625913 100644 --- a/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java @@ -30,9 +30,9 @@ import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.LinkedFile; -import org.jabref.model.pdf.search.LuceneSearchResults; -import org.jabref.model.pdf.search.SearchResult; -import org.jabref.model.search.rules.SearchRules.SearchFlags; +import org.jabref.model.search.LuceneSearchResults; +import org.jabref.model.search.SearchResult; +import org.jabref.model.search.SearchFlags; import org.jabref.preferences.PreferencesService; import org.slf4j.Logger; diff --git a/src/main/java/org/jabref/gui/exporter/SaveDatabaseAction.java b/src/main/java/org/jabref/gui/exporter/SaveDatabaseAction.java index 1c2d8c810ab..7678b30d824 100644 --- a/src/main/java/org/jabref/gui/exporter/SaveDatabaseAction.java +++ b/src/main/java/org/jabref/gui/exporter/SaveDatabaseAction.java @@ -34,7 +34,6 @@ import org.jabref.logic.exporter.SelfContainedSaveConfiguration; import org.jabref.logic.l10n.Encodings; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.pdf.search.PdfIndexerManager; import org.jabref.logic.shared.DatabaseLocation; import org.jabref.logic.shared.prefs.SharedDatabasePreferences; import org.jabref.logic.util.StandardFileType; @@ -143,7 +142,7 @@ boolean saveAs(Path file, SaveDatabaseMode mode) { // Close AutosaveManager, BackupManager, and PdfIndexer for original library AutosaveManager.shutdown(context); BackupManager.shutdown(context, this.preferences.getFilePreferences().getBackupDirectory(), preferences.getFilePreferences().shouldCreateBackup()); - PdfIndexerManager.shutdownIndexer(context); + // PdfIndexerManager.shutdownIndexer(context); } // Set new location diff --git a/src/main/java/org/jabref/gui/externalfiles/ExternalFilesEntryLinker.java b/src/main/java/org/jabref/gui/externalfiles/ExternalFilesEntryLinker.java index 53e68855429..d9d99ef4822 100644 --- a/src/main/java/org/jabref/gui/externalfiles/ExternalFilesEntryLinker.java +++ b/src/main/java/org/jabref/gui/externalfiles/ExternalFilesEntryLinker.java @@ -15,8 +15,8 @@ import org.jabref.logic.cleanup.MoveFilesCleanup; import org.jabref.logic.cleanup.RenamePdfCleanup; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.pdf.search.IndexingTaskManager; -import org.jabref.logic.pdf.search.LuceneIndexer; +import org.jabref.logic.search.indexing.IndexingTaskManager; +import org.jabref.logic.search.indexing.LuceneIndexer; import org.jabref.logic.util.io.FileNameCleaner; import org.jabref.logic.util.io.FileUtil; import org.jabref.model.database.BibDatabaseContext; diff --git a/src/main/java/org/jabref/gui/groups/GroupDialogView.java b/src/main/java/org/jabref/gui/groups/GroupDialogView.java index 0e237c054ae..0e6ead1b40e 100644 --- a/src/main/java/org/jabref/gui/groups/GroupDialogView.java +++ b/src/main/java/org/jabref/gui/groups/GroupDialogView.java @@ -45,7 +45,7 @@ import org.jabref.model.groups.AbstractGroup; import org.jabref.model.groups.GroupHierarchyType; import org.jabref.model.groups.GroupTreeNode; -import org.jabref.model.search.rules.SearchRules.SearchFlags; +import org.jabref.model.search.SearchFlags; import org.jabref.model.util.FileUpdateMonitor; import org.jabref.preferences.PreferencesService; diff --git a/src/main/java/org/jabref/gui/groups/GroupDialogViewModel.java b/src/main/java/org/jabref/gui/groups/GroupDialogViewModel.java index f1b57a6fb55..ef24ec4f790 100644 --- a/src/main/java/org/jabref/gui/groups/GroupDialogViewModel.java +++ b/src/main/java/org/jabref/gui/groups/GroupDialogViewModel.java @@ -47,7 +47,7 @@ import org.jabref.model.groups.TexGroup; import org.jabref.model.groups.WordKeywordGroup; import org.jabref.model.metadata.MetaData; -import org.jabref.model.search.rules.SearchRules.SearchFlags; +import org.jabref.model.search.SearchFlags; import org.jabref.model.strings.StringUtil; import org.jabref.model.util.FileUpdateMonitor; import org.jabref.preferences.PreferencesService; diff --git a/src/main/java/org/jabref/gui/groups/GroupViewMode.java b/src/main/java/org/jabref/gui/groups/GroupViewMode.java index b9b96cce436..711dc9c8bd7 100644 --- a/src/main/java/org/jabref/gui/groups/GroupViewMode.java +++ b/src/main/java/org/jabref/gui/groups/GroupViewMode.java @@ -1,4 +1,6 @@ package org.jabref.gui.groups; -public enum GroupViewMode { INTERSECTION, FILTER, INVERT } +public enum GroupViewMode { + INTERSECTION, UNION, FILTER, INVERT +} diff --git a/src/main/java/org/jabref/gui/maintable/BibEntryTableViewModel.java b/src/main/java/org/jabref/gui/maintable/BibEntryTableViewModel.java index d8395ce1bed..2c3a8bfde66 100644 --- a/src/main/java/org/jabref/gui/maintable/BibEntryTableViewModel.java +++ b/src/main/java/org/jabref/gui/maintable/BibEntryTableViewModel.java @@ -33,7 +33,7 @@ import org.jabref.model.entry.field.StandardField; import org.jabref.model.groups.AbstractGroup; import org.jabref.model.groups.GroupTreeNode; -import org.jabref.model.pdf.search.LuceneSearchResults; +import org.jabref.model.search.LuceneSearchResults; import com.tobiasdiez.easybind.EasyBind; import com.tobiasdiez.easybind.EasyBinding; @@ -50,7 +50,6 @@ public class BibEntryTableViewModel { private final Binding> matchedGroups; private final BibDatabaseContext bibDatabaseContext; private final StateManager stateManager; - private final FloatProperty searchScore = new SimpleFloatProperty(0); public BibEntryTableViewModel(BibEntry entry, BibDatabaseContext bibDatabaseContext, ObservableValue fieldValueFormatter, StateManager stateManager) { diff --git a/src/main/java/org/jabref/gui/maintable/MainTable.css b/src/main/java/org/jabref/gui/maintable/MainTable.css index 25c86e1eae9..89669dbc819 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTable.css +++ b/src/main/java/org/jabref/gui/maintable/MainTable.css @@ -66,4 +66,3 @@ .rating > .container > .button:hover { -fx-effect: dropshadow(three-pass-box, rgba(0, 0, 0, 0.6), 8, 0.0, 0, 0); } - diff --git a/src/main/java/org/jabref/gui/maintable/MainTable.java b/src/main/java/org/jabref/gui/maintable/MainTable.java index 67ac2f80672..5e4ef3df87d 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTable.java +++ b/src/main/java/org/jabref/gui/maintable/MainTable.java @@ -55,7 +55,7 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.BibEntryTypesManager; import org.jabref.model.entry.BibtexString; -import org.jabref.model.search.rules.SearchRules.SearchFlags; +import org.jabref.model.search.SearchFlags; import org.jabref.model.util.FileUpdateMonitor; import org.jabref.preferences.PreferencesService; diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index 3e4f2cd3e98..7bdd51a9a00 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -15,15 +15,15 @@ import org.jabref.gui.groups.GroupViewMode; import org.jabref.gui.groups.GroupsPreferences; import org.jabref.gui.util.BindingsHelper; -import org.jabref.logic.pdf.search.LuceneSearcher; import org.jabref.logic.search.SearchQuery; +import org.jabref.logic.search.retrieval.LuceneSearcher; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.groups.GroupTreeNode; import org.jabref.model.groups.SearchGroup; +import org.jabref.model.search.SearchFlags; import org.jabref.model.search.matchers.MatcherSet; import org.jabref.model.search.matchers.MatcherSets; -import org.jabref.model.search.rules.SearchRules; import org.jabref.preferences.PreferencesService; import com.tobiasdiez.easybind.EasyBind; @@ -114,7 +114,7 @@ private boolean isMatched(ObservableList groups, Optional query, BibEntryTableViewModel entry) { - if (query.isEmpty() || !query.get().getSearchFlags().contains(SearchRules.SearchFlags.FILTERING_SEARCH)) { + if (query.isEmpty() || !query.get().getSearchFlags().contains(SearchFlags.FILTERING_SEARCH)) { return true; } return entry.getSearchScore() > 0; diff --git a/src/main/java/org/jabref/gui/preview/PreviewPanel.java b/src/main/java/org/jabref/gui/preview/PreviewPanel.java index c0daa04e31f..08ed593c14d 100644 --- a/src/main/java/org/jabref/gui/preview/PreviewPanel.java +++ b/src/main/java/org/jabref/gui/preview/PreviewPanel.java @@ -25,8 +25,8 @@ import org.jabref.gui.theme.ThemeManager; import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.pdf.search.IndexingTaskManager; import org.jabref.logic.preview.PreviewLayout; +import org.jabref.logic.search.indexing.IndexingTaskManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.preferences.PreferencesService; diff --git a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java index bbec25421b0..98b9f3687a9 100644 --- a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java +++ b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java @@ -65,8 +65,8 @@ import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.Author; import org.jabref.model.entry.BibEntry; -import org.jabref.model.pdf.search.LuceneSearchResults; -import org.jabref.model.search.rules.SearchRules; +import org.jabref.model.search.LuceneSearchResults; +import org.jabref.model.search.SearchFlags; import org.jabref.preferences.PreferencesService; import org.jabref.preferences.SearchPreferences; @@ -244,7 +244,7 @@ public GlobalSearchBar(LibraryTabContainer tabContainer, // Async update searchTask.restart(); }, - query -> setSearchTerm(query.orElse(new SearchQuery("", EnumSet.noneOf(SearchRules.SearchFlags.class))))); + query -> setSearchTerm(query.orElse(new SearchQuery("", EnumSet.noneOf(SearchFlags.class))))); this.stateManager.getSearchResults().addListener((MapChangeListener>) change -> { stateManager.activeSearchQueryProperty().get().ifPresent(this::updateSearchResultsForQuery); @@ -273,7 +273,7 @@ private void initSearchModifierButtons() { regularExpressionButton.setTooltip(new Tooltip(Localization.lang("regular expression"))); initSearchModifierButton(regularExpressionButton); regularExpressionButton.setOnAction(event -> { - searchPreferences.setSearchFlag(SearchRules.SearchFlags.REGULAR_EXPRESSION, regularExpressionButton.isSelected()); + searchPreferences.setSearchFlag(SearchFlags.REGULAR_EXPRESSION, regularExpressionButton.isSelected()); updateSearchQuery(); }); @@ -281,7 +281,7 @@ private void initSearchModifierButtons() { fulltextButton.setTooltip(new Tooltip(Localization.lang("Fulltext search"))); initSearchModifierButton(fulltextButton); fulltextButton.setOnAction(event -> { - searchPreferences.setSearchFlag(SearchRules.SearchFlags.FULLTEXT, fulltextButton.isSelected()); + searchPreferences.setSearchFlag(SearchFlags.FULLTEXT, fulltextButton.isSelected()); updateSearchQuery(); }); @@ -289,7 +289,7 @@ private void initSearchModifierButtons() { keepSearchString.setTooltip(new Tooltip(Localization.lang("Keep search string across libraries"))); initSearchModifierButton(keepSearchString); keepSearchString.setOnAction(evt -> { - searchPreferences.setSearchFlag(SearchRules.SearchFlags.KEEP_SEARCH_STRING, keepSearchString.isSelected()); + searchPreferences.setSearchFlag(SearchFlags.KEEP_SEARCH_STRING, keepSearchString.isSelected()); updateSearchQuery(); }); @@ -297,7 +297,7 @@ private void initSearchModifierButtons() { filterModeButton.setTooltip(new Tooltip(Localization.lang("Filter search results"))); initSearchModifierButton(filterModeButton); filterModeButton.setOnAction(event -> { - searchPreferences.setSearchFlag(SearchRules.SearchFlags.FILTERING_SEARCH, filterModeButton.isSelected()); + searchPreferences.setSearchFlag(SearchFlags.FILTERING_SEARCH, filterModeButton.isSelected()); updateSearchQuery(); }); @@ -305,7 +305,7 @@ private void initSearchModifierButtons() { sortByScoreButton.setTooltip(new Tooltip(Localization.lang("Always sort by score"))); initSearchModifierButton(sortByScoreButton); sortByScoreButton.setOnAction(event -> { - searchPreferences.setSearchFlag(SearchRules.SearchFlags.SORT_BY_SCORE, sortByScoreButton.isSelected()); + searchPreferences.setSearchFlag(SearchFlags.SORT_BY_SCORE, sortByScoreButton.isSelected()); updateSearchQuery(); }); diff --git a/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java b/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java index e3949ff67f9..adbbfbcc897 100644 --- a/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java +++ b/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java @@ -9,7 +9,7 @@ import org.jabref.gui.util.BackgroundTask; import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.pdf.search.LuceneIndexer; +import org.jabref.logic.search.indexing.LuceneIndexer; import org.jabref.model.database.BibDatabaseContext; import org.jabref.preferences.PreferencesService; diff --git a/src/main/java/org/jabref/logic/exporter/GroupSerializer.java b/src/main/java/org/jabref/logic/exporter/GroupSerializer.java index 802eb62cdd5..efec40fe062 100644 --- a/src/main/java/org/jabref/logic/exporter/GroupSerializer.java +++ b/src/main/java/org/jabref/logic/exporter/GroupSerializer.java @@ -18,7 +18,7 @@ import org.jabref.model.groups.RegexKeywordGroup; import org.jabref.model.groups.SearchGroup; import org.jabref.model.groups.TexGroup; -import org.jabref.model.search.rules.SearchRules.SearchFlags; +import org.jabref.model.search.SearchFlags; import org.jabref.model.strings.StringUtil; public class GroupSerializer { diff --git a/src/main/java/org/jabref/logic/importer/util/GroupsParser.java b/src/main/java/org/jabref/logic/importer/util/GroupsParser.java index 0e785d24f85..c1c49eea457 100644 --- a/src/main/java/org/jabref/logic/importer/util/GroupsParser.java +++ b/src/main/java/org/jabref/logic/importer/util/GroupsParser.java @@ -27,8 +27,7 @@ import org.jabref.model.groups.TexGroup; import org.jabref.model.groups.WordKeywordGroup; import org.jabref.model.metadata.MetaData; -import org.jabref.model.search.rules.SearchRules; -import org.jabref.model.search.rules.SearchRules.SearchFlags; +import org.jabref.model.search.SearchFlags; import org.jabref.model.strings.StringUtil; import org.jabref.model.util.FileUpdateMonitor; @@ -280,7 +279,7 @@ private static AbstractGroup searchGroupFromString(String s) { EnumSet searchFlags = EnumSet.noneOf(SearchFlags.class); tok.nextToken(); // This used to be the flag for CASE_SENSITIVE search. Skip it for backwards-compatibility if (Integer.parseInt(tok.nextToken()) == 1) { - searchFlags.add(SearchRules.SearchFlags.REGULAR_EXPRESSION); + searchFlags.add(SearchFlags.REGULAR_EXPRESSION); } // version 0 contained 4 additional booleans to specify search // fields; these are ignored now, all fields are always searched diff --git a/src/main/java/org/jabref/logic/pdf/search/PdfIndexer.java b/src/main/java/org/jabref/logic/pdf/search/PdfIndexer.java deleted file mode 100644 index 45750ac5b84..00000000000 --- a/src/main/java/org/jabref/logic/pdf/search/PdfIndexer.java +++ /dev/null @@ -1,351 +0,0 @@ -package org.jabref.logic.pdf.search; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -import org.jabref.logic.util.StandardFileType; -import org.jabref.model.database.BibDatabaseContext; -import org.jabref.model.entry.BibEntry; -import org.jabref.model.entry.LinkedFile; -import org.jabref.model.pdf.search.SearchFieldConstants; -import org.jabref.preferences.FilePreferences; - -import com.google.common.annotations.VisibleForTesting; -import org.apache.lucene.analysis.en.EnglishAnalyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexNotFoundException; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.MatchAllDocsQuery; -import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TopDocs; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.NIOFSDirectory; -import org.jooq.lambda.Unchecked; -import org.jspecify.annotations.Nullable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Indexes the text of PDF files and adds it into the lucene search index. - */ -public class PdfIndexer { - - private static final Logger LOGGER = LoggerFactory.getLogger(PdfIndexer.class); - - @VisibleForTesting - @Nullable // null might happen if lock is held by another JabRef instance - IndexWriter indexWriter; - - private final BibDatabaseContext databaseContext; - - private final FilePreferences filePreferences; - - @Nullable - private final Directory indexDirectory; - - private IndexReader reader; - - private PdfIndexer(BibDatabaseContext databaseContext, Directory indexDirectory, FilePreferences filePreferences) { - this.databaseContext = databaseContext; - if (indexDirectory == null) { - // FIXME: This should never happen, but was reported at https://github.com/JabRef/jabref/issues/10781. - String tmpDir = System.getProperty("java.io.tmpdir"); - LOGGER.info("Index directory must not be null. Falling back to {}", tmpDir); - Directory tmpIndexDirectory = null; - try { - tmpIndexDirectory = new NIOFSDirectory(Path.of(tmpDir)); - } catch (IOException e) { - LOGGER.info("Could not use {}. Indexing unavailable.", tmpDir, e); - } - this.indexDirectory = tmpIndexDirectory; - } else { - this.indexDirectory = indexDirectory; - } - this.filePreferences = filePreferences; - } - - /** - * Method is public, because DatabaseSearcherWithBibFilesTest resides in another package - */ - @VisibleForTesting - public static PdfIndexer of(BibDatabaseContext databaseContext, Path indexDirectory, FilePreferences filePreferences) throws IOException { - return new PdfIndexer(databaseContext, new NIOFSDirectory(indexDirectory), filePreferences); - } - - /** - * Method is public, because DatabaseSearcherWithBibFilesTest resides in another package - */ - public static PdfIndexer of(BibDatabaseContext databaseContext, FilePreferences filePreferences) throws IOException { - return new PdfIndexer(databaseContext, new NIOFSDirectory(databaseContext.getFulltextIndexPath()), filePreferences); - } - - /** - * Creates (and thus resets) the PDF index. No re-indexing will be done. - * Any previous state of the Lucene search is deleted. - */ - public void createIndex() { - if (indexDirectory == null) { - LOGGER.info("Index directory must not be null. Returning."); - return; - } - LOGGER.debug("Creating new index for directory {}.", indexDirectory); - initializeIndexWriterAndReader(IndexWriterConfig.OpenMode.CREATE); - } - - /** - * Needs to be accessed by {@link PdfSearcher} - */ - Optional getIndexWriter() { - LOGGER.trace("Getting the index writer"); - if (indexWriter == null) { - LOGGER.trace("Initializing the index writer"); - initializeIndexWriterAndReader(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); - } else { - LOGGER.trace("Using existing index writer"); - } - return Optional.ofNullable(indexWriter); - } - - private void initializeIndexWriterAndReader(IndexWriterConfig.OpenMode mode) { - if (indexDirectory == null) { - LOGGER.info("Index directory must not be null. Returning."); - return; - } - try { - indexWriter = new IndexWriter( - indexDirectory, - new IndexWriterConfig( - new EnglishAnalyzer()).setOpenMode(mode)); - } catch (IOException e) { - LOGGER.error("Could not initialize the IndexWriter", e); - // FIXME: This can also happen if another instance of JabRef is launched in parallel. - // We could implement a read-only access to the index in this case. - // This requires a major rewrite of the code, though. - // Accessing the index using a permanent writer object is (much) faster than always - // closing and opening the writer and reader on demand. - return; - } - try { - reader = DirectoryReader.open(indexWriter); - } catch (IOException e) { - LOGGER.error("Could not initialize the IndexReader", e); - } - } - - /** - * Rebuilds the PDF index. All PDF files linked to entries in the database will be re-indexed. - */ - public void rebuildIndex() { - LOGGER.debug("Rebuilding index."); - createIndex(); - addToIndex(databaseContext.getEntries()); - } - - public void addToIndex(List entries) { - int count = 0; - for (BibEntry entry : entries) { - addToIndex(entry, false); - count++; - if (count % 100 == 0) { - doCommit(); - } - } - doCommit(); - LOGGER.debug("Added {} documents to the index.", count); - } - - /** - * Adds all PDF files linked to one entry in the database to an existing (or new) Lucene search index - * - * @param entry a bibtex entry to link the pdf files to - */ - public void addToIndex(BibEntry entry) { - addToIndex(entry, entry.getFiles(), true); - } - - private void addToIndex(BibEntry entry, boolean shouldCommit) { - addToIndex(entry, entry.getFiles(), false); - if (shouldCommit) { - doCommit(); - } - } - - /** - * Adds a list of pdf files linked to one entry in the database to an existing (or new) Lucene search index - * - * @param entry a bibtex entry to link the pdf files to - */ - public void addToIndex(BibEntry entry, Collection linkedFiles) { - addToIndex(entry, linkedFiles, true); - } - - public void addToIndex(BibEntry entry, Collection linkedFiles, boolean shouldCommit) { - for (LinkedFile linkedFile : linkedFiles) { - addToIndex(entry, linkedFile, false); - } - if (shouldCommit) { - doCommit(); - } - } - - private void doCommit() { - try { - getIndexWriter().ifPresent(Unchecked.consumer(IndexWriter::commit)); - } catch (UncheckedIOException e) { - LOGGER.warn("Could not commit changes to the index.", e); - } - } - - /** - * Removes a pdf file identified by its path from the index - * - * @param linkedFilePath the path to the file to be removed - */ - public void removeFromIndex(String linkedFilePath) { - try { - getIndexWriter().ifPresent(Unchecked.consumer(writer -> { - writer.deleteDocuments(new Term(SearchFieldConstants.PATH, linkedFilePath)); - writer.commit(); - })); - } catch (UncheckedIOException e) { - LOGGER.debug("Could not remove document {} from the index.", linkedFilePath, e); - } - } - - /** - * Removes all files linked to a bib-entry from the index - * - * @param entry the entry documents are linked to - */ - public void removeFromIndex(BibEntry entry) { - removeFromIndex(entry.getFiles()); - } - - /** - * Removes a list of files linked to a bib-entry from the index - */ - public void removeFromIndex(Collection linkedFiles) { - for (LinkedFile linkedFile : linkedFiles) { - removeFromIndex(linkedFile.getLink()); - } - } - - public void removePathsFromIndex(Collection linkedFiles) { - for (String linkedFile : linkedFiles) { - removeFromIndex(linkedFile); - } - } - - /** - * Writes the file to the index if the file is not yet in the index or the file on the fs is newer than the one in - * the index. - * - * @param entry the entry associated with the file - * @param linkedFile the file to write to the index - */ - public void addToIndex(BibEntry entry, LinkedFile linkedFile) { - this.addToIndex(entry, linkedFile, true); - } - - private void addToIndex(BibEntry entry, LinkedFile linkedFile, boolean shouldCommit) { - if (linkedFile.isOnlineLink() || - (!StandardFileType.PDF.getName().equals(linkedFile.getFileType()) && - // We do not require the file type to be set - (!linkedFile.getLink().endsWith(".pdf") && !linkedFile.getLink().endsWith(".PDF")))) { - return; - } - Optional resolvedPath = linkedFile.findIn(databaseContext, filePreferences); - if (resolvedPath.isEmpty()) { - LOGGER.debug("Could not find {}", linkedFile.getLink()); - return; - } - try { - // Check if a document with this path is already in the index - try { - IndexSearcher searcher = new IndexSearcher(reader); - TermQuery query = new TermQuery(new Term(SearchFieldConstants.PATH, linkedFile.getLink())); - TopDocs topDocs = searcher.search(query, 1); - // If a document was found, check if is less current than the one in the FS - if (topDocs.scoreDocs.length > 0) { - Document doc = reader.storedFields().document(topDocs.scoreDocs[0].doc); - long indexModificationTime = Long.parseLong(doc.getField(SearchFieldConstants.MODIFIED).stringValue()); - BasicFileAttributes attributes = Files.readAttributes(resolvedPath.get(), BasicFileAttributes.class); - if (indexModificationTime >= attributes.lastModifiedTime().to(TimeUnit.SECONDS)) { - LOGGER.debug("File {} is already indexed and up-to-date.", linkedFile.getLink()); - return; - } else { - LOGGER.debug("File {} is already indexed but outdated. Removing from index.", linkedFile.getLink()); - removeFromIndex(linkedFile.getLink()); - } - } - } catch (IndexNotFoundException e) { - LOGGER.debug("Index not found. Continuing.", e); - } - LOGGER.debug("Adding {} to index", linkedFile.getLink()); - // If no document was found, add the new one - Optional> pages = new DocumentReader(entry, filePreferences).readLinkedPdf(this.databaseContext, linkedFile); - if (pages.isPresent()) { - getIndexWriter().ifPresent(Unchecked.consumer(writer -> { - writer.addDocuments(pages.get()); - if (shouldCommit) { - writer.commit(); - } - })); - } else { - LOGGER.debug("No content found in file {}", linkedFile.getLink()); - } - } catch (UncheckedIOException | IOException e) { - LOGGER.warn("Could not add document {} to the index.", linkedFile.getLink(), e); - } - } - - /** - * Lists the paths of all the files that are stored in the index - * - * @return all file paths - */ - public Set getListOfFilePaths() { - Set paths = new HashSet<>(); - Optional optionalIndexWriter = getIndexWriter(); - if (optionalIndexWriter.isEmpty()) { - LOGGER.debug("IndexWriter is empty. Returning empty list."); - return paths; - } - try (IndexReader reader = DirectoryReader.open(optionalIndexWriter.get())) { - IndexSearcher searcher = new IndexSearcher(reader); - MatchAllDocsQuery query = new MatchAllDocsQuery(); - TopDocs allDocs = searcher.search(query, Integer.MAX_VALUE); - for (ScoreDoc scoreDoc : allDocs.scoreDocs) { - Document doc = reader.storedFields().document(scoreDoc.doc); - paths.add(doc.getField(SearchFieldConstants.PATH).stringValue()); - } - } catch (IOException e) { - LOGGER.debug("Could not read from index. Returning intermediate result.", e); - return paths; - } - return paths; - } - - public void close() throws IOException { - if (indexWriter == null) { - LOGGER.debug("IndexWriter is null."); - return; - } - indexWriter.close(); - } -} diff --git a/src/main/java/org/jabref/logic/pdf/search/PdfIndexerManager.java b/src/main/java/org/jabref/logic/pdf/search/PdfIndexerManager.java deleted file mode 100644 index 8b620c015c9..00000000000 --- a/src/main/java/org/jabref/logic/pdf/search/PdfIndexerManager.java +++ /dev/null @@ -1,83 +0,0 @@ -package org.jabref.logic.pdf.search; - -import java.io.IOException; -import java.nio.file.Path; -import java.util.HashMap; -import java.util.Map; - -import org.jabref.model.database.BibDatabaseContext; -import org.jabref.preferences.FilePreferences; - -import org.jspecify.annotations.NonNull; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * A PdfIndexer takes a long time to build. Caching it speeds up. - *

- * The PdfIndexer is related to the BibDatabaseContext and the FilePreferences. If the user changes the path of the library - * or the file preferences, we need to create a new PdfIndexer. Otherwise, we can reuse the existing one. - *

- * This manager implements a Object Pool pattern for {@link PdfIndexer}. - */ -public class PdfIndexerManager { - - private static final Logger LOGGER = LoggerFactory.getLogger(PdfIndexerManager.class); - - // Map from the path of the library index to the respective indexer - private static Map indexerMap = new HashMap<>(); - - // We store the file preferences for each path, so that we can update the indexer when the preferences change - private static Map pathFilePreferencesMap = new HashMap<>(); - - public static @NonNull PdfIndexer getIndexer(BibDatabaseContext context, FilePreferences filePreferences) throws IOException { - Path fulltextIndexPath = context.getFulltextIndexPath(); - PdfIndexer indexer = indexerMap.get(fulltextIndexPath); - if (indexer != null) { - // Check if the file preferences have changed - FilePreferences storedFilePreferences = pathFilePreferencesMap.get(fulltextIndexPath); - if (storedFilePreferences.equals(filePreferences)) { - LOGGER.trace("Found existing indexer for context {}", context); - return indexer; - } - LOGGER.debug("File preferences have changed, updating indexer"); - indexer.close(); - indexer = PdfIndexer.of(context, filePreferences); - indexerMap.put(fulltextIndexPath, indexer); - pathFilePreferencesMap.put(fulltextIndexPath, filePreferences); - return indexer; - } - LOGGER.debug("No indexer found for context {}, creating new one", context); - indexer = PdfIndexer.of(context, filePreferences); - indexerMap.put(fulltextIndexPath, indexer); - pathFilePreferencesMap.put(fulltextIndexPath, filePreferences); - return indexer; - } - - public static void shutdownAllIndexers() { - indexerMap.values().forEach(indexer -> { - try { - indexer.close(); - } catch (Exception e) { - LOGGER.debug("Problem closing PDF indexer", e); - } - }); - indexerMap.clear(); - pathFilePreferencesMap.clear(); - } - - public static void shutdownIndexer(BibDatabaseContext context) { - Path fulltextIndexPath = context.getFulltextIndexPath(); - PdfIndexer indexer = indexerMap.remove(fulltextIndexPath); - if (indexer != null) { - try { - indexer.close(); - } catch (IOException e) { - LOGGER.debug("Could not close indexer", e); - } - pathFilePreferencesMap.remove(fulltextIndexPath); - } else { - LOGGER.debug("No indexer found for context {}", context); - } - } -} diff --git a/src/main/java/org/jabref/logic/search/SearchQuery.java b/src/main/java/org/jabref/logic/search/SearchQuery.java index 7301ab1ed8e..5683a97e69c 100644 --- a/src/main/java/org/jabref/logic/search/SearchQuery.java +++ b/src/main/java/org/jabref/logic/search/SearchQuery.java @@ -7,39 +7,39 @@ import java.util.Set; import java.util.stream.Collectors; -import org.jabref.model.pdf.search.SearchFieldConstants; -import org.jabref.model.search.rules.SearchRules; +import org.jabref.model.search.SearchFieldConstants; +import org.jabref.model.search.SearchFlags; import org.apache.lucene.analysis.en.EnglishAnalyzer; import org.apache.lucene.queryparser.classic.MultiFieldQueryParser; import org.apache.lucene.queryparser.classic.ParseException; import org.apache.lucene.search.Query; -import static org.jabref.model.search.rules.SearchRules.SearchFlags.FILTERING_SEARCH; -import static org.jabref.model.search.rules.SearchRules.SearchFlags.KEEP_SEARCH_STRING; -import static org.jabref.model.search.rules.SearchRules.SearchFlags.SORT_BY_SCORE; +import static org.jabref.model.search.SearchFlags.FILTERING_SEARCH; +import static org.jabref.model.search.SearchFlags.KEEP_SEARCH_STRING; +import static org.jabref.model.search.SearchFlags.SORT_BY_SCORE; public class SearchQuery { protected final String query; protected Query parsedQuery; protected String parseError; - protected EnumSet searchFlags; + protected EnumSet searchFlags; - public SearchQuery(String query, EnumSet searchFlags) { + public SearchQuery(String query, EnumSet searchFlags) { this.query = Objects.requireNonNull(query); this.searchFlags = searchFlags; HashMap boosts = new HashMap<>(); SearchFieldConstants.searchableBibFields.forEach(field -> boosts.put(field, Float.valueOf(4))); - if (searchFlags.contains(SearchRules.SearchFlags.FULLTEXT)) { + if (searchFlags.contains(SearchFlags.FULLTEXT)) { Arrays.stream(SearchFieldConstants.PDF_FIELDS).forEach(field -> boosts.put(field, Float.valueOf(1))); } String[] fieldsToSearchArray = new String[boosts.size()]; boosts.keySet().toArray(fieldsToSearchArray); - if (searchFlags.contains(SearchRules.SearchFlags.REGULAR_EXPRESSION)) { + if (searchFlags.contains(SearchFlags.REGULAR_EXPRESSION)) { if (query.length() > 0 && !(query.startsWith("/") && query.endsWith("/"))) { query = "/" + query + "/"; } @@ -75,9 +75,9 @@ public boolean equals(Object other) { if (!searchQuery.query.equals(this.query)) { return false; } - Set thisSearchRulesWithoutFilterAndSort = this.searchFlags.clone(); - Set otherSearchRulesWithoutFilterAndSort = searchQuery.searchFlags.clone(); - Set filterAndSortFlags = EnumSet.of(SORT_BY_SCORE, FILTERING_SEARCH, KEEP_SEARCH_STRING); + Set thisSearchRulesWithoutFilterAndSort = this.searchFlags.clone(); + Set otherSearchRulesWithoutFilterAndSort = searchQuery.searchFlags.clone(); + Set filterAndSortFlags = EnumSet.of(SORT_BY_SCORE, FILTERING_SEARCH, KEEP_SEARCH_STRING); thisSearchRulesWithoutFilterAndSort.removeAll(filterAndSortFlags); otherSearchRulesWithoutFilterAndSort.removeAll(filterAndSortFlags); return thisSearchRulesWithoutFilterAndSort.equals(otherSearchRulesWithoutFilterAndSort); @@ -98,7 +98,7 @@ public Query getQuery() { return parsedQuery; } - public EnumSet getSearchFlags() { + public EnumSet getSearchFlags() { return searchFlags; } } diff --git a/src/main/java/org/jabref/logic/pdf/search/DocumentReader.java b/src/main/java/org/jabref/logic/search/indexing/DocumentReader.java similarity index 94% rename from src/main/java/org/jabref/logic/pdf/search/DocumentReader.java rename to src/main/java/org/jabref/logic/search/indexing/DocumentReader.java index 425d4d57a8b..f945e0dac86 100644 --- a/src/main/java/org/jabref/logic/pdf/search/DocumentReader.java +++ b/src/main/java/org/jabref/logic/search/indexing/DocumentReader.java @@ -1,4 +1,4 @@ -package org.jabref.logic.pdf.search; +package org.jabref.logic.search.indexing; import java.io.IOException; import java.nio.file.Files; @@ -30,11 +30,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.jabref.model.pdf.search.SearchFieldConstants.ANNOTATIONS; -import static org.jabref.model.pdf.search.SearchFieldConstants.CONTENT; -import static org.jabref.model.pdf.search.SearchFieldConstants.MODIFIED; -import static org.jabref.model.pdf.search.SearchFieldConstants.PAGE_NUMBER; -import static org.jabref.model.pdf.search.SearchFieldConstants.PATH; +import static org.jabref.model.search.SearchFieldConstants.ANNOTATIONS; +import static org.jabref.model.search.SearchFieldConstants.CONTENT; +import static org.jabref.model.search.SearchFieldConstants.MODIFIED; +import static org.jabref.model.search.SearchFieldConstants.PAGE_NUMBER; +import static org.jabref.model.search.SearchFieldConstants.PATH; /** * Utility class for reading the data from LinkedFiles of a BibEntry for Lucene. diff --git a/src/main/java/org/jabref/logic/pdf/search/IndexingTaskManager.java b/src/main/java/org/jabref/logic/search/indexing/IndexingTaskManager.java similarity index 99% rename from src/main/java/org/jabref/logic/pdf/search/IndexingTaskManager.java rename to src/main/java/org/jabref/logic/search/indexing/IndexingTaskManager.java index 6e4f39dd935..5fd8a41075f 100644 --- a/src/main/java/org/jabref/logic/pdf/search/IndexingTaskManager.java +++ b/src/main/java/org/jabref/logic/search/indexing/IndexingTaskManager.java @@ -1,4 +1,4 @@ -package org.jabref.logic.pdf.search; +package org.jabref.logic.search.indexing; import java.util.List; import java.util.Set; diff --git a/src/main/java/org/jabref/logic/pdf/search/LuceneIndexer.java b/src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java similarity index 99% rename from src/main/java/org/jabref/logic/pdf/search/LuceneIndexer.java rename to src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java index 91ae488fb7d..5e07ec1af23 100644 --- a/src/main/java/org/jabref/logic/pdf/search/LuceneIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java @@ -1,4 +1,4 @@ -package org.jabref.logic.pdf.search; +package org.jabref.logic.search.indexing; import java.io.IOException; import java.nio.file.Files; @@ -22,7 +22,7 @@ import org.jabref.model.entry.LinkedFile; import org.jabref.model.entry.field.Field; import org.jabref.model.entry.field.StandardField; -import org.jabref.model.pdf.search.SearchFieldConstants; +import org.jabref.model.search.SearchFieldConstants; import org.jabref.preferences.FilePreferences; import org.jabref.preferences.PreferencesService; diff --git a/src/main/java/org/jabref/logic/pdf/search/LuceneSearcher.java b/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java similarity index 94% rename from src/main/java/org/jabref/logic/pdf/search/LuceneSearcher.java rename to src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java index a51b0bbd3b2..0c4c32e235a 100644 --- a/src/main/java/org/jabref/logic/pdf/search/LuceneSearcher.java +++ b/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java @@ -1,4 +1,4 @@ -package org.jabref.logic.pdf.search; +package org.jabref.logic.search.retrieval; import java.io.IOException; import java.util.HashMap; @@ -7,8 +7,8 @@ import org.jabref.logic.search.SearchQuery; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; -import org.jabref.model.pdf.search.LuceneSearchResults; -import org.jabref.model.pdf.search.SearchResult; +import org.jabref.model.search.LuceneSearchResults; +import org.jabref.model.search.SearchResult; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; diff --git a/src/main/java/org/jabref/model/groups/SearchGroup.java b/src/main/java/org/jabref/model/groups/SearchGroup.java index 855d4079310..9b83c3addbc 100644 --- a/src/main/java/org/jabref/model/groups/SearchGroup.java +++ b/src/main/java/org/jabref/model/groups/SearchGroup.java @@ -6,12 +6,11 @@ import java.util.Objects; import java.util.Set; -import org.jabref.logic.pdf.search.LuceneSearcher; +import org.jabref.logic.search.retrieval.LuceneSearcher; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.search.GroupSearchQuery; -import org.jabref.model.search.rules.SearchRules; -import org.jabref.model.search.rules.SearchRules.SearchFlags; +import org.jabref.model.search.SearchFlags; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,7 +54,7 @@ public boolean contains(BibEntry entry) { return matches.contains(entry); } - public EnumSet getSearchFlags() { + public EnumSet getSearchFlags() { return query.getSearchFlags(); } diff --git a/src/main/java/org/jabref/model/search/GroupSearchQuery.java b/src/main/java/org/jabref/model/search/GroupSearchQuery.java index cbc999204a0..8b2b09551d9 100644 --- a/src/main/java/org/jabref/model/search/GroupSearchQuery.java +++ b/src/main/java/org/jabref/model/search/GroupSearchQuery.java @@ -3,7 +3,6 @@ import java.util.EnumSet; import org.jabref.logic.search.SearchQuery; -import org.jabref.model.search.rules.SearchRules.SearchFlags; public class GroupSearchQuery extends SearchQuery { diff --git a/src/main/java/org/jabref/model/pdf/search/LuceneSearchResults.java b/src/main/java/org/jabref/model/search/LuceneSearchResults.java similarity index 97% rename from src/main/java/org/jabref/model/pdf/search/LuceneSearchResults.java rename to src/main/java/org/jabref/model/search/LuceneSearchResults.java index 36738440228..fbdfe6f1448 100644 --- a/src/main/java/org/jabref/model/pdf/search/LuceneSearchResults.java +++ b/src/main/java/org/jabref/model/search/LuceneSearchResults.java @@ -1,4 +1,4 @@ -package org.jabref.model.pdf.search; +package org.jabref.model.search; import java.util.ArrayList; import java.util.Collections; diff --git a/src/main/java/org/jabref/model/pdf/search/SearchFieldConstants.java b/src/main/java/org/jabref/model/search/SearchFieldConstants.java similarity index 95% rename from src/main/java/org/jabref/model/pdf/search/SearchFieldConstants.java rename to src/main/java/org/jabref/model/search/SearchFieldConstants.java index 20893d89e3b..8da81395c18 100644 --- a/src/main/java/org/jabref/model/pdf/search/SearchFieldConstants.java +++ b/src/main/java/org/jabref/model/search/SearchFieldConstants.java @@ -1,4 +1,4 @@ -package org.jabref.model.pdf.search; +package org.jabref.model.search; import java.util.HashSet; import java.util.Set; diff --git a/src/main/java/org/jabref/model/search/SearchFlags.java b/src/main/java/org/jabref/model/search/SearchFlags.java new file mode 100644 index 00000000000..1b5ce59db24 --- /dev/null +++ b/src/main/java/org/jabref/model/search/SearchFlags.java @@ -0,0 +1,10 @@ +package org.jabref.model.search; + +/** + * This is a factory to instantiate the matching SearchRule implementation matching a given query + */ + +public enum SearchFlags { + REGULAR_EXPRESSION, FULLTEXT, KEEP_SEARCH_STRING, FILTERING_SEARCH, SORT_BY_SCORE +} + diff --git a/src/main/java/org/jabref/model/pdf/search/SearchResult.java b/src/main/java/org/jabref/model/search/SearchResult.java similarity index 86% rename from src/main/java/org/jabref/model/pdf/search/SearchResult.java rename to src/main/java/org/jabref/model/search/SearchResult.java index 8e3a838a39e..edc04846cc6 100644 --- a/src/main/java/org/jabref/model/pdf/search/SearchResult.java +++ b/src/main/java/org/jabref/model/search/SearchResult.java @@ -1,4 +1,4 @@ -package org.jabref.model.pdf.search; +package org.jabref.model.search; import java.io.IOException; import java.util.Arrays; @@ -22,13 +22,6 @@ import org.apache.lucene.search.highlight.SimpleHTMLFormatter; import org.apache.lucene.search.highlight.TextFragment; -import static org.jabref.model.pdf.search.SearchFieldConstants.ANNOTATIONS; -import static org.jabref.model.pdf.search.SearchFieldConstants.BIB_ENTRY_ID_HASH; -import static org.jabref.model.pdf.search.SearchFieldConstants.CONTENT; -import static org.jabref.model.pdf.search.SearchFieldConstants.MODIFIED; -import static org.jabref.model.pdf.search.SearchFieldConstants.PAGE_NUMBER; -import static org.jabref.model.pdf.search.SearchFieldConstants.PATH; - public final class SearchResult { private final String path; @@ -43,22 +36,22 @@ public final class SearchResult { private List annotationsResultStringsHtml = List.of(); public SearchResult(IndexSearcher searcher, Query query, ScoreDoc scoreDoc) throws IOException { - this.path = getFieldContents(searcher, scoreDoc, PATH); + this.path = getFieldContents(searcher, scoreDoc, SearchFieldConstants.PATH); this.luceneScore = scoreDoc.score; if (this.path.length() > 0) { // pdf result - this.pageNumber = Integer.parseInt(getFieldContents(searcher, scoreDoc, PAGE_NUMBER)); - this.modified = Long.parseLong(getFieldContents(searcher, scoreDoc, MODIFIED)); + this.pageNumber = Integer.parseInt(getFieldContents(searcher, scoreDoc, SearchFieldConstants.PAGE_NUMBER)); + this.modified = Long.parseLong(getFieldContents(searcher, scoreDoc, SearchFieldConstants.MODIFIED)); this.hash = 0; - String content = getFieldContents(searcher, scoreDoc, CONTENT); - String annotations = getFieldContents(searcher, scoreDoc, ANNOTATIONS); + String content = getFieldContents(searcher, scoreDoc, SearchFieldConstants.CONTENT); + String annotations = getFieldContents(searcher, scoreDoc, SearchFieldConstants.ANNOTATIONS); this.hasFulltextResults = !(content.isEmpty() && annotations.isEmpty()); Highlighter highlighter = new Highlighter(new SimpleHTMLFormatter("", ""), new QueryScorer(query)); try (Analyzer analyzer = new EnglishAnalyzer(); - TokenStream contentStream = analyzer.tokenStream(CONTENT, content)) { + TokenStream contentStream = analyzer.tokenStream(SearchFieldConstants.CONTENT, content)) { TextFragment[] frags = highlighter.getBestTextFragments(contentStream, content, true, 10); this.contentResultStringsHtml = Arrays.stream(frags).map(TextFragment::toString).collect(Collectors.toList()); } catch (InvalidTokenOffsetsException e) { @@ -66,7 +59,7 @@ public SearchResult(IndexSearcher searcher, Query query, ScoreDoc scoreDoc) thro } try (Analyzer analyzer = new EnglishAnalyzer(); - TokenStream annotationStream = analyzer.tokenStream(ANNOTATIONS, annotations)) { + TokenStream annotationStream = analyzer.tokenStream(SearchFieldConstants.ANNOTATIONS, annotations)) { TextFragment[] frags = highlighter.getBestTextFragments(annotationStream, annotations, true, 10); this.annotationsResultStringsHtml = Arrays.stream(frags).map(TextFragment::toString).collect(Collectors.toList()); } catch (InvalidTokenOffsetsException e) { @@ -74,7 +67,7 @@ public SearchResult(IndexSearcher searcher, Query query, ScoreDoc scoreDoc) thro } } else { // Found somewhere in the bib entry - this.hash = Integer.parseInt(getFieldContents(searcher, scoreDoc, BIB_ENTRY_ID_HASH)); + this.hash = Integer.parseInt(getFieldContents(searcher, scoreDoc, SearchFieldConstants.BIB_ENTRY_ID_HASH)); this.pageNumber = -1; this.modified = -1; this.hasFulltextResults = false; diff --git a/src/main/java/org/jabref/model/search/rules/SearchRules.java b/src/main/java/org/jabref/model/search/rules/SearchRules.java deleted file mode 100644 index 708ca1df316..00000000000 --- a/src/main/java/org/jabref/model/search/rules/SearchRules.java +++ /dev/null @@ -1,11 +0,0 @@ -package org.jabref.model.search.rules; - -/** - * This is a factory to instantiate the matching SearchRule implementation matching a given query - */ -public class SearchRules { - - public enum SearchFlags { - REGULAR_EXPRESSION, FULLTEXT, KEEP_SEARCH_STRING, FILTERING_SEARCH, SORT_BY_SCORE; - } -} diff --git a/src/main/java/org/jabref/preferences/JabRefPreferences.java b/src/main/java/org/jabref/preferences/JabRefPreferences.java index c8fd91b12d5..84a444e3f26 100644 --- a/src/main/java/org/jabref/preferences/JabRefPreferences.java +++ b/src/main/java/org/jabref/preferences/JabRefPreferences.java @@ -122,7 +122,7 @@ import org.jabref.model.groups.GroupHierarchyType; import org.jabref.model.metadata.SaveOrder; import org.jabref.model.metadata.SelfContainedSaveOrder; -import org.jabref.model.search.rules.SearchRules.SearchFlags; +import org.jabref.model.search.SearchFlags; import org.jabref.model.strings.StringUtil; import com.github.javakeyring.Keyring; diff --git a/src/main/java/org/jabref/preferences/SearchPreferences.java b/src/main/java/org/jabref/preferences/SearchPreferences.java index dd4b23b7b97..4a4b0424e6f 100644 --- a/src/main/java/org/jabref/preferences/SearchPreferences.java +++ b/src/main/java/org/jabref/preferences/SearchPreferences.java @@ -9,11 +9,11 @@ import javafx.collections.FXCollections; import javafx.collections.ObservableSet; -import org.jabref.model.search.rules.SearchRules; +import org.jabref.model.search.SearchFlags; public class SearchPreferences { - private final ObservableSet searchFlags; + private final ObservableSet searchFlags; private final BooleanProperty keepWindowOnTop; private final DoubleProperty searchWindowHeight = new SimpleDoubleProperty(); @@ -22,46 +22,46 @@ public class SearchPreferences { public SearchPreferences(boolean isRegularExpression, boolean isFulltext, boolean isKeepSearchString, boolean isFilteringMode, boolean isSortByScore, boolean keepWindowOnTop, double searchWindowHeight, double searchWindowWidth) { this.keepWindowOnTop = new SimpleBooleanProperty(keepWindowOnTop); - searchFlags = FXCollections.observableSet(EnumSet.noneOf(SearchRules.SearchFlags.class)); + searchFlags = FXCollections.observableSet(EnumSet.noneOf(SearchFlags.class)); if (isRegularExpression) { - searchFlags.add(SearchRules.SearchFlags.REGULAR_EXPRESSION); + searchFlags.add(SearchFlags.REGULAR_EXPRESSION); } if (isFulltext) { - searchFlags.add(SearchRules.SearchFlags.FULLTEXT); + searchFlags.add(SearchFlags.FULLTEXT); } if (isKeepSearchString) { - searchFlags.add(SearchRules.SearchFlags.KEEP_SEARCH_STRING); + searchFlags.add(SearchFlags.KEEP_SEARCH_STRING); } if (isFilteringMode) { - searchFlags.add(SearchRules.SearchFlags.FILTERING_SEARCH); + searchFlags.add(SearchFlags.FILTERING_SEARCH); } if (isSortByScore) { - searchFlags.add(SearchRules.SearchFlags.SORT_BY_SCORE); + searchFlags.add(SearchFlags.SORT_BY_SCORE); } this.setSearchWindowHeight(searchWindowHeight); this.setSearchWindowWidth(searchWindowWidth); } - public SearchPreferences(EnumSet searchFlags, boolean keepWindowOnTop) { + public SearchPreferences(EnumSet searchFlags, boolean keepWindowOnTop) { this.keepWindowOnTop = new SimpleBooleanProperty(keepWindowOnTop); this.searchFlags = FXCollections.observableSet(searchFlags); } - public EnumSet getSearchFlags() { + public EnumSet getSearchFlags() { // copy of returns an exception when the EnumSet is empty if (searchFlags.isEmpty()) { - return EnumSet.noneOf(SearchRules.SearchFlags.class); + return EnumSet.noneOf(SearchFlags.class); } return EnumSet.copyOf(searchFlags); } - public ObservableSet getObservableSearchFlags() { + public ObservableSet getObservableSearchFlags() { return searchFlags; } - public void setSearchFlag(SearchRules.SearchFlags flag, boolean value) { + public void setSearchFlag(SearchFlags flag, boolean value) { if (searchFlags.contains(flag) && !value) { searchFlags.remove(flag); } else if (!searchFlags.contains(flag) && value) { @@ -70,23 +70,23 @@ public void setSearchFlag(SearchRules.SearchFlags flag, boolean value) { } public boolean isRegularExpression() { - return searchFlags.contains(SearchRules.SearchFlags.REGULAR_EXPRESSION); + return searchFlags.contains(SearchFlags.REGULAR_EXPRESSION); } public boolean isFulltext() { - return searchFlags.contains(SearchRules.SearchFlags.FULLTEXT); + return searchFlags.contains(SearchFlags.FULLTEXT); } public boolean shouldKeepSearchString() { - return searchFlags.contains(SearchRules.SearchFlags.KEEP_SEARCH_STRING); + return searchFlags.contains(SearchFlags.KEEP_SEARCH_STRING); } public boolean isFilteringMode() { - return searchFlags.contains(SearchRules.SearchFlags.FILTERING_SEARCH); + return searchFlags.contains(SearchFlags.FILTERING_SEARCH); } public boolean isSortByScore() { - return searchFlags.contains(SearchRules.SearchFlags.SORT_BY_SCORE); + return searchFlags.contains(SearchFlags.SORT_BY_SCORE); } public boolean shouldKeepWindowOnTop() { diff --git a/src/test/java/org/jabref/cli/ArgumentProcessorTest.java b/src/test/java/org/jabref/cli/ArgumentProcessorTest.java index a7e8028f6b6..227874d5065 100644 --- a/src/test/java/org/jabref/cli/ArgumentProcessorTest.java +++ b/src/test/java/org/jabref/cli/ArgumentProcessorTest.java @@ -19,7 +19,7 @@ import org.jabref.model.entry.BibEntryTypesManager; import org.jabref.model.metadata.SaveOrder; import org.jabref.model.metadata.SelfContainedSaveOrder; -import org.jabref.model.search.rules.SearchRules; +import org.jabref.model.search.SearchFlags; import org.jabref.model.util.DummyFileUpdateMonitor; import org.jabref.model.util.FileUpdateMonitor; import org.jabref.preferences.ExportPreferences; @@ -49,7 +49,7 @@ void setup() { when(preferencesService.getImporterPreferences()).thenReturn(importerPreferences); when(preferencesService.getImportFormatPreferences()).thenReturn(importFormatPreferences); when(preferencesService.getSearchPreferences()).thenReturn( - new SearchPreferences(EnumSet.noneOf(SearchRules.SearchFlags.class), false) + new SearchPreferences(EnumSet.noneOf(SearchFlags.class), false) ); } diff --git a/src/test/java/org/jabref/gui/entryeditor/CommentsTabTest.java b/src/test/java/org/jabref/gui/entryeditor/CommentsTabTest.java index 84e9e5e8b38..87d9dfbcd77 100644 --- a/src/test/java/org/jabref/gui/entryeditor/CommentsTabTest.java +++ b/src/test/java/org/jabref/gui/entryeditor/CommentsTabTest.java @@ -12,8 +12,8 @@ import org.jabref.gui.theme.ThemeManager; import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.journals.JournalAbbreviationRepository; -import org.jabref.logic.pdf.search.IndexingTaskManager; import org.jabref.logic.preferences.OwnerPreferences; +import org.jabref.logic.search.indexing.IndexingTaskManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.database.BibDatabaseMode; import org.jabref.model.entry.BibEntry; @@ -97,9 +97,10 @@ void setUp() { dialogService, stateManager, themeManager, + indexingTaskManager, taskExecutor, - journalAbbreviationRepository, - indexingTaskManager); + journalAbbreviationRepository + ); } @Test diff --git a/src/test/java/org/jabref/gui/groups/GroupTreeViewModelTest.java b/src/test/java/org/jabref/gui/groups/GroupTreeViewModelTest.java index 13cdfa58e60..3446d21a8fd 100644 --- a/src/test/java/org/jabref/gui/groups/GroupTreeViewModelTest.java +++ b/src/test/java/org/jabref/gui/groups/GroupTreeViewModelTest.java @@ -47,9 +47,9 @@ void setUp() { dialogService = mock(DialogService.class, Answers.RETURNS_DEEP_STUBS); when(preferencesService.getGroupsPreferences()).thenReturn(new GroupsPreferences( - EnumSet.of(GroupViewMode.FILTER), - true, - true, + EnumSet.of(GroupViewMode.FILTER), + true, + true, GroupHierarchyType.INDEPENDENT)); groupTree = new GroupTreeViewModel(stateManager, mock(DialogService.class), preferencesService, taskExecutor, new CustomLocalDragboard()); } diff --git a/src/test/java/org/jabref/gui/search/GlobalSearchBarTest.java b/src/test/java/org/jabref/gui/search/GlobalSearchBarTest.java index fa35a3ec606..958a476e65c 100644 --- a/src/test/java/org/jabref/gui/search/GlobalSearchBarTest.java +++ b/src/test/java/org/jabref/gui/search/GlobalSearchBarTest.java @@ -15,7 +15,7 @@ import org.jabref.gui.undo.CountingUndoManager; import org.jabref.gui.util.DefaultTaskExecutor; import org.jabref.model.database.BibDatabaseContext; -import org.jabref.model.search.rules.SearchRules; +import org.jabref.model.search.SearchFlags; import org.jabref.preferences.PreferencesService; import org.jabref.preferences.SearchPreferences; import org.jabref.testutils.category.GUITest; @@ -43,7 +43,7 @@ public class GlobalSearchBarTest { @Start public void onStart(Stage stage) { SearchPreferences searchPreferences = mock(SearchPreferences.class); - when(searchPreferences.getSearchFlags()).thenReturn(EnumSet.noneOf(SearchRules.SearchFlags.class)); + when(searchPreferences.getSearchFlags()).thenReturn(EnumSet.noneOf(SearchFlags.class)); PreferencesService prefs = mock(PreferencesService.class, Answers.RETURNS_DEEP_STUBS); when(prefs.getSearchPreferences()).thenReturn(searchPreferences); diff --git a/src/test/java/org/jabref/logic/exporter/GroupSerializerTest.java b/src/test/java/org/jabref/logic/exporter/GroupSerializerTest.java index 17a79e94ce0..e396f7c4c9e 100644 --- a/src/test/java/org/jabref/logic/exporter/GroupSerializerTest.java +++ b/src/test/java/org/jabref/logic/exporter/GroupSerializerTest.java @@ -25,7 +25,7 @@ import org.jabref.model.groups.TexGroup; import org.jabref.model.groups.WordKeywordGroup; import org.jabref.model.metadata.MetaData; -import org.jabref.model.search.rules.SearchRules; +import org.jabref.model.search.SearchFlags; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -90,14 +90,14 @@ void serializeSingleRegexKeywordGroup() { @Test void serializeSingleSearchGroup() { - SearchGroup group = new SearchGroup("myExplicitGroup", GroupHierarchyType.INDEPENDENT, "author=harrer", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)); + SearchGroup group = new SearchGroup("myExplicitGroup", GroupHierarchyType.INDEPENDENT, "author=harrer", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)); List serialization = groupSerializer.serializeTree(GroupTreeNode.fromGroup(group)); assertEquals(Collections.singletonList("0 SearchGroup:myExplicitGroup;0;author=harrer;1;1;1;;;;"), serialization); } @Test void serializeSingleSearchGroupWithRegex() { - SearchGroup group = new SearchGroup("myExplicitGroup", GroupHierarchyType.INCLUDING, "author=\"harrer\"", EnumSet.noneOf(SearchRules.SearchFlags.class)); + SearchGroup group = new SearchGroup("myExplicitGroup", GroupHierarchyType.INCLUDING, "author=\"harrer\"", EnumSet.noneOf(SearchFlags.class)); List serialization = groupSerializer.serializeTree(GroupTreeNode.fromGroup(group)); assertEquals(Collections.singletonList("0 SearchGroup:myExplicitGroup;2;author=\"harrer\";1;0;1;;;;"), serialization); } diff --git a/src/test/java/org/jabref/logic/importer/util/GroupsParserTest.java b/src/test/java/org/jabref/logic/importer/util/GroupsParserTest.java index 886657e7125..a2e7f667b87 100644 --- a/src/test/java/org/jabref/logic/importer/util/GroupsParserTest.java +++ b/src/test/java/org/jabref/logic/importer/util/GroupsParserTest.java @@ -21,7 +21,7 @@ import org.jabref.model.groups.SearchGroup; import org.jabref.model.groups.TexGroup; import org.jabref.model.metadata.MetaData; -import org.jabref.model.search.rules.SearchRules; +import org.jabref.model.search.SearchFlags; import org.jabref.model.util.DummyFileUpdateMonitor; import org.jabref.model.util.FileUpdateMonitor; @@ -136,7 +136,7 @@ void fromStringUnknownGroupThrowsException() throws Exception { @Test void fromStringParsesSearchGroup() throws Exception { - SearchGroup expected = new SearchGroup("Data", GroupHierarchyType.INCLUDING, "project=data|number|quant*", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)); + SearchGroup expected = new SearchGroup("Data", GroupHierarchyType.INCLUDING, "project=data|number|quant*", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)); AbstractGroup parsed = GroupsParser.fromString("SearchGroup:Data;2;project=data|number|quant*;0;1;1;;;;;", ',', fileMonitor, metaData); assertEquals(expected, parsed); } diff --git a/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java b/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java index d9d50fc51b9..12d228daef0 100644 --- a/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java +++ b/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java @@ -8,7 +8,7 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.types.StandardEntryType; -import org.jabref.model.search.rules.SearchRules; +import org.jabref.model.search.SearchFlags; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -17,7 +17,7 @@ public class DatabaseSearcherTest { - public static final SearchQuery INVALID_SEARCH_QUERY = new SearchQuery("\\asd123{}asdf", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)); + public static final SearchQuery INVALID_SEARCH_QUERY = new SearchQuery("\\asd123{}asdf", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)); private BibDatabase database; @@ -28,7 +28,7 @@ public void setUp() { @Test public void noMatchesFromEmptyDatabase() { - List matches = new DatabaseSearcher(new SearchQuery("whatever", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)), database).getMatches(); + List matches = new DatabaseSearcher(new SearchQuery("whatever", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)), database).getMatches(); assertEquals(Collections.emptyList(), matches); } @@ -41,7 +41,7 @@ public void noMatchesFromEmptyDatabaseWithInvalidSearchExpression() { @Test public void getDatabaseFromMatchesDatabaseWithEmptyEntries() { database.insertEntry(new BibEntry()); - List matches = new DatabaseSearcher(new SearchQuery("whatever", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)), database).getMatches(); + List matches = new DatabaseSearcher(new SearchQuery("whatever", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)), database).getMatches(); assertEquals(Collections.emptyList(), matches); } @@ -50,7 +50,7 @@ public void noMatchesFromDatabaseWithArticleTypeEntry() { BibEntry entry = new BibEntry(StandardEntryType.Article); entry.setField(StandardField.AUTHOR, "harrer"); database.insertEntry(entry); - List matches = new DatabaseSearcher(new SearchQuery("whatever", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)), database).getMatches(); + List matches = new DatabaseSearcher(new SearchQuery("whatever", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)), database).getMatches(); assertEquals(Collections.emptyList(), matches); } @@ -59,13 +59,13 @@ public void correctMatchFromDatabaseWithArticleTypeEntry() { BibEntry entry = new BibEntry(StandardEntryType.Article); entry.setField(StandardField.AUTHOR, "harrer"); database.insertEntry(entry); - List matches = new DatabaseSearcher(new SearchQuery("harrer", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)), database).getMatches(); + List matches = new DatabaseSearcher(new SearchQuery("harrer", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)), database).getMatches(); assertEquals(Collections.singletonList(entry), matches); } @Test public void noMatchesFromEmptyDatabaseWithInvalidQuery() { - SearchQuery query = new SearchQuery("asdf[", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)); + SearchQuery query = new SearchQuery("asdf[", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)); DatabaseSearcher databaseSearcher = new DatabaseSearcher(query, database); assertEquals(Collections.emptyList(), databaseSearcher.getMatches()); } @@ -76,7 +76,7 @@ public void correctMatchFromDatabaseWithIncollectionTypeEntry() { entry.setField(StandardField.AUTHOR, "tonho"); database.insertEntry(entry); - SearchQuery query = new SearchQuery("tonho", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)); + SearchQuery query = new SearchQuery("tonho", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)); List matches = new DatabaseSearcher(query, database).getMatches(); assertEquals(Collections.singletonList(entry), matches); @@ -91,7 +91,7 @@ public void noMatchesFromDatabaseWithTwoEntries() { entry.setField(StandardField.AUTHOR, "tonho"); database.insertEntry(entry); - SearchQuery query = new SearchQuery("tonho", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)); + SearchQuery query = new SearchQuery("tonho", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)); DatabaseSearcher databaseSearcher = new DatabaseSearcher(query, database); assertEquals(Collections.singletonList(entry), databaseSearcher.getMatches()); @@ -103,7 +103,7 @@ public void noMatchesFromDabaseWithIncollectionTypeEntry() { entry.setField(StandardField.AUTHOR, "tonho"); database.insertEntry(entry); - SearchQuery query = new SearchQuery("asdf", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)); + SearchQuery query = new SearchQuery("asdf", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)); DatabaseSearcher databaseSearcher = new DatabaseSearcher(query, database); assertEquals(Collections.emptyList(), databaseSearcher.getMatches()); @@ -114,7 +114,7 @@ public void noMatchFromDatabaseWithEmptyEntry() { BibEntry entry = new BibEntry(); database.insertEntry(entry); - SearchQuery query = new SearchQuery("tonho", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)); + SearchQuery query = new SearchQuery("tonho", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)); DatabaseSearcher databaseSearcher = new DatabaseSearcher(query, database); assertEquals(Collections.emptyList(), databaseSearcher.getMatches()); diff --git a/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java b/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java index 269699c49b0..47dc8468ef5 100644 --- a/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java +++ b/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java @@ -1,170 +1,131 @@ package org.jabref.logic.search; -import java.nio.file.Path; -import java.util.Collections; -import java.util.EnumSet; -import java.util.List; -import java.util.Objects; -import java.util.stream.Stream; - -import org.jabref.gui.Globals; -import org.jabref.logic.importer.ImportFormatPreferences; -import org.jabref.logic.importer.ParserResult; -import org.jabref.logic.importer.fileformat.BibtexImporter; -import org.jabref.logic.pdf.search.PdfIndexer; -import org.jabref.logic.pdf.search.PdfIndexerManager; -import org.jabref.logic.util.StandardFileType; -import org.jabref.model.database.BibDatabase; -import org.jabref.model.database.BibDatabaseContext; -import org.jabref.model.entry.BibEntry; -import org.jabref.model.entry.LinkedFile; -import org.jabref.model.entry.field.StandardField; -import org.jabref.model.entry.types.StandardEntryType; -import org.jabref.model.search.rules.SearchRules; -import org.jabref.model.util.DummyFileUpdateMonitor; -import org.jabref.preferences.FilePreferences; -import org.jabref.preferences.PreferencesService; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.io.TempDir; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import org.mockito.Answers; -import org.mockito.Mockito; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - public class DatabaseSearcherWithBibFilesTest { - private static BibEntry titleSentenceCased = new BibEntry(StandardEntryType.Misc) - .withCitationKey("title-sentence-cased") - .withField(StandardField.TITLE, "Title Sentence Cased"); - private static BibEntry titleMixedCased = new BibEntry(StandardEntryType.Misc) - .withCitationKey("title-mixed-cased") - .withField(StandardField.TITLE, "TiTle MiXed CaSed"); - private static BibEntry titleUpperCased = new BibEntry(StandardEntryType.Misc) - .withCitationKey("title-upper-cased") - .withField(StandardField.TITLE, "TITLE UPPER CASED"); - - private static BibEntry mininimalSentenceCase = new BibEntry(StandardEntryType.Misc) - .withCitationKey("minimal-sentence-case") - .withFiles(Collections.singletonList(new LinkedFile("", "minimal-sentence-case.pdf", StandardFileType.PDF.getName()))); - private static BibEntry minimalAllUpperCase = new BibEntry(StandardEntryType.Misc) - .withCitationKey("minimal-all-upper-case") - .withFiles(Collections.singletonList(new LinkedFile("", "minimal-all-upper-case.pdf", StandardFileType.PDF.getName()))); - private static BibEntry minimalMixedCase = new BibEntry(StandardEntryType.Misc) - .withCitationKey("minimal-mixed-case") - .withFiles(Collections.singletonList(new LinkedFile("", "minimal-mixed-case.pdf", StandardFileType.PDF.getName()))); - private static BibEntry minimalNoteSentenceCase = new BibEntry(StandardEntryType.Misc) - .withCitationKey("minimal-note-sentence-case") - .withFiles(Collections.singletonList(new LinkedFile("", "minimal-note-sentence-case.pdf", StandardFileType.PDF.getName()))); - private static BibEntry minimalNoteAllUpperCase = new BibEntry(StandardEntryType.Misc) - .withCitationKey("minimal-note-all-upper-case") - .withFiles(Collections.singletonList(new LinkedFile("", "minimal-note-all-upper-case.pdf", StandardFileType.PDF.getName()))); - private static BibEntry minimalNoteMixedCase = new BibEntry(StandardEntryType.Misc) - .withCitationKey("minimal-note-mixed-case") - .withFiles(Collections.singletonList(new LinkedFile("", "minimal-note-mixed-case.pdf", StandardFileType.PDF.getName()))); - - FilePreferences filePreferences = mock(FilePreferences.class); - - @TempDir - private Path indexDir; - private PdfIndexer pdfIndexer; - - private BibDatabase initializeDatabaseFromPath(String testFile) throws Exception { - return initializeDatabaseFromPath(Path.of(Objects.requireNonNull(DatabaseSearcherWithBibFilesTest.class.getResource(testFile)).toURI())); - } - - private BibDatabase initializeDatabaseFromPath(Path testFile) throws Exception { - ParserResult result = new BibtexImporter(mock(ImportFormatPreferences.class, Answers.RETURNS_DEEP_STUBS), new DummyFileUpdateMonitor()).importDatabase(testFile); - BibDatabase database = result.getDatabase(); - - BibDatabaseContext context = mock(BibDatabaseContext.class); - when(context.getFileDirectories(Mockito.any())).thenReturn(List.of(testFile.getParent())); - when(context.getFulltextIndexPath()).thenReturn(indexDir); - when(context.getDatabase()).thenReturn(database); - when(context.getEntries()).thenReturn(database.getEntries()); - - // Required because of {@Link org.jabref.model.search.rules.FullTextSearchRule.FullTextSearchRule} - Globals.stateManager.setActiveDatabase(context); - PreferencesService preferencesService = mock(PreferencesService.class); - when(preferencesService.getFilePreferences()).thenReturn(filePreferences); - Globals.prefs = preferencesService; - - pdfIndexer = PdfIndexerManager.getIndexer(context, filePreferences); - // Alternative - For debugging with Luke (part of the Apache Lucene distribution) - // pdfIndexer = PdfIndexer.of(context, Path.of("C:\\temp\\index"), filePreferences); - - pdfIndexer.rebuildIndex(); - return database; - } - - @AfterEach - public void tearDown() throws Exception { - pdfIndexer.close(); - } - - private static Stream searchLibrary() { - return Stream.of( - // empty library - Arguments.of(List.of(), "empty.bib", "Test", EnumSet.noneOf(SearchRules.SearchFlags.class)), - - // test-library-title-casing - - Arguments.of(List.of(), "test-library-title-casing.bib", "NotExisting", EnumSet.noneOf(SearchRules.SearchFlags.class)), - Arguments.of(List.of(titleSentenceCased, titleMixedCased, titleUpperCased), "test-library-title-casing.bib", "Title", EnumSet.noneOf(SearchRules.SearchFlags.class)), - - Arguments.of(List.of(), "test-library-title-casing.bib", "title=NotExisting", EnumSet.noneOf(SearchRules.SearchFlags.class)), - Arguments.of(List.of(titleSentenceCased, titleMixedCased, titleUpperCased), "test-library-title-casing.bib", "title=Title", EnumSet.noneOf(SearchRules.SearchFlags.class)), - - Arguments.of(List.of(), "test-library-title-casing.bib", "title=TiTLE", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE)), - Arguments.of(List.of(titleSentenceCased), "test-library-title-casing.bib", "title=Title", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE)), - - Arguments.of(List.of(), "test-library-title-casing.bib", "TiTLE", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE)), - Arguments.of(List.of(titleMixedCased), "test-library-title-casing.bib", "TiTle", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE)), - - Arguments.of(List.of(), "test-library-title-casing.bib", "title=NotExisting", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE)), - Arguments.of(List.of(titleMixedCased), "test-library-title-casing.bib", "title=TiTle", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE)), - - Arguments.of(List.of(), "test-library-title-casing.bib", "[Y]", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)), - Arguments.of(List.of(titleUpperCased), "test-library-title-casing.bib", "[U]", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)), - - // Word boundaries - Arguments.of(List.of(), "test-library-title-casing.bib", "\\bTit\\b", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION, SearchRules.SearchFlags.CASE_SENSITIVE)), - Arguments.of(List.of(titleSentenceCased), "test-library-title-casing.bib", "\\bTitle\\b", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION, SearchRules.SearchFlags.CASE_SENSITIVE)), - - // test-library-with-attached-files - - Arguments.of(List.of(), "test-library-with-attached-files.bib", "This is a test.", EnumSet.of(SearchRules.SearchFlags.FULLTEXT, SearchRules.SearchFlags.CASE_SENSITIVE)), - - Arguments.of(List.of(mininimalSentenceCase, minimalAllUpperCase, minimalMixedCase), "test-library-with-attached-files.bib", "This is a short sentence, comma included.", EnumSet.of(SearchRules.SearchFlags.FULLTEXT)), - Arguments.of(List.of(mininimalSentenceCase, minimalAllUpperCase, minimalMixedCase), "test-library-with-attached-files.bib", "comma", EnumSet.of(SearchRules.SearchFlags.FULLTEXT)), - // TODO: PDF search does not support case sensitive search (yet) - // Arguments.of(List.of(minimalAllUpperCase, minimalMixedCase), "test-library-with-attached-files.bib", "THIS", EnumSet.of(SearchRules.SearchFlags.FULLTEXT, SearchRules.SearchFlags.CASE_SENSITIVE)), - // Arguments.of(List.of(minimalAllUpperCase), "test-library-with-attached-files.bib", "THIS is a short sentence, comma included.", EnumSet.of(SearchRules.SearchFlags.FULLTEXT, SearchRules.SearchFlags.CASE_SENSITIVE)), - // Arguments.of(List.of(minimalSentenceCase, minimalAllUpperCase, minimalMixedCase), "test-library-with-attached-files.bib", "comma", EnumSet.of(SearchRules.SearchFlags.FULLTEXT, SearchRules.SearchFlags.CASE_SENSITIVE)), - // Arguments.of(List.of(minimalNoteAllUpperCase), "test-library-with-attached-files.bib", "THIS IS A SHORT SENTENCE, COMMA INCLUDED.", EnumSet.of(SearchRules.SearchFlags.FULLTEXT, SearchRules.SearchFlags.CASE_SENSITIVE)), - - Arguments.of(List.of(), "test-library-with-attached-files.bib", "NotExisting", EnumSet.of(SearchRules.SearchFlags.FULLTEXT)), - - Arguments.of(List.of(minimalNoteSentenceCase, minimalNoteAllUpperCase, minimalNoteMixedCase), "test-library-with-attached-files.bib", "world", EnumSet.of(SearchRules.SearchFlags.FULLTEXT)), - Arguments.of(List.of(minimalNoteSentenceCase, minimalNoteAllUpperCase, minimalNoteMixedCase), "test-library-with-attached-files.bib", "Hello World", EnumSet.of(SearchRules.SearchFlags.FULLTEXT)), - // TODO: PDF search does not support case sensitive search (yet) - // Arguments.of(List.of(minimalNoteAllUpperCase), "test-library-with-attached-files.bib", "HELLO WORLD", EnumSet.of(SearchRules.SearchFlags.FULLTEXT, SearchRules.SearchFlags.CASE_SENSITIVE)), - Arguments.of(List.of(), "test-library-with-attached-files.bib", "NotExisting", EnumSet.of(SearchRules.SearchFlags.FULLTEXT, SearchRules.SearchFlags.CASE_SENSITIVE)) - ); - } - - @ParameterizedTest(name = "{index} => query={2}, searchFlags={3}, testFile={1}, expected={0}") - @MethodSource - public void searchLibrary(List expected, String testFile, String query, EnumSet searchFlags) throws Exception { - BibDatabase database = initializeDatabaseFromPath(testFile); - List matches = new DatabaseSearcher(new SearchQuery(query, searchFlags), database).getMatches(); - assertEquals(expected, matches); - } +// private static BibEntry titleSentenceCased = new BibEntry(StandardEntryType.Misc) +// .withCitationKey("title-sentence-cased") +// .withField(StandardField.TITLE, "Title Sentence Cased"); +// private static BibEntry titleMixedCased = new BibEntry(StandardEntryType.Misc) +// .withCitationKey("title-mixed-cased") +// .withField(StandardField.TITLE, "TiTle MiXed CaSed"); +// private static BibEntry titleUpperCased = new BibEntry(StandardEntryType.Misc) +// .withCitationKey("title-upper-cased") +// .withField(StandardField.TITLE, "TITLE UPPER CASED"); +// +// private static BibEntry mininimalSentenceCase = new BibEntry(StandardEntryType.Misc) +// .withCitationKey("minimal-sentence-case") +// .withFiles(Collections.singletonList(new LinkedFile("", "minimal-sentence-case.pdf", StandardFileType.PDF.getName()))); +// private static BibEntry minimalAllUpperCase = new BibEntry(StandardEntryType.Misc) +// .withCitationKey("minimal-all-upper-case") +// .withFiles(Collections.singletonList(new LinkedFile("", "minimal-all-upper-case.pdf", StandardFileType.PDF.getName()))); +// private static BibEntry minimalMixedCase = new BibEntry(StandardEntryType.Misc) +// .withCitationKey("minimal-mixed-case") +// .withFiles(Collections.singletonList(new LinkedFile("", "minimal-mixed-case.pdf", StandardFileType.PDF.getName()))); +// private static BibEntry minimalNoteSentenceCase = new BibEntry(StandardEntryType.Misc) +// .withCitationKey("minimal-note-sentence-case") +// .withFiles(Collections.singletonList(new LinkedFile("", "minimal-note-sentence-case.pdf", StandardFileType.PDF.getName()))); +// private static BibEntry minimalNoteAllUpperCase = new BibEntry(StandardEntryType.Misc) +// .withCitationKey("minimal-note-all-upper-case") +// .withFiles(Collections.singletonList(new LinkedFile("", "minimal-note-all-upper-case.pdf", StandardFileType.PDF.getName()))); +// private static BibEntry minimalNoteMixedCase = new BibEntry(StandardEntryType.Misc) +// .withCitationKey("minimal-note-mixed-case") +// .withFiles(Collections.singletonList(new LinkedFile("", "minimal-note-mixed-case.pdf", StandardFileType.PDF.getName()))); +// +// FilePreferences filePreferences = mock(FilePreferences.class); +// +// @TempDir +// private Path indexDir; +// private PdfIndexer pdfIndexer; +// +// private BibDatabase initializeDatabaseFromPath(String testFile) throws Exception { +// return initializeDatabaseFromPath(Path.of(Objects.requireNonNull(DatabaseSearcherWithBibFilesTest.class.getResource(testFile)).toURI())); +// } +// +// private BibDatabase initializeDatabaseFromPath(Path testFile) throws Exception { +// ParserResult result = new BibtexImporter(mock(ImportFormatPreferences.class, Answers.RETURNS_DEEP_STUBS), new DummyFileUpdateMonitor()).importDatabase(testFile); +// BibDatabase database = result.getDatabase(); +// +// BibDatabaseContext context = mock(BibDatabaseContext.class); +// when(context.getFileDirectories(Mockito.any())).thenReturn(List.of(testFile.getParent())); +// when(context.getFulltextIndexPath()).thenReturn(indexDir); +// when(context.getDatabase()).thenReturn(database); +// when(context.getEntries()).thenReturn(database.getEntries()); +// +//// Required because of {@Link org.jabref.model.search.FullTextSearchRule.FullTextSearchRule} +// Globals.stateManager.setActiveDatabase(context); +// PreferencesService preferencesService = mock(PreferencesService.class); +// when(preferencesService.getFilePreferences()).thenReturn(filePreferences); +// Globals.prefs = preferencesService; +// +// pdfIndexer = PdfIndexerManager.getIndexer(context, filePreferences); +// // Alternative - For debugging with Luke (part of the Apache Lucene distribution) +// // pdfIndexer = PdfIndexer.of(context, Path.of("C:\\temp\\index"), filePreferences); +// +// pdfIndexer.rebuildIndex(); +// return database; +// } +// +// @AfterEach +// public void tearDown() throws Exception { +// pdfIndexer.close(); +// } +// +// private static Stream searchLibrary() { +// return Stream.of( +// // empty library +// Arguments.of(List.of(), "empty.bib", "Test", EnumSet.noneOf(SearchFlags.class)), +// +// // test-library-title-casing +// +// Arguments.of(List.of(), "test-library-title-casing.bib", "NotExisting", EnumSet.noneOf(SearchFlags.class)), +// Arguments.of(List.of(titleSentenceCased, titleMixedCased, titleUpperCased), "test-library-title-casing.bib", "Title", EnumSet.noneOf(SearchFlags.SearchFlags.class)), +// +// Arguments.of(List.of(), "test-library-title-casing.bib", "title=NotExisting", EnumSet.noneOf(SearchFlags.class)), +// Arguments.of(List.of(titleSentenceCased, titleMixedCased, titleUpperCased), "test-library-title-casing.bib", "title=Title", EnumSet.noneOf(SearchFlags.SearchFlags.class)), +// +// Arguments.of(List.of(), "test-library-title-casing.bib", "title=TiTLE", EnumSet.of(SearchFlags.CASE_SENSITIVE)), +// Arguments.of(List.of(titleSentenceCased), "test-library-title-casing.bib", "title=Title", EnumSet.of(SearchFlags.CASE_SENSITIVE)), +// +// Arguments.of(List.of(), "test-library-title-casing.bib", "TiTLE", EnumSet.of(SearchFlags.CASE_SENSITIVE)), +// Arguments.of(List.of(titleMixedCased), "test-library-title-casing.bib", "TiTle", EnumSet.of(SearchFlags.CASE_SENSITIVE)), +// +// Arguments.of(List.of(), "test-library-title-casing.bib", "title=NotExisting", EnumSet.of(SearchFlags.CASE_SENSITIVE)), +// Arguments.of(List.of(titleMixedCased), "test-library-title-casing.bib", "title=TiTle", EnumSet.of(SearchFlags.CASE_SENSITIVE)), +// +// Arguments.of(List.of(), "test-library-title-casing.bib", "[Y]", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)), +// Arguments.of(List.of(titleUpperCased), "test-library-title-casing.bib", "[U]", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)), +// +// // Word boundaries +// Arguments.of(List.of(), "test-library-title-casing.bib", "\\bTit\\b", EnumSet.of(SearchFlags.REGULAR_EXPRESSION, SearchFlags.CASE_SENSITIVE)), +// Arguments.of(List.of(titleSentenceCased), "test-library-title-casing.bib", "\\bTitle\\b", EnumSet.of(SearchFlags.REGULAR_EXPRESSION, SearchFlags.CASE_SENSITIVE)), +// +// // test-library-with-attached-files +// +// Arguments.of(List.of(), "test-library-with-attached-files.bib", "This is a test.", EnumSet.of(SearchFlags.FULLTEXT, SearchFlags.CASE_SENSITIVE)), +// +// Arguments.of(List.of(mininimalSentenceCase, minimalAllUpperCase, minimalMixedCase), "test-library-with-attached-files.bib", "This is a short sentence, comma included.", EnumSet.of(SearchFlags.FULLTEXT)), +// Arguments.of(List.of(mininimalSentenceCase, minimalAllUpperCase, minimalMixedCase), "test-library-with-attached-files.bib", "comma", EnumSet.of(SearchFlags.FULLTEXT)), +// // TODO: PDF search does not support case sensitive search (yet) +// Arguments.of(List.of(minimalAllUpperCase, minimalMixedCase), "test-library-with-attached-files.bib", "THIS", EnumSet.of(SearchFlags.FULLTEXT, SearchFlags.CASE_SENSITIVE)), +// Arguments.of(List.of(minimalAllUpperCase), "test-library-with-attached-files.bib", "THIS is a short sentence, comma included.", EnumSet.of(SearchFlags.FULLTEXT, SearchFlags.CASE_SENSITIVE)), +// Arguments.of(List.of(minimalSentenceCase, minimalAllUpperCase, minimalMixedCase), "test-library-with-attached-files.bib", "comma", EnumSet.of(SearchFlags.FULLTEXT, SearchFlags.CASE_SENSITIVE)), +// Arguments.of(List.of(minimalNoteAllUpperCase), "test-library-with-attached-files.bib", "THIS IS A SHORT SENTENCE, COMMA INCLUDED.", EnumSet.of(SearchFlags.FULLTEXT, SearchFlags.CASE_SENSITIVE)), +// +// Arguments.of(List.of(), "test-library-with-attached-files.bib", "NotExisting", EnumSet.of(SearchFlags.FULLTEXT)), +// +// Arguments.of(List.of(minimalNoteSentenceCase, minimalNoteAllUpperCase, minimalNoteMixedCase), "test-library-with-attached-files.bib", "world", EnumSet.of(SearchFlags.FULLTEXT)), +// Arguments.of(List.of(minimalNoteSentenceCase, minimalNoteAllUpperCase, minimalNoteMixedCase), "test-library-with-attached-files.bib", "Hello World", EnumSet.of(SearchFlags.FULLTEXT)), +// // TODO: PDF search does not support case sensitive search (yet) +// Arguments.of(List.of(minimalNoteAllUpperCase), "test-library-with-attached-files.bib", "HELLO WORLD", EnumSet.of(SearchFlags.FULLTEXT, SearchFlags.CASE_SENSITIVE)), +// Arguments.of(List.of(), "test-library-with-attached-files.bib", "NotExisting", EnumSet.of(SearchFlags.FULLTEXT, SearchFlags.CASE_SENSITIVE)) +// ); +// } +// +// @ParameterizedTest(name = "{index} => query={2}, searchFlags={3}, testFile={1}, expected={0}") +// @MethodSource +// public void searchLibrary(List expected, String testFile, String query, EnumSet searchFlags) throws Exception { +// BibDatabase database = initializeDatabaseFromPath(testFile); +// List matches = new DatabaseSearcher(new SearchQuery(query, searchFlags), database).getMatches(); +// assertEquals(expected, matches); +// } } - - diff --git a/src/test/java/org/jabref/logic/search/LuceneTest.java b/src/test/java/org/jabref/logic/search/LuceneTest.java index 1acfa23a1c4..1e006c4e718 100644 --- a/src/test/java/org/jabref/logic/search/LuceneTest.java +++ b/src/test/java/org/jabref/logic/search/LuceneTest.java @@ -1,8 +1,7 @@ package org.jabref.logic.search; -import org.jabref.model.pdf.search.EnglishStemAnalyzer; - import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.TextField; @@ -19,7 +18,7 @@ public class LuceneTest { public static void main(String[] args) throws Exception { // Setup the analyzer - Analyzer analyzer = new EnglishStemAnalyzer(); + Analyzer analyzer = new StandardAnalyzer(); // Store the index in memory Directory directory = new ByteBuffersDirectory(); @@ -47,7 +46,7 @@ public static void search(String queryString, Directory directory, Analyzer anal for (ScoreDoc scoreDoc : hits) { Document doc = searcher.doc(scoreDoc.doc); - System.out.println(doc.get("content")); +// System.out.println(doc.get("content")); } } } diff --git a/src/test/java/org/jabref/logic/search/SearchQueryTest.java b/src/test/java/org/jabref/logic/search/SearchQueryTest.java index 16501f0c7f7..2e775bbec8f 100644 --- a/src/test/java/org/jabref/logic/search/SearchQueryTest.java +++ b/src/test/java/org/jabref/logic/search/SearchQueryTest.java @@ -5,11 +5,11 @@ import java.util.Collections; import java.util.EnumSet; -import org.jabref.logic.pdf.search.LuceneIndexer; -import org.jabref.logic.pdf.search.LuceneSearcher; +import org.jabref.logic.search.indexing.LuceneIndexer; +import org.jabref.logic.search.retrieval.LuceneSearcher; import org.jabref.model.database.BibDatabase; import org.jabref.model.database.BibDatabaseContext; -import org.jabref.model.search.rules.SearchRules; +import org.jabref.model.search.SearchFlags; import org.jabref.preferences.FilePreferences; import org.jabref.preferences.PreferencesService; @@ -49,7 +49,7 @@ public void setUp(@TempDir Path indexDir) throws IOException { @Test public void nullWhenQueryBlank() { - assertNull(new SearchQuery("", EnumSet.noneOf(SearchRules.SearchFlags.class)).getQuery()); + assertNull(new SearchQuery("", EnumSet.noneOf(SearchFlags.class)).getQuery()); } // @Test diff --git a/src/test/java/org/jabref/logic/pdf/search/DocumentReaderTest.java b/src/test/java/org/jabref/logic/search/indexing/DocumentReaderTest.java similarity index 96% rename from src/test/java/org/jabref/logic/pdf/search/DocumentReaderTest.java rename to src/test/java/org/jabref/logic/search/indexing/DocumentReaderTest.java index adca115c789..a6d7548c3bb 100644 --- a/src/test/java/org/jabref/logic/pdf/search/DocumentReaderTest.java +++ b/src/test/java/org/jabref/logic/search/indexing/DocumentReaderTest.java @@ -1,4 +1,4 @@ -package org.jabref.logic.pdf.search; +package org.jabref.logic.search.indexing; import java.nio.file.Path; import java.util.Collections; @@ -6,7 +6,6 @@ import java.util.Optional; import java.util.stream.Stream; -import org.jabref.logic.pdf.search.DocumentReader; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.LinkedFile; diff --git a/src/test/java/org/jabref/logic/pdf/search/LuceneIndexerTest.java b/src/test/java/org/jabref/logic/search/indexing/LuceneIndexerTest.java similarity index 98% rename from src/test/java/org/jabref/logic/pdf/search/LuceneIndexerTest.java rename to src/test/java/org/jabref/logic/search/indexing/LuceneIndexerTest.java index 1645445fa9c..f4c845d9189 100644 --- a/src/test/java/org/jabref/logic/pdf/search/LuceneIndexerTest.java +++ b/src/test/java/org/jabref/logic/search/indexing/LuceneIndexerTest.java @@ -1,4 +1,4 @@ -package org.jabref.logic.pdf.search; +package org.jabref.logic.search.indexing; import java.io.IOException; import java.nio.file.Path; @@ -152,7 +152,7 @@ public void metaDataIndex() throws IOException { } @Test - public void testFlushIndex() throws IOException { + public void flushIndex() throws IOException { // given BibEntry entry = new BibEntry(StandardEntryType.PhdThesis); entry.setCitationKey("Example2017"); diff --git a/src/test/java/org/jabref/logic/search/retrieval/LuceneSearcherTest.java b/src/test/java/org/jabref/logic/search/retrieval/LuceneSearcherTest.java new file mode 100644 index 00000000000..7418d2151c6 --- /dev/null +++ b/src/test/java/org/jabref/logic/search/retrieval/LuceneSearcherTest.java @@ -0,0 +1,144 @@ +package org.jabref.logic.search.retrieval; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; + +import org.jabref.logic.search.SearchQuery; +import org.jabref.logic.search.indexing.LuceneIndexer; +import org.jabref.logic.util.StandardFileType; +import org.jabref.model.database.BibDatabase; +import org.jabref.model.database.BibDatabaseContext; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.LinkedFile; +import org.jabref.model.entry.types.StandardEntryType; +import org.jabref.model.search.LuceneSearchResults; +import org.jabref.model.search.SearchFlags; +import org.jabref.preferences.FilePreferences; +import org.jabref.preferences.PreferencesService; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mockito; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class LuceneSearcherTest { + + private LuceneSearcher searcher; + private PreferencesService preferencesService; + private BibDatabase bibDatabase; + private BibDatabaseContext bibDatabaseContext; + + @BeforeEach + public void setUp(@TempDir Path indexDir) throws IOException { + preferencesService = mock(PreferencesService.class); + when(preferencesService.getFilePreferences()).thenReturn(mock(FilePreferences.class)); + + bibDatabase = new BibDatabase(); + bibDatabaseContext = mock(BibDatabaseContext.class); + when(bibDatabaseContext.getFileDirectories(Mockito.any())).thenReturn(Collections.singletonList(Path.of("src/test/resources/pdfs"))); + when(bibDatabaseContext.getFulltextIndexPath()).thenReturn(indexDir); + when(bibDatabaseContext.getDatabase()).thenReturn(bibDatabase); + when(bibDatabaseContext.getEntries()).thenReturn(bibDatabase.getEntries()); + } + + private void initIndexer() throws IOException { + LuceneIndexer indexer = LuceneIndexer.of(bibDatabaseContext, preferencesService); + searcher = LuceneSearcher.of(bibDatabaseContext); + + for (BibEntry bibEntry : bibDatabaseContext.getEntries()) { + indexer.addBibFieldsToIndex(bibEntry); + indexer.addLinkedFilesToIndex(bibEntry); + } + } + + @Test + public void searchForTest() throws IOException { + insertPdfsForSearch(); + initIndexer(); + + HashMap searchResults = searcher.search(new SearchQuery("", EnumSet.noneOf(SearchFlags.class))); + int hits = searchResults.keySet().stream().mapToInt((key) -> searchResults.get(key).numSearchResults()).sum(); + assertEquals(8, hits); + } + + @Test + public void searchForUniversity() throws IOException { + insertPdfsForSearch(); + initIndexer(); + + HashMap searchResults = searcher.search(new SearchQuery("University", EnumSet.noneOf(SearchFlags.class))); + int hits = searchResults.keySet().stream().mapToInt((key) -> searchResults.get(key).numSearchResults()).sum(); + assertEquals(1, hits); + } + + @Test + public void searchForStopWord() throws IOException { + insertPdfsForSearch(); + initIndexer(); + + HashMap searchResults = searcher.search(new SearchQuery("and", EnumSet.noneOf(SearchFlags.class))); + int hits = searchResults.keySet().stream().mapToInt((key) -> searchResults.get(key).numSearchResults()).sum(); + assertEquals(0, hits); + } + + @Test + public void searchForSecond() throws IOException { + insertPdfsForSearch(); + initIndexer(); + + HashMap searchResults = searcher.search(new SearchQuery("second", EnumSet.noneOf(SearchFlags.class))); + int hits = searchResults.keySet().stream().mapToInt((key) -> searchResults.get(key).numSearchResults()).sum(); + assertEquals(4, hits); + } + + @Test + public void searchForAnnotation() throws IOException { + insertPdfsForSearch(); + initIndexer(); + + HashMap searchResults = searcher.search(new SearchQuery("annotation", EnumSet.noneOf(SearchFlags.class))); + int hits = searchResults.keySet().stream().mapToInt((key) -> searchResults.get(key).numSearchResults()).sum(); + assertEquals(2, hits); + } + + @Test + public void searchForEmptyString() throws IOException { + insertPdfsForSearch(); + initIndexer(); + + HashMap searchResults = searcher.search(new SearchQuery("", EnumSet.noneOf(SearchFlags.class))); + int hits = searchResults.keySet().stream().mapToInt((key) -> searchResults.get(key).numSearchResults()).sum(); + assertEquals(0, hits); + } + + @Test + public void searchWithNullString() { + assertThrows(NullPointerException.class, () -> searcher.search(null)); + } + + private void insertPdfsForSearch() { + when(preferencesService.getFilePreferences().shouldFulltextIndexLinkedFiles()).thenReturn(true); + + BibEntry examplePdf = new BibEntry(StandardEntryType.Article); + examplePdf.setFiles(Collections.singletonList(new LinkedFile("Example Entry", "example.pdf", StandardFileType.PDF.getName()))); + bibDatabase.insertEntry(examplePdf); + + BibEntry metaDataEntry = new BibEntry(StandardEntryType.Article); + metaDataEntry.setFiles(Collections.singletonList(new LinkedFile("Metadata Entry", "metaData.pdf", StandardFileType.PDF.getName()))); + metaDataEntry.setCitationKey("MetaData2017"); + bibDatabase.insertEntry(metaDataEntry); + + BibEntry exampleThesis = new BibEntry(StandardEntryType.PhdThesis); + exampleThesis.setFiles(Collections.singletonList(new LinkedFile("Example Thesis", "thesis-example.pdf", StandardFileType.PDF.getName()))); + exampleThesis.setCitationKey("ExampleThesis"); + bibDatabase.insertEntry(exampleThesis); + } +} diff --git a/src/test/java/org/jabref/model/groups/GroupTreeNodeTest.java b/src/test/java/org/jabref/model/groups/GroupTreeNodeTest.java index 142c2a0fedf..c9be98f60f2 100644 --- a/src/test/java/org/jabref/model/groups/GroupTreeNodeTest.java +++ b/src/test/java/org/jabref/model/groups/GroupTreeNodeTest.java @@ -10,9 +10,9 @@ import org.jabref.model.FieldChange; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; +import org.jabref.model.search.SearchFlags; import org.jabref.model.search.matchers.AndMatcher; import org.jabref.model.search.matchers.OrMatcher; -import org.jabref.model.search.rules.SearchRules; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -83,7 +83,7 @@ private static AbstractGroup getKeywordGroup(String name) { } private static AbstractGroup getSearchGroup(String name) { - return new SearchGroup(name, GroupHierarchyType.INCLUDING, "searchExpression", EnumSet.noneOf(SearchRules.SearchFlags.class)); + return new SearchGroup(name, GroupHierarchyType.INCLUDING, "searchExpression", EnumSet.noneOf(SearchFlags.class)); } private static AbstractGroup getExplict(String name) { @@ -255,7 +255,7 @@ void setGroupExplicitToSearchDoesNotKeepPreviousAssignments() { ExplicitGroup oldGroup = new ExplicitGroup("OldGroup", GroupHierarchyType.INDEPENDENT, ','); oldGroup.add(entry); GroupTreeNode node = GroupTreeNode.fromGroup(oldGroup); - AbstractGroup newGroup = new SearchGroup("NewGroup", GroupHierarchyType.INDEPENDENT, "test", EnumSet.noneOf(SearchRules.SearchFlags.class)); + AbstractGroup newGroup = new SearchGroup("NewGroup", GroupHierarchyType.INDEPENDENT, "test", EnumSet.noneOf(SearchFlags.class)); node.setGroup(newGroup, true, true, entries); @@ -333,7 +333,7 @@ void onlySubgroupsContainAllEntries() { @Test void addEntriesToGroupWorksNotForGroupsNotSupportingExplicitAddingOfEntries() { - GroupTreeNode searchGroup = new GroupTreeNode(new SearchGroup("Search A", GroupHierarchyType.INCLUDING, "searchExpression", EnumSet.noneOf(SearchRules.SearchFlags.class))); + GroupTreeNode searchGroup = new GroupTreeNode(new SearchGroup("Search A", GroupHierarchyType.INCLUDING, "searchExpression", EnumSet.noneOf(SearchFlags.class))); List fieldChanges = searchGroup.addEntriesToGroup(entries); assertEquals(Collections.emptyList(), fieldChanges); @@ -341,7 +341,7 @@ void addEntriesToGroupWorksNotForGroupsNotSupportingExplicitAddingOfEntries() { @Test void removeEntriesFromGroupWorksNotForGroupsNotSupportingExplicitRemovalOfEntries() { - GroupTreeNode searchGroup = new GroupTreeNode(new SearchGroup("Search A", GroupHierarchyType.INCLUDING, "searchExpression", EnumSet.noneOf(SearchRules.SearchFlags.class))); + GroupTreeNode searchGroup = new GroupTreeNode(new SearchGroup("Search A", GroupHierarchyType.INCLUDING, "searchExpression", EnumSet.noneOf(SearchFlags.class))); List fieldChanges = searchGroup.removeEntriesFromGroup(entries); assertEquals(Collections.emptyList(), fieldChanges); diff --git a/src/test/java/org/jabref/model/groups/SearchGroupTest.java b/src/test/java/org/jabref/model/groups/SearchGroupTest.java index 470b18c597f..806c3afe9f9 100644 --- a/src/test/java/org/jabref/model/groups/SearchGroupTest.java +++ b/src/test/java/org/jabref/model/groups/SearchGroupTest.java @@ -6,7 +6,7 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.types.StandardEntryType; -import org.jabref.model.search.rules.SearchRules; +import org.jabref.model.search.SearchFlags; import org.junit.jupiter.api.Test; @@ -29,21 +29,21 @@ public class SearchGroupTest { @Test public void containsFindsWords() { - SearchGroup groupPositive = new SearchGroup("A", GroupHierarchyType.INDEPENDENT, "Test", EnumSet.noneOf(SearchRules.SearchFlags.class)); + SearchGroup groupPositive = new SearchGroup("A", GroupHierarchyType.INDEPENDENT, "Test", EnumSet.noneOf(SearchFlags.class)); List positiveResult = List.of(entry1D, entry2D); assertTrue(groupPositive.containsAll(positiveResult)); } @Test public void containsDoesNotFindWords() { - SearchGroup groupNegative = new SearchGroup("A", GroupHierarchyType.INDEPENDENT, "Unknown", EnumSet.noneOf(SearchRules.SearchFlags.class)); + SearchGroup groupNegative = new SearchGroup("A", GroupHierarchyType.INDEPENDENT, "Unknown", EnumSet.noneOf(SearchFlags.class)); List positiveResult = List.of(entry1D, entry2D); assertFalse(groupNegative.containsAny(positiveResult)); } @Test public void containsFindsWordWithRegularExpression() { - SearchGroup group = new SearchGroup("myExplicitGroup", GroupHierarchyType.INDEPENDENT, "anyfield=rev*", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)); + SearchGroup group = new SearchGroup("myExplicitGroup", GroupHierarchyType.INDEPENDENT, "anyfield=rev*", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)); BibEntry entry = new BibEntry(); entry.addKeyword("review", ','); @@ -52,7 +52,7 @@ public void containsFindsWordWithRegularExpression() { @Test public void containsDoesNotFindsWordWithInvalidRegularExpression() { - SearchGroup group = new SearchGroup("myExplicitGroup", GroupHierarchyType.INDEPENDENT, "anyfield=*rev*", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)); + SearchGroup group = new SearchGroup("myExplicitGroup", GroupHierarchyType.INDEPENDENT, "anyfield=*rev*", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)); BibEntry entry = new BibEntry(); entry.addKeyword("review", ','); @@ -61,7 +61,7 @@ public void containsDoesNotFindsWordWithInvalidRegularExpression() { @Test public void notQueryWorksWithLeftPartOfQuery() { - SearchGroup groupToBeClassified = new SearchGroup("to-be-classified", GroupHierarchyType.INDEPENDENT, "NOT(groups=alpha) AND NOT(groups=beta)", EnumSet.noneOf(SearchRules.SearchFlags.class)); + SearchGroup groupToBeClassified = new SearchGroup("to-be-classified", GroupHierarchyType.INDEPENDENT, "NOT(groups=alpha) AND NOT(groups=beta)", EnumSet.noneOf(SearchFlags.class)); BibEntry alphaEntry = new BibEntry() .withCitationKey("alpha") @@ -71,7 +71,7 @@ public void notQueryWorksWithLeftPartOfQuery() { @Test public void notQueryWorksWithLRightPartOfQuery() { - SearchGroup groupToBeClassified = new SearchGroup("to-be-classified", GroupHierarchyType.INDEPENDENT, "NOT(groups=alpha) AND NOT(groups=beta)", EnumSet.noneOf(SearchRules.SearchFlags.class)); + SearchGroup groupToBeClassified = new SearchGroup("to-be-classified", GroupHierarchyType.INDEPENDENT, "NOT(groups=alpha) AND NOT(groups=beta)", EnumSet.noneOf(SearchFlags.class)); BibEntry betaEntry = new BibEntry() .withCitationKey("beta") From d5c87df194478e6f983d63d99d8d44d9acde715e Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Fri, 24 May 2024 07:41:31 +0300 Subject: [PATCH 085/256] rewriteRun --- .../fileannotationtab/FulltextSearchResultsTab.java | 2 +- .../org/jabref/gui/maintable/columns/FileColumn.java | 2 +- .../logic/search/retrieval/LuceneSearcherTest.java | 12 ++++++------ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java b/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java index 3dd3b625913..2e0f3d86936 100644 --- a/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java @@ -31,8 +31,8 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.LinkedFile; import org.jabref.model.search.LuceneSearchResults; -import org.jabref.model.search.SearchResult; import org.jabref.model.search.SearchFlags; +import org.jabref.model.search.SearchResult; import org.jabref.preferences.PreferencesService; import org.slf4j.Logger; diff --git a/src/main/java/org/jabref/gui/maintable/columns/FileColumn.java b/src/main/java/org/jabref/gui/maintable/columns/FileColumn.java index dda17d460b1..2dbbb58bf58 100644 --- a/src/main/java/org/jabref/gui/maintable/columns/FileColumn.java +++ b/src/main/java/org/jabref/gui/maintable/columns/FileColumn.java @@ -148,7 +148,7 @@ private ContextMenu createFileMenu(BibEntryTableViewModel entry, List linkedFiles) { - if (linkedFiles.size() > 0 && stateManager.getSearchResults().containsKey(database) && stateManager.getSearchResults().get(database).containsKey(entry.getEntry()) && stateManager.getSearchResults().get(database).get(entry.getEntry()).hasFulltextResults()) { + if (!linkedFiles.isEmpty() && stateManager.getSearchResults().containsKey(database) && stateManager.getSearchResults().get(database).containsKey(entry.getEntry()) && stateManager.getSearchResults().get(database).get(entry.getEntry()).hasFulltextResults()) { return IconTheme.JabRefIcons.FILE_SEARCH.getGraphicNode(); } if (linkedFiles.size() > 1) { diff --git a/src/test/java/org/jabref/logic/search/retrieval/LuceneSearcherTest.java b/src/test/java/org/jabref/logic/search/retrieval/LuceneSearcherTest.java index 7418d2151c6..8415c9ae86f 100644 --- a/src/test/java/org/jabref/logic/search/retrieval/LuceneSearcherTest.java +++ b/src/test/java/org/jabref/logic/search/retrieval/LuceneSearcherTest.java @@ -65,7 +65,7 @@ public void searchForTest() throws IOException { initIndexer(); HashMap searchResults = searcher.search(new SearchQuery("", EnumSet.noneOf(SearchFlags.class))); - int hits = searchResults.keySet().stream().mapToInt((key) -> searchResults.get(key).numSearchResults()).sum(); + int hits = searchResults.keySet().stream().mapToInt(key -> searchResults.get(key).numSearchResults()).sum(); assertEquals(8, hits); } @@ -75,7 +75,7 @@ public void searchForUniversity() throws IOException { initIndexer(); HashMap searchResults = searcher.search(new SearchQuery("University", EnumSet.noneOf(SearchFlags.class))); - int hits = searchResults.keySet().stream().mapToInt((key) -> searchResults.get(key).numSearchResults()).sum(); + int hits = searchResults.keySet().stream().mapToInt(key -> searchResults.get(key).numSearchResults()).sum(); assertEquals(1, hits); } @@ -85,7 +85,7 @@ public void searchForStopWord() throws IOException { initIndexer(); HashMap searchResults = searcher.search(new SearchQuery("and", EnumSet.noneOf(SearchFlags.class))); - int hits = searchResults.keySet().stream().mapToInt((key) -> searchResults.get(key).numSearchResults()).sum(); + int hits = searchResults.keySet().stream().mapToInt(key -> searchResults.get(key).numSearchResults()).sum(); assertEquals(0, hits); } @@ -95,7 +95,7 @@ public void searchForSecond() throws IOException { initIndexer(); HashMap searchResults = searcher.search(new SearchQuery("second", EnumSet.noneOf(SearchFlags.class))); - int hits = searchResults.keySet().stream().mapToInt((key) -> searchResults.get(key).numSearchResults()).sum(); + int hits = searchResults.keySet().stream().mapToInt(key -> searchResults.get(key).numSearchResults()).sum(); assertEquals(4, hits); } @@ -105,7 +105,7 @@ public void searchForAnnotation() throws IOException { initIndexer(); HashMap searchResults = searcher.search(new SearchQuery("annotation", EnumSet.noneOf(SearchFlags.class))); - int hits = searchResults.keySet().stream().mapToInt((key) -> searchResults.get(key).numSearchResults()).sum(); + int hits = searchResults.keySet().stream().mapToInt(key -> searchResults.get(key).numSearchResults()).sum(); assertEquals(2, hits); } @@ -115,7 +115,7 @@ public void searchForEmptyString() throws IOException { initIndexer(); HashMap searchResults = searcher.search(new SearchQuery("", EnumSet.noneOf(SearchFlags.class))); - int hits = searchResults.keySet().stream().mapToInt((key) -> searchResults.get(key).numSearchResults()).sum(); + int hits = searchResults.keySet().stream().mapToInt(key -> searchResults.get(key).numSearchResults()).sum(); assertEquals(0, hits); } From bfc33c85e6a8d1a054a51748a610e8e4d97ffe48 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Fri, 31 May 2024 10:19:58 +0300 Subject: [PATCH 086/256] Remove bibEntry from DocumentReader --- .../logic/search/indexing/DocumentReader.java | 88 +++++++++---------- .../logic/search/indexing/LuceneIndexer.java | 13 ++- .../model/search/SearchFieldConstants.java | 13 ++- .../search/indexing/DocumentReaderTest.java | 2 +- 4 files changed, 54 insertions(+), 62 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/indexing/DocumentReader.java b/src/main/java/org/jabref/logic/search/indexing/DocumentReader.java index f945e0dac86..34af15d521b 100644 --- a/src/main/java/org/jabref/logic/search/indexing/DocumentReader.java +++ b/src/main/java/org/jabref/logic/search/indexing/DocumentReader.java @@ -6,12 +6,12 @@ import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; import java.util.stream.Collectors; -import org.jabref.gui.LibraryTab; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.LinkedFile; @@ -41,26 +41,19 @@ */ public final class DocumentReader { - private static final Logger LOGGER = LoggerFactory.getLogger(LibraryTab.class); + private static final Logger LOGGER = LoggerFactory.getLogger(DocumentReader.class); private static final Pattern HYPHEN_LINEBREAK_PATTERN = Pattern.compile("\\-\n"); private static final Pattern LINEBREAK_WITHOUT_PERIOD_PATTERN = Pattern.compile("([^\\\\.])\\n"); - private final BibEntry entry; private final FilePreferences filePreferences; /** * Creates a new DocumentReader using a BibEntry. * - * @param bibEntry Must not be null and must have at least one LinkedFile. */ - public DocumentReader(BibEntry bibEntry, FilePreferences filePreferences) { + public DocumentReader(FilePreferences filePreferences) { this.filePreferences = filePreferences; - if (bibEntry.getFiles().isEmpty()) { - throw new IllegalStateException("There are no linked PDF files to this BibEntry."); - } - - this.entry = bibEntry; } /** @@ -70,10 +63,7 @@ public DocumentReader(BibEntry bibEntry, FilePreferences filePreferences) { */ public Optional> readLinkedPdf(BibDatabaseContext databaseContext, LinkedFile pdf) { Optional pdfPath = pdf.findIn(databaseContext, filePreferences); - if (pdfPath.isPresent()) { - return Optional.of(readPdfContents(pdf, pdfPath.get())); - } - return Optional.empty(); + return pdfPath.map(path -> readPdfContents(pdf, path)); } /** @@ -81,7 +71,7 @@ public Optional> readLinkedPdf(BibDatabaseContext databaseContext * * @return A List of Documents with the (meta)data. Can be empty if there is a problem reading the LinkedFile. */ - public List readLinkedPdfs(BibDatabaseContext databaseContext) { + public List readLinkedPdfs(BibDatabaseContext databaseContext, BibEntry entry) { return entry.getFiles().stream() .map(pdf -> readLinkedPdf(databaseContext, pdf)) .filter(Optional::isPresent) @@ -90,18 +80,16 @@ public List readLinkedPdfs(BibDatabaseContext databaseContext) { .collect(Collectors.toList()); } - private List readPdfContents(LinkedFile pdf, Path resolvedPdfPath) { + public List readPdfContents(LinkedFile pdf, Path resolvedPdfPath) { List pages = new ArrayList<>(); try (PDDocument pdfDocument = Loader.loadPDF(resolvedPdfPath.toFile())) { - for (int pageNumber = 0; pageNumber < pdfDocument.getNumberOfPages(); pageNumber++) { + int numberOfPages = pdfDocument.getNumberOfPages(); + for (int pageNumber = 1; pageNumber <= numberOfPages; pageNumber++) { Document newDocument = new Document(); addIdentifiers(newDocument, pdf.getLink()); addMetaData(newDocument, resolvedPdfPath, pageNumber); - try { - addContentIfNotEmpty(pdfDocument, newDocument, pageNumber); - } catch (IOException e) { - LOGGER.warn("Could not read page {} of {}", pageNumber, resolvedPdfPath.toAbsolutePath(), e); - } + addContentIfNotEmpty(pdfDocument, newDocument, resolvedPdfPath, pageNumber); + pages.add(newDocument); } } catch (IOException e) { @@ -110,22 +98,12 @@ private List readPdfContents(LinkedFile pdf, Path resolvedPdfPath) { if (pages.isEmpty()) { Document newDocument = new Document(); addIdentifiers(newDocument, pdf.getLink()); - addMetaData(newDocument, resolvedPdfPath, 0); + addMetaData(newDocument, resolvedPdfPath, 1); pages.add(newDocument); } return pages; } - private void addMetaData(Document newDocument, Path resolvedPdfPath, int pageNumber) { - try { - BasicFileAttributes attributes = Files.readAttributes(resolvedPdfPath, BasicFileAttributes.class); - addStringField(newDocument, MODIFIED, String.valueOf(attributes.lastModifiedTime().to(TimeUnit.SECONDS))); - } catch (IOException e) { - LOGGER.error("Could not read timestamp for {}", resolvedPdfPath, e); - } - addStringField(newDocument, PAGE_NUMBER, String.valueOf(pageNumber)); - } - private void addStringField(Document newDocument, String field, String value) { if (!isValidField(value)) { return; @@ -142,21 +120,41 @@ public static String mergeLines(String text) { return LINEBREAK_WITHOUT_PERIOD_PATTERN.matcher(mergedHyphenNewlines).replaceAll("$1 "); } - private void addContentIfNotEmpty(PDDocument pdfDocument, Document newDocument, int pageNumber) throws IOException { + private void addMetaData(Document newDocument, Path resolvedPdfPath, int pageNumber) { + try { + BasicFileAttributes attributes = Files.readAttributes(resolvedPdfPath, BasicFileAttributes.class); + addStringField(newDocument, MODIFIED, String.valueOf(attributes.lastModifiedTime().to(TimeUnit.SECONDS))); + } catch (IOException e) { + LOGGER.error("Could not read timestamp for {}", resolvedPdfPath, e); + } + addStringField(newDocument, PAGE_NUMBER, String.valueOf(pageNumber)); + } + + private void addContentIfNotEmpty(PDDocument pdfDocument, Document newDocument, Path resolvedPath, int pageNumber) { PDFTextStripper pdfTextStripper = new PDFTextStripper(); pdfTextStripper.setLineSeparator("\n"); - // Apache PDFTextStripper is 1-based. See {@link org.apache.pdfbox.text.PDFTextStripper.processPages} - pdfTextStripper.setStartPage(pageNumber + 1); - pdfTextStripper.setEndPage(pageNumber + 1); + pdfTextStripper.setStartPage(pageNumber); + pdfTextStripper.setEndPage(pageNumber); - String pdfContent = pdfTextStripper.getText(pdfDocument); - if (StringUtil.isNotBlank(pdfContent)) { - newDocument.add(new TextField(CONTENT, mergeLines(pdfContent), Field.Store.YES)); - } - PDPage page = pdfDocument.getPage(pageNumber); - List annotations = page.getAnnotations().stream().filter(annotation -> annotation.getContents() != null).map(PDAnnotation::getContents).collect(Collectors.toList()); - if (!annotations.isEmpty()) { - newDocument.add(new TextField(ANNOTATIONS, annotations.stream().collect(Collectors.joining("\n")), Field.Store.YES)); + try { + String pdfContent = pdfTextStripper.getText(pdfDocument); + if (StringUtil.isNotBlank(pdfContent)) { + newDocument.add(new TextField(CONTENT, mergeLines(pdfContent), Field.Store.YES)); + } + + // Apache PDFTextStripper is 1-based. See {@link org.apache.pdfbox.text.PDFTextStripper.processPages} + PDPage page = pdfDocument.getPage(pageNumber - 1); + List annotations = page.getAnnotations() + .stream() + .map(PDAnnotation::getContents) + .filter(Objects::nonNull) + .toList(); + + if (!annotations.isEmpty()) { + newDocument.add(new TextField(ANNOTATIONS, String.join("\n", annotations), Field.Store.YES)); + } + } catch (IOException e) { + LOGGER.warn("Could not read page {} of {}", pageNumber, resolvedPath.toAbsolutePath(), e); } } diff --git a/src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java b/src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java index 5e07ec1af23..83096afe3a3 100644 --- a/src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java @@ -13,7 +13,6 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import org.jabref.gui.LibraryTab; import org.jabref.logic.util.StandardFileType; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; @@ -48,22 +47,20 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * Indexes the text of PDF files and adds it into the lucene search index. - */ public class LuceneIndexer { - private static final Logger LOGGER = LoggerFactory.getLogger(LibraryTab.class); + private static final Logger LOGGER = LoggerFactory.getLogger(LuceneIndexer.class); private final Directory directoryToIndex; private final BibDatabaseContext databaseContext; - private final PreferencesService preferences; + private final DocumentReader documentReader; public LuceneIndexer(BibDatabaseContext databaseContext, PreferencesService preferences) throws IOException { this.databaseContext = databaseContext; - this.directoryToIndex = new NIOFSDirectory(databaseContext.getFulltextIndexPath()); this.preferences = preferences; + documentReader = new DocumentReader(preferences.getFilePreferences()); + this.directoryToIndex = new NIOFSDirectory(databaseContext.getFulltextIndexPath()); } public static LuceneIndexer of(BibDatabaseContext databaseContext, PreferencesService preferences) throws IOException { @@ -227,7 +224,7 @@ private void writeFileToIndex(BibEntry entry, LinkedFile linkedFile) { // if there is no index yet, don't need to check anything! } // If no document was found, add the new one - Optional> pages = new DocumentReader(entry, preferences.getFilePreferences()).readLinkedPdf(this.databaseContext, linkedFile); + Optional> pages = documentReader.readLinkedPdf(this.databaseContext, linkedFile); if (pages.isPresent()) { try (IndexWriter indexWriter = new IndexWriter(directoryToIndex, new IndexWriterConfig( diff --git a/src/main/java/org/jabref/model/search/SearchFieldConstants.java b/src/main/java/org/jabref/model/search/SearchFieldConstants.java index 8da81395c18..455175e74ae 100644 --- a/src/main/java/org/jabref/model/search/SearchFieldConstants.java +++ b/src/main/java/org/jabref/model/search/SearchFieldConstants.java @@ -5,17 +5,14 @@ public class SearchFieldConstants { + public static final String VERSION = "99"; public static final String FILE_FIELDS_PREFIX = "f_"; - public static final String BIB_ENTRY_ID_HASH = "id_hash"; - public static final String PATH = FILE_FIELDS_PREFIX + "path"; - public static final String CONTENT = "content"; + public static final String CONTENT = FILE_FIELDS_PREFIX + "content"; + public static final String ANNOTATIONS = FILE_FIELDS_PREFIX + "annotations"; public static final String PAGE_NUMBER = FILE_FIELDS_PREFIX + "pageNumber"; - public static final String ANNOTATIONS = "annotations"; public static final String MODIFIED = FILE_FIELDS_PREFIX + "modified"; - - public static final String[] PDF_FIELDS = new String[]{PATH, CONTENT, ANNOTATIONS}; + public static final String BIB_ENTRY_ID_HASH = "id_hash"; + public static final String[] PDF_FIELDS = {PATH, CONTENT, ANNOTATIONS}; public static Set searchableBibFields = new HashSet<>(); - - public static final String VERSION = "99"; } diff --git a/src/test/java/org/jabref/logic/search/indexing/DocumentReaderTest.java b/src/test/java/org/jabref/logic/search/indexing/DocumentReaderTest.java index a6d7548c3bb..ac44f90c4fa 100644 --- a/src/test/java/org/jabref/logic/search/indexing/DocumentReaderTest.java +++ b/src/test/java/org/jabref/logic/search/indexing/DocumentReaderTest.java @@ -45,7 +45,7 @@ public void unknownFileTestShouldReturnEmptyList() { entry.setFiles(Collections.singletonList(new LinkedFile("Wrong path", "NOT_PRESENT.pdf", "Type"))); // when - final List emptyDocumentList = new DocumentReader(entry, filePreferences).readLinkedPdfs(databaseContext); + final List emptyDocumentList = new DocumentReader(filePreferences).readLinkedPdfs(databaseContext, entry); // then assertEquals(Collections.emptyList(), emptyDocumentList); From c0098c95902445969c51ff2a9baef2f213120b27 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Mon, 3 Jun 2024 20:07:16 +0300 Subject: [PATCH 087/256] Rewrite LuceneIndexer --- src/main/java/org/jabref/gui/LibraryTab.java | 94 ++- .../ExternalFilesEntryLinker.java | 41 +- .../RebuildFulltextSearchIndexAction.java | 11 +- .../org/jabref/gui/util/BackgroundTask.java | 13 +- .../search/indexing/IndexingTaskManager.java | 137 +---- .../logic/search/indexing/LuceneIndexer.java | 549 +++++++++++------- .../model/database/BibDatabaseContext.java | 2 +- src/main/resources/l10n/JabRef_en.properties | 6 +- 8 files changed, 419 insertions(+), 434 deletions(-) diff --git a/src/main/java/org/jabref/gui/LibraryTab.java b/src/main/java/org/jabref/gui/LibraryTab.java index 61bdb2d3dfe..f4717fec690 100644 --- a/src/main/java/org/jabref/gui/LibraryTab.java +++ b/src/main/java/org/jabref/gui/LibraryTab.java @@ -1,13 +1,13 @@ package org.jabref.gui; -import java.io.IOException; import java.nio.file.Path; -import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Random; +import java.util.Set; import java.util.stream.Collectors; import javax.swing.undo.UndoManager; @@ -90,6 +90,7 @@ import com.google.common.eventbus.Subscribe; import com.tobiasdiez.easybind.EasyBind; import com.tobiasdiez.easybind.Subscription; +import org.apache.lucene.index.IndexWriterConfig; import org.controlsfx.control.NotificationPane; import org.controlsfx.control.action.Action; import org.slf4j.Logger; @@ -148,20 +149,19 @@ private enum PanelMode { MAIN_TABLE, MAIN_TABLE_AND_ENTRY_EDITOR } private Optional changeMonitor = Optional.empty(); private BackgroundTask dataLoadingTask; - - private final IndexingTaskManager indexingTaskManager; + private LuceneIndexer luceneIndexer; private final TaskExecutor taskExecutor; private final DirectoryMonitorManager directoryMonitorManager; private LibraryTab(BibDatabaseContext bibDatabaseContext, - LibraryTabContainer tabContainer, - DialogService dialogService, - PreferencesService preferencesService, - StateManager stateManager, - FileUpdateMonitor fileUpdateMonitor, - BibEntryTypesManager entryTypesManager, - CountingUndoManager undoManager, - TaskExecutor taskExecutor) { + LibraryTabContainer tabContainer, + DialogService dialogService, + PreferencesService preferencesService, + StateManager stateManager, + FileUpdateMonitor fileUpdateMonitor, + BibEntryTypesManager entryTypesManager, + CountingUndoManager undoManager, + TaskExecutor taskExecutor) { this.tabContainer = Objects.requireNonNull(tabContainer); this.bibDatabaseContext = Objects.requireNonNull(bibDatabaseContext); this.undoManager = undoManager; @@ -170,7 +170,6 @@ private LibraryTab(BibDatabaseContext bibDatabaseContext, this.stateManager = Objects.requireNonNull(stateManager); this.fileUpdateMonitor = fileUpdateMonitor; this.entryTypesManager = entryTypesManager; - this.indexingTaskManager = new IndexingTaskManager(taskExecutor); this.taskExecutor = taskExecutor; this.directoryMonitorManager = new DirectoryMonitorManager(Globals.getDirectoryMonitor()); @@ -255,14 +254,9 @@ private void onDatabaseLoadingSucceed(ParserResult result) { setDatabaseContext(context); - if (preferencesService.getFilePreferences().shouldFulltextIndexLinkedFiles()) { - try { - indexingTaskManager.manageFulltextIndexAccordingToPrefs(LuceneIndexer.of(bibDatabaseContext, preferencesService)); - indexingTaskManager.updateIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService)); - } catch (IOException e) { - LOGGER.error("Cannot access lucene index", e); - } - } + luceneIndexer = new LuceneIndexer(context, taskExecutor, preferencesService); + luceneIndexer.initializeIndexWriterAndReader(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); + luceneIndexer.updateIndex(); LOGGER.trace("loading.set(false);"); loading.set(false); @@ -422,10 +416,6 @@ public void updateTabTitle(boolean isChanged) { textProperty().setValue(tabTitle.toString()); setTooltip(new Tooltip(toolTipText.toString())); }); - - if (preferencesService.getFilePreferences().shouldFulltextIndexLinkedFiles()) { - indexingTaskManager.updateDatabaseName(tabTitle.toString()); - } } @Subscribe @@ -845,9 +835,9 @@ private void onClosed(Event event) { LOGGER.error("Problem when closing directory monitor", e); } try { - // TODO: Close Lucene indexer + luceneIndexer.close(); } catch (RuntimeException e) { - LOGGER.error("Problem when shutting down PDF indexer", e); + LOGGER.error("Problem when closing lucene indexer", e); } try { AutosaveManager.shutdown(bibDatabaseContext); @@ -1057,47 +1047,41 @@ private class IndexUpdateListener { @Subscribe public void listen(EntriesAddedEvent addedEntryEvent) { - try { - LuceneIndexer luceneIndexer = LuceneIndexer.of(bibDatabaseContext, preferencesService); - for (BibEntry addedEntry : addedEntryEvent.getBibEntries()) { - indexingTaskManager.addToIndex(luceneIndexer, addedEntry); - } - } catch (IOException e) { - LOGGER.error("Cannot access lucene index", e); - } + luceneIndexer.indexEntries(addedEntryEvent.getBibEntries()); } @Subscribe public void listen(EntriesRemovedEvent removedEntriesEvent) { - try { - LuceneIndexer luceneIndexer = LuceneIndexer.of(bibDatabaseContext, preferencesService); - for (BibEntry removedEntry : removedEntriesEvent.getBibEntries()) { - indexingTaskManager.removeFromIndex(luceneIndexer, removedEntry); - } - } catch (IOException e) { - LOGGER.error("Cannot access lucene index", e); - } + luceneIndexer.removeEntries(removedEntriesEvent.getBibEntries()); } @Subscribe public void listen(FieldChangedEvent fieldChangedEvent) { - for (BibEntry bibEntry : fieldChangedEvent.getBibEntries()) { - try { - List toRemoveList = new ArrayList<>(); - if (fieldChangedEvent.getField().equals(StandardField.FILE)) { - toRemoveList.addAll(FileFieldParser.parse(fieldChangedEvent.getOldValue())); - toRemoveList.removeAll(FileFieldParser.parse(fieldChangedEvent.getNewValue())); - } - indexingTaskManager.updateIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService), bibEntry, toRemoveList); - } catch (IOException e) { - LOGGER.warn("I/O error when writing lucene index", e); - } + if (!fieldChangedEvent.getField().equals(StandardField.FILE)) { + luceneIndexer.removeBibFieldsFromIndex(fieldChangedEvent.getBibEntries()); + luceneIndexer.indexBibFields(fieldChangedEvent.getBibEntries()); + } else { + List oldFiles = FileFieldParser.parse(fieldChangedEvent.getOldValue()); + List newFiles = FileFieldParser.parse(fieldChangedEvent.getNewValue()); + + Set toRemove = new HashSet<>(oldFiles); + Set toAdd = new HashSet<>(newFiles); + + newFiles.forEach(toRemove::remove); + oldFiles.forEach(toAdd::remove); + + luceneIndexer.removeLinkedFilesByLink(toRemove.stream().map(LinkedFile::getLink).collect(Collectors.toSet())); + luceneIndexer.indexLinkedFiles(toAdd); } } } public IndexingTaskManager getIndexingTaskManager() { - return indexingTaskManager; + return null; + } + + public LuceneIndexer getLuceneIndexer() { + return luceneIndexer; } public static class DatabaseNotification extends NotificationPane { diff --git a/src/main/java/org/jabref/gui/externalfiles/ExternalFilesEntryLinker.java b/src/main/java/org/jabref/gui/externalfiles/ExternalFilesEntryLinker.java index b8b89bc4502..ec062ae87fc 100644 --- a/src/main/java/org/jabref/gui/externalfiles/ExternalFilesEntryLinker.java +++ b/src/main/java/org/jabref/gui/externalfiles/ExternalFilesEntryLinker.java @@ -16,7 +16,6 @@ import org.jabref.logic.cleanup.RenamePdfCleanup; import org.jabref.logic.l10n.Localization; import org.jabref.logic.search.indexing.IndexingTaskManager; -import org.jabref.logic.search.indexing.LuceneIndexer; import org.jabref.logic.util.io.FileNameCleaner; import org.jabref.logic.util.io.FileUtil; import org.jabref.model.database.BibDatabaseContext; @@ -81,37 +80,37 @@ public void addFilesToEntry(BibEntry entry, List files) { } public void moveFilesToFileDirRenameAndAddToEntry(BibEntry entry, List files, IndexingTaskManager indexingTaskManager) { - try (AutoCloseable blocker = indexingTaskManager.blockNewTasks()) { +// try (AutoCloseable blocker = indexingTaskManager.blockNewTasks()) { addFilesToEntry(entry, files); moveLinkedFilesToFileDir(entry); renameLinkedFilesToPattern(entry); - } catch (Exception e) { - LOGGER.error("Could not block IndexingTaskManager", e); - } - - try { - indexingTaskManager.addToIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService), entry); - } catch (IOException e) { - LOGGER.error("Could not access Fulltext-Index", e); - } +// } catch (Exception e) { +// LOGGER.error("Could not block IndexingTaskManager", e); +// } + +// try { +// indexingTaskManager.addToIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService), entry); +// } catch (IOException e) { +// LOGGER.error("Could not access Fulltext-Index", e); +// } } public void copyFilesToFileDirAndAddToEntry(BibEntry entry, List files, IndexingTaskManager indexingTaskManager) { - try (AutoCloseable blocker = indexingTaskManager.blockNewTasks()) { +// try (AutoCloseable blocker = indexingTaskManager.blockNewTasks()) { for (Path file : files) { copyFileToFileDir(file) .ifPresent(copiedFile -> addFilesToEntry(entry, Collections.singletonList(copiedFile))); } renameLinkedFilesToPattern(entry); - } catch (Exception e) { - LOGGER.error("Could not block IndexingTaskManager", e); - } - - try { - indexingTaskManager.addToIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService), entry); - } catch (IOException e) { - LOGGER.error("Could not access fulltext index", e); - } +// } catch (Exception e) { +// LOGGER.error("Could not block IndexingTaskManager", e); +// } + +// try { +// indexingTaskManager.addToIndex(LuceneIndexer.of(bibDatabaseContext, preferencesService), entry); +// } catch (IOException e) { +// LOGGER.error("Could not access fulltext index", e); +// } } private List getValidFileNames(List filesToAdd) { diff --git a/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java b/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java index adbbfbcc897..d2c6bfe6c45 100644 --- a/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java +++ b/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java @@ -1,7 +1,5 @@ package org.jabref.gui.search; -import java.io.IOException; - import org.jabref.gui.DialogService; import org.jabref.gui.LibraryTab; import org.jabref.gui.StateManager; @@ -9,7 +7,6 @@ import org.jabref.gui.util.BackgroundTask; import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.search.indexing.LuceneIndexer; import org.jabref.model.database.BibDatabaseContext; import org.jabref.preferences.PreferencesService; @@ -73,13 +70,7 @@ private void rebuildIndex() { if (!shouldContinue || stateManager.getActiveDatabase().isEmpty()) { return; } - try { - currentLibraryTab.get().getIndexingTaskManager().createIndex(LuceneIndexer.of(databaseContext, preferencesService)); - currentLibraryTab.get().getIndexingTaskManager().updateIndex(LuceneIndexer.of(databaseContext, preferencesService)); - } catch (IOException e) { - dialogService.notify(Localization.lang("Failed to access fulltext search index")); - LOGGER.error("Failed to access fulltext search index", e); - } + currentLibraryTab.get().getLuceneIndexer().rebuildIndex(); } public interface GetCurrentLibraryTab { diff --git a/src/main/java/org/jabref/gui/util/BackgroundTask.java b/src/main/java/org/jabref/gui/util/BackgroundTask.java index 0262e54cb61..b9c9c4a17e6 100644 --- a/src/main/java/org/jabref/gui/util/BackgroundTask.java +++ b/src/main/java/org/jabref/gui/util/BackgroundTask.java @@ -116,6 +116,11 @@ public StringProperty titleProperty() { return title; } + public BackgroundTask setTitle(String title) { + this.title.set(title); + return this; + } + public double getWorkDonePercentage() { return workDonePercentage.get(); } @@ -136,16 +141,18 @@ public boolean showToUser() { return showToUser.get(); } - public void showToUser(boolean show) { + public BackgroundTask showToUser(boolean show) { showToUser.set(show); + return this; } public boolean willBeRecoveredAutomatically() { return willBeRecoveredAutomatically.get(); } - public void willBeRecoveredAutomatically(boolean willBeRecoveredAutomatically) { + public BackgroundTask willBeRecoveredAutomatically(boolean willBeRecoveredAutomatically) { this.willBeRecoveredAutomatically.set(willBeRecoveredAutomatically); + return this; } /** @@ -262,7 +269,7 @@ protected void updateProgress(BackgroundProgress newProgress) { progress.setValue(newProgress); } - protected void updateProgress(double workDone, double max) { + public void updateProgress(double workDone, double max) { updateProgress(new BackgroundProgress(workDone, max)); } diff --git a/src/main/java/org/jabref/logic/search/indexing/IndexingTaskManager.java b/src/main/java/org/jabref/logic/search/indexing/IndexingTaskManager.java index 5fd8a41075f..a51efd3207e 100644 --- a/src/main/java/org/jabref/logic/search/indexing/IndexingTaskManager.java +++ b/src/main/java/org/jabref/logic/search/indexing/IndexingTaskManager.java @@ -1,143 +1,10 @@ package org.jabref.logic.search.indexing; -import java.util.List; -import java.util.Set; -import java.util.concurrent.ConcurrentLinkedDeque; - -import org.jabref.gui.util.BackgroundTask; -import org.jabref.gui.util.DefaultTaskExecutor; -import org.jabref.gui.util.TaskExecutor; -import org.jabref.logic.l10n.Localization; -import org.jabref.model.entry.BibEntry; -import org.jabref.model.entry.LinkedFile; - /** * Wrapper around {@link LuceneIndexer} to execute all operations in the background. */ -public class IndexingTaskManager extends BackgroundTask { - - private final ConcurrentLinkedDeque taskQueue = new ConcurrentLinkedDeque<>(); - private final TaskExecutor taskExecutor; - private int numOfIndexedFiles = 0; - - private final Object lock = new Object(); - private boolean isRunning = false; - private boolean isBlockingNewTasks = false; - - public IndexingTaskManager(TaskExecutor taskExecutor) { - this.taskExecutor = taskExecutor; - showToUser(true); - willBeRecoveredAutomatically(true); - DefaultTaskExecutor.runInJavaFXThread(() -> { - this.updateProgress(1, 1); - this.titleProperty().set(Localization.lang("Indexing pdf files")); - }); - } - - @Override - protected Void call() throws Exception { - synchronized (lock) { - isRunning = true; - } - updateProgress(); - while (!taskQueue.isEmpty() && !isCanceled()) { - taskQueue.pollFirst().run(); - numOfIndexedFiles++; - updateProgress(); - } - synchronized (lock) { - isRunning = false; - } - return null; - } - - private void updateProgress() { - DefaultTaskExecutor.runInJavaFXThread(() -> { - updateMessage(Localization.lang("%0 of %1 entries added to the index", numOfIndexedFiles, numOfIndexedFiles + taskQueue.size())); - updateProgress(numOfIndexedFiles, numOfIndexedFiles + taskQueue.size()); - }); - } - - private void enqueueTask(Runnable indexingTask, boolean skipToFront) { - if (!isBlockingNewTasks) { - if (skipToFront) { - taskQueue.addFirst(indexingTask); - } else { - taskQueue.addLast(indexingTask); - } - // What if already running? - synchronized (lock) { - if (!isRunning) { - isRunning = true; - this.executeWith(taskExecutor); - showToUser(false); - } - } - } - } - - public AutoCloseable blockNewTasks() { - synchronized (lock) { - isBlockingNewTasks = true; - } - return () -> { - synchronized (lock) { - isBlockingNewTasks = false; - } - }; - } - - public void createIndex(LuceneIndexer indexer) { - enqueueTask(indexer::createIndex, true); - } - - public void manageFulltextIndexAccordingToPrefs(LuceneIndexer indexer) { - indexer.getFilePreferences().fulltextIndexLinkedFilesProperty().addListener((observable, oldValue, newValue) -> { - if (newValue) { - for (BibEntry bibEntry : indexer.getDatabaseContext().getEntries()) { - enqueueTask(() -> indexer.updateLinkedFilesInIndex(bibEntry, List.of()), false); - } - } else { - enqueueTask(indexer::deleteLinkedFilesIndex, true); - } - }); - } - - public void updateIndex(LuceneIndexer indexer) { - Set pathsToRemove = indexer.getListOfFilePaths(); - Set hashesOfEntriesToRemove = indexer.getListOfHashes(); - indexer.getDatabaseContext().getEntries().forEach(BibEntry::updateAndGetIndexHash); - for (BibEntry entry : indexer.getDatabaseContext().getEntries()) { - enqueueTask(() -> indexer.addBibFieldsToIndex(entry), true); - enqueueTask(() -> indexer.addLinkedFilesToIndex(entry), false); - hashesOfEntriesToRemove.removeIf(hash -> Integer.valueOf(entry.getLastIndexHash()).equals(hash)); - for (LinkedFile file : entry.getFiles()) { - pathsToRemove.remove(file.getLink()); - } - } - for (String pathToRemove : pathsToRemove) { - enqueueTask(() -> indexer.removeFromIndex(pathToRemove), true); - } - for (int hashToRemove : hashesOfEntriesToRemove) { - enqueueTask(() -> indexer.removeFromIndex(hashToRemove), true); - } - } - - public void addToIndex(LuceneIndexer indexer, BibEntry entry) { - enqueueTask(() -> indexer.addBibFieldsToIndex(entry), true); - enqueueTask(() -> indexer.addLinkedFilesToIndex(entry), false); - } - - public void removeFromIndex(LuceneIndexer indexer, BibEntry entry) { - enqueueTask(() -> indexer.removeFromIndex(entry), false); - } - - public void updateIndex(LuceneIndexer indexer, BibEntry entry, List removedFiles) { - enqueueTask(() -> indexer.updateBibFieldsInIndex(entry), true); - enqueueTask(() -> indexer.updateLinkedFilesInIndex(entry, removedFiles), false); - } +public class IndexingTaskManager { - public void updateDatabaseName(String name) { - DefaultTaskExecutor.runInJavaFXThread(() -> this.titleProperty().set(Localization.lang("Indexing for %0", name))); + public IndexingTaskManager() { } } diff --git a/src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java b/src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java index 83096afe3a3..644f9996fd0 100644 --- a/src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/LuceneIndexer.java @@ -5,6 +5,8 @@ import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -13,308 +15,443 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import javafx.beans.property.BooleanProperty; +import javafx.util.Pair; + +import org.jabref.gui.util.BackgroundTask; +import org.jabref.gui.util.DefaultTaskExecutor; +import org.jabref.gui.util.TaskExecutor; +import org.jabref.logic.importer.util.FileFieldParser; +import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.StandardFileType; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; -import org.jabref.model.entry.Keyword; import org.jabref.model.entry.KeywordList; import org.jabref.model.entry.LinkedFile; import org.jabref.model.entry.field.Field; -import org.jabref.model.entry.field.StandardField; import org.jabref.model.search.SearchFieldConstants; -import org.jabref.preferences.FilePreferences; import org.jabref.preferences.PreferencesService; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexNotFoundException; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.StoredFields; import org.apache.lucene.index.Term; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; import org.apache.lucene.queryparser.classic.ParseException; import org.apache.lucene.queryparser.classic.QueryParser; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.store.NIOFSDirectory; +import org.apache.lucene.util.BytesRef; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.jabref.model.entry.field.StandardField.FILE; +import static org.jabref.model.entry.field.StandardField.GROUPS; +import static org.jabref.model.entry.field.StandardField.KEYWORDS; + public class LuceneIndexer { private static final Logger LOGGER = LoggerFactory.getLogger(LuceneIndexer.class); + private static final Analyzer ANALYZER = new StandardAnalyzer(); - private final Directory directoryToIndex; private final BibDatabaseContext databaseContext; private final PreferencesService preferences; + private final TaskExecutor taskExecutor; private final DocumentReader documentReader; + private final String databaseName; + private final BooleanProperty shouldIndexLinkedFiles; + private final Directory directoryToIndex; + private DirectoryReader reader; + private IndexWriter indexWriter; - public LuceneIndexer(BibDatabaseContext databaseContext, PreferencesService preferences) throws IOException { + public LuceneIndexer(BibDatabaseContext databaseContext, TaskExecutor taskExecutor, PreferencesService preferences) { this.databaseContext = databaseContext; + this.taskExecutor = taskExecutor; this.preferences = preferences; - documentReader = new DocumentReader(preferences.getFilePreferences()); - this.directoryToIndex = new NIOFSDirectory(databaseContext.getFulltextIndexPath()); - } + this.documentReader = new DocumentReader(preferences.getFilePreferences()); + this.databaseName = databaseContext.getDatabasePath().map(path -> path.getFileName().toString()).orElse(""); + this.shouldIndexLinkedFiles = preferences.getFilePreferences().fulltextIndexLinkedFilesProperty(); - public static LuceneIndexer of(BibDatabaseContext databaseContext, PreferencesService preferences) throws IOException { - return new LuceneIndexer(databaseContext, preferences); - } + Directory tmpDirectory = null; + try { + tmpDirectory = new NIOFSDirectory(databaseContext.getFulltextIndexPath()); + } catch (IOException e) { + LOGGER.error("Could not initialize the index directory.", e); + } + this.directoryToIndex = tmpDirectory; - public BibDatabaseContext getDatabaseContext() { - return databaseContext; + bindToPreferences(); } - /** - * Adds all PDF files linked to an entry in the database to new Lucene search index. Any previous state of the - * Lucene search index will be deleted! - */ - public void createIndex() { - // Create new index by creating IndexWriter but not writing anything. - try (IndexWriter indexWriter = new IndexWriter(directoryToIndex, new IndexWriterConfig(new StandardAnalyzer()).setOpenMode(IndexWriterConfig.OpenMode.CREATE))) { - // empty comment for checkstyle + public void initializeIndexWriterAndReader(IndexWriterConfig.OpenMode openMode) { + if (directoryToIndex == null) { + LOGGER.info("Index directory must not be null."); + return; + } + + try { + IndexWriterConfig config = new IndexWriterConfig(ANALYZER); + config.setOpenMode(openMode); + indexWriter = new IndexWriter(directoryToIndex, config); } catch (IOException e) { - LOGGER.warn("Could not create new Index!", e); + LOGGER.error("Could not initialize the IndexWriter.", e); + return; + } + try { + reader = DirectoryReader.open(indexWriter, true, false); + } catch (IOException e) { + LOGGER.error("Could not initialize the IndexReader.", e); } } - /** - * Adds all the pdf files linked to one entry in the database to an existing (or new) Lucene search index - * - * @param entry a bibtex entry to link the pdf files to - */ - public void addLinkedFilesToIndex(BibEntry entry) { - for (LinkedFile file : entry.getFiles()) { - writeFileToIndex(entry, file); + public void close() { + try { + if (indexWriter != null) { + indexWriter.close(); + } + } catch (IOException e) { + LOGGER.warn("Could not close the index writer.", e); + } + try { + if (reader != null) { + reader.close(); + } + } catch (IOException e) { + LOGGER.warn("Could not close the index reader.", e); } } - /** - * Removes an entry identified by its hash from the index - * - * @param hash the hash to be removed - */ - public void removeFromIndex(int hash) { - try (IndexWriter indexWriter = new IndexWriter( - directoryToIndex, - new IndexWriterConfig( - new StandardAnalyzer()).setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND))) { - indexWriter.deleteDocuments(new Term(SearchFieldConstants.BIB_ENTRY_ID_HASH, String.valueOf(hash))); - indexWriter.commit(); + public void commit() { + try { + if (indexWriter != null) { + indexWriter.commit(); + } } catch (IOException e) { - LOGGER.warn("Could not initialize the IndexWriter!", e); + LOGGER.warn("Could not commit changes to the index.", e); } } - /** - * Removes a pdf file identified by its path from the index - * - * @param linkedFilePath the path to the file to be removed - */ - public void removeFromIndex(String linkedFilePath) { - try (IndexWriter indexWriter = new IndexWriter( - directoryToIndex, - new IndexWriterConfig( - new StandardAnalyzer()).setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND))) { - indexWriter.deleteDocuments(new Term(SearchFieldConstants.PATH, linkedFilePath)); - indexWriter.commit(); - } catch (IOException e) { - LOGGER.warn("Could not initialize the IndexWriter!", e); + public void flushIndex() { + close(); + initializeIndexWriterAndReader(IndexWriterConfig.OpenMode.CREATE); + } + + private void bindToPreferences() { + shouldIndexLinkedFiles.addListener((observable, oldValue, newValue) -> { + if (newValue) { + indexLinkedFiles(getLinkedFilesFromEntries(databaseContext.getEntries())); + } else { + deleteAllLinkedFilesFromIndex(); + } + }); + } + + public void rebuildIndex() { + flushIndex(); + indexEntries(databaseContext.getEntries()); + } + + public void indexEntries(Collection entries) { + indexBibFields(entries); + + if (shouldIndexLinkedFiles.get()) { + indexLinkedFiles(getLinkedFilesFromEntries(entries)); } } - /** - * Removes a list of files linked to a bib-entry from the index - * - * @param entry the entry documents are linked to - */ - public void removeFromIndex(BibEntry entry) { - for (LinkedFile linkedFile : entry.getFiles()) { - removeFromIndex(linkedFile.getLink()); + public void updateIndex() { + Set indexedHashes = getIndexedHashes(); + + Set currentHashes = getHashes(databaseContext.getEntries(), true); + + Pair, Set> diffHashes = calculateDifferences(indexedHashes, currentHashes); + Set hashesToAdd = diffHashes.getKey(); + Set hashesToRemove = diffHashes.getValue(); + + removeBibFieldsByHash(hashesToRemove); + + indexBibFields(databaseContext.getEntries() + .stream() + .filter(entry -> hashesToAdd.contains(String.valueOf(entry.getLastIndexHash()))) + .toList()); + + if (shouldIndexLinkedFiles.get()) { + Set indexedFiles = getIndexedFiles().keySet(); + + Set currentFiles = getLinkedFilesFromEntries(databaseContext.getEntries()); + Set currentFilesLinks = currentFiles.stream() + .map(LinkedFile::getLink) + .collect(Collectors.toSet()); + + Pair, Set> diffFiles = calculateDifferences(indexedFiles, currentFilesLinks); + Set filesToAdd = diffFiles.getKey(); + Set filesToRemove = diffFiles.getValue(); + + removeLinkedFilesByLink(filesToRemove); + + indexLinkedFiles(currentFiles.stream() + .filter(linkedFile -> filesToAdd.contains(linkedFile.getLink())) + .collect(Collectors.toSet())); + } else { + deleteAllLinkedFilesFromIndex(); } } - /** - * Deletes all entries from the Lucene search index. - */ - public void flushIndex() { - IndexWriterConfig config = new IndexWriterConfig(); - config.setOpenMode(IndexWriterConfig.OpenMode.CREATE); - try (IndexWriter deleter = new IndexWriter(directoryToIndex, config)) { - // Do nothing. Index is deleted. - } catch (IOException e) { - LOGGER.warn("The IndexWriter could not be initialized", e); + public void indexBibFields(Collection entries) { + if (entries.isEmpty()) { + return; } + + DefaultTaskExecutor.runInJavaFXThread(() -> + indexBibFieldsTask(entries) + .showToUser(true) + .setTitle(Localization.lang("Indexing bib fields for %0", databaseName)) + .executeWith(taskExecutor)); } - public void addBibFieldsToIndex(BibEntry bibEntry) { + private BackgroundTask indexBibFieldsTask(Collection entries) { + return new BackgroundTask<>() { + @Override + protected Void call() { + updateProgress(0, entries.size()); + updateMessage(Localization.lang("%0 of %1 entries added to the index", 0, entries.size())); + + int i = 1; + for (BibEntry entry : entries) { + indexBibFields(entry); + updateProgress(i, entries.size()); + updateMessage(Localization.lang("%0 of %1 entries added to the index", i, entries.size())); + i++; + } + commit(); + return null; + } + }; + } + + private void indexBibFields(BibEntry bibEntry) { try { - try (IndexWriter indexWriter = new IndexWriter(directoryToIndex, - new IndexWriterConfig( - new StandardAnalyzer()).setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND))) { - Document document = new Document(); - document.add(new StringField(SearchFieldConstants.BIB_ENTRY_ID_HASH, String.valueOf(bibEntry.getLastIndexHash()), org.apache.lucene.document.Field.Store.YES)); - for (Map.Entry field : bibEntry.getFieldMap().entrySet()) { - SearchFieldConstants.searchableBibFields.add(field.getKey().getName()); - if (field.getKey() == StandardField.KEYWORDS) { - KeywordList keywords = KeywordList.parse(field.getValue(), preferences.getBibEntryPreferences().getKeywordSeparator()); - for (Keyword keyword : keywords) { - document.add(new StringField(field.getKey().getName(), keyword.toString(), org.apache.lucene.document.Field.Store.YES)); - } - } else if (field.getKey() == StandardField.GROUPS) { - List groups = Arrays.stream(field.getValue().split(preferences.getBibEntryPreferences().getKeywordSeparator().toString())).map(String::trim).toList(); - for (String group : groups) { - document.add(new StringField(field.getKey().getName(), group, org.apache.lucene.document.Field.Store.YES)); - } - } else { - document.add(new TextField(field.getKey().getName(), field.getValue(), org.apache.lucene.document.Field.Store.YES)); - } + Document document = new Document(); + org.apache.lucene.document.Field.Store store = org.apache.lucene.document.Field.Store.YES; + + document.add(new StringField(SearchFieldConstants.BIB_ENTRY_ID_HASH, String.valueOf(bibEntry.updateAndGetIndexHash()), store)); + + for (Map.Entry field : bibEntry.getFieldMap().entrySet()) { + String fieldValue = field.getValue(); + String fieldName = field.getKey().getName(); + SearchFieldConstants.searchableBibFields.add(fieldName); + + switch (field.getKey()) { + case KEYWORDS -> + KeywordList.parse(fieldValue, preferences.getBibEntryPreferences().getKeywordSeparator()) + .forEach(keyword -> document.add(new StringField(fieldName, keyword.toString(), store))); + case GROUPS -> + Arrays.stream(fieldValue.split(preferences.getBibEntryPreferences().getKeywordSeparator().toString())) + .forEach(group -> document.add(new StringField(fieldName, group, store))); + case FILE -> + FileFieldParser.parse(fieldValue).stream() + .map(LinkedFile::getLink) + .forEach(link -> document.add(new StringField(fieldName, link, store))); + default -> + document.add(new TextField(fieldName, fieldValue, store)); } - indexWriter.addDocument(document); - indexWriter.commit(); } + indexWriter.addDocument(document); } catch (IOException e) { - LOGGER.warn("Could not add an entry to the index!", e); + LOGGER.warn("Could not add an entry to the index.", e); } } - /** - * Writes the file to the index if the file is not yet in the index or the file on the fs is newer than the one in - * the index. - * - * @param entry the entry associated with the file - * @param linkedFile the file to write to the index - */ - private void writeFileToIndex(BibEntry entry, LinkedFile linkedFile) { - if (!preferences.getFilePreferences().shouldFulltextIndexLinkedFiles()) { - return; - } - if (linkedFile.isOnlineLink() || !StandardFileType.PDF.getName().equals(linkedFile.getFileType())) { + public void indexLinkedFiles(Collection linkedFiles) { + if (!shouldIndexLinkedFiles.get() || linkedFiles.isEmpty()) { return; } - Optional resolvedPath = linkedFile.findIn(databaseContext, preferences.getFilePreferences()); - if (resolvedPath.isEmpty()) { - LOGGER.debug("Could not find {}", linkedFile.getLink()); - return; - } - try { - // Check if a document with this path is already in the index - try (IndexReader reader = DirectoryReader.open(directoryToIndex)) { - IndexSearcher searcher = new IndexSearcher(reader); - TermQuery query = new TermQuery(new Term(SearchFieldConstants.PATH, linkedFile.getLink())); - TopDocs topDocs = searcher.search(query, 1); - // If a document was found, check if is less current than the one in the FS - if (topDocs.scoreDocs.length > 0) { - Document doc = reader.document(topDocs.scoreDocs[0].doc); - long indexModificationTime = Long.parseLong(doc.getField(SearchFieldConstants.MODIFIED).stringValue()); - - BasicFileAttributes attributes = Files.readAttributes(resolvedPath.get(), BasicFileAttributes.class); - - if (indexModificationTime >= attributes.lastModifiedTime().to(TimeUnit.SECONDS)) { - return; + DefaultTaskExecutor.runInJavaFXThread(() -> + indexLinkedFilesTask(linkedFiles) + .willBeRecoveredAutomatically(true) + .showToUser(true) + .setTitle(Localization.lang("Indexing pdf files for %0", databaseName)) + .executeWith(taskExecutor)); + } + + private BackgroundTask indexLinkedFilesTask(Collection linkedFiles) { + return new BackgroundTask<>() { + @Override + protected Void call() { + updateProgress(0, linkedFiles.size()); + updateMessage(Localization.lang("%0 of %1 files added to the index", 0, linkedFiles.size())); + + int i = 1; + Map indexedFiles = getIndexedFiles(); + + for (LinkedFile linkedFile : linkedFiles) { + Optional resolvedPath = linkedFile.findIn(databaseContext, preferences.getFilePreferences()); + if (resolvedPath.isEmpty()) { + LOGGER.debug("Could not find {}", linkedFile.getLink()); + } else { + if (indexedFiles.containsKey(linkedFile.getLink())) { + long indexModificationTime = indexedFiles.get(linkedFile.getLink()); + try { + BasicFileAttributes attributes = Files.readAttributes(resolvedPath.get(), BasicFileAttributes.class); + if (attributes.lastModifiedTime().to(TimeUnit.SECONDS) > indexModificationTime) { + removeLinkedFileByLink(linkedFile.getLink()); + indexLinkedFile(linkedFile, resolvedPath.get()); + } + } catch (IOException e) { + LOGGER.warn("Could not check the modification time of the file {}.", linkedFile.getLink(), e); + } + } else { + indexLinkedFile(linkedFile, resolvedPath.get()); + } } + updateProgress(i, linkedFiles.size()); + updateMessage(Localization.lang("%0 of %1 files added to the index", i, linkedFiles.size())); + i++; } - } catch (IndexNotFoundException e) { - // if there is no index yet, don't need to check anything! - } - // If no document was found, add the new one - Optional> pages = documentReader.readLinkedPdf(this.databaseContext, linkedFile); - if (pages.isPresent()) { - try (IndexWriter indexWriter = new IndexWriter(directoryToIndex, - new IndexWriterConfig( - new StandardAnalyzer()).setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND))) { - indexWriter.addDocuments(pages.get()); - indexWriter.commit(); - } + commit(); + return null; } + }; + } + + private void indexLinkedFile(LinkedFile linkedFile, Path resolvedPath) { + List pages = documentReader.readPdfContents(linkedFile, resolvedPath); + try { + indexWriter.addDocuments(pages); } catch (IOException e) { - LOGGER.warn("Could not add the document {} to the index!", linkedFile.getLink(), e); + LOGGER.warn("Could not add the document {} to the index.", linkedFile.getLink(), e); } } - public void updateBibFieldsInIndex(BibEntry entry) { - int oldHash = entry.getLastIndexHash(); - int newHash = entry.updateAndGetIndexHash(); - if (oldHash != newHash) { - addBibFieldsToIndex(entry); - removeFromIndex(oldHash); + public void removeEntries(Collection entries) { + removeBibFieldsFromIndex(entries); + removeLinkedFilesByLink(getLinkedFilesFromEntries(entries).stream() + .map(LinkedFile::getLink) + .collect(Collectors.toSet())); + } + + public void removeBibFieldsFromIndex(Collection entries) { + removeBibFieldsByHash(getHashes(entries, false)); + } + + private void removeBibFieldsByHash(Collection hashes) { + BackgroundTask.wrap(() -> { + hashes.forEach(this::removeBibFieldsByHash); + commit(); + }).executeWith(taskExecutor); + } + + private void removeBibFieldsByHash(String hash) { + try { + indexWriter.deleteDocuments(new Term(SearchFieldConstants.BIB_ENTRY_ID_HASH, hash)); + } catch (IOException e) { + LOGGER.warn("Could not remove the entry from the index.", e); } } - public void updateLinkedFilesInIndex(BibEntry entry, List removedFiles) { - for (LinkedFile removedFile : removedFiles) { - removeFromIndex(removedFile.getLink()); + public void removeLinkedFilesByLink(Collection linkedFiles) { + BackgroundTask.wrap(() -> { + linkedFiles.forEach(this::removeLinkedFileByLink); + commit(); + }).executeWith(taskExecutor); + } + + private void removeLinkedFileByLink(String linkedFile) { + try { + indexWriter.deleteDocuments(new Term(SearchFieldConstants.PATH, linkedFile)); + } catch (IOException e) { + LOGGER.warn("Could not remove the linked file {} from the index.", linkedFile, e); } - for (LinkedFile linkedFile : entry.getFiles()) { - writeFileToIndex(entry, linkedFile); + } + + private void deleteAllLinkedFilesFromIndex() { + BackgroundTask.wrap(() -> { + QueryParser queryParser = new QueryParser(SearchFieldConstants.PATH, ANALYZER); + queryParser.setAllowLeadingWildcard(true); + + try { + indexWriter.deleteDocuments(queryParser.parse("*")); + commit(); + } catch (ParseException | IOException e) { + LOGGER.warn("Could not delete linked files from the index.", e); + } + }).executeWith(taskExecutor); + } + + private Set getIndexedHashes() { + Set hashes = new HashSet<>(); + for (LeafReaderContext leaf : reader.leaves()) { + try { + Terms terms = leaf.reader().terms(SearchFieldConstants.BIB_ENTRY_ID_HASH); + if (terms != null) { + TermsEnum termsEnum = terms.iterator(); + BytesRef term; + while ((term = termsEnum.next()) != null) { + hashes.add(term.utf8ToString()); + } + } + } catch (IOException e) { + LOGGER.warn("Could not retrieve the indexed entries.", e); + } } + return hashes; } - /** - * Lists all values of a given field stored in the index - * - * @param field the field to get the values for - * @return all values for this field - */ - private Set getListOfField(String field) { - Set values = new HashSet<>(); - try (IndexReader reader = DirectoryReader.open(directoryToIndex)) { - IndexSearcher searcher = new IndexSearcher(reader); - MatchAllDocsQuery query = new MatchAllDocsQuery(); - TopDocs allDocs = searcher.search(query, Integer.MAX_VALUE); + private Map getIndexedFiles() { + Map paths = new HashMap<>(); + try { + TermQuery query = new TermQuery(new Term(SearchFieldConstants.PAGE_NUMBER, "1")); + TopDocs allDocs = new IndexSearcher(reader).search(query, Integer.MAX_VALUE); + StoredFields storedFields = reader.storedFields(); for (ScoreDoc scoreDoc : allDocs.scoreDocs) { - Document doc = reader.document(scoreDoc.doc); - if (doc.getField(field) != null) { - values.add(doc.getField(field).stringValue()); + Document doc = storedFields.document(scoreDoc.doc); + var pathField = doc.getField(SearchFieldConstants.PATH); + var modifiedField = doc.getField(SearchFieldConstants.MODIFIED); + if (pathField != null && modifiedField != null) { + paths.put(pathField.stringValue(), Long.valueOf(modifiedField.stringValue())); } } } catch (IOException e) { - return values; + LOGGER.warn("Could not retrieve the indexed files.", e); } - return values; + return paths; } - /** - * Lists the paths of all the files that are stored in the index - * - * @return all file paths - */ - public Set getListOfFilePaths() { - return getListOfField(SearchFieldConstants.PATH); + private static Set getHashes(Collection entries, boolean update) { + return entries.stream() + .map(update ? BibEntry::updateAndGetIndexHash : BibEntry::getLastIndexHash) + .map(String::valueOf) + .collect(Collectors.toSet()); } - /** - * Lists the hashes of all the entries that are stored in the index - * - * @return all entry hashes - */ - public Set getListOfHashes() { - return getListOfField(SearchFieldConstants.BIB_ENTRY_ID_HASH).stream().map(Integer::valueOf).collect(Collectors.toSet()); + private static Set getLinkedFilesFromEntries(Collection entries) { + return entries.stream() + .flatMap(entry -> entry.getFiles().stream()) + .filter(linkedFile -> !linkedFile.isOnlineLink() && StandardFileType.PDF.getName().equals(linkedFile.getFileType())) + .collect(Collectors.toSet()); } - public void deleteLinkedFilesIndex() { - try (IndexWriter indexWriter = new IndexWriter( - directoryToIndex, - new IndexWriterConfig( - new StandardAnalyzer()).setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND))) { - QueryParser queryParser = new QueryParser(SearchFieldConstants.PATH, new StandardAnalyzer()); - queryParser.setAllowLeadingWildcard(true); - indexWriter.deleteDocuments(queryParser.parse("*")); - indexWriter.commit(); - } catch (IOException e) { - LOGGER.warn("Could not initialize the IndexWriter", e); - } catch (ParseException e) { - LOGGER.error("Could not parse", e); - } - } + private static Pair, Set> calculateDifferences(Set indexedSet, Set currentSet) { + Set toAdd = currentSet.stream() + .filter(element -> !indexedSet.contains(element)) + .collect(Collectors.toSet()); + + Set toRemove = indexedSet.stream() + .filter(element -> !currentSet.contains(element)) + .collect(Collectors.toSet()); - public FilePreferences getFilePreferences() { - return preferences.getFilePreferences(); + return new Pair<>(toAdd, toRemove); } } diff --git a/src/main/java/org/jabref/model/database/BibDatabaseContext.java b/src/main/java/org/jabref/model/database/BibDatabaseContext.java index 058a238187d..65eb1ee8816 100644 --- a/src/main/java/org/jabref/model/database/BibDatabaseContext.java +++ b/src/main/java/org/jabref/model/database/BibDatabaseContext.java @@ -256,7 +256,7 @@ public Path getFulltextIndexPath() { if (getDatabasePath().isPresent()) { Path databasePath = getDatabasePath().get(); - String fileName = BackupFileUtil.getUniqueFilePrefix(databasePath) + "--" + databasePath.getFileName(); + String fileName = BackupFileUtil.getUniqueFilePrefix(databasePath) + "-" + databasePath.getFileName(); indexPath = appData.resolve(fileName); LOGGER.debug("Index path for {} is {}", getDatabasePath().get(), indexPath); return indexPath; diff --git a/src/main/resources/l10n/JabRef_en.properties b/src/main/resources/l10n/JabRef_en.properties index 1f82582d3d2..c564fd98dcc 100644 --- a/src/main/resources/l10n/JabRef_en.properties +++ b/src/main/resources/l10n/JabRef_en.properties @@ -474,9 +474,10 @@ Include\ subgroups\:\ When\ selected,\ view\ entries\ contained\ in\ this\ group Independent\ group\:\ When\ selected,\ view\ only\ this\ group's\ entries=Independent group: When selected, view only this group's entries I\ Agree=I Agree -Indexing\ pdf\ files=Indexing pdf files -Indexing\ for\ %0=Indexing for %0 +Indexing\ pdf\ files\ for\ %0=Indexing pdf files for %0 +Indexing\ bib\ fields\ for\ %0=Indexing bib fields for %0 %0\ of\ %1\ entries\ added\ to\ the\ index=%0 of %1 entries added to the index +%0\ of\ %1\ files\ added\ to\ the\ index=%0 of %1 files added to the index Invalid\ URL=Invalid URL @@ -2454,7 +2455,6 @@ Automatically\ index\ all\ linked\ files\ for\ fulltext\ search=Automatically in Rebuild\ fulltext\ search\ index=Rebuild fulltext search index Rebuild\ fulltext\ search\ index\ for\ current\ library?=Rebuild fulltext search index for current library? Rebuilding\ fulltext\ search\ index...=Rebuilding fulltext search index... -Failed\ to\ access\ fulltext\ search\ index=Failed to access fulltext search index Found\ match\ in\ %0=Found match in %0 On\ page\ %0=On page %0 Found\ matches\ in\ Annotations\:=Found matches in Annotations: From cebaa1c104ee1f344bf307c6ec21e9416faa7829 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Mon, 3 Jun 2024 20:18:37 +0300 Subject: [PATCH 088/256] Remove IndexingTaskManager --- src/main/java/org/jabref/gui/LibraryTab.java | 5 ----- .../jabref/gui/entryeditor/CommentsTab.java | 5 +---- .../gui/entryeditor/DeprecatedFieldsTab.java | 4 +--- .../entryeditor/DetailOptionalFieldsTab.java | 3 --- .../jabref/gui/entryeditor/EntryEditor.java | 20 +++++++++---------- .../gui/entryeditor/FieldsEditorTab.java | 10 +++------- .../ImportantOptionalFieldsTab.java | 3 --- .../entryeditor/OptionalFieldsTabBase.java | 6 ++---- .../gui/entryeditor/OtherFieldsTab.java | 6 ++---- .../jabref/gui/entryeditor/PreviewTab.java | 6 +----- .../gui/entryeditor/RequiredFieldsTab.java | 4 +--- .../gui/entryeditor/UserDefinedFieldsTab.java | 4 +--- .../ExternalFilesEntryLinker.java | 5 ++--- .../org/jabref/gui/maintable/MainTable.java | 4 ++-- .../org/jabref/gui/preview/PreviewPanel.java | 6 ++---- .../search/indexing/IndexingTaskManager.java | 10 ---------- .../gui/entryeditor/CommentsTabTest.java | 4 ---- 17 files changed, 28 insertions(+), 77 deletions(-) delete mode 100644 src/main/java/org/jabref/logic/search/indexing/IndexingTaskManager.java diff --git a/src/main/java/org/jabref/gui/LibraryTab.java b/src/main/java/org/jabref/gui/LibraryTab.java index f4717fec690..6a9303b6553 100644 --- a/src/main/java/org/jabref/gui/LibraryTab.java +++ b/src/main/java/org/jabref/gui/LibraryTab.java @@ -63,7 +63,6 @@ import org.jabref.logic.l10n.Localization; import org.jabref.logic.pdf.FileAnnotationCache; import org.jabref.logic.search.SearchQuery; -import org.jabref.logic.search.indexing.IndexingTaskManager; import org.jabref.logic.search.indexing.LuceneIndexer; import org.jabref.logic.shared.DatabaseLocation; import org.jabref.logic.util.UpdateField; @@ -1076,10 +1075,6 @@ public void listen(FieldChangedEvent fieldChangedEvent) { } } - public IndexingTaskManager getIndexingTaskManager() { - return null; - } - public LuceneIndexer getLuceneIndexer() { return luceneIndexer; } diff --git a/src/main/java/org/jabref/gui/entryeditor/CommentsTab.java b/src/main/java/org/jabref/gui/entryeditor/CommentsTab.java index 51374f7a1cd..30a98001f36 100644 --- a/src/main/java/org/jabref/gui/entryeditor/CommentsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/CommentsTab.java @@ -26,7 +26,6 @@ import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.journals.JournalAbbreviationRepository; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.search.indexing.IndexingTaskManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.Field; @@ -49,7 +48,6 @@ public CommentsTab(PreferencesService preferences, DialogService dialogService, StateManager stateManager, ThemeManager themeManager, - IndexingTaskManager indexingTaskManager, TaskExecutor taskExecutor, JournalAbbreviationRepository journalAbbreviationRepository) { super( @@ -62,8 +60,7 @@ public CommentsTab(PreferencesService preferences, stateManager, themeManager, taskExecutor, - journalAbbreviationRepository, - indexingTaskManager + journalAbbreviationRepository ); this.defaultOwner = preferences.getOwnerPreferences().getDefaultOwner().toLowerCase(Locale.ROOT).replaceAll("[^a-z0-9]", "-"); setText(Localization.lang("Comments")); diff --git a/src/main/java/org/jabref/gui/entryeditor/DeprecatedFieldsTab.java b/src/main/java/org/jabref/gui/entryeditor/DeprecatedFieldsTab.java index 3a98ae79fc2..3e18325c970 100644 --- a/src/main/java/org/jabref/gui/entryeditor/DeprecatedFieldsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/DeprecatedFieldsTab.java @@ -17,7 +17,6 @@ import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.journals.JournalAbbreviationRepository; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.search.indexing.IndexingTaskManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.database.BibDatabaseMode; import org.jabref.model.entry.BibEntry; @@ -40,11 +39,10 @@ public DeprecatedFieldsTab(BibDatabaseContext databaseContext, PreferencesService preferences, StateManager stateManager, ThemeManager themeManager, - IndexingTaskManager indexingTaskManager, BibEntryTypesManager entryTypesManager, TaskExecutor taskExecutor, JournalAbbreviationRepository journalAbbreviationRepository) { - super(false, databaseContext, suggestionProviders, undoManager, dialogService, preferences, stateManager, themeManager, taskExecutor, journalAbbreviationRepository, indexingTaskManager); + super(false, databaseContext, suggestionProviders, undoManager, dialogService, preferences, stateManager, themeManager, taskExecutor, journalAbbreviationRepository); this.entryTypesManager = entryTypesManager; setText(Localization.lang("Deprecated fields")); diff --git a/src/main/java/org/jabref/gui/entryeditor/DetailOptionalFieldsTab.java b/src/main/java/org/jabref/gui/entryeditor/DetailOptionalFieldsTab.java index dcee79c13c4..3c4433307e8 100644 --- a/src/main/java/org/jabref/gui/entryeditor/DetailOptionalFieldsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/DetailOptionalFieldsTab.java @@ -9,7 +9,6 @@ import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.journals.JournalAbbreviationRepository; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.search.indexing.IndexingTaskManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntryTypesManager; import org.jabref.preferences.PreferencesService; @@ -25,7 +24,6 @@ public DetailOptionalFieldsTab(BibDatabaseContext databaseContext, PreferencesService preferences, StateManager stateManager, ThemeManager themeManager, - IndexingTaskManager indexingTaskManager, BibEntryTypesManager entryTypesManager, TaskExecutor taskExecutor, JournalAbbreviationRepository journalAbbreviationRepository) { @@ -39,7 +37,6 @@ public DetailOptionalFieldsTab(BibDatabaseContext databaseContext, preferences, stateManager, themeManager, - indexingTaskManager, entryTypesManager, taskExecutor, journalAbbreviationRepository diff --git a/src/main/java/org/jabref/gui/entryeditor/EntryEditor.java b/src/main/java/org/jabref/gui/entryeditor/EntryEditor.java index 516b1014f35..8d3602b0498 100644 --- a/src/main/java/org/jabref/gui/entryeditor/EntryEditor.java +++ b/src/main/java/org/jabref/gui/entryeditor/EntryEditor.java @@ -161,11 +161,11 @@ public EntryEditor(LibraryTab libraryTab) { switch (event.getTransferMode()) { case COPY -> { LOGGER.debug("Mode COPY"); - fileLinker.copyFilesToFileDirAndAddToEntry(entry, draggedFiles, libraryTab.getIndexingTaskManager()); + fileLinker.copyFilesToFileDirAndAddToEntry(entry, draggedFiles); } case MOVE -> { LOGGER.debug("Mode MOVE"); - fileLinker.moveFilesToFileDirRenameAndAddToEntry(entry, draggedFiles, libraryTab.getIndexingTaskManager()); + fileLinker.moveFilesToFileDirRenameAndAddToEntry(entry, draggedFiles); } case LINK -> { LOGGER.debug("Mode LINK"); @@ -260,17 +260,17 @@ private void navigateToNextEntry() { } private List createTabs() { - entryEditorTabs.add(new PreviewTab(databaseContext, dialogService, preferencesService, stateManager, themeManager, libraryTab.getIndexingTaskManager(), taskExecutor)); + entryEditorTabs.add(new PreviewTab(databaseContext, dialogService, preferencesService, stateManager, themeManager, taskExecutor)); // Required, optional (important+detail), deprecated, and "other" fields - entryEditorTabs.add(new RequiredFieldsTab(databaseContext, libraryTab.getSuggestionProviders(), undoManager, dialogService, preferencesService, stateManager, themeManager, libraryTab.getIndexingTaskManager(), bibEntryTypesManager, taskExecutor, journalAbbreviationRepository)); - entryEditorTabs.add(new ImportantOptionalFieldsTab(databaseContext, libraryTab.getSuggestionProviders(), undoManager, dialogService, preferencesService, stateManager, themeManager, libraryTab.getIndexingTaskManager(), bibEntryTypesManager, taskExecutor, journalAbbreviationRepository)); - entryEditorTabs.add(new DetailOptionalFieldsTab(databaseContext, libraryTab.getSuggestionProviders(), undoManager, dialogService, preferencesService, stateManager, themeManager, libraryTab.getIndexingTaskManager(), bibEntryTypesManager, taskExecutor, journalAbbreviationRepository)); - entryEditorTabs.add(new DeprecatedFieldsTab(databaseContext, libraryTab.getSuggestionProviders(), undoManager, dialogService, preferencesService, stateManager, themeManager, libraryTab.getIndexingTaskManager(), bibEntryTypesManager, taskExecutor, journalAbbreviationRepository)); - entryEditorTabs.add(new OtherFieldsTab(databaseContext, libraryTab.getSuggestionProviders(), undoManager, dialogService, preferencesService, stateManager, themeManager, libraryTab.getIndexingTaskManager(), bibEntryTypesManager, taskExecutor, journalAbbreviationRepository)); + entryEditorTabs.add(new RequiredFieldsTab(databaseContext, libraryTab.getSuggestionProviders(), undoManager, dialogService, preferencesService, stateManager, themeManager, bibEntryTypesManager, taskExecutor, journalAbbreviationRepository)); + entryEditorTabs.add(new ImportantOptionalFieldsTab(databaseContext, libraryTab.getSuggestionProviders(), undoManager, dialogService, preferencesService, stateManager, themeManager, bibEntryTypesManager, taskExecutor, journalAbbreviationRepository)); + entryEditorTabs.add(new DetailOptionalFieldsTab(databaseContext, libraryTab.getSuggestionProviders(), undoManager, dialogService, preferencesService, stateManager, themeManager, bibEntryTypesManager, taskExecutor, journalAbbreviationRepository)); + entryEditorTabs.add(new DeprecatedFieldsTab(databaseContext, libraryTab.getSuggestionProviders(), undoManager, dialogService, preferencesService, stateManager, themeManager, bibEntryTypesManager, taskExecutor, journalAbbreviationRepository)); + entryEditorTabs.add(new OtherFieldsTab(databaseContext, libraryTab.getSuggestionProviders(), undoManager, dialogService, preferencesService, stateManager, themeManager, bibEntryTypesManager, taskExecutor, journalAbbreviationRepository)); // Comment Tab: Tab for general and user-specific comments - entryEditorTabs.add(new CommentsTab(preferencesService, databaseContext, libraryTab.getSuggestionProviders(), undoManager, dialogService, stateManager, themeManager, libraryTab.getIndexingTaskManager(), taskExecutor, journalAbbreviationRepository)); + entryEditorTabs.add(new CommentsTab(preferencesService, databaseContext, libraryTab.getSuggestionProviders(), undoManager, dialogService, stateManager, themeManager, taskExecutor, journalAbbreviationRepository)); // The preferences allow to configure tabs to show (e.g.,"General", "Abstract") // These should be shown. Already hard-coded ones should be removed. @@ -292,7 +292,7 @@ private List createTabs() { entryEditorTabList.remove(FulltextSearchResultsTab.NAME); for (Map.Entry> tab : entryEditorTabList.entrySet()) { - entryEditorTabs.add(new UserDefinedFieldsTab(tab.getKey(), tab.getValue(), databaseContext, libraryTab.getSuggestionProviders(), undoManager, dialogService, preferencesService, stateManager, themeManager, libraryTab.getIndexingTaskManager(), taskExecutor, journalAbbreviationRepository)); + entryEditorTabs.add(new UserDefinedFieldsTab(tab.getKey(), tab.getValue(), databaseContext, libraryTab.getSuggestionProviders(), undoManager, dialogService, preferencesService, stateManager, themeManager, taskExecutor, journalAbbreviationRepository)); } entryEditorTabs.add(new MathSciNetTab()); diff --git a/src/main/java/org/jabref/gui/entryeditor/FieldsEditorTab.java b/src/main/java/org/jabref/gui/entryeditor/FieldsEditorTab.java index e0f98d538e3..0687770a879 100644 --- a/src/main/java/org/jabref/gui/entryeditor/FieldsEditorTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/FieldsEditorTab.java @@ -34,7 +34,6 @@ import org.jabref.gui.theme.ThemeManager; import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.journals.JournalAbbreviationRepository; -import org.jabref.logic.search.indexing.IndexingTaskManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.Field; @@ -58,7 +57,6 @@ abstract class FieldsEditorTab extends EntryEditorTab { private final TaskExecutor taskExecutor; private final JournalAbbreviationRepository journalAbbreviationRepository; private final StateManager stateManager; - private final IndexingTaskManager indexingTaskManager; private PreviewPanel previewPanel; private final UndoManager undoManager; private Collection fields = new ArrayList<>(); @@ -73,8 +71,7 @@ public FieldsEditorTab(boolean compressed, StateManager stateManager, ThemeManager themeManager, TaskExecutor taskExecutor, - JournalAbbreviationRepository journalAbbreviationRepository, - IndexingTaskManager indexingTaskManager) { + JournalAbbreviationRepository journalAbbreviationRepository) { this.isCompressed = compressed; this.databaseContext = Objects.requireNonNull(databaseContext); this.suggestionProviders = Objects.requireNonNull(suggestionProviders); @@ -85,7 +82,6 @@ public FieldsEditorTab(boolean compressed, this.taskExecutor = Objects.requireNonNull(taskExecutor); this.journalAbbreviationRepository = Objects.requireNonNull(journalAbbreviationRepository); this.stateManager = stateManager; - this.indexingTaskManager = indexingTaskManager; } private static void addColumn(GridPane gridPane, int columnIndex, List

%s
- """.formatted(JS_HIGHLIGHT_FUNCTION, text); - previewView.getEngine().setJavaScriptEnabled(true); - previewView.getEngine().loadContent(myText); - + """.formatted(text); + + Optional searchQuery = stateManager.activeSearchQuery(SearchType.NORMAL_SEARCH).get(); + if (searchQuery.isPresent()) { + LOGGER.info("Before highlighting {}", myText); + String highlightedContent = LuceneHighlighter.highlightPreviewViewer(myText, searchQuery.get().getParsedQuery()); + LOGGER.info("After highlighting {}", highlightedContent); + previewView.getEngine().loadContent(highlightedContent); + } else { + previewView.getEngine().loadContent(myText); + } this.setHvalue(0); } diff --git a/src/main/java/org/jabref/logic/search/LuceneHighlighter.java b/src/main/java/org/jabref/logic/search/LuceneHighlighter.java new file mode 100644 index 00000000000..b1622b5b14b --- /dev/null +++ b/src/main/java/org/jabref/logic/search/LuceneHighlighter.java @@ -0,0 +1,50 @@ +package org.jabref.logic.search; + +import java.io.IOException; + +import org.jabref.model.search.SearchFieldConstants; + +import org.apache.lucene.search.Query; +import org.apache.lucene.search.highlight.Highlighter; +import org.apache.lucene.search.highlight.InvalidTokenOffsetsException; +import org.apache.lucene.search.highlight.NullFragmenter; +import org.apache.lucene.search.highlight.QueryScorer; +import org.apache.lucene.search.highlight.SimpleHTMLFormatter; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.nodes.Node; +import org.jsoup.nodes.TextNode; + +public class LuceneHighlighter { + + public static String highlightPreviewViewer(String htmlText, Query query) { + Highlighter highlighter = new Highlighter( + new SimpleHTMLFormatter("", ""), + new QueryScorer(query)); + highlighter.setTextFragmenter(new NullFragmenter()); + + try { + Document doc = Jsoup.parse(htmlText); + highlightTextNodes(doc.body(), highlighter); + return doc.outerHtml(); + } catch (InvalidTokenOffsetsException | IOException e) { + return htmlText; + } + } + + private static void highlightTextNodes(Element element, Highlighter highlighter) throws InvalidTokenOffsetsException, IOException { + for (Node node : element.childNodes()) { + if (node instanceof TextNode textNode) { + String originalText = textNode.text(); + String highlightedText = highlighter.getBestFragment(SearchFieldConstants.NGram_ANALYZER, "", originalText); + if (highlightedText != null) { + textNode.text(""); + textNode.after(highlightedText); + } + } else if (node instanceof Element) { + highlightTextNodes((Element) node, highlighter); + } + } + } +} From 76297a08e4be7504da9d1cb0e5498a82f821547a Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Thu, 8 Aug 2024 01:58:18 +0300 Subject: [PATCH 124/256] Delete IndexingTaskManager.java --- .../logic/pdf/search/IndexingTaskManager.java | 133 ------------------ 1 file changed, 133 deletions(-) delete mode 100644 src/main/java/org/jabref/logic/pdf/search/IndexingTaskManager.java diff --git a/src/main/java/org/jabref/logic/pdf/search/IndexingTaskManager.java b/src/main/java/org/jabref/logic/pdf/search/IndexingTaskManager.java deleted file mode 100644 index c1f1430f39c..00000000000 --- a/src/main/java/org/jabref/logic/pdf/search/IndexingTaskManager.java +++ /dev/null @@ -1,133 +0,0 @@ -package org.jabref.logic.pdf.search; - -import java.util.List; -import java.util.Queue; -import java.util.Set; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; - -import org.jabref.gui.util.BackgroundTask; -import org.jabref.gui.util.TaskExecutor; -import org.jabref.gui.util.UiTaskExecutor; -import org.jabref.logic.l10n.Localization; -import org.jabref.model.database.BibDatabaseContext; -import org.jabref.model.entry.BibEntry; -import org.jabref.model.entry.LinkedFile; - -/** - * Wrapper around {@link PdfIndexer} to execute all operations in the background. - */ -public class IndexingTaskManager extends BackgroundTask { - - private final Queue taskQueue = new ConcurrentLinkedQueue<>(); - private TaskExecutor taskExecutor; - private int numOfIndexedFiles = 0; - - private final Object lock = new Object(); - private boolean isRunning = false; - private boolean isBlockingNewTasks = false; - - public IndexingTaskManager(TaskExecutor taskExecutor) { - this.taskExecutor = taskExecutor; - showToUser(true); - willBeRecoveredAutomatically(true); - // runs on fx thread, no need to wrap - this.updateProgress(1, 1); - this.titleProperty().set(Localization.lang("Indexing PDF files")); - } - - @Override - protected Void call() throws Exception { - synchronized (lock) { - isRunning = true; - } - updateProgress(); - while (!taskQueue.isEmpty() && !isCanceled()) { - taskQueue.poll().run(); - numOfIndexedFiles++; - updateProgress(); - } - synchronized (lock) { - isRunning = false; - } - return null; - } - - private void updateProgress() { - updateMessage(Localization.lang("%0 of %1 linked files added to the index", numOfIndexedFiles, numOfIndexedFiles + taskQueue.size())); - updateProgress(numOfIndexedFiles, numOfIndexedFiles + taskQueue.size()); - } - - private void enqueueTask(Runnable indexingTask) { - if (!isBlockingNewTasks) { - taskQueue.add(indexingTask); - // What if already running? - synchronized (lock) { - if (!isRunning) { - isRunning = true; - this.executeWith(taskExecutor); - showToUser(false); - } - } - } - } - - public AutoCloseable blockNewTasks() { - synchronized (lock) { - isBlockingNewTasks = true; - } - return () -> { - synchronized (lock) { - isBlockingNewTasks = false; - } - }; - } - - public void rebuildIndex(PdfIndexer indexer) { - enqueueTask(indexer::rebuildIndex); - } - - /** - * Updates the index by performing a delta analysis of the files already existing in the index and the files in the library. - */ - public void updateIndex(PdfIndexer indexer, BibDatabaseContext databaseContext) { - Set pathsToRemove = indexer.getListOfFilePaths(); - databaseContext.getEntries().stream() - .flatMap(entry -> entry.getFiles().stream()) - .map(LinkedFile::getLink) - .forEach(pathsToRemove::remove); - // The indexer checks the attached PDFs for modifications (based on the timestamp of the PDF) and reindexes the PDF if it is newer than the index. Therefore, we need to pass the whole library to the indexer for re-indexing. - addToIndex(indexer, databaseContext.getEntries()); - enqueueTask(() -> indexer.removePathsFromIndex(pathsToRemove)); - } - - public void addToIndex(PdfIndexer indexer, List entries) { - AtomicInteger counter = new AtomicInteger(); - // To enable seeing progress in the UI, we group the entries in chunks of 50 - // Solution inspired by https://stackoverflow.com/a/27595803/873282 - entries.stream().collect(Collectors.groupingBy(x -> counter.getAndIncrement() / 50)) - .values() - .forEach(list -> enqueueTask(() -> indexer.addToIndex(list))); - } - - public void addToIndex(PdfIndexer indexer, BibEntry entry) { - enqueueTask(() -> indexer.addToIndex(entry)); - } - - public void addToIndex(PdfIndexer indexer, BibEntry entry, List linkedFiles) { - enqueueTask(() -> indexer.addToIndex(entry, linkedFiles)); - } - - public void removeFromIndex(PdfIndexer indexer, BibEntry entry) { - enqueueTask(() -> indexer.removeFromIndex(entry)); - } - - public void removeFromIndex(PdfIndexer indexer, List linkedFiles) { - enqueueTask(() -> indexer.removeFromIndex(linkedFiles)); - } - - public void updateDatabaseName(String name) { - UiTaskExecutor.runInJavaFXThread(() -> this.titleProperty().set(Localization.lang("Indexing for %0", name))); - } -} From 6d4e99299e2215bb3794ec6317eafe01b705153a Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Mon, 12 Aug 2024 00:17:00 +0300 Subject: [PATCH 125/256] SourceTab highlighting with Lucene --- .../org/jabref/gui/entryeditor/SourceTab.java | 26 ++++++----- .../org/jabref/gui/preview/PreviewViewer.java | 2 - .../logic/search/LuceneHighlighter.java | 46 ++++++++++++++++++- .../search/indexing/BibFieldsIndexer.java | 2 +- .../jabref/model/search/NGramAnalyzer.java | 10 +++- .../model/search/SearchFieldConstants.java | 3 +- 6 files changed, 70 insertions(+), 19 deletions(-) diff --git a/src/main/java/org/jabref/gui/entryeditor/SourceTab.java b/src/main/java/org/jabref/gui/entryeditor/SourceTab.java index 77b2ab46356..85f448602b7 100644 --- a/src/main/java/org/jabref/gui/entryeditor/SourceTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/SourceTab.java @@ -42,6 +42,7 @@ import org.jabref.logic.importer.ParserResult; import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.logic.l10n.Localization; +import org.jabref.logic.search.LuceneHighlighter; import org.jabref.logic.util.OS; import org.jabref.model.database.BibDatabase; import org.jabref.model.database.BibDatabaseContext; @@ -73,9 +74,8 @@ public class SourceTab extends EntryEditorTab { private final DialogService dialogService; private final BibEntryTypesManager entryTypesManager; private final KeyBindingRepository keyBindingRepository; - private Optional searchHighlightPattern = Optional.empty(); + private final OptionalObjectProperty searchQueryProperty; private CodeArea codeArea; - private BibEntry previousEntry; private class EditAction extends SimpleCommand { @@ -118,21 +118,23 @@ public SourceTab(BibDatabaseContext bibDatabaseContext, this.dialogService = dialogService; this.entryTypesManager = entryTypesManager; this.keyBindingRepository = keyBindingRepository; - - searchQueryProperty.addListener((observable, oldValue, newValue) -> { - // searchHighlightPattern = newValue.flatMap(SearchQuery::getPatternForWords); - // highlightSearchPattern(); - }); + this.searchQueryProperty = searchQueryProperty; + searchQueryProperty.addListener((observable, oldValue, newValue) -> highlightSearchPattern()); } private void highlightSearchPattern() { if (codeArea != null) { codeArea.setStyleClass(0, codeArea.getLength(), "text"); - if (searchHighlightPattern.isPresent()) { - Matcher matcher = searchHighlightPattern.get().matcher(codeArea.getText()); - while (matcher.find()) { - for (int i = 0; i <= matcher.groupCount(); i++) { - codeArea.setStyleClass(matcher.start(), matcher.end(), "search"); + if (searchQueryProperty.get().isPresent() && searchQueryProperty.get().get().isValid()) { + String text = codeArea.getText(); + Optional searchHighlightPattern = LuceneHighlighter.getHighlightingPattern(text, searchQueryProperty.get().get().getParsedQuery()); + if (searchHighlightPattern.isPresent()) { + LOGGER.debug("Highlighting pattern: {}", searchHighlightPattern); + Matcher matcher = searchHighlightPattern.get().matcher(text); + while (matcher.find()) { + int start = matcher.start(); + int end = matcher.end(); + codeArea.setStyleClass(start, end, "search"); } } } diff --git a/src/main/java/org/jabref/gui/preview/PreviewViewer.java b/src/main/java/org/jabref/gui/preview/PreviewViewer.java index 7e812152a07..35d07aab6e8 100644 --- a/src/main/java/org/jabref/gui/preview/PreviewViewer.java +++ b/src/main/java/org/jabref/gui/preview/PreviewViewer.java @@ -204,9 +204,7 @@ private void setPreviewText(String text) { Optional searchQuery = stateManager.activeSearchQuery(SearchType.NORMAL_SEARCH).get(); if (searchQuery.isPresent()) { - LOGGER.info("Before highlighting {}", myText); String highlightedContent = LuceneHighlighter.highlightPreviewViewer(myText, searchQuery.get().getParsedQuery()); - LOGGER.info("After highlighting {}", highlightedContent); previewView.getEngine().loadContent(highlightedContent); } else { previewView.getEngine().loadContent(myText); diff --git a/src/main/java/org/jabref/logic/search/LuceneHighlighter.java b/src/main/java/org/jabref/logic/search/LuceneHighlighter.java index b1622b5b14b..917403f606e 100644 --- a/src/main/java/org/jabref/logic/search/LuceneHighlighter.java +++ b/src/main/java/org/jabref/logic/search/LuceneHighlighter.java @@ -1,10 +1,19 @@ package org.jabref.logic.search; import java.io.IOException; +import java.util.Comparator; +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; import org.jabref.model.search.SearchFieldConstants; import org.apache.lucene.search.Query; +import org.apache.lucene.search.highlight.Formatter; +import org.apache.lucene.search.highlight.Fragmenter; import org.apache.lucene.search.highlight.Highlighter; import org.apache.lucene.search.highlight.InvalidTokenOffsetsException; import org.apache.lucene.search.highlight.NullFragmenter; @@ -15,8 +24,14 @@ import org.jsoup.nodes.Element; import org.jsoup.nodes.Node; import org.jsoup.nodes.TextNode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class LuceneHighlighter { + private static final Logger LOGGER = LoggerFactory.getLogger(LuceneHighlighter.class); + private static final Formatter FORMATTER = new SimpleHTMLFormatter("", ""); + private static final Fragmenter FRAGMENTER = new NullFragmenter(); + private static final Pattern HIGHLIGHTED_TERM_PATTERN = Pattern.compile("(.*?)"); public static String highlightPreviewViewer(String htmlText, Query query) { Highlighter highlighter = new Highlighter( @@ -37,7 +52,7 @@ private static void highlightTextNodes(Element element, Highlighter highlighter) for (Node node : element.childNodes()) { if (node instanceof TextNode textNode) { String originalText = textNode.text(); - String highlightedText = highlighter.getBestFragment(SearchFieldConstants.NGram_ANALYZER, "", originalText); + String highlightedText = highlighter.getBestFragment(SearchFieldConstants.NGram_Analyzer_For_HIGHLIGHING, "", originalText); if (highlightedText != null) { textNode.text(""); textNode.after(highlightedText); @@ -47,4 +62,33 @@ private static void highlightTextNodes(Element element, Highlighter highlighter) } } } + + public static Optional getHighlightingPattern(String content, Query query) { + try { + Highlighter highlighter = new Highlighter(FORMATTER, new QueryScorer(query)); + highlighter.setTextFragmenter(FRAGMENTER); + String highlightedText = highlighter.getBestFragment(SearchFieldConstants.NGram_Analyzer_For_HIGHLIGHING, null, content); + if (highlightedText == null) { + return Optional.empty(); + } + LOGGER.debug("Highlighted text: {}", highlightedText); + Set matchedTerms = getMatchedTerms(highlightedText); + return Optional.of(Pattern.compile( + matchedTerms.stream() + .sorted(Comparator.comparing(String::length)) + .collect(Collectors.joining("|")), + Pattern.CASE_INSENSITIVE)); + } catch (InvalidTokenOffsetsException | IOException e) { + return Optional.empty(); + } + } + + private static Set getMatchedTerms(String highlightedText) { + Set matchedTerms = new HashSet<>(); + Matcher matcher = HIGHLIGHTED_TERM_PATTERN.matcher(highlightedText); + while (matcher.find()) { + matchedTerms.add(matcher.group(1)); + } + return matchedTerms; + } } diff --git a/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java b/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java index e0cd47f8160..d4e0e614b9b 100644 --- a/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java @@ -46,7 +46,7 @@ public BibFieldsIndexer(BibDatabaseContext databaseContext, TaskExecutor executo this.taskExecutor = executor; this.libraryName = databaseContext.getDatabasePath().map(path -> path.getFileName().toString()).orElseGet(() -> "unsaved"); - IndexWriterConfig config = new IndexWriterConfig(SearchFieldConstants.NGram_ANALYZER); + IndexWriterConfig config = new IndexWriterConfig(SearchFieldConstants.NGram_Analyzer_For_INDEXING); this.indexDirectory = new ByteBuffersDirectory(); this.indexWriter = new IndexWriter(indexDirectory, config); diff --git a/src/main/java/org/jabref/model/search/NGramAnalyzer.java b/src/main/java/org/jabref/model/search/NGramAnalyzer.java index 859be2d60f1..d226b887d4b 100644 --- a/src/main/java/org/jabref/model/search/NGramAnalyzer.java +++ b/src/main/java/org/jabref/model/search/NGramAnalyzer.java @@ -6,13 +6,18 @@ import org.apache.lucene.analysis.StopFilter; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter; import org.apache.lucene.analysis.ngram.NGramTokenFilter; import org.apache.lucene.analysis.standard.StandardTokenizer; public class NGramAnalyzer extends Analyzer { + private final int minGram; + private final int maxGram; private final CharArraySet stopWords; - public NGramAnalyzer(CharArraySet stopWords) { + public NGramAnalyzer(int minGram, int maxGram, CharArraySet stopWords) { + this.minGram = minGram; + this.maxGram = maxGram; this.stopWords = stopWords; } @@ -21,7 +26,8 @@ protected TokenStreamComponents createComponents(String fieldName) { Tokenizer source = new StandardTokenizer(); TokenStream result = new LowerCaseFilter(source); result = new StopFilter(result, stopWords); - result = new NGramTokenFilter(result, 1, 5, true); + result = new ASCIIFoldingFilter(result); + result = new NGramTokenFilter(result, minGram, maxGram, true); return new TokenStreamComponents(source, result); } } diff --git a/src/main/java/org/jabref/model/search/SearchFieldConstants.java b/src/main/java/org/jabref/model/search/SearchFieldConstants.java index 9d1bfe74509..90e86732cd8 100644 --- a/src/main/java/org/jabref/model/search/SearchFieldConstants.java +++ b/src/main/java/org/jabref/model/search/SearchFieldConstants.java @@ -19,7 +19,8 @@ public enum SearchFieldConstants { MODIFIED("modified"); public static final Analyzer Standard_ANALYZER = new StandardAnalyzer(EnglishAnalyzer.ENGLISH_STOP_WORDS_SET); - public static final Analyzer NGram_ANALYZER = new NGramAnalyzer(EnglishAnalyzer.ENGLISH_STOP_WORDS_SET); + public static final Analyzer NGram_Analyzer_For_INDEXING = new NGramAnalyzer(1, 5, EnglishAnalyzer.ENGLISH_STOP_WORDS_SET); + public static final Analyzer NGram_Analyzer_For_HIGHLIGHING = new NGramAnalyzer(1, Integer.MAX_VALUE, EnglishAnalyzer.ENGLISH_STOP_WORDS_SET); public static final List PDF_FIELDS = List.of(PATH.toString(), CONTENT.toString(), ANNOTATIONS.toString()); private final String field; From e71087e7f6a87b9d2b1b5ed409c1ce40350d9abc Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Mon, 12 Aug 2024 12:02:29 +0300 Subject: [PATCH 126/256] Fix non-ASCII characters --- .../model/search/SearchFieldConstants.java | 4 +-- .../jabref/model/search/StandardAnalyzer.java | 26 +++++++++++++++++++ 2 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 src/main/java/org/jabref/model/search/StandardAnalyzer.java diff --git a/src/main/java/org/jabref/model/search/SearchFieldConstants.java b/src/main/java/org/jabref/model/search/SearchFieldConstants.java index 90e86732cd8..77b5f7cb957 100644 --- a/src/main/java/org/jabref/model/search/SearchFieldConstants.java +++ b/src/main/java/org/jabref/model/search/SearchFieldConstants.java @@ -4,7 +4,6 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.en.EnglishAnalyzer; -import org.apache.lucene.analysis.standard.StandardAnalyzer; public enum SearchFieldConstants { @@ -19,8 +18,7 @@ public enum SearchFieldConstants { MODIFIED("modified"); public static final Analyzer Standard_ANALYZER = new StandardAnalyzer(EnglishAnalyzer.ENGLISH_STOP_WORDS_SET); - public static final Analyzer NGram_Analyzer_For_INDEXING = new NGramAnalyzer(1, 5, EnglishAnalyzer.ENGLISH_STOP_WORDS_SET); - public static final Analyzer NGram_Analyzer_For_HIGHLIGHING = new NGramAnalyzer(1, Integer.MAX_VALUE, EnglishAnalyzer.ENGLISH_STOP_WORDS_SET); + public static final Analyzer NGram_Analyzer_For_INDEXING = new NGramAnalyzer(1, Integer.MAX_VALUE, EnglishAnalyzer.ENGLISH_STOP_WORDS_SET); public static final List PDF_FIELDS = List.of(PATH.toString(), CONTENT.toString(), ANNOTATIONS.toString()); private final String field; diff --git a/src/main/java/org/jabref/model/search/StandardAnalyzer.java b/src/main/java/org/jabref/model/search/StandardAnalyzer.java new file mode 100644 index 00000000000..cf50b518ced --- /dev/null +++ b/src/main/java/org/jabref/model/search/StandardAnalyzer.java @@ -0,0 +1,26 @@ +package org.jabref.model.search; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.CharArraySet; +import org.apache.lucene.analysis.LowerCaseFilter; +import org.apache.lucene.analysis.StopFilter; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter; +import org.apache.lucene.analysis.standard.StandardTokenizer; + +public class StandardAnalyzer extends Analyzer { + private final CharArraySet stopWords; + public StandardAnalyzer(CharArraySet stopWords) { + this.stopWords = stopWords; + } + + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer source = new StandardTokenizer(); + TokenStream result = new LowerCaseFilter(source); + result = new StopFilter(result, stopWords); + result = new ASCIIFoldingFilter(result); + return new TokenStreamComponents(source, result); + } +} From f4d24f63607d4527849450d553ad58abdb5e114e Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Mon, 12 Aug 2024 12:03:43 +0300 Subject: [PATCH 127/256] Extract query terms from search query --- .../org/jabref/gui/entryeditor/SourceTab.java | 25 ++++++------- .../logic/search/LuceneHighlighter.java | 37 +------------------ .../org/jabref/model/search/SearchQuery.java | 25 +++++++++++++ 3 files changed, 37 insertions(+), 50 deletions(-) diff --git a/src/main/java/org/jabref/gui/entryeditor/SourceTab.java b/src/main/java/org/jabref/gui/entryeditor/SourceTab.java index 85f448602b7..981d1a2f897 100644 --- a/src/main/java/org/jabref/gui/entryeditor/SourceTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/SourceTab.java @@ -42,7 +42,6 @@ import org.jabref.logic.importer.ParserResult; import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.search.LuceneHighlighter; import org.jabref.logic.util.OS; import org.jabref.model.database.BibDatabase; import org.jabref.model.database.BibDatabaseContext; @@ -74,7 +73,7 @@ public class SourceTab extends EntryEditorTab { private final DialogService dialogService; private final BibEntryTypesManager entryTypesManager; private final KeyBindingRepository keyBindingRepository; - private final OptionalObjectProperty searchQueryProperty; + private Optional searchHighlightPattern = Optional.empty(); private CodeArea codeArea; private BibEntry previousEntry; @@ -118,23 +117,21 @@ public SourceTab(BibDatabaseContext bibDatabaseContext, this.dialogService = dialogService; this.entryTypesManager = entryTypesManager; this.keyBindingRepository = keyBindingRepository; - this.searchQueryProperty = searchQueryProperty; - searchQueryProperty.addListener((observable, oldValue, newValue) -> highlightSearchPattern()); + + searchQueryProperty.addListener((observable, oldValue, newValue) -> { + searchHighlightPattern = newValue.flatMap(SearchQuery::getPatternForWords); + highlightSearchPattern(); + }); } private void highlightSearchPattern() { if (codeArea != null) { codeArea.setStyleClass(0, codeArea.getLength(), "text"); - if (searchQueryProperty.get().isPresent() && searchQueryProperty.get().get().isValid()) { - String text = codeArea.getText(); - Optional searchHighlightPattern = LuceneHighlighter.getHighlightingPattern(text, searchQueryProperty.get().get().getParsedQuery()); - if (searchHighlightPattern.isPresent()) { - LOGGER.debug("Highlighting pattern: {}", searchHighlightPattern); - Matcher matcher = searchHighlightPattern.get().matcher(text); - while (matcher.find()) { - int start = matcher.start(); - int end = matcher.end(); - codeArea.setStyleClass(start, end, "search"); + if (searchHighlightPattern.isPresent()) { + Matcher matcher = searchHighlightPattern.get().matcher(codeArea.getText()); + while (matcher.find()) { + for (int i = 0; i <= matcher.groupCount(); i++) { + codeArea.setStyleClass(matcher.start(), matcher.end(), "search"); } } } diff --git a/src/main/java/org/jabref/logic/search/LuceneHighlighter.java b/src/main/java/org/jabref/logic/search/LuceneHighlighter.java index 917403f606e..1239793bca0 100644 --- a/src/main/java/org/jabref/logic/search/LuceneHighlighter.java +++ b/src/main/java/org/jabref/logic/search/LuceneHighlighter.java @@ -1,13 +1,7 @@ package org.jabref.logic.search; import java.io.IOException; -import java.util.Comparator; -import java.util.HashSet; -import java.util.Optional; -import java.util.Set; -import java.util.regex.Matcher; import java.util.regex.Pattern; -import java.util.stream.Collectors; import org.jabref.model.search.SearchFieldConstants; @@ -52,7 +46,7 @@ private static void highlightTextNodes(Element element, Highlighter highlighter) for (Node node : element.childNodes()) { if (node instanceof TextNode textNode) { String originalText = textNode.text(); - String highlightedText = highlighter.getBestFragment(SearchFieldConstants.NGram_Analyzer_For_HIGHLIGHING, "", originalText); + String highlightedText = highlighter.getBestFragment(SearchFieldConstants.Standard_ANALYZER, "", originalText); if (highlightedText != null) { textNode.text(""); textNode.after(highlightedText); @@ -62,33 +56,4 @@ private static void highlightTextNodes(Element element, Highlighter highlighter) } } } - - public static Optional getHighlightingPattern(String content, Query query) { - try { - Highlighter highlighter = new Highlighter(FORMATTER, new QueryScorer(query)); - highlighter.setTextFragmenter(FRAGMENTER); - String highlightedText = highlighter.getBestFragment(SearchFieldConstants.NGram_Analyzer_For_HIGHLIGHING, null, content); - if (highlightedText == null) { - return Optional.empty(); - } - LOGGER.debug("Highlighted text: {}", highlightedText); - Set matchedTerms = getMatchedTerms(highlightedText); - return Optional.of(Pattern.compile( - matchedTerms.stream() - .sorted(Comparator.comparing(String::length)) - .collect(Collectors.joining("|")), - Pattern.CASE_INSENSITIVE)); - } catch (InvalidTokenOffsetsException | IOException e) { - return Optional.empty(); - } - } - - private static Set getMatchedTerms(String highlightedText) { - Set matchedTerms = new HashSet<>(); - Matcher matcher = HIGHLIGHTED_TERM_PATTERN.matcher(highlightedText); - while (matcher.find()) { - matchedTerms.add(matcher.group(1)); - } - return matchedTerms; - } } diff --git a/src/main/java/org/jabref/model/search/SearchQuery.java b/src/main/java/org/jabref/model/search/SearchQuery.java index bec417ec505..669b9c345ac 100644 --- a/src/main/java/org/jabref/model/search/SearchQuery.java +++ b/src/main/java/org/jabref/model/search/SearchQuery.java @@ -1,13 +1,21 @@ package org.jabref.model.search; +import java.util.Arrays; import java.util.EnumSet; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.lucene.queryparser.classic.MultiFieldQueryParser; import org.apache.lucene.queryparser.classic.ParseException; import org.apache.lucene.search.Query; +import org.apache.lucene.search.highlight.QueryTermExtractor; +import org.apache.lucene.search.highlight.WeightedTerm; public class SearchQuery { @@ -79,4 +87,21 @@ public Query getParsedQuery() { public EnumSet getSearchFlags() { return searchFlags; } + + public Optional getPatternForWords() { + List words = getSearchWords(); + if ((words == null) || words.isEmpty() || words.getFirst().isEmpty()) { + return Optional.empty(); + } + // compile the words to a regular expression in the form (w1)|(w2)|(w3) + Stream joiner = words.stream(); + String searchPattern = joiner.collect(Collectors.joining(")|(", "(", ")")); + return Optional.of(Pattern.compile(searchPattern, Pattern.CASE_INSENSITIVE)); + } + + private List getSearchWords() { + return Arrays.stream(QueryTermExtractor.getTerms(parsedQuery)) + .map(WeightedTerm::getTerm) + .toList(); + } } From 805a5a0135de017cef268fb22654f1c3e5606256 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Mon, 12 Aug 2024 14:37:09 +0300 Subject: [PATCH 128/256] Highlight regex queries --- .../org/jabref/model/search/SearchQuery.java | 83 +++++++++++++++++-- 1 file changed, 77 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/jabref/model/search/SearchQuery.java b/src/main/java/org/jabref/model/search/SearchQuery.java index 669b9c345ac..839c622a9ac 100644 --- a/src/main/java/org/jabref/model/search/SearchQuery.java +++ b/src/main/java/org/jabref/model/search/SearchQuery.java @@ -1,6 +1,7 @@ package org.jabref.model.search; import java.util.Arrays; +import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; import java.util.List; @@ -16,9 +17,49 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.highlight.QueryTermExtractor; import org.apache.lucene.search.highlight.WeightedTerm; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class SearchQuery { + /** + * The mode of escaping special characters in regular expressions + */ + private enum EscapeMode { + /** + * using \Q and \E marks + */ + JAVA { + @Override + String format(String regex) { + return Pattern.quote(regex); + } + }, + /** + * escaping all javascript regex special characters separately + */ + JAVASCRIPT { + @Override + String format(String regex) { + return JAVASCRIPT_ESCAPED_CHARS_PATTERN.matcher(regex).replaceAll("\\\\$0"); + } + }; + + /** + * Regex pattern for escaping special characters in javascript regular expressions + */ + private static final Pattern JAVASCRIPT_ESCAPED_CHARS_PATTERN = Pattern.compile("[.*+?^${}()|\\[\\]\\\\/]"); + + /** + * Attempt to escape all regex special characters. + * + * @param regex a string containing a regex expression + * @return a regex with all special characters escaped + */ + abstract String format(String regex); + } + + private final static Logger LOGGER = LoggerFactory.getLogger(SearchQuery.class); protected final String query; protected Query parsedQuery; protected String parseError; @@ -88,20 +129,50 @@ public EnumSet getSearchFlags() { return searchFlags; } + /** + * Returns a list of words this query searches for. The returned strings can be a regular expression. + */ + public List getSearchWords() { + if (searchFlags.contains(SearchFlags.REGULAR_EXPRESSION)) { + return Collections.singletonList(query); + } + if (!isValid()) { + return List.of(); + } + return Arrays.stream(QueryTermExtractor.getTerms(parsedQuery)) + .map(WeightedTerm::getTerm) + .toList(); + } + + // Returns a regular expression pattern in the form (w1)|(w2)| ... wi are escaped if no regular expression search is enabled public Optional getPatternForWords() { + return joinWordsToPattern(EscapeMode.JAVA); + } + + // Returns a regular expression pattern in the form (w1)|(w2)| ... wi are escaped for javascript if no regular expression search is enabled + public Optional getJavaScriptPatternForWords() { + return joinWordsToPattern(EscapeMode.JAVASCRIPT); + } + + /** + * Returns a regular expression pattern in the form (w1)|(w2)| ... wi are escaped if no regular expression search is enabled + * + * @param escapeMode the mode of escaping special characters in wi + */ + private Optional joinWordsToPattern(EscapeMode escapeMode) { List words = getSearchWords(); + if ((words == null) || words.isEmpty() || words.getFirst().isEmpty()) { return Optional.empty(); } + // compile the words to a regular expression in the form (w1)|(w2)|(w3) Stream joiner = words.stream(); + if (!searchFlags.contains(SearchFlags.REGULAR_EXPRESSION)) { + // Reformat string when we are looking for a literal match + joiner = joiner.map(escapeMode::format); + } String searchPattern = joiner.collect(Collectors.joining(")|(", "(", ")")); return Optional.of(Pattern.compile(searchPattern, Pattern.CASE_INSENSITIVE)); } - - private List getSearchWords() { - return Arrays.stream(QueryTermExtractor.getTerms(parsedQuery)) - .map(WeightedTerm::getTerm) - .toList(); - } } From 2446ccd5af100b41c027a1238b69f2cfda92a5b9 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Mon, 12 Aug 2024 14:42:57 +0300 Subject: [PATCH 129/256] return js highlight function --- .../org/jabref/gui/preview/PreviewViewer.java | 106 ++++++++++++++++-- .../logic/search/LuceneHighlighter.java | 59 ---------- src/main/resources/tinylog.properties | 1 + 3 files changed, 95 insertions(+), 71 deletions(-) delete mode 100644 src/main/java/org/jabref/logic/search/LuceneHighlighter.java diff --git a/src/main/java/org/jabref/gui/preview/PreviewViewer.java b/src/main/java/org/jabref/gui/preview/PreviewViewer.java index 35d07aab6e8..1d0e4554a9c 100644 --- a/src/main/java/org/jabref/gui/preview/PreviewViewer.java +++ b/src/main/java/org/jabref/gui/preview/PreviewViewer.java @@ -6,9 +6,11 @@ import java.net.MalformedURLException; import java.util.Objects; import java.util.Optional; +import java.util.regex.Pattern; import javafx.beans.InvalidationListener; import javafx.beans.Observable; +import javafx.beans.value.ChangeListener; import javafx.concurrent.Worker; import javafx.print.PrinterJob; import javafx.scene.control.ScrollPane; @@ -26,7 +28,6 @@ import org.jabref.logic.l10n.Localization; import org.jabref.logic.layout.format.Number; import org.jabref.logic.preview.PreviewLayout; -import org.jabref.logic.search.LuceneHighlighter; import org.jabref.logic.util.WebViewStore; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; @@ -71,9 +72,58 @@ function getSelectionHtml() { } getSelectionHtml(); """; + private static final String JS_HIGHLIGHT_FUNCTION = + """ + + + + + + """; + // This is a string format, and it takes a variable name as an argument to pass to the markInstance.markRegExp() Javascript method. + private static final String JS_MARK_REG_EXP_CALLBACK = + """ + {done: function(){ + markInstance.markRegExp(%s);} + }"""; + + // This is a string format, and it takes a variable name as an argument to pass to the markInstance.unmark() Javascript method. + private static final String JS_UNMARK_WITH_CALLBACK = + """ + var markInstance = new Mark(document.getElementById("content")); + markInstance.unmark(%s);"""; + private static final Pattern UNESCAPED_FORWARD_SLASH = Pattern.compile("(? entry = Optional.empty(); + private Optional searchHighlightPattern = Optional.empty(); private final BibDatabaseContext database; + private boolean registered; + + private final ChangeListener> listener = (queryObservable, queryOldValue, queryNewValue) -> { + searchHighlightPattern = queryNewValue.flatMap(SearchQuery::getJavaScriptPatternForWords); + highlightSearchPattern(); + }; /** * @param database Used for resolving strings and pdf directories for links. @@ -96,7 +153,6 @@ public PreviewViewer(BibDatabaseContext database, TaskExecutor taskExecutor) { this.database = Objects.requireNonNull(database); this.dialogService = dialogService; - this.stateManager = stateManager; this.clipBoardManager = Injector.instantiateModelOrService(ClipBoardManager.class); this.taskExecutor = taskExecutor; @@ -111,6 +167,12 @@ public PreviewViewer(BibDatabaseContext database, if (newValue != Worker.State.SUCCEEDED) { return; } + if (!registered) { + stateManager.activeSearchQuery(SearchType.NORMAL_SEARCH).addListener(listener); + stateManager.activeSearchQuery(SearchType.GLOBAL_SEARCH).addListener(listener); + registered = true; + } + highlightSearchPattern(); // https://stackoverflow.com/questions/15555510/javafx-stop-opening-url-in-webview-open-in-browser-instead NodeList anchorList = previewView.getEngine().getDocument().getElementsByTagName("a"); @@ -138,6 +200,30 @@ public PreviewViewer(BibDatabaseContext database, themeManager.installCss(previewView.getEngine()); } + private void highlightSearchPattern() { + String callbackForUnmark = ""; + if (searchHighlightPattern.isPresent()) { + String javaScriptRegex = createJavaScriptRegex(searchHighlightPattern.get()); + callbackForUnmark = JS_MARK_REG_EXP_CALLBACK.formatted(javaScriptRegex); + } + String unmarkInstance = JS_UNMARK_WITH_CALLBACK.formatted(callbackForUnmark); + previewView.getEngine().executeScript(unmarkInstance); + } + + /** + * Returns the String representation of a JavaScript regex object. The method does not take into account differences between the regex implementations in Java and JavaScript. + * + * @param regex Java regex to print as a JavaScript regex + * @return JavaScript regex object + */ + private static String createJavaScriptRegex(Pattern regex) { + String pattern = regex.pattern(); + // Create a JavaScript regular expression literal (https://ecma-international.org/ecma-262/10.0/index.html#sec-literals-regular-expression-literals) + // Forward slashes are reserved to delimit the regular expression body. Hence, they must be escaped. + pattern = UNESCAPED_FORWARD_SLASH.matcher(pattern).replaceAll("\\\\/"); + return "/" + pattern + "/gmi"; + } + public void setLayout(PreviewLayout newLayout) { // Change listeners might set the layout to null while the update method is executing, therefore we need to prevent this here if (newLayout == null) { @@ -196,19 +282,15 @@ private void update() { private void setPreviewText(String text) { String myText = """ + %s
%s
- """.formatted(text); - - Optional searchQuery = stateManager.activeSearchQuery(SearchType.NORMAL_SEARCH).get(); - if (searchQuery.isPresent()) { - String highlightedContent = LuceneHighlighter.highlightPreviewViewer(myText, searchQuery.get().getParsedQuery()); - previewView.getEngine().loadContent(highlightedContent); - } else { - previewView.getEngine().loadContent(myText); - } + """.formatted(JS_HIGHLIGHT_FUNCTION, text); + previewView.getEngine().setJavaScriptEnabled(true); + previewView.getEngine().loadContent(myText); + this.setHvalue(0); } diff --git a/src/main/java/org/jabref/logic/search/LuceneHighlighter.java b/src/main/java/org/jabref/logic/search/LuceneHighlighter.java deleted file mode 100644 index 1239793bca0..00000000000 --- a/src/main/java/org/jabref/logic/search/LuceneHighlighter.java +++ /dev/null @@ -1,59 +0,0 @@ -package org.jabref.logic.search; - -import java.io.IOException; -import java.util.regex.Pattern; - -import org.jabref.model.search.SearchFieldConstants; - -import org.apache.lucene.search.Query; -import org.apache.lucene.search.highlight.Formatter; -import org.apache.lucene.search.highlight.Fragmenter; -import org.apache.lucene.search.highlight.Highlighter; -import org.apache.lucene.search.highlight.InvalidTokenOffsetsException; -import org.apache.lucene.search.highlight.NullFragmenter; -import org.apache.lucene.search.highlight.QueryScorer; -import org.apache.lucene.search.highlight.SimpleHTMLFormatter; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.nodes.Node; -import org.jsoup.nodes.TextNode; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class LuceneHighlighter { - private static final Logger LOGGER = LoggerFactory.getLogger(LuceneHighlighter.class); - private static final Formatter FORMATTER = new SimpleHTMLFormatter("", ""); - private static final Fragmenter FRAGMENTER = new NullFragmenter(); - private static final Pattern HIGHLIGHTED_TERM_PATTERN = Pattern.compile("(.*?)"); - - public static String highlightPreviewViewer(String htmlText, Query query) { - Highlighter highlighter = new Highlighter( - new SimpleHTMLFormatter("", ""), - new QueryScorer(query)); - highlighter.setTextFragmenter(new NullFragmenter()); - - try { - Document doc = Jsoup.parse(htmlText); - highlightTextNodes(doc.body(), highlighter); - return doc.outerHtml(); - } catch (InvalidTokenOffsetsException | IOException e) { - return htmlText; - } - } - - private static void highlightTextNodes(Element element, Highlighter highlighter) throws InvalidTokenOffsetsException, IOException { - for (Node node : element.childNodes()) { - if (node instanceof TextNode textNode) { - String originalText = textNode.text(); - String highlightedText = highlighter.getBestFragment(SearchFieldConstants.Standard_ANALYZER, "", originalText); - if (highlightedText != null) { - textNode.text(""); - textNode.after(highlightedText); - } - } else if (node instanceof Element) { - highlightTextNodes((Element) node, highlighter); - } - } - } -} diff --git a/src/main/resources/tinylog.properties b/src/main/resources/tinylog.properties index d6862e6d277..cff0c4c9364 100644 --- a/src/main/resources/tinylog.properties +++ b/src/main/resources/tinylog.properties @@ -10,6 +10,7 @@ exception = strip: jdk.internal #level@org.jabref.model.entry.BibEntry = debug level@org.jabref.gui.maintable = debug level@org.jabref.logic.search = debug +level@org.jabref.model.search = debug level@org.jabref.http.server.Server = debug #level@org.jabref.gui.JabRefGUI = debug From 20d7096cd4450eedeb660937bc4cd8c3b318fb28 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Mon, 12 Aug 2024 14:44:30 +0300 Subject: [PATCH 130/256] Fix invalid search query throw exception --- .../FulltextSearchResultsTab.java | 4 +- .../jabref/gui/search/GlobalSearchBar.java | 75 ++++++------------- 2 files changed, 23 insertions(+), 56 deletions(-) diff --git a/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java b/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java index 0c5631b2bbd..32b93921f44 100644 --- a/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java @@ -80,9 +80,7 @@ public FulltextSearchResultsTab(StateManager stateManager, @Override public boolean shouldShow(BibEntry entry) { - return searchQueryProperty.get().map(query -> - !query.getParsedQuery().toString().isEmpty() && query.getSearchFlags().contains(SearchFlags.FULLTEXT) - ).orElse(false); + return searchQueryProperty.get().map(query -> query.isValid() && query.getSearchFlags().contains(SearchFlags.FULLTEXT)).orElse(false); } @Override diff --git a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java index 9536827542d..e28eddebb13 100644 --- a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java +++ b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java @@ -6,8 +6,6 @@ import java.util.List; import java.util.Objects; import java.util.Optional; -import java.util.regex.Pattern; -import java.util.regex.PatternSyntaxException; import javax.swing.undo.UndoManager; @@ -64,6 +62,7 @@ import org.jabref.preferences.PreferencesService; import org.jabref.preferences.SearchPreferences; +import com.tobiasdiez.easybind.EasyBind; import impl.org.controlsfx.skin.AutoCompletePopup; import org.controlsfx.control.textfield.AutoCompletionBinding; import org.controlsfx.control.textfield.CustomTextField; @@ -88,7 +87,6 @@ public class GlobalSearchBar extends HBox { private final ToggleButton keepSearchString; private final ToggleButton filterModeButton; private final Tooltip searchFieldTooltip = new Tooltip(); - private final Label currentResults = new Label(""); private final StateManager stateManager; private final PreferencesService preferencesService; private final UndoManager undoManager; @@ -120,18 +118,23 @@ public GlobalSearchBar(LibraryTabContainer tabContainer, searchField.disableProperty().bind(needsDatabase(stateManager).not()); // fits the standard "found x entries"-message thus hinders the searchbar to jump around while searching if the tabContainer width is too small + Label currentResults = new Label(""); currentResults.setPrefWidth(150); currentResults.visibleProperty().bind(stateManager.activeSearchQuery(searchType).isPresent()); - currentResults.textProperty().bind(Bindings.createStringBinding(() -> { - int matched = stateManager.searchResultSize(searchType).get(); - if (matched == 0) { - searchField.pseudoClassStateChanged(CLASS_NO_RESULTS, true); - return Localization.lang("No results found."); - } else { - searchField.pseudoClassStateChanged(CLASS_RESULTS_FOUND, true); - return Localization.lang("Found %0 results.", String.valueOf(matched)); - } - }, stateManager.searchResultSize(searchType))); + currentResults.textProperty().bind(EasyBind.combine(stateManager.searchResultSize(searchType), stateManager.activeSearchQuery(searchType), + (size, query) -> { + if (query.isPresent() && !query.get().isValid()) { + searchField.pseudoClassStateChanged(CLASS_NO_RESULTS, true); + return Localization.lang("Search failed: illegal search expression"); + } + if (size.intValue() == 0) { + searchField.pseudoClassStateChanged(CLASS_NO_RESULTS, true); + return Localization.lang("No results found."); + } else { + searchField.pseudoClassStateChanged(CLASS_RESULTS_FOUND, true); + return Localization.lang("Found %0 results.", String.valueOf(size)); + } + })); searchField.setTooltip(searchFieldTooltip); searchFieldTooltip.setContentDisplay(ContentDisplay.GRAPHIC_ONLY); @@ -211,9 +214,6 @@ public GlobalSearchBar(LibraryTabContainer tabContainer, }, query -> setSearchTerm(query.orElseGet(() -> new SearchQuery("", EnumSet.noneOf(SearchFlags.class))))); - // TODO search describer - // stateManager.activeSearchQuery(searchType).addListener((obs, oldValue, newValue) -> - // newValue.ifPresent(query -> setSearchFieldHintTooltip(SearchDescribers.getSearchDescriberFor(query).getDescription()))); /* * The listener tracks a change on the focus property value. * This happens, from active (user types a query) to inactive / focus @@ -309,40 +309,15 @@ public void updateSearchQuery() { // An empty search field should cause the search to be cleared. if (searchField.getText().isEmpty()) { - setSearchFieldHintTooltip(null); + setSearchFieldHintTooltip(); stateManager.activeSearchQuery(searchType).set(Optional.empty()); return; } - // Invalid regular expression - if (regularExpressionButton.isSelected() && !validRegex()) { - currentResults.setText(Localization.lang("Invalid regular expression")); - return; - } - SearchQuery searchQuery = new SearchQuery(this.searchField.getText(), searchPreferences.getSearchFlags()); - if (!searchQuery.isValid()) { - informUserAboutInvalidSearchQuery(); - return; - } stateManager.activeSearchQuery(searchType).set(Optional.of(searchQuery)); } - private boolean validRegex() { - try { - Pattern.compile(searchField.getText()); - } catch (PatternSyntaxException e) { - LOGGER.debug(e.getMessage()); - return false; - } - return true; - } - - private void informUserAboutInvalidSearchQuery() { - searchField.pseudoClassStateChanged(CLASS_NO_RESULTS, true); - currentResults.setText(Localization.lang("Search failed: illegal search expression")); - } - public void setAutoCompleter(SuggestionProvider searchCompleter) { if (preferencesService.getAutoCompletePreferences().shouldAutoComplete()) { AutoCompletionTextInputBinding autoComplete = AutoCompletionTextInputBinding.autoComplete(searchField, @@ -370,25 +345,19 @@ private AutoCompletePopup getPopup(AutoCompletionBinding autoCompletio } } - private void setSearchFieldHintTooltip(TextFlow description) { + private void setSearchFieldHintTooltip() { if (preferencesService.getWorkspacePreferences().shouldShowAdvancedHints()) { String genericDescription = Localization.lang("Hint:\n\nTo search all fields for Smith, enter:\nsmith\n\nTo search the field author for Smith and the field title for electrical, enter:\nauthor:Smith AND title:electrical"); List genericDescriptionTexts = TooltipTextUtil.createTextsFromHtml(genericDescription); - if (description == null) { - TextFlow emptyHintTooltip = new TextFlow(); - emptyHintTooltip.getChildren().setAll(genericDescriptionTexts); - searchFieldTooltip.setGraphic(emptyHintTooltip); - } else { - description.getChildren().add(new Text("\n\n")); - description.getChildren().addAll(genericDescriptionTexts); - searchFieldTooltip.setGraphic(description); - } + TextFlow emptyHintTooltip = new TextFlow(); + emptyHintTooltip.getChildren().setAll(genericDescriptionTexts); + searchFieldTooltip.setGraphic(emptyHintTooltip); } } public void updateHintVisibility() { - setSearchFieldHintTooltip(null); + setSearchFieldHintTooltip(); } public void setSearchTerm(SearchQuery searchQuery) { From 793dd59fc99b5a601d10de9731ef3241a559361c Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Mon, 12 Aug 2024 18:51:56 +0300 Subject: [PATCH 131/256] Refactor Lucene indexer classes --- .../jabref/logic/search/LuceneManager.java | 80 +++++++++++++------ .../search/indexing/BibFieldsIndexer.java | 80 +++++++++---------- .../indexing/DefaultLinkedFilesIndexer.java | 21 +++-- .../indexing/ReadOnlyLinkedFilesIndexer.java | 19 +++-- .../jabref/model/search/LuceneIndexer.java | 11 +-- .../model/search/envent/IndexAddedEvent.java | 4 + .../search/envent/IndexRemovedEvent.java | 4 + .../search/envent/IndexStartedEvent.java | 4 + .../search/envent/IndexUpdatedEvent.java | 4 + src/main/resources/l10n/JabRef_en.properties | 11 +-- 10 files changed, 136 insertions(+), 102 deletions(-) create mode 100644 src/main/java/org/jabref/model/search/envent/IndexAddedEvent.java create mode 100644 src/main/java/org/jabref/model/search/envent/IndexRemovedEvent.java create mode 100644 src/main/java/org/jabref/model/search/envent/IndexStartedEvent.java create mode 100644 src/main/java/org/jabref/model/search/envent/IndexUpdatedEvent.java diff --git a/src/main/java/org/jabref/logic/search/LuceneManager.java b/src/main/java/org/jabref/logic/search/LuceneManager.java index 399d3f02880..81afcd24d23 100644 --- a/src/main/java/org/jabref/logic/search/LuceneManager.java +++ b/src/main/java/org/jabref/logic/search/LuceneManager.java @@ -2,7 +2,6 @@ import java.io.IOException; import java.util.Collection; -import java.util.List; import javafx.beans.property.BooleanProperty; import javafx.beans.property.SimpleBooleanProperty; @@ -11,8 +10,6 @@ import org.jabref.gui.util.BackgroundTask; import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.search.indexing.BibFieldsIndexer; -import org.jabref.logic.search.indexing.DefaultLinkedFilesIndexer; -import org.jabref.logic.search.indexing.ReadOnlyLinkedFilesIndexer; import org.jabref.logic.search.retrieval.LuceneSearcher; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; @@ -55,7 +52,7 @@ public LuceneManager(BibDatabaseContext databaseContext, TaskExecutor executor, private void initializeIndexers() { try { - bibFieldsIndexer = new BibFieldsIndexer(databaseContext, taskExecutor); + bibFieldsIndexer = new BibFieldsIndexer(databaseContext); } catch (IOException e) { LOGGER.error("Error initializing bib fields index", e); } @@ -68,19 +65,19 @@ private void initializeIndexers() { } private void initializeLinkedFilesIndexer() { - try { - linkedFilesIndexer = new DefaultLinkedFilesIndexer(databaseContext, taskExecutor, preferences.getFilePreferences()); - } catch (IOException e) { - LOGGER.debug("Error initializing linked files index - using read only index"); - linkedFilesIndexer = new ReadOnlyLinkedFilesIndexer(databaseContext); - } +// try { +// linkedFilesIndexer = new DefaultLinkedFilesIndexer(databaseContext, taskExecutor, preferences.getFilePreferences()); +// } catch (IOException e) { +// LOGGER.debug("Error initializing linked files index - using read only index"); +// linkedFilesIndexer = new ReadOnlyLinkedFilesIndexer(databaseContext); +// } } private void bindToPreferences(boolean newValue) { BackgroundTask.wrap(() -> { if (newValue) { initializeLinkedFilesIndexer(); - linkedFilesIndexer.updateOnStart(); + // linkedFilesIndexer.updateOnStart(); } else { linkedFilesIndexer.removeAllFromIndex(); linkedFilesIndexer.close(); @@ -90,50 +87,87 @@ private void bindToPreferences(boolean newValue) { } public void updateOnStart() { - BackgroundTask.wrap(bibFieldsIndexer::updateOnStart).executeWith(taskExecutor); + new BackgroundTask<>() { + @Override + protected Object call() { + bibFieldsIndexer.updateOnStart(this); + return null; + } + }.showToUser(true).executeWith(taskExecutor); if (shouldIndexLinkedFiles.get()) { - BackgroundTask.wrap(linkedFilesIndexer::updateOnStart).executeWith(taskExecutor); + // BackgroundTask.wrap(linkedFilesIndexer::updateOnStart).executeWith(taskExecutor); } } public void addToIndex(Collection entries) { - BackgroundTask.wrap(() -> bibFieldsIndexer.addToIndex(entries)).executeWith(taskExecutor); + new BackgroundTask<>() { + @Override + protected Object call() { + bibFieldsIndexer.addToIndex(entries, this); + return null; + } + }.onSuccess(r -> LOGGER.debug("OnSuccess")) + .showToUser(true).executeWith(taskExecutor); if (shouldIndexLinkedFiles.get() && !isLinkedFilesIndexerBlocked.get()) { - BackgroundTask.wrap(() -> linkedFilesIndexer.addToIndex(entries)).executeWith(taskExecutor); + // BackgroundTask.wrap(() -> linkedFilesIndexer.addToIndex(entries)).executeWith(taskExecutor); } } public void removeFromIndex(Collection entries) { - BackgroundTask.wrap(() -> bibFieldsIndexer.removeFromIndex(entries)).executeWith(taskExecutor); + new BackgroundTask<>() { + @Override + protected Object call() { + bibFieldsIndexer.removeFromIndex(entries, this); + return null; + } + }.showToUser(true).executeWith(taskExecutor); if (shouldIndexLinkedFiles.get()) { - BackgroundTask.wrap(() -> linkedFilesIndexer.removeFromIndex(entries)).executeWith(taskExecutor); + // BackgroundTask.wrap(() -> linkedFilesIndexer.removeFromIndex(entries)).executeWith(taskExecutor); } } public void updateEntry(BibEntry entry, String oldValue, String newValue, boolean isLinkedFile) { - BackgroundTask.wrap(() -> bibFieldsIndexer.updateEntry(entry, oldValue, newValue)).executeWith(taskExecutor); + new BackgroundTask<>() { + @Override + protected Object call() { + bibFieldsIndexer.updateEntry(entry, oldValue, newValue, this); + return null; + } + }.showToUser(true).executeWith(taskExecutor); if (isLinkedFile && shouldIndexLinkedFiles.get() && !isLinkedFilesIndexerBlocked.get()) { - BackgroundTask.wrap(() -> linkedFilesIndexer.updateEntry(entry, oldValue, newValue)).executeWith(taskExecutor); + // BackgroundTask.wrap(() -> linkedFilesIndexer.updateEntry(entry, oldValue, newValue)).executeWith(taskExecutor); } } public void updateAfterDropFiles(BibEntry entry) { - BackgroundTask.wrap(() -> bibFieldsIndexer.updateEntry(entry, "", "")).executeWith(taskExecutor); + new BackgroundTask<>() { + @Override + protected Object call() { + bibFieldsIndexer.updateEntry(entry, "", "", this); + return null; + } + }.showToUser(true).executeWith(taskExecutor); if (shouldIndexLinkedFiles.get() && !isLinkedFilesIndexerBlocked.get()) { - BackgroundTask.wrap(() -> linkedFilesIndexer.addToIndex(List.of(entry))).executeWith(taskExecutor); + // BackgroundTask.wrap(() -> linkedFilesIndexer.addToIndex(List.of(entry))).executeWith(taskExecutor); } } public void rebuildIndex() { - BackgroundTask.wrap(bibFieldsIndexer::rebuildIndex).executeWith(taskExecutor); + new BackgroundTask<>() { + @Override + protected Object call() { + bibFieldsIndexer.rebuildIndex(this); + return null; + } + }.showToUser(true).executeWith(taskExecutor); if (shouldIndexLinkedFiles.get()) { - BackgroundTask.wrap(linkedFilesIndexer::rebuildIndex).executeWith(taskExecutor); + // BackgroundTask.wrap(linkedFilesIndexer::rebuildIndex).executeWith(taskExecutor); } } diff --git a/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java b/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java index d4e0e614b9b..57b7e3edd87 100644 --- a/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java @@ -2,12 +2,9 @@ import java.io.IOException; import java.util.Collection; -import java.util.List; import java.util.Map; import org.jabref.gui.util.BackgroundTask; -import org.jabref.gui.util.TaskExecutor; -import org.jabref.gui.util.UiTaskExecutor; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.HeadlessExecutorService; import org.jabref.model.database.BibDatabaseContext; @@ -34,16 +31,14 @@ public class BibFieldsIndexer implements LuceneIndexer { private static final Logger LOGGER = LoggerFactory.getLogger(BibFieldsIndexer.class); private final BibDatabaseContext databaseContext; - private final TaskExecutor taskExecutor; private final String libraryName; private final Directory indexDirectory; private final IndexWriter indexWriter; private final SearcherManager searcherManager; private IndexSearcher indexSearcher; - public BibFieldsIndexer(BibDatabaseContext databaseContext, TaskExecutor executor) throws IOException { + public BibFieldsIndexer(BibDatabaseContext databaseContext) throws IOException { this.databaseContext = databaseContext; - this.taskExecutor = executor; this.libraryName = databaseContext.getDatabasePath().map(path -> path.getFileName().toString()).orElseGet(() -> "unsaved"); IndexWriterConfig config = new IndexWriterConfig(SearchFieldConstants.NGram_Analyzer_For_INDEXING); @@ -54,37 +49,27 @@ public BibFieldsIndexer(BibDatabaseContext databaseContext, TaskExecutor executo } @Override - public void updateOnStart() { - addToIndex(databaseContext.getDatabase().getEntries()); + public void updateOnStart(BackgroundTask task) { + addToIndex(databaseContext.getDatabase().getEntries(), task); } @Override - public void addToIndex(Collection entries) { - UiTaskExecutor.runInJavaFXThread(() -> { - new BackgroundTask<>() { - @Override - protected Void call() { - int i = 1; - long startTime = System.currentTimeMillis(); - LOGGER.debug("Adding {} entries to index", entries.size()); - for (BibEntry entry : entries) { - if (isCanceled()) { - updateMessage(Localization.lang("Indexing canceled: %0 of %1 entries added to the index.", i, entries.size())); - break; - } - addToIndex(entry); - updateProgress(i, entries.size()); - updateMessage(Localization.lang("%0 of %1 entries added to the index.", i, entries.size())); - i++; - } - updateMessage(Localization.lang("Indexing completed: %0 entries added to the index.", entries.size())); - LOGGER.debug("Added {} entries to index in {} ms", entries.size(), System.currentTimeMillis() - startTime); - return null; - } - }.showToUser(entries.size() > 1) - .setTitle(Localization.lang("Indexing bib fields for %0", libraryName)) - .executeWith(taskExecutor); - }); + public void addToIndex(Collection entries, BackgroundTask task) { + task.setTitle(Localization.lang("Indexing bib fields for %0", libraryName)); + int i = 1; + long startTime = System.currentTimeMillis(); + LOGGER.debug("Adding {} entries to index", entries.size()); + for (BibEntry entry : entries) { + if (task.isCanceled()) { + LOGGER.debug("Indexing canceled"); + return; + } + addToIndex(entry); + task.updateProgress(i, entries.size()); + task.updateMessage(Localization.lang("%0 of %1 entries added to the index.", i, entries.size())); + i++; + } + LOGGER.debug("Added {} entries to index in {} ms", entries.size(), System.currentTimeMillis() - startTime); } private void addToIndex(BibEntry bibEntry) { @@ -113,13 +98,23 @@ private void addToIndex(BibEntry bibEntry) { } @Override - public void removeFromIndex(Collection entries) { - entries.forEach(this::removeFromIndex); + public void removeFromIndex(Collection entries, BackgroundTask task) { + task.setTitle(Localization.lang("Removing entries from index for %0", libraryName)); + int i = 1; + for (BibEntry entry : entries) { + if (task.isCanceled()) { + LOGGER.debug("Removing entries canceled"); + return; + } + removeFromIndex(entry); + task.updateProgress(i, entries.size()); + task.updateMessage(Localization.lang("%0 of %1 entries removed from the index.", i, entries.size())); + i++; + } } private void removeFromIndex(BibEntry entry) { try { - LOGGER.debug("Removing entry {} from index", entry.getId()); indexWriter.deleteDocuments((new Term(SearchFieldConstants.ENTRY_ID.toString(), entry.getId()))); LOGGER.debug("Entry {} removed from index", entry.getId()); } catch (IOException e) { @@ -128,10 +123,10 @@ private void removeFromIndex(BibEntry entry) { } @Override - public void updateEntry(BibEntry entry, String oldValue, String newValue) { + public void updateEntry(BibEntry entry, String oldValue, String newValue, BackgroundTask task) { LOGGER.debug("Updating entry {} in index", entry.getId()); - removeFromIndex(List.of(entry)); - addToIndex(List.of(entry)); + removeFromIndex(entry); + addToIndex(entry); } @Override @@ -146,16 +141,15 @@ public void removeAllFromIndex() { } @Override - public void rebuildIndex() { + public void rebuildIndex(BackgroundTask task) { removeAllFromIndex(); - addToIndex(databaseContext.getDatabase().getEntries()); + addToIndex(databaseContext.getDatabase().getEntries(), task); } @Override public IndexSearcher getIndexSearcher() { LOGGER.debug("Getting index searcher for bib fields index"); try { - IndexSearcher oldSearcher = indexSearcher; if (indexSearcher != null) { LOGGER.debug("Releasing bib fields index searcher"); searcherManager.release(indexSearcher); diff --git a/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java b/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java index 4268e0fc268..06fed78a31f 100644 --- a/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java @@ -26,7 +26,6 @@ import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.LinkedFile; -import org.jabref.model.search.LuceneIndexer; import org.jabref.model.search.SearchFieldConstants; import org.jabref.preferences.FilePreferences; @@ -46,7 +45,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class DefaultLinkedFilesIndexer implements LuceneIndexer { +public class DefaultLinkedFilesIndexer { private static final Logger LOGGER = LoggerFactory.getLogger(DefaultLinkedFilesIndexer.class); private static final DocumentReader DOCUMENT_READER = new DocumentReader(); private static int NUMBER_OF_UNSAVED_LIBRARIES = 1; @@ -81,7 +80,7 @@ public DefaultLinkedFilesIndexer(BibDatabaseContext databaseContext, TaskExecuto this.searcherManager = new SearcherManager(indexWriter, null); } - @Override + // @Override public void updateOnStart() { indexedFiles = getLinkedFilesFromIndex(); Map> currentFiles = getLinkedFilesFromEntries(databaseContext.getEntries()); @@ -112,7 +111,7 @@ public void updateOnStart() { addToIndex(filesToAdd); } - @Override + // @Override public void addToIndex(Collection entries) { addToIndex(getLinkedFilesFromEntries(entries)); } @@ -147,7 +146,6 @@ protected Void call() { LOGGER.debug("Adding {} files to index", linkedFiles.size()); for (Map.Entry> entry : linkedFiles.entrySet()) { if (isCanceled()) { - updateMessage(Localization.lang("Indexing canceled: %0 of %1 files added to the index.", i, linkedFiles.size())); break; } addToIndex(entry.getKey(), entry.getValue().getKey(), entry.getValue().getValue()); @@ -155,7 +153,6 @@ protected Void call() { updateMessage(Localization.lang("Indexing %0. %1 of %2 files added to the index.", entry.getValue().getValue().getFileName(), i, linkedFiles.size())); i++; } - updateMessage(Localization.lang("Indexing completed: %0 files added to the index.", linkedFiles.size())); return null; } }.willBeRecoveredAutomatically(true) @@ -176,7 +173,7 @@ private void addToIndex(String fileLink, long modifiedTime, Path resolvedPath) { } } - @Override + // @Override public void removeFromIndex(Collection entries) { Map> linkedFiles = getLinkedFilesFromEntries(entries); removeUnlinkedFiles(entries, linkedFiles.keySet()); @@ -217,7 +214,7 @@ private void removeFromIndex(String fileLink) { } } - @Override + // @Override public void updateEntry(BibEntry entry, String oldValue, String newValue) { Set oldFiles = new HashSet<>(FileFieldParser.parse(oldValue)); Set newFiles = new HashSet<>(FileFieldParser.parse(newValue)); @@ -231,7 +228,7 @@ public void updateEntry(BibEntry entry, String oldValue, String newValue) { addToIndex(toAdd); } - @Override + // @Override public void removeAllFromIndex() { try { LOGGER.debug("Removing all linked files from index."); @@ -243,7 +240,7 @@ public void removeAllFromIndex() { } } - @Override + // @Override public void rebuildIndex() { removeAllFromIndex(); addToIndex(getLinkedFilesFromEntries(databaseContext.getEntries())); @@ -303,7 +300,7 @@ private Pair getLinkedFileInfo(LinkedFile linkedFile) { } } - @Override + // @Override public IndexSearcher getIndexSearcher() { LOGGER.debug("Getting index searcher for linked files index"); try { @@ -337,7 +334,7 @@ private void optimizeIndex() { } } - @Override + // @Override public void close() { HeadlessExecutorService.INSTANCE.execute(() -> { try { diff --git a/src/main/java/org/jabref/logic/search/indexing/ReadOnlyLinkedFilesIndexer.java b/src/main/java/org/jabref/logic/search/indexing/ReadOnlyLinkedFilesIndexer.java index dac590d6af3..0ed842ec6ff 100644 --- a/src/main/java/org/jabref/logic/search/indexing/ReadOnlyLinkedFilesIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/ReadOnlyLinkedFilesIndexer.java @@ -6,7 +6,6 @@ import org.jabref.logic.util.HeadlessExecutorService; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; -import org.jabref.model.search.LuceneIndexer; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.SearcherManager; @@ -15,7 +14,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class ReadOnlyLinkedFilesIndexer implements LuceneIndexer { +public class ReadOnlyLinkedFilesIndexer { private static final Logger LOGGER = LoggerFactory.getLogger(ReadOnlyLinkedFilesIndexer.class); private Directory indexDirectory; private SearcherManager searcherManager; @@ -30,31 +29,31 @@ public ReadOnlyLinkedFilesIndexer(BibDatabaseContext databaseContext) { } } - @Override +// @Override public void updateOnStart() { } - @Override +// @Override public void addToIndex(Collection entries) { } - @Override +// @Override public void removeFromIndex(Collection entries) { } - @Override +// @Override public void updateEntry(BibEntry entry, String oldValue, String newValue) { } - @Override +// @Override public void removeAllFromIndex() { } - @Override +// @Override public void rebuildIndex() { } - @Override +// @Override public IndexSearcher getIndexSearcher() { try { if (indexSearcher != null) { @@ -68,7 +67,7 @@ public IndexSearcher getIndexSearcher() { return indexSearcher; } - @Override +// @Override public void close() { HeadlessExecutorService.INSTANCE.execute(() -> { try { diff --git a/src/main/java/org/jabref/model/search/LuceneIndexer.java b/src/main/java/org/jabref/model/search/LuceneIndexer.java index eb6143ee840..f8ed054a2c9 100644 --- a/src/main/java/org/jabref/model/search/LuceneIndexer.java +++ b/src/main/java/org/jabref/model/search/LuceneIndexer.java @@ -2,22 +2,23 @@ import java.util.Collection; +import org.jabref.gui.util.BackgroundTask; import org.jabref.model.entry.BibEntry; import org.apache.lucene.search.IndexSearcher; public interface LuceneIndexer { - void updateOnStart(); + void updateOnStart(BackgroundTask task); - void addToIndex(Collection entries); + void addToIndex(Collection entries, BackgroundTask task); - void removeFromIndex(Collection entries); + void removeFromIndex(Collection entries, BackgroundTask task); - void updateEntry(BibEntry entry, String oldValue, String newValue); + void updateEntry(BibEntry entry, String oldValue, String newValue, BackgroundTask task); void removeAllFromIndex(); - void rebuildIndex(); + void rebuildIndex(BackgroundTask task); IndexSearcher getIndexSearcher(); diff --git a/src/main/java/org/jabref/model/search/envent/IndexAddedEvent.java b/src/main/java/org/jabref/model/search/envent/IndexAddedEvent.java new file mode 100644 index 00000000000..c8f01b04c27 --- /dev/null +++ b/src/main/java/org/jabref/model/search/envent/IndexAddedEvent.java @@ -0,0 +1,4 @@ +package org.jabref.model.search.envent; + +public class IndexAddedEvent { +} diff --git a/src/main/java/org/jabref/model/search/envent/IndexRemovedEvent.java b/src/main/java/org/jabref/model/search/envent/IndexRemovedEvent.java new file mode 100644 index 00000000000..dd00c496c0d --- /dev/null +++ b/src/main/java/org/jabref/model/search/envent/IndexRemovedEvent.java @@ -0,0 +1,4 @@ +package org.jabref.model.search.envent; + +public class IndexRemovedEvent { +} diff --git a/src/main/java/org/jabref/model/search/envent/IndexStartedEvent.java b/src/main/java/org/jabref/model/search/envent/IndexStartedEvent.java new file mode 100644 index 00000000000..0bd7f738167 --- /dev/null +++ b/src/main/java/org/jabref/model/search/envent/IndexStartedEvent.java @@ -0,0 +1,4 @@ +package org.jabref.model.search.envent; + +public class IndexStartedEvent { +} diff --git a/src/main/java/org/jabref/model/search/envent/IndexUpdatedEvent.java b/src/main/java/org/jabref/model/search/envent/IndexUpdatedEvent.java new file mode 100644 index 00000000000..e43c8677011 --- /dev/null +++ b/src/main/java/org/jabref/model/search/envent/IndexUpdatedEvent.java @@ -0,0 +1,4 @@ +package org.jabref.model.search.envent; + +public class IndexUpdatedEvent { +} diff --git a/src/main/resources/l10n/JabRef_en.properties b/src/main/resources/l10n/JabRef_en.properties index 2c29df57c37..110f0365e3c 100644 --- a/src/main/resources/l10n/JabRef_en.properties +++ b/src/main/resources/l10n/JabRef_en.properties @@ -498,18 +498,11 @@ Include\ subgroups\:\ When\ selected,\ view\ entries\ contained\ in\ this\ group Independent\ group\:\ When\ selected,\ view\ only\ this\ group's\ entries=Independent group: When selected, view only this group's entries I\ Agree=I Agree -# library name (Title of the task) Indexing\ bib\ fields\ for\ %0=Indexing bib fields for %0 -# indexing canceled -Indexing\ canceled\:\ %0\ of\ %1\ entries\ added\ to\ the\ index.=Indexing canceled: %0 of %1 entries added to the index. -Indexing\ canceled\:\ %0\ of\ %1\ files\ added\ to\ the\ index.=Indexing canceled: %0 of %1 files added to the index. -# indexing running %0\ of\ %1\ entries\ added\ to\ the\ index.=%0 of %1 entries added to the index. +%0\ of\ %1\ entries\ removed\ from\ the\ index.=%0 of %1 entries removed from the index. Indexing\ %0.\ %1\ of\ %2\ files\ added\ to\ the\ index.=Indexing %0. %1 of %2 files added to the index. -# indexing finished -Indexing\ completed\:\ %0\ entries\ added\ to\ the\ index.=Indexing completed: %0 entries added to the index. -Indexing\ completed\:\ %0\ files\ added\ to\ the\ index.=Indexing completed: %0 files added to the index. - +Removing\ entries\ from\ index\ for\ %0=Removing entries from index for %0 Invalid\ URL=Invalid URL Online\ help=Online help From 1b1cdbb8031e7af17634819307fcfd6b0c87a548 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Mon, 12 Aug 2024 19:59:05 +0300 Subject: [PATCH 132/256] Refactor linked files indexer --- .../jabref/logic/search/LuceneManager.java | 102 ++++++++++++----- .../indexing/DefaultLinkedFilesIndexer.java | 105 ++++++++---------- .../indexing/ReadOnlyLinkedFilesIndexer.java | 30 ++--- 3 files changed, 134 insertions(+), 103 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/LuceneManager.java b/src/main/java/org/jabref/logic/search/LuceneManager.java index 81afcd24d23..a2cb1cba5dc 100644 --- a/src/main/java/org/jabref/logic/search/LuceneManager.java +++ b/src/main/java/org/jabref/logic/search/LuceneManager.java @@ -2,6 +2,7 @@ import java.io.IOException; import java.util.Collection; +import java.util.List; import javafx.beans.property.BooleanProperty; import javafx.beans.property.SimpleBooleanProperty; @@ -10,6 +11,8 @@ import org.jabref.gui.util.BackgroundTask; import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.search.indexing.BibFieldsIndexer; +import org.jabref.logic.search.indexing.DefaultLinkedFilesIndexer; +import org.jabref.logic.search.indexing.ReadOnlyLinkedFilesIndexer; import org.jabref.logic.search.retrieval.LuceneSearcher; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; @@ -65,25 +68,29 @@ private void initializeIndexers() { } private void initializeLinkedFilesIndexer() { -// try { -// linkedFilesIndexer = new DefaultLinkedFilesIndexer(databaseContext, taskExecutor, preferences.getFilePreferences()); -// } catch (IOException e) { -// LOGGER.debug("Error initializing linked files index - using read only index"); -// linkedFilesIndexer = new ReadOnlyLinkedFilesIndexer(databaseContext); -// } + try { + linkedFilesIndexer = new DefaultLinkedFilesIndexer(databaseContext, preferences.getFilePreferences()); + } catch (IOException e) { + LOGGER.debug("Error initializing linked files index - using read only index"); + linkedFilesIndexer = new ReadOnlyLinkedFilesIndexer(databaseContext); + } } private void bindToPreferences(boolean newValue) { - BackgroundTask.wrap(() -> { - if (newValue) { - initializeLinkedFilesIndexer(); - // linkedFilesIndexer.updateOnStart(); - } else { - linkedFilesIndexer.removeAllFromIndex(); - linkedFilesIndexer.close(); - linkedFilesIndexer = null; - } - }).executeWith(taskExecutor); + if (newValue) { + initializeLinkedFilesIndexer(); + new BackgroundTask<>() { + @Override + protected Object call() { + linkedFilesIndexer.updateOnStart(this); + return null; + } + }.showToUser(true).executeWith(taskExecutor); + } else { + linkedFilesIndexer.removeAllFromIndex(); + linkedFilesIndexer.close(); + linkedFilesIndexer = null; + } } public void updateOnStart() { @@ -95,8 +102,14 @@ protected Object call() { } }.showToUser(true).executeWith(taskExecutor); - if (shouldIndexLinkedFiles.get()) { - // BackgroundTask.wrap(linkedFilesIndexer::updateOnStart).executeWith(taskExecutor); + if (shouldIndexLinkedFiles.get() && linkedFilesIndexer != null) { + new BackgroundTask<>() { + @Override + protected Object call() { + linkedFilesIndexer.updateOnStart(this); + return null; + } + }.showToUser(true).executeWith(taskExecutor); } } @@ -107,11 +120,16 @@ protected Object call() { bibFieldsIndexer.addToIndex(entries, this); return null; } - }.onSuccess(r -> LOGGER.debug("OnSuccess")) - .showToUser(true).executeWith(taskExecutor); + }.showToUser(true).executeWith(taskExecutor); - if (shouldIndexLinkedFiles.get() && !isLinkedFilesIndexerBlocked.get()) { - // BackgroundTask.wrap(() -> linkedFilesIndexer.addToIndex(entries)).executeWith(taskExecutor); + if (shouldIndexLinkedFiles.get() && !isLinkedFilesIndexerBlocked.get() && linkedFilesIndexer != null) { + new BackgroundTask<>() { + @Override + protected Object call() { + linkedFilesIndexer.addToIndex(entries, this); + return null; + } + }.showToUser(true).executeWith(taskExecutor); } } @@ -124,8 +142,14 @@ protected Object call() { } }.showToUser(true).executeWith(taskExecutor); - if (shouldIndexLinkedFiles.get()) { - // BackgroundTask.wrap(() -> linkedFilesIndexer.removeFromIndex(entries)).executeWith(taskExecutor); + if (shouldIndexLinkedFiles.get() && linkedFilesIndexer != null) { + new BackgroundTask<>() { + @Override + protected Object call() { + linkedFilesIndexer.removeFromIndex(entries, this); + return null; + } + }.showToUser(true).executeWith(taskExecutor); } } @@ -138,8 +162,14 @@ protected Object call() { } }.showToUser(true).executeWith(taskExecutor); - if (isLinkedFile && shouldIndexLinkedFiles.get() && !isLinkedFilesIndexerBlocked.get()) { - // BackgroundTask.wrap(() -> linkedFilesIndexer.updateEntry(entry, oldValue, newValue)).executeWith(taskExecutor); + if (isLinkedFile && shouldIndexLinkedFiles.get() && !isLinkedFilesIndexerBlocked.get() && linkedFilesIndexer != null) { + new BackgroundTask<>() { + @Override + protected Object call() { + linkedFilesIndexer.updateEntry(entry, oldValue, newValue, this); + return null; + } + }.showToUser(true).executeWith(taskExecutor); } } @@ -152,8 +182,14 @@ protected Object call() { } }.showToUser(true).executeWith(taskExecutor); - if (shouldIndexLinkedFiles.get() && !isLinkedFilesIndexerBlocked.get()) { - // BackgroundTask.wrap(() -> linkedFilesIndexer.addToIndex(List.of(entry))).executeWith(taskExecutor); + if (shouldIndexLinkedFiles.get() && !isLinkedFilesIndexerBlocked.get() && linkedFilesIndexer != null) { + new BackgroundTask<>() { + @Override + protected Object call() { + linkedFilesIndexer.addToIndex(List.of(entry), this); + return null; + } + }.showToUser(true).executeWith(taskExecutor); } } @@ -166,8 +202,14 @@ protected Object call() { } }.showToUser(true).executeWith(taskExecutor); - if (shouldIndexLinkedFiles.get()) { - // BackgroundTask.wrap(linkedFilesIndexer::rebuildIndex).executeWith(taskExecutor); + if (shouldIndexLinkedFiles.get() && linkedFilesIndexer != null) { + new BackgroundTask<>() { + @Override + protected Object call() { + linkedFilesIndexer.rebuildIndex(this); + return null; + } + }.showToUser(true).executeWith(taskExecutor); } } diff --git a/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java b/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java index 06fed78a31f..5c637350318 100644 --- a/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java @@ -17,8 +17,6 @@ import javafx.util.Pair; import org.jabref.gui.util.BackgroundTask; -import org.jabref.gui.util.TaskExecutor; -import org.jabref.gui.util.UiTaskExecutor; import org.jabref.logic.importer.util.FileFieldParser; import org.jabref.logic.l10n.Localization; import org.jabref.logic.util.HeadlessExecutorService; @@ -26,6 +24,7 @@ import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.LinkedFile; +import org.jabref.model.search.LuceneIndexer; import org.jabref.model.search.SearchFieldConstants; import org.jabref.preferences.FilePreferences; @@ -45,13 +44,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class DefaultLinkedFilesIndexer { +public class DefaultLinkedFilesIndexer implements LuceneIndexer { private static final Logger LOGGER = LoggerFactory.getLogger(DefaultLinkedFilesIndexer.class); private static final DocumentReader DOCUMENT_READER = new DocumentReader(); private static int NUMBER_OF_UNSAVED_LIBRARIES = 1; private final BibDatabaseContext databaseContext; - private final TaskExecutor taskExecutor; private final FilePreferences filePreferences; private final String libraryName; private final Directory indexDirectory; @@ -61,9 +59,8 @@ public class DefaultLinkedFilesIndexer { private Map indexedFiles; private IndexSearcher indexSearcher; - public DefaultLinkedFilesIndexer(BibDatabaseContext databaseContext, TaskExecutor executor, FilePreferences filePreferences) throws IOException { + public DefaultLinkedFilesIndexer(BibDatabaseContext databaseContext, FilePreferences filePreferences) throws IOException { this.databaseContext = databaseContext; - this.taskExecutor = executor; this.filePreferences = filePreferences; this.libraryName = databaseContext.getDatabasePath().map(path -> path.getFileName().toString()).orElseGet(() -> "untitled"); this.indexedFiles = new ConcurrentHashMap<>(); @@ -80,8 +77,8 @@ public DefaultLinkedFilesIndexer(BibDatabaseContext databaseContext, TaskExecuto this.searcherManager = new SearcherManager(indexWriter, null); } - // @Override - public void updateOnStart() { + @Override + public void updateOnStart(BackgroundTask task) { indexedFiles = getLinkedFilesFromIndex(); Map> currentFiles = getLinkedFilesFromEntries(databaseContext.getEntries()); @@ -102,21 +99,20 @@ public void updateOnStart() { Map> filesToAdd = new HashMap<>(); for (Map.Entry> entry : currentFiles.entrySet()) { String fileLink = entry.getKey(); - long modification = entry.getValue().getKey(); if (!indexedFiles.containsKey(fileLink)) { LOGGER.debug("File {} has been added to the library. Will be added to the index.", fileLink); filesToAdd.put(fileLink, entry.getValue()); } } - addToIndex(filesToAdd); + addToIndex(filesToAdd, task); } - // @Override - public void addToIndex(Collection entries) { - addToIndex(getLinkedFilesFromEntries(entries)); + @Override + public void addToIndex(Collection entries, BackgroundTask task) { + addToIndex(getLinkedFilesFromEntries(entries), task); } - private void addToIndex(Set linkedFiles) { + private void addToIndex(Set linkedFiles, BackgroundTask task) { Map> filesToAdd = new HashMap<>(); for (LinkedFile linkedFile : linkedFiles) { Pair fileInfo = getLinkedFileInfo(linkedFile); @@ -124,10 +120,10 @@ private void addToIndex(Set linkedFiles) { filesToAdd.put(linkedFile.getLink(), fileInfo); } } - addToIndex(filesToAdd); + addToIndex(filesToAdd, task); } - private void addToIndex(Map> linkedFiles) { + private void addToIndex(Map> linkedFiles, BackgroundTask task) { for (String fileLink : linkedFiles.keySet()) { if (indexedFiles.containsKey(fileLink)) { LOGGER.debug("File {} is already indexed.", fileLink); @@ -138,28 +134,21 @@ private void addToIndex(Map> linkedFiles) { return; } - UiTaskExecutor.runInJavaFXThread(() -> { - new BackgroundTask<>() { - @Override - protected Void call() { - int i = 1; - LOGGER.debug("Adding {} files to index", linkedFiles.size()); - for (Map.Entry> entry : linkedFiles.entrySet()) { - if (isCanceled()) { - break; - } - addToIndex(entry.getKey(), entry.getValue().getKey(), entry.getValue().getValue()); - updateProgress(i, linkedFiles.size()); - updateMessage(Localization.lang("Indexing %0. %1 of %2 files added to the index.", entry.getValue().getValue().getFileName(), i, linkedFiles.size())); - i++; - } - return null; - } - }.willBeRecoveredAutomatically(true) - .showToUser(true) - .setTitle(Localization.lang("Indexing pdf files for %0", libraryName)) - .executeWith(taskExecutor); - }); + task.willBeRecoveredAutomatically(true); + task.setTitle(Localization.lang("Indexing pdf files for %0", libraryName)); + LOGGER.debug("Adding {} files to index", linkedFiles.size()); + int i = 1; + for (Map.Entry> entry : linkedFiles.entrySet()) { + if (task.isCanceled()) { + LOGGER.debug("Adding files to index canceled"); + return; + } + addToIndex(entry.getKey(), entry.getValue().getKey(), entry.getValue().getValue()); + task.updateProgress(i, linkedFiles.size()); + task.updateMessage(Localization.lang("Indexing %0. %1 of %2 files added to the index.", entry.getValue().getValue().getFileName(), i, linkedFiles.size())); + i++; + } + LOGGER.debug("Added {} files to index", linkedFiles.size()); } private void addToIndex(String fileLink, long modifiedTime, Path resolvedPath) { @@ -173,8 +162,8 @@ private void addToIndex(String fileLink, long modifiedTime, Path resolvedPath) { } } - // @Override - public void removeFromIndex(Collection entries) { + @Override + public void removeFromIndex(Collection entries, BackgroundTask task) { Map> linkedFiles = getLinkedFilesFromEntries(entries); removeUnlinkedFiles(entries, linkedFiles.keySet()); } @@ -201,21 +190,19 @@ private void removeUnlinkedFiles(Collection entriesToRemove, Collectio } private void removeFromIndex(Set links) { - links.forEach(this::removeFromIndex); - } - - private void removeFromIndex(String fileLink) { - try { - LOGGER.debug("Removing file {} from index.", fileLink); - indexWriter.deleteDocuments(new Term(SearchFieldConstants.PATH.toString(), fileLink)); - indexedFiles.remove(fileLink); - } catch (IOException e) { - LOGGER.warn("Could not remove linked file {} from index.", fileLink, e); + for (String fileLink : links) { + try { + LOGGER.debug("Removing file {} from index.", fileLink); + indexWriter.deleteDocuments(new Term(SearchFieldConstants.PATH.toString(), fileLink)); + indexedFiles.remove(fileLink); + } catch (IOException e) { + LOGGER.warn("Could not remove linked file {} from index.", fileLink, e); + } } } - // @Override - public void updateEntry(BibEntry entry, String oldValue, String newValue) { + @Override + public void updateEntry(BibEntry entry, String oldValue, String newValue, BackgroundTask task) { Set oldFiles = new HashSet<>(FileFieldParser.parse(oldValue)); Set newFiles = new HashSet<>(FileFieldParser.parse(newValue)); @@ -225,10 +212,10 @@ public void updateEntry(BibEntry entry, String oldValue, String newValue) { Set toAdd = new HashSet<>(newFiles); toAdd.removeAll(oldFiles); - addToIndex(toAdd); + addToIndex(toAdd, task); } - // @Override + @Override public void removeAllFromIndex() { try { LOGGER.debug("Removing all linked files from index."); @@ -240,10 +227,10 @@ public void removeAllFromIndex() { } } - // @Override - public void rebuildIndex() { + @Override + public void rebuildIndex(BackgroundTask task) { removeAllFromIndex(); - addToIndex(getLinkedFilesFromEntries(databaseContext.getEntries())); + addToIndex(getLinkedFilesFromEntries(databaseContext.getEntries()), task); } private Map getLinkedFilesFromIndex() { @@ -300,7 +287,7 @@ private Pair getLinkedFileInfo(LinkedFile linkedFile) { } } - // @Override + @Override public IndexSearcher getIndexSearcher() { LOGGER.debug("Getting index searcher for linked files index"); try { @@ -334,7 +321,7 @@ private void optimizeIndex() { } } - // @Override + @Override public void close() { HeadlessExecutorService.INSTANCE.execute(() -> { try { diff --git a/src/main/java/org/jabref/logic/search/indexing/ReadOnlyLinkedFilesIndexer.java b/src/main/java/org/jabref/logic/search/indexing/ReadOnlyLinkedFilesIndexer.java index 0ed842ec6ff..19e63e7ab16 100644 --- a/src/main/java/org/jabref/logic/search/indexing/ReadOnlyLinkedFilesIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/ReadOnlyLinkedFilesIndexer.java @@ -3,9 +3,11 @@ import java.io.IOException; import java.util.Collection; +import org.jabref.gui.util.BackgroundTask; import org.jabref.logic.util.HeadlessExecutorService; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; +import org.jabref.model.search.LuceneIndexer; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.SearcherManager; @@ -14,7 +16,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class ReadOnlyLinkedFilesIndexer { +public class ReadOnlyLinkedFilesIndexer implements LuceneIndexer { private static final Logger LOGGER = LoggerFactory.getLogger(ReadOnlyLinkedFilesIndexer.class); private Directory indexDirectory; private SearcherManager searcherManager; @@ -29,31 +31,31 @@ public ReadOnlyLinkedFilesIndexer(BibDatabaseContext databaseContext) { } } -// @Override - public void updateOnStart() { + @Override + public void updateOnStart(BackgroundTask task) { } -// @Override - public void addToIndex(Collection entries) { + @Override + public void addToIndex(Collection entries, BackgroundTask task) { } -// @Override - public void removeFromIndex(Collection entries) { + @Override + public void removeFromIndex(Collection entries, BackgroundTask task) { } -// @Override - public void updateEntry(BibEntry entry, String oldValue, String newValue) { + @Override + public void updateEntry(BibEntry entry, String oldValue, String newValue, BackgroundTask task) { } -// @Override + @Override public void removeAllFromIndex() { } -// @Override - public void rebuildIndex() { + @Override + public void rebuildIndex(BackgroundTask task) { } -// @Override + @Override public IndexSearcher getIndexSearcher() { try { if (indexSearcher != null) { @@ -67,7 +69,7 @@ public IndexSearcher getIndexSearcher() { return indexSearcher; } -// @Override + @Override public void close() { HeadlessExecutorService.INSTANCE.execute(() -> { try { From 43c8d9de585d9c084cc78eec4e653be39fe0cf25 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Tue, 13 Aug 2024 21:51:40 +0300 Subject: [PATCH 133/256] Update search matches when entries are added or updated --- .../gui/maintable/MainTableColumnFactory.java | 8 +-- .../gui/maintable/MainTableDataModel.java | 63 ++++++++++++++----- .../jabref/logic/search/LuceneManager.java | 37 ++++++++--- .../jabref/model/search/GroupSearchQuery.java | 4 -- .../org/jabref/model/search/SearchQuery.java | 4 ++ .../model/search/envent/IndexAddedEvent.java | 4 -- .../envent/IndexAddedOrUpdatedEvent.java | 8 +++ 7 files changed, 92 insertions(+), 36 deletions(-) delete mode 100644 src/main/java/org/jabref/model/search/envent/IndexAddedEvent.java create mode 100644 src/main/java/org/jabref/model/search/envent/IndexAddedOrUpdatedEvent.java diff --git a/src/main/java/org/jabref/gui/maintable/MainTableColumnFactory.java b/src/main/java/org/jabref/gui/maintable/MainTableColumnFactory.java index 9c31ad40cd1..12892812f78 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableColumnFactory.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableColumnFactory.java @@ -162,15 +162,15 @@ private TableColumn createMatchCategoryCo return column; } - private TableColumn createScoreColumn(MainTableColumnModel columnModel) { - TableColumn column = new MainTableColumn<>(columnModel); + private TableColumn createScoreColumn(MainTableColumnModel columnModel) { + TableColumn column = new MainTableColumn<>(columnModel); Node header = new Text(Localization.lang("Score")); header.getStyleClass().add("mainTable-header"); Tooltip.install(header, new Tooltip(MainTableColumnModel.Type.SCORE.getDisplayName())); column.setGraphic(header); column.setStyle("-fx-alignment: CENTER-RIGHT;"); - column.setCellValueFactory(cellData -> new ReadOnlyObjectWrapper<>(cellData.getValue().searchScoreProperty().getValue())); - new ValueTableCellFactory().withText(String::valueOf).install(column); + column.setCellValueFactory(cellData -> cellData.getValue().searchScoreProperty()); + new ValueTableCellFactory().withText(String::valueOf).install(column); column.setSortable(true); column.setReorderable(false); column.visibleProperty().bind(stateManager.activeSearchQuery(SearchType.NORMAL_SEARCH).isPresent()); diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index 22b736cea61..5831beaead4 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -8,7 +8,6 @@ import javafx.beans.property.ListProperty; import javafx.beans.property.ObjectProperty; import javafx.beans.property.SimpleObjectProperty; -import javafx.collections.ListChangeListener; import javafx.collections.ObservableList; import javafx.collections.transformation.FilteredList; import javafx.collections.transformation.SortedList; @@ -27,18 +26,24 @@ import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.groups.GroupTreeNode; +import org.jabref.model.groups.SearchGroup; +import org.jabref.model.search.SearchFieldConstants; import org.jabref.model.search.SearchQuery; import org.jabref.model.search.SearchResults; +import org.jabref.model.search.envent.IndexAddedOrUpdatedEvent; +import org.jabref.model.search.envent.IndexStartedEvent; import org.jabref.model.search.matchers.MatcherSet; import org.jabref.model.search.matchers.MatcherSets; import org.jabref.preferences.PreferencesService; import org.jabref.preferences.SearchPreferences; +import com.google.common.eventbus.Subscribe; import com.tobiasdiez.easybind.EasyBind; import com.tobiasdiez.easybind.Subscription; public class MainTableDataModel { + private final ObservableList allEntries; private final ObservableList entriesViewModel; private final FilteredList entriesFiltered; private final SortedList entriesFilteredAndSorted; @@ -54,6 +59,8 @@ public class MainTableDataModel { private final Subscription searchDisplayModeSubscription; private final Subscription selectedGroupsSubscription; private final Subscription groupViewModeSubscription; + private final LuceneIndexListener indexUpdatedListener; + private final OptionalObjectProperty searchQueryProperty; private Optional groupsMatcher; public MainTableDataModel(BibDatabaseContext context, @@ -72,26 +79,18 @@ public MainTableDataModel(BibDatabaseContext context, this.luceneManager = luceneManager; this.bibDatabaseContext = context; this.groupsMatcher = createGroupMatcher(selectedGroupsProperty.get(), groupsPreferences); + this.searchQueryProperty = searchQueryProperty; + this.indexUpdatedListener = new LuceneIndexListener(); + if (luceneManager != null) { + this.luceneManager.registerListener(indexUpdatedListener); + } resetFieldFormatter(); - ObservableList allEntries = BindingsHelper.forUI(context.getDatabase().getEntries()); + allEntries = BindingsHelper.forUI(context.getDatabase().getEntries()); entriesViewModel = EasyBind.mapBacked(allEntries, entry -> new BibEntryTableViewModel(entry, bibDatabaseContext, fieldValueFormatter), false); entriesFiltered = new FilteredList<>(entriesViewModel, BibEntryTableViewModel::isVisible); - entriesViewModel.addListener((ListChangeListener.Change change) -> { - while (change.next()) { - if (change.wasAdded() || change.wasUpdated()) { - BackgroundTask.wrap(() -> { - for (BibEntryTableViewModel entry : change.getList().subList(change.getFrom(), change.getTo())) { - // updateEntrySearchMatch(searchQueryProperty.get(), entry, searchPreferences.getSearchDisplayMode() == SearchDisplayMode.FLOAT); - updateEntryGroupMatch(entry, groupsMatcher, groupsPreferences.getGroupViewMode().contains(GroupViewMode.INVERT), !groupsPreferences.getGroupViewMode().contains(GroupViewMode.FILTER)); - } - }).onSuccess(result -> FilteredListProxy.refilterListReflection(entriesFiltered, change.getFrom(), change.getTo())).executeWith(taskExecutor); - } - } - }); - searchQuerySubscription = EasyBind.listen(searchQueryProperty, (observable, oldValue, newValue) -> updateSearchMatches(newValue)); searchDisplayModeSubscription = EasyBind.listen(searchPreferences.searchDisplayModeProperty(), (observable, oldValue, newValue) -> updateSearchDisplayMode(newValue)); selectedGroupsSubscription = EasyBind.listen(selectedGroupsProperty, (observable, oldValue, newValue) -> updateGroupMatches(newValue)); @@ -187,6 +186,9 @@ public void unbind() { searchDisplayModeSubscription.unsubscribe(); selectedGroupsSubscription.unsubscribe(); groupViewModeSubscription.unsubscribe(); + if (luceneManager != null) { + luceneManager.unregisterListener(indexUpdatedListener); + } } public SortedList getEntriesFilteredAndSorted() { @@ -196,4 +198,35 @@ public SortedList getEntriesFilteredAndSorted() { public void resetFieldFormatter() { this.fieldValueFormatter.setValue(new MainTableFieldValueFormatter(nameDisplayPreferences, bibDatabaseContext)); } + + class LuceneIndexListener { + @Subscribe + public void listen(IndexAddedOrUpdatedEvent indexAddedOrUpdatedEvent) { + indexAddedOrUpdatedEvent.addedEntries().forEach(entry -> { + BackgroundTask.wrap(() -> { + int index = allEntries.indexOf(entry); + if (index >= 0) { + BibEntryTableViewModel viewModel = entriesViewModel.get(index); + boolean isFloatingMode = searchPreferences.getSearchDisplayMode() == SearchDisplayMode.FLOAT; + if (searchQueryProperty.get().isPresent()) { + SearchQuery searchQuery = searchQueryProperty.get().get(); + String newSearchExpression = "+" + SearchFieldConstants.ENTRY_ID + ":" + entry.getId() + " +" + searchQuery.getSearchExpression(); + SearchQuery entryQuery = new SearchQuery(newSearchExpression, searchQuery.getSearchFlags()); + SearchResults results = luceneManager.search(entryQuery); + + viewModel.searchScoreProperty().set(results.getSearchScoreForEntry(entry)); + viewModel.hasFullTextResultsProperty().set(results.hasFulltextResults(entry)); + updateEntrySearchMatch(viewModel, viewModel.searchScoreProperty().get() > 0, isFloatingMode); + } else { + viewModel.searchScoreProperty().set(0); + viewModel.hasFullTextResultsProperty().set(false); + updateEntrySearchMatch(viewModel, true, isFloatingMode); + } + updateEntryGroupMatch(viewModel, groupsMatcher, groupsPreferences.getGroupViewMode().contains(GroupViewMode.INVERT), !groupsPreferences.getGroupViewMode().contains(GroupViewMode.FILTER)); + } + return index; + }).onSuccess(index -> FilteredListProxy.refilterListReflection(entriesFiltered, index, index + 1)).executeWith(taskExecutor); + }); + } + } } diff --git a/src/main/java/org/jabref/logic/search/LuceneManager.java b/src/main/java/org/jabref/logic/search/LuceneManager.java index a2cb1cba5dc..ef90d488e23 100644 --- a/src/main/java/org/jabref/logic/search/LuceneManager.java +++ b/src/main/java/org/jabref/logic/search/LuceneManager.java @@ -1,7 +1,6 @@ package org.jabref.logic.search; import java.io.IOException; -import java.util.Collection; import java.util.List; import javafx.beans.property.BooleanProperty; @@ -20,8 +19,12 @@ import org.jabref.model.search.SearchFlags; import org.jabref.model.search.SearchQuery; import org.jabref.model.search.SearchResults; +import org.jabref.model.search.envent.IndexAddedOrUpdatedEvent; +import org.jabref.model.search.envent.IndexRemovedEvent; +import org.jabref.model.search.envent.IndexStartedEvent; import org.jabref.preferences.PreferencesService; +import com.google.common.eventbus.EventBus; import org.apache.lucene.index.MultiReader; import org.apache.lucene.search.IndexSearcher; import org.slf4j.Logger; @@ -30,6 +33,7 @@ public class LuceneManager { private static final Logger LOGGER = LoggerFactory.getLogger(LuceneManager.class); + private final EventBus eventBus = new EventBus(); private final BibDatabaseContext databaseContext; private final TaskExecutor taskExecutor; private final PreferencesService preferences; @@ -100,7 +104,9 @@ protected Object call() { bibFieldsIndexer.updateOnStart(this); return null; } - }.showToUser(true).executeWith(taskExecutor); + }.showToUser(true) + .onFinished(() -> this.eventBus.post(new IndexStartedEvent())) + .executeWith(taskExecutor); if (shouldIndexLinkedFiles.get() && linkedFilesIndexer != null) { new BackgroundTask<>() { @@ -113,14 +119,15 @@ protected Object call() { } } - public void addToIndex(Collection entries) { + public void addToIndex(List entries) { new BackgroundTask<>() { @Override protected Object call() { bibFieldsIndexer.addToIndex(entries, this); return null; } - }.showToUser(true).executeWith(taskExecutor); + }.onFinished(() -> this.eventBus.post(new IndexAddedOrUpdatedEvent(entries))) + .showToUser(true).executeWith(taskExecutor); if (shouldIndexLinkedFiles.get() && !isLinkedFilesIndexerBlocked.get() && linkedFilesIndexer != null) { new BackgroundTask<>() { @@ -133,14 +140,15 @@ protected Object call() { } } - public void removeFromIndex(Collection entries) { + public void removeFromIndex(List entries) { new BackgroundTask<>() { @Override protected Object call() { bibFieldsIndexer.removeFromIndex(entries, this); return null; } - }.showToUser(true).executeWith(taskExecutor); + }.onFinished(() -> this.eventBus.post(new IndexRemovedEvent())) + .showToUser(true).executeWith(taskExecutor); if (shouldIndexLinkedFiles.get() && linkedFilesIndexer != null) { new BackgroundTask<>() { @@ -160,7 +168,8 @@ protected Object call() { bibFieldsIndexer.updateEntry(entry, oldValue, newValue, this); return null; } - }.showToUser(true).executeWith(taskExecutor); + }.onFinished(() -> this.eventBus.post(new IndexAddedOrUpdatedEvent(List.of(entry)))) + .showToUser(true).executeWith(taskExecutor); if (isLinkedFile && shouldIndexLinkedFiles.get() && !isLinkedFilesIndexerBlocked.get() && linkedFilesIndexer != null) { new BackgroundTask<>() { @@ -180,7 +189,8 @@ protected Object call() { bibFieldsIndexer.updateEntry(entry, "", "", this); return null; } - }.showToUser(true).executeWith(taskExecutor); + }.onFinished(() -> this.eventBus.post(new IndexAddedOrUpdatedEvent(List.of(entry)))) + .showToUser(true).executeWith(taskExecutor); if (shouldIndexLinkedFiles.get() && !isLinkedFilesIndexerBlocked.get() && linkedFilesIndexer != null) { new BackgroundTask<>() { @@ -200,7 +210,8 @@ protected Object call() { bibFieldsIndexer.rebuildIndex(this); return null; } - }.showToUser(true).executeWith(taskExecutor); + }.onFinished(() -> this.eventBus.post(new IndexStartedEvent())) + .showToUser(true).executeWith(taskExecutor); if (shouldIndexLinkedFiles.get() && linkedFilesIndexer != null) { new BackgroundTask<>() { @@ -244,4 +255,12 @@ public IndexSearcher getIndexSearcher(SearchQuery query) { public SearchResults search(SearchQuery query) { return luceneSearcher.search(query, getIndexSearcher(query)); } + + public void registerListener(Object listener) { + this.eventBus.register(listener); + } + + public void unregisterListener(Object listener) { + eventBus.unregister(listener); + } } diff --git a/src/main/java/org/jabref/model/search/GroupSearchQuery.java b/src/main/java/org/jabref/model/search/GroupSearchQuery.java index 26d08dab0cc..a7813697412 100644 --- a/src/main/java/org/jabref/model/search/GroupSearchQuery.java +++ b/src/main/java/org/jabref/model/search/GroupSearchQuery.java @@ -7,8 +7,4 @@ public class GroupSearchQuery extends SearchQuery { public GroupSearchQuery(String query, EnumSet searchFlags) { super(query, searchFlags); } - - public String getSearchExpression() { - return query; - } } diff --git a/src/main/java/org/jabref/model/search/SearchQuery.java b/src/main/java/org/jabref/model/search/SearchQuery.java index 839c622a9ac..4ef1f6a0d46 100644 --- a/src/main/java/org/jabref/model/search/SearchQuery.java +++ b/src/main/java/org/jabref/model/search/SearchQuery.java @@ -95,6 +95,10 @@ public SearchQuery(String query, EnumSet searchFlags) { } } + public String getSearchExpression() { + return query; + } + @Override public String toString() { return query; diff --git a/src/main/java/org/jabref/model/search/envent/IndexAddedEvent.java b/src/main/java/org/jabref/model/search/envent/IndexAddedEvent.java deleted file mode 100644 index c8f01b04c27..00000000000 --- a/src/main/java/org/jabref/model/search/envent/IndexAddedEvent.java +++ /dev/null @@ -1,4 +0,0 @@ -package org.jabref.model.search.envent; - -public class IndexAddedEvent { -} diff --git a/src/main/java/org/jabref/model/search/envent/IndexAddedOrUpdatedEvent.java b/src/main/java/org/jabref/model/search/envent/IndexAddedOrUpdatedEvent.java new file mode 100644 index 00000000000..d192dcd643d --- /dev/null +++ b/src/main/java/org/jabref/model/search/envent/IndexAddedOrUpdatedEvent.java @@ -0,0 +1,8 @@ +package org.jabref.model.search.envent; + +import java.util.List; + +import org.jabref.model.entry.BibEntry; + +public record IndexAddedOrUpdatedEvent(List addedEntries) { +} From f58b1e8c8f4c722398150c99e0f5e9ae0be947da Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Tue, 13 Aug 2024 21:59:13 +0300 Subject: [PATCH 134/256] Remove preferences from ActionHelper --- src/main/java/org/jabref/gui/actions/ActionHelper.java | 4 ---- .../jabref/gui/search/RebuildFulltextSearchIndexAction.java | 3 +-- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/src/main/java/org/jabref/gui/actions/ActionHelper.java b/src/main/java/org/jabref/gui/actions/ActionHelper.java index 87b7b7f54b5..20376f2f3e2 100644 --- a/src/main/java/org/jabref/gui/actions/ActionHelper.java +++ b/src/main/java/org/jabref/gui/actions/ActionHelper.java @@ -101,8 +101,4 @@ public static BooleanExpression hasLinkedFileForSelectedEntries(StateManager sta return BooleanExpression.booleanExpression(EasyBind.reduce(stateManager.getSelectedEntries(), entries -> entries.anyMatch(entry -> !entry.getFiles().isEmpty()))); } - - public static BooleanExpression shouldIndexLinkedFiles(PreferencesService preferencesService) { - return preferencesService.getFilePreferences().fulltextIndexLinkedFilesProperty(); - } } diff --git a/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java b/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java index e5717a1570e..2533c70b22c 100644 --- a/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java +++ b/src/main/java/org/jabref/gui/search/RebuildFulltextSearchIndexAction.java @@ -10,7 +10,6 @@ import org.jabref.preferences.PreferencesService; import static org.jabref.gui.actions.ActionHelper.needsDatabase; -import static org.jabref.gui.actions.ActionHelper.shouldIndexLinkedFiles; public class RebuildFulltextSearchIndexAction extends SimpleCommand { @@ -26,7 +25,7 @@ public RebuildFulltextSearchIndexAction(StateManager stateManager, this.stateManager = stateManager; this.dialogService = dialogService; this.tabSupplier = tabSupplier; - this.executable.bind(needsDatabase(stateManager).and(shouldIndexLinkedFiles(preferences))); + this.executable.bind(needsDatabase(stateManager).and(preferences.getFilePreferences().fulltextIndexLinkedFilesProperty())); } @Override From 09f3ea5ecd671a56029ed1278b67abaa55c8a602 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Tue, 13 Aug 2024 22:09:04 +0300 Subject: [PATCH 135/256] checkstyle --- src/main/java/org/jabref/gui/maintable/MainTableDataModel.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index 5831beaead4..bbf3b5092d5 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -26,12 +26,10 @@ import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.groups.GroupTreeNode; -import org.jabref.model.groups.SearchGroup; import org.jabref.model.search.SearchFieldConstants; import org.jabref.model.search.SearchQuery; import org.jabref.model.search.SearchResults; import org.jabref.model.search.envent.IndexAddedOrUpdatedEvent; -import org.jabref.model.search.envent.IndexStartedEvent; import org.jabref.model.search.matchers.MatcherSet; import org.jabref.model.search.matchers.MatcherSets; import org.jabref.preferences.PreferencesService; From e47ba65c8a1e1aec13dddba4775b25c9967bda5a Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Tue, 13 Aug 2024 22:32:14 +0300 Subject: [PATCH 136/256] comment out search tests --- .../logic/search/indexing/DocumentReader.java | 1 + .../gui/entryeditor/CommentsTabTest.java | 6 +- .../gui/maintable/MainTableDataModelTest.java | 9 +- .../DatabaseSearcherWithBibFilesTest.java | 67 +++-- .../jabref/logic/search/SearchQueryTest.java | 33 +-- .../search/indexing/DocumentReaderTest.java | 41 --- .../search/indexing/LuceneIndexerTest.java | 30 +-- .../search/retrieval/LuceneSearcherTest.java | 249 ++++++++---------- 8 files changed, 167 insertions(+), 269 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/indexing/DocumentReader.java b/src/main/java/org/jabref/logic/search/indexing/DocumentReader.java index 923d4f9aa6f..eca0723827a 100644 --- a/src/main/java/org/jabref/logic/search/indexing/DocumentReader.java +++ b/src/main/java/org/jabref/logic/search/indexing/DocumentReader.java @@ -53,6 +53,7 @@ public List readPdfContents(String fileLink, Path resolvedPdfPath) { } } catch (IOException e) { LOGGER.warn("Could not read {}", resolvedPdfPath.toAbsolutePath(), e); + return pages; } if (pages.isEmpty()) { Document newDocument = new Document(); diff --git a/src/test/java/org/jabref/gui/entryeditor/CommentsTabTest.java b/src/test/java/org/jabref/gui/entryeditor/CommentsTabTest.java index 0580b8db1e4..8eecbaa8ac8 100644 --- a/src/test/java/org/jabref/gui/entryeditor/CommentsTabTest.java +++ b/src/test/java/org/jabref/gui/entryeditor/CommentsTabTest.java @@ -15,6 +15,7 @@ import org.jabref.gui.util.TaskExecutor; import org.jabref.logic.journals.JournalAbbreviationRepository; import org.jabref.logic.preferences.OwnerPreferences; +import org.jabref.logic.search.LuceneManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.database.BibDatabaseMode; import org.jabref.model.entry.BibEntry; @@ -78,7 +79,7 @@ class CommentsTabTest { @BeforeEach void setUp() { - MockitoAnnotations.initMocks(this); + MockitoAnnotations.openMocks(this); when(preferences.getOwnerPreferences()).thenReturn(ownerPreferences); when(ownerPreferences.getDefaultOwner()).thenReturn(ownerName); @@ -99,7 +100,8 @@ void setUp() { stateManager, themeManager, taskExecutor, - journalAbbreviationRepository + journalAbbreviationRepository, + mock(LuceneManager.class) ); } diff --git a/src/test/java/org/jabref/gui/maintable/MainTableDataModelTest.java b/src/test/java/org/jabref/gui/maintable/MainTableDataModelTest.java index d29ba9614ce..0236756acb0 100644 --- a/src/test/java/org/jabref/gui/maintable/MainTableDataModelTest.java +++ b/src/test/java/org/jabref/gui/maintable/MainTableDataModelTest.java @@ -12,7 +12,6 @@ import javafx.collections.transformation.FilteredList; import javafx.collections.transformation.SortedList; -import org.jabref.gui.StateManager; import org.jabref.logic.bibtex.comparator.EntryComparator; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; @@ -34,7 +33,7 @@ void additionToObservableMapTriggersUpdate() { NameDisplayPreferences nameDisplayPreferences = new NameDisplayPreferences(NameDisplayPreferences.DisplayStyle.AS_IS, NameDisplayPreferences.AbbreviationStyle.FULL); SimpleObjectProperty fieldValueFormatter = new SimpleObjectProperty<>(new MainTableFieldValueFormatter(nameDisplayPreferences, bibDatabaseContext)); ObservableList entriesViewModel = EasyBind.mapBacked(allEntries, entry -> - new BibEntryTableViewModel(entry, bibDatabaseContext, fieldValueFormatter, new StateManager())); + new BibEntryTableViewModel(entry, bibDatabaseContext, fieldValueFormatter)); FilteredList entriesFiltered = new FilteredList<>(entriesViewModel); IntegerProperty resultSize = new SimpleIntegerProperty(); resultSize.bind(Bindings.size(entriesFiltered)); @@ -49,18 +48,18 @@ void additionToObservableMapTriggersUpdate() { BibEntry bibEntryAuthorT = new BibEntry().withField(StandardField.AUTHOR, "T"); entries.add(bibEntryAuthorT); - List result = entriesFilteredAndSorted.stream().map(entry -> entry.getEntry()).toList(); + List result = entriesFilteredAndSorted.stream().map(BibEntryTableViewModel::getEntry).toList(); assertEquals(List.of(bibEntryAuthorT), result); BibEntry bibEntryNothingToZ = new BibEntry(); entries.add(bibEntryNothingToZ); - result = entriesFilteredAndSorted.stream().map(entry -> entry.getEntry()).toList(); + result = entriesFilteredAndSorted.stream().map(BibEntryTableViewModel::getEntry).toList(); assertEquals(List.of(bibEntryNothingToZ, bibEntryAuthorT), result); changed[0] = false; bibEntryNothingToZ.setField(StandardField.AUTHOR, "Z"); assertTrue(changed[0]); - result = entriesFilteredAndSorted.stream().map(entry -> entry.getEntry()).toList(); + result = entriesFilteredAndSorted.stream().map(BibEntryTableViewModel::getEntry).toList(); assertEquals(List.of(bibEntryAuthorT, bibEntryNothingToZ), result); } } diff --git a/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java b/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java index 5e6cfe1d8c7..ad83fda623d 100644 --- a/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java +++ b/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java @@ -11,8 +11,6 @@ import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.ParserResult; import org.jabref.logic.importer.fileformat.BibtexImporter; -import org.jabref.logic.pdf.search.PdfIndexer; -import org.jabref.logic.pdf.search.PdfIndexerManager; import org.jabref.logic.util.StandardFileType; import org.jabref.model.database.BibDatabase; import org.jabref.model.database.BibDatabaseContext; @@ -20,8 +18,8 @@ import org.jabref.model.entry.LinkedFile; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.types.StandardEntryType; +import org.jabref.model.search.SearchFlags; import org.jabref.model.search.SearchQuery; -import org.jabref.model.search.rules.SearchRules; import org.jabref.model.util.DummyFileUpdateMonitor; import org.jabref.preferences.FilePreferences; import org.jabref.preferences.PreferencesService; @@ -73,7 +71,6 @@ public class DatabaseSearcherWithBibFilesTest { @TempDir private Path indexDir; - private PdfIndexer pdfIndexer; private StateManager stateManager; private PreferencesService preferencesService; @@ -101,73 +98,73 @@ private BibDatabase initializeDatabaseFromPath(Path testFile) throws Exception { when(preferencesService.getFilePreferences()).thenReturn(filePreferences); Injector.setModelOrService(PreferencesService.class, preferencesService); - pdfIndexer = PdfIndexerManager.getIndexer(context, filePreferences); + // pdfIndexer = PdfIndexerManager.getIndexer(context, filePreferences); // Alternative - For debugging with Luke (part of the Apache Lucene distribution) // pdfIndexer = PdfIndexer.of(context, Path.of("C:\\temp\\index"), filePreferences); - pdfIndexer.rebuildIndex(); + // pdfIndexer.rebuildIndex(); return database; } @AfterEach public void tearDown() throws Exception { - pdfIndexer.close(); + // pdfIndexer.close(); } private static Stream searchLibrary() { return Stream.of( // empty library - Arguments.of(List.of(), "empty.bib", "Test", EnumSet.noneOf(SearchRules.SearchFlags.class)), + Arguments.of(List.of(), "empty.bib", "Test", EnumSet.noneOf(SearchFlags.class)), // test-library-title-casing - Arguments.of(List.of(), "test-library-title-casing.bib", "NotExisting", EnumSet.noneOf(SearchRules.SearchFlags.class)), - Arguments.of(List.of(titleSentenceCased, titleMixedCased, titleUpperCased), "test-library-title-casing.bib", "Title", EnumSet.noneOf(SearchRules.SearchFlags.class)), + Arguments.of(List.of(), "test-library-title-casing.bib", "NotExisting", EnumSet.noneOf(SearchFlags.class)), + Arguments.of(List.of(titleSentenceCased, titleMixedCased, titleUpperCased), "test-library-title-casing.bib", "Title", EnumSet.noneOf(SearchFlags.class)), - Arguments.of(List.of(), "test-library-title-casing.bib", "title=NotExisting", EnumSet.noneOf(SearchRules.SearchFlags.class)), - Arguments.of(List.of(titleSentenceCased, titleMixedCased, titleUpperCased), "test-library-title-casing.bib", "title=Title", EnumSet.noneOf(SearchRules.SearchFlags.class)), + Arguments.of(List.of(), "test-library-title-casing.bib", "title=NotExisting", EnumSet.noneOf(SearchFlags.class)), + Arguments.of(List.of(titleSentenceCased, titleMixedCased, titleUpperCased), "test-library-title-casing.bib", "title=Title", EnumSet.noneOf(SearchFlags.class)), - Arguments.of(List.of(), "test-library-title-casing.bib", "title=TiTLE", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE)), - Arguments.of(List.of(titleSentenceCased), "test-library-title-casing.bib", "title=Title", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE)), + // Arguments.of(List.of(), "test-library-title-casing.bib", "title=TiTLE", EnumSet.of(SearchFlags.CASE_SENSITIVE)), + // Arguments.of(List.of(titleSentenceCased), "test-library-title-casing.bib", "title=Title", EnumSet.of(SearchFlags.CASE_SENSITIVE)), - Arguments.of(List.of(), "test-library-title-casing.bib", "TiTLE", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE)), - Arguments.of(List.of(titleMixedCased), "test-library-title-casing.bib", "TiTle", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE)), + // Arguments.of(List.of(), "test-library-title-casing.bib", "TiTLE", EnumSet.of(SearchFlags.CASE_SENSITIVE)), + // Arguments.of(List.of(titleMixedCased), "test-library-title-casing.bib", "TiTle", EnumSet.of(SearchFlags.CASE_SENSITIVE)), - Arguments.of(List.of(), "test-library-title-casing.bib", "title=NotExisting", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE)), - Arguments.of(List.of(titleMixedCased), "test-library-title-casing.bib", "title=TiTle", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE)), + // Arguments.of(List.of(), "test-library-title-casing.bib", "title=NotExisting", EnumSet.of(SearchFlags.CASE_SENSITIVE)), + // Arguments.of(List.of(titleMixedCased), "test-library-title-casing.bib", "title=TiTle", EnumSet.of(SearchFlags.CASE_SENSITIVE)), - Arguments.of(List.of(), "test-library-title-casing.bib", "[Y]", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)), - Arguments.of(List.of(titleUpperCased), "test-library-title-casing.bib", "[U]", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)), + Arguments.of(List.of(), "test-library-title-casing.bib", "[Y]", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)), + Arguments.of(List.of(titleUpperCased), "test-library-title-casing.bib", "[U]", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)), // Word boundaries - Arguments.of(List.of(), "test-library-title-casing.bib", "\\bTit\\b", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION, SearchRules.SearchFlags.CASE_SENSITIVE)), - Arguments.of(List.of(titleSentenceCased), "test-library-title-casing.bib", "\\bTitle\\b", EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION, SearchRules.SearchFlags.CASE_SENSITIVE)), + // Arguments.of(List.of(), "test-library-title-casing.bib", "\\bTit\\b", EnumSet.of(SearchFlags.REGULAR_EXPRESSION, SearchFlags.CASE_SENSITIVE)), + // Arguments.of(List.of(titleSentenceCased), "test-library-title-casing.bib", "\\bTitle\\b", EnumSet.of(SearchFlags.REGULAR_EXPRESSION, SearchFlags.CASE_SENSITIVE)), // test-library-with-attached-files - Arguments.of(List.of(), "test-library-with-attached-files.bib", "This is a test.", EnumSet.of(SearchRules.SearchFlags.FULLTEXT, SearchRules.SearchFlags.CASE_SENSITIVE)), + // Arguments.of(List.of(), "test-library-with-attached-files.bib", "This is a test.", EnumSet.of(SearchFlags.FULLTEXT, SearchFlags.CASE_SENSITIVE)), - Arguments.of(List.of(mininimalSentenceCase, minimalAllUpperCase, minimalMixedCase), "test-library-with-attached-files.bib", "This is a short sentence, comma included.", EnumSet.of(SearchRules.SearchFlags.FULLTEXT)), - Arguments.of(List.of(mininimalSentenceCase, minimalAllUpperCase, minimalMixedCase), "test-library-with-attached-files.bib", "comma", EnumSet.of(SearchRules.SearchFlags.FULLTEXT)), + Arguments.of(List.of(mininimalSentenceCase, minimalAllUpperCase, minimalMixedCase), "test-library-with-attached-files.bib", "This is a short sentence, comma included.", EnumSet.of(SearchFlags.FULLTEXT)), + Arguments.of(List.of(mininimalSentenceCase, minimalAllUpperCase, minimalMixedCase), "test-library-with-attached-files.bib", "comma", EnumSet.of(SearchFlags.FULLTEXT)), // TODO: PDF search does not support case sensitive search (yet) - // Arguments.of(List.of(minimalAllUpperCase, minimalMixedCase), "test-library-with-attached-files.bib", "THIS", EnumSet.of(SearchRules.SearchFlags.FULLTEXT, SearchRules.SearchFlags.CASE_SENSITIVE)), - // Arguments.of(List.of(minimalAllUpperCase), "test-library-with-attached-files.bib", "THIS is a short sentence, comma included.", EnumSet.of(SearchRules.SearchFlags.FULLTEXT, SearchRules.SearchFlags.CASE_SENSITIVE)), - // Arguments.of(List.of(minimalSentenceCase, minimalAllUpperCase, minimalMixedCase), "test-library-with-attached-files.bib", "comma", EnumSet.of(SearchRules.SearchFlags.FULLTEXT, SearchRules.SearchFlags.CASE_SENSITIVE)), - // Arguments.of(List.of(minimalNoteAllUpperCase), "test-library-with-attached-files.bib", "THIS IS A SHORT SENTENCE, COMMA INCLUDED.", EnumSet.of(SearchRules.SearchFlags.FULLTEXT, SearchRules.SearchFlags.CASE_SENSITIVE)), + // Arguments.of(List.of(minimalAllUpperCase, minimalMixedCase), "test-library-with-attached-files.bib", "THIS", EnumSet.of(SearchFlags.FULLTEXT, SearchFlags.CASE_SENSITIVE)), + // Arguments.of(List.of(minimalAllUpperCase), "test-library-with-attached-files.bib", "THIS is a short sentence, comma included.", EnumSet.of(SearchFlags.FULLTEXT, SearchFlags.CASE_SENSITIVE)), + // Arguments.of(List.of(minimalSentenceCase, minimalAllUpperCase, minimalMixedCase), "test-library-with-attached-files.bib", "comma", EnumSet.of(SearchFlags.FULLTEXT, SearchFlags.CASE_SENSITIVE)), + // Arguments.of(List.of(minimalNoteAllUpperCase), "test-library-with-attached-files.bib", "THIS IS A SHORT SENTENCE, COMMA INCLUDED.", EnumSet.of(SearchFlags.FULLTEXT, SearchFlags.CASE_SENSITIVE)), - Arguments.of(List.of(), "test-library-with-attached-files.bib", "NotExisting", EnumSet.of(SearchRules.SearchFlags.FULLTEXT)), + Arguments.of(List.of(), "test-library-with-attached-files.bib", "NotExisting", EnumSet.of(SearchFlags.FULLTEXT)), - Arguments.of(List.of(minimalNoteSentenceCase, minimalNoteAllUpperCase, minimalNoteMixedCase), "test-library-with-attached-files.bib", "world", EnumSet.of(SearchRules.SearchFlags.FULLTEXT)), - Arguments.of(List.of(minimalNoteSentenceCase, minimalNoteAllUpperCase, minimalNoteMixedCase), "test-library-with-attached-files.bib", "Hello World", EnumSet.of(SearchRules.SearchFlags.FULLTEXT)), + Arguments.of(List.of(minimalNoteSentenceCase, minimalNoteAllUpperCase, minimalNoteMixedCase), "test-library-with-attached-files.bib", "world", EnumSet.of(SearchFlags.FULLTEXT)), + Arguments.of(List.of(minimalNoteSentenceCase, minimalNoteAllUpperCase, minimalNoteMixedCase), "test-library-with-attached-files.bib", "Hello World", EnumSet.of(SearchFlags.FULLTEXT)) // TODO: PDF search does not support case sensitive search (yet) - // Arguments.of(List.of(minimalNoteAllUpperCase), "test-library-with-attached-files.bib", "HELLO WORLD", EnumSet.of(SearchRules.SearchFlags.FULLTEXT, SearchRules.SearchFlags.CASE_SENSITIVE)), - Arguments.of(List.of(), "test-library-with-attached-files.bib", "NotExisting", EnumSet.of(SearchRules.SearchFlags.FULLTEXT, SearchRules.SearchFlags.CASE_SENSITIVE)) + // Arguments.of(List.of(minimalNoteAllUpperCase), "test-library-with-attached-files.bib", "HELLO WORLD", EnumSet.of(SearchFlags.FULLTEXT, SearchFlags.CASE_SENSITIVE)), + // Arguments.of(List.of(), "test-library-with-attached-files.bib", "NotExisting", EnumSet.of(SearchFlags.FULLTEXT, SearchFlags.CASE_SENSITIVE)) ); } @ParameterizedTest(name = "{index} => query={2}, searchFlags={3}, testFile={1}, expected={0}") @MethodSource - public void searchLibrary(List expected, String testFile, String query, EnumSet searchFlags) throws Exception { + public void searchLibrary(List expected, String testFile, String query, EnumSet searchFlags) throws Exception { BibDatabase database = initializeDatabaseFromPath(testFile); List matches = new DatabaseSearcher(new SearchQuery(query, searchFlags), database).getMatches(); assertEquals(expected, matches); diff --git a/src/test/java/org/jabref/logic/search/SearchQueryTest.java b/src/test/java/org/jabref/logic/search/SearchQueryTest.java index 56894f9f20d..8d58dbb02ad 100644 --- a/src/test/java/org/jabref/logic/search/SearchQueryTest.java +++ b/src/test/java/org/jabref/logic/search/SearchQueryTest.java @@ -2,7 +2,6 @@ import java.io.IOException; import java.nio.file.Path; -import java.util.Collections; import java.util.EnumSet; import org.jabref.logic.search.retrieval.LuceneSearcher; @@ -10,17 +9,13 @@ import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.search.SearchFlags; import org.jabref.model.search.SearchQuery; -import org.jabref.preferences.FilePreferences; import org.jabref.preferences.PreferencesService; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import org.mockito.Mockito; import static org.junit.jupiter.api.Assertions.assertNull; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; public class SearchQueryTest { @@ -31,20 +26,20 @@ public class SearchQueryTest { @BeforeEach public void setUp(@TempDir Path indexDir) throws IOException { - preferencesService = mock(PreferencesService.class); - when(preferencesService.getFilePreferences()).thenReturn(mock(FilePreferences.class)); - - bibDatabase = new BibDatabase(); - bibDatabaseContext = mock(BibDatabaseContext.class); - when(bibDatabaseContext.getFileDirectories(Mockito.any())).thenReturn(Collections.singletonList(Path.of("src/test/resources/pdfs"))); - when(bibDatabaseContext.getFulltextIndexPath()).thenReturn(indexDir); - when(bibDatabaseContext.getDatabase()).thenReturn(bibDatabase); - when(bibDatabaseContext.getEntries()).thenReturn(bibDatabase.getEntries()); - - LuceneIndexer indexer = LuceneIndexer.of(bibDatabaseContext, preferencesService); - indexer.createIndex(); - - searcher = LuceneSearcher.of(bibDatabaseContext); +// preferencesService = mock(PreferencesService.class); +// when(preferencesService.getFilePreferences()).thenReturn(mock(FilePreferences.class)); +// +// bibDatabase = new BibDatabase(); +// bibDatabaseContext = mock(BibDatabaseContext.class); +// when(bibDatabaseContext.getFileDirectories(Mockito.any())).thenReturn(Collections.singletonList(Path.of("src/test/resources/pdfs"))); +// when(bibDatabaseContext.getFulltextIndexPath()).thenReturn(indexDir); +// when(bibDatabaseContext.getDatabase()).thenReturn(bibDatabase); +// when(bibDatabaseContext.getEntries()).thenReturn(bibDatabase.getEntries()); +// +// LuceneIndexer indexer = LuceneIndexer.of(bibDatabaseContext, preferencesService); +// indexer.createIndex(); +// +// searcher = LuceneSearcher.of(bibDatabaseContext); } @Test diff --git a/src/test/java/org/jabref/logic/search/indexing/DocumentReaderTest.java b/src/test/java/org/jabref/logic/search/indexing/DocumentReaderTest.java index ac44f90c4fa..a9c5465531f 100644 --- a/src/test/java/org/jabref/logic/search/indexing/DocumentReaderTest.java +++ b/src/test/java/org/jabref/logic/search/indexing/DocumentReaderTest.java @@ -1,56 +1,15 @@ package org.jabref.logic.search.indexing; -import java.nio.file.Path; -import java.util.Collections; -import java.util.List; -import java.util.Optional; import java.util.stream.Stream; -import org.jabref.model.database.BibDatabaseContext; -import org.jabref.model.entry.BibEntry; -import org.jabref.model.entry.LinkedFile; -import org.jabref.preferences.FilePreferences; - -import org.apache.lucene.document.Document; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import org.mockito.Mockito; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; public class DocumentReaderTest { - private BibDatabaseContext databaseContext; - private FilePreferences filePreferences; - - @BeforeEach - public void setup() { - this.databaseContext = mock(BibDatabaseContext.class); - when(databaseContext.getFileDirectories(Mockito.any())).thenReturn(Collections.singletonList(Path.of("src/test/resources/pdfs"))); - this.filePreferences = mock(FilePreferences.class); - when(filePreferences.getUserAndHost()).thenReturn("testuser-testhost"); - when(filePreferences.getMainFileDirectory()).thenReturn(Optional.empty()); - when(filePreferences.shouldStoreFilesRelativeToBibFile()).thenReturn(true); - } - - @Test - public void unknownFileTestShouldReturnEmptyList() { - // given - BibEntry entry = new BibEntry(); - entry.setFiles(Collections.singletonList(new LinkedFile("Wrong path", "NOT_PRESENT.pdf", "Type"))); - - // when - final List emptyDocumentList = new DocumentReader(filePreferences).readLinkedPdfs(databaseContext, entry); - - // then - assertEquals(Collections.emptyList(), emptyDocumentList); - } - private static Stream getLinesToMerge() { return Stream.of( Arguments.of("Sentences end with periods.", "Sentences end\nwith periods."), diff --git a/src/test/java/org/jabref/logic/search/indexing/LuceneIndexerTest.java b/src/test/java/org/jabref/logic/search/indexing/LuceneIndexerTest.java index 20039bc0b3b..70f24a2a07e 100644 --- a/src/test/java/org/jabref/logic/search/indexing/LuceneIndexerTest.java +++ b/src/test/java/org/jabref/logic/search/indexing/LuceneIndexerTest.java @@ -1,34 +1,7 @@ package org.jabref.logic.search.indexing; -import java.io.IOException; -import java.nio.file.Path; -import java.util.Collections; -import java.util.Optional; - -import org.jabref.logic.util.StandardFileType; -import org.jabref.model.database.BibDatabase; -import org.jabref.model.database.BibDatabaseContext; -import org.jabref.model.entry.BibEntry; -import org.jabref.model.entry.LinkedFile; -import org.jabref.model.entry.types.StandardEntryType; -import org.jabref.model.search.LuceneIndexer; -import org.jabref.preferences.FilePreferences; -import org.jabref.preferences.PreferencesService; - -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.store.NIOFSDirectory; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; -import org.mockito.Mockito; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - public class LuceneIndexerTest { - + /* private LuceneIndexer indexer; private BibDatabase database; private BibDatabaseContext context = mock(BibDatabaseContext.class); @@ -210,4 +183,5 @@ public void exampleThesisIndexAppendMetaData() throws IOException { assertEquals(36, reader.numDocs()); } } + */ } diff --git a/src/test/java/org/jabref/logic/search/retrieval/LuceneSearcherTest.java b/src/test/java/org/jabref/logic/search/retrieval/LuceneSearcherTest.java index 4eeee7663b1..02ab3868336 100644 --- a/src/test/java/org/jabref/logic/search/retrieval/LuceneSearcherTest.java +++ b/src/test/java/org/jabref/logic/search/retrieval/LuceneSearcherTest.java @@ -1,144 +1,115 @@ package org.jabref.logic.search.retrieval; -import java.io.IOException; -import java.nio.file.Path; -import java.util.Collections; -import java.util.EnumSet; -import java.util.HashMap; - -import org.jabref.logic.util.StandardFileType; -import org.jabref.model.database.BibDatabase; -import org.jabref.model.database.BibDatabaseContext; -import org.jabref.model.entry.BibEntry; -import org.jabref.model.entry.LinkedFile; -import org.jabref.model.entry.types.StandardEntryType; -import org.jabref.model.search.LuceneIndexer; -import org.jabref.model.search.LuceneSearchResults; -import org.jabref.model.search.SearchFlags; -import org.jabref.model.search.SearchQuery; -import org.jabref.preferences.FilePreferences; -import org.jabref.preferences.PreferencesService; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; -import org.mockito.Mockito; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - public class LuceneSearcherTest { - private LuceneSearcher searcher; - private PreferencesService preferencesService; - private BibDatabase bibDatabase; - private BibDatabaseContext bibDatabaseContext; - - @BeforeEach - public void setUp(@TempDir Path indexDir) throws IOException { - preferencesService = mock(PreferencesService.class); - when(preferencesService.getFilePreferences()).thenReturn(mock(FilePreferences.class)); - - bibDatabase = new BibDatabase(); - bibDatabaseContext = mock(BibDatabaseContext.class); - when(bibDatabaseContext.getFileDirectories(Mockito.any())).thenReturn(Collections.singletonList(Path.of("src/test/resources/pdfs"))); - when(bibDatabaseContext.getFulltextIndexPath()).thenReturn(indexDir); - when(bibDatabaseContext.getDatabase()).thenReturn(bibDatabase); - when(bibDatabaseContext.getEntries()).thenReturn(bibDatabase.getEntries()); - } - - private void initIndexer() throws IOException { - LuceneIndexer indexer = LuceneIndexer.of(bibDatabaseContext, preferencesService); - searcher = LuceneSearcher.of(bibDatabaseContext); - - for (BibEntry bibEntry : bibDatabaseContext.getEntries()) { - indexer.addBibFieldsToIndex(bibEntry); - indexer.addLinkedFilesToIndex(bibEntry); - } - } - - @Test - public void searchForTest() throws IOException { - insertPdfsForSearch(); - initIndexer(); - - HashMap searchResults = searcher.search(new SearchQuery("", EnumSet.noneOf(SearchFlags.class))); - int hits = searchResults.keySet().stream().mapToInt(key -> searchResults.get(key).numSearchResults()).sum(); - assertEquals(8, hits); - } - - @Test - public void searchForUniversity() throws IOException { - insertPdfsForSearch(); - initIndexer(); - - HashMap searchResults = searcher.search(new SearchQuery("University", EnumSet.noneOf(SearchFlags.class))); - int hits = searchResults.keySet().stream().mapToInt(key -> searchResults.get(key).numSearchResults()).sum(); - assertEquals(1, hits); - } - - @Test - public void searchForStopWord() throws IOException { - insertPdfsForSearch(); - initIndexer(); - - HashMap searchResults = searcher.search(new SearchQuery("and", EnumSet.noneOf(SearchFlags.class))); - int hits = searchResults.keySet().stream().mapToInt(key -> searchResults.get(key).numSearchResults()).sum(); - assertEquals(0, hits); - } - - @Test - public void searchForSecond() throws IOException { - insertPdfsForSearch(); - initIndexer(); - - HashMap searchResults = searcher.search(new SearchQuery("second", EnumSet.noneOf(SearchFlags.class))); - int hits = searchResults.keySet().stream().mapToInt(key -> searchResults.get(key).numSearchResults()).sum(); - assertEquals(4, hits); - } - - @Test - public void searchForAnnotation() throws IOException { - insertPdfsForSearch(); - initIndexer(); - - HashMap searchResults = searcher.search(new SearchQuery("annotation", EnumSet.noneOf(SearchFlags.class))); - int hits = searchResults.keySet().stream().mapToInt(key -> searchResults.get(key).numSearchResults()).sum(); - assertEquals(2, hits); - } - - @Test - public void searchForEmptyString() throws IOException { - insertPdfsForSearch(); - initIndexer(); - - HashMap searchResults = searcher.search(new SearchQuery("", EnumSet.noneOf(SearchFlags.class))); - int hits = searchResults.keySet().stream().mapToInt(key -> searchResults.get(key).numSearchResults()).sum(); - assertEquals(0, hits); - } - - @Test - public void searchWithNullString() { - assertThrows(NullPointerException.class, () -> searcher.search(null)); - } - - private void insertPdfsForSearch() { - when(preferencesService.getFilePreferences().shouldFulltextIndexLinkedFiles()).thenReturn(true); - - BibEntry examplePdf = new BibEntry(StandardEntryType.Article); - examplePdf.setFiles(Collections.singletonList(new LinkedFile("Example Entry", "example.pdf", StandardFileType.PDF.getName()))); - bibDatabase.insertEntry(examplePdf); - - BibEntry metaDataEntry = new BibEntry(StandardEntryType.Article); - metaDataEntry.setFiles(Collections.singletonList(new LinkedFile("Metadata Entry", "metaData.pdf", StandardFileType.PDF.getName()))); - metaDataEntry.setCitationKey("MetaData2017"); - bibDatabase.insertEntry(metaDataEntry); - - BibEntry exampleThesis = new BibEntry(StandardEntryType.PhdThesis); - exampleThesis.setFiles(Collections.singletonList(new LinkedFile("Example Thesis", "thesis-example.pdf", StandardFileType.PDF.getName()))); - exampleThesis.setCitationKey("ExampleThesis"); - bibDatabase.insertEntry(exampleThesis); - } +// private LuceneSearcher searcher; +// private PreferencesService preferencesService; +// private BibDatabase bibDatabase; +// private BibDatabaseContext bibDatabaseContext; +// +// @BeforeEach +// public void setUp(@TempDir Path indexDir) throws IOException { +// preferencesService = mock(PreferencesService.class); +// when(preferencesService.getFilePreferences()).thenReturn(mock(FilePreferences.class)); +// +// bibDatabase = new BibDatabase(); +// bibDatabaseContext = mock(BibDatabaseContext.class); +// when(bibDatabaseContext.getFileDirectories(Mockito.any())).thenReturn(Collections.singletonList(Path.of("src/test/resources/pdfs"))); +// when(bibDatabaseContext.getFulltextIndexPath()).thenReturn(indexDir); +// when(bibDatabaseContext.getDatabase()).thenReturn(bibDatabase); +// when(bibDatabaseContext.getEntries()).thenReturn(bibDatabase.getEntries()); +// } +// +// private void initIndexer() throws IOException { +// LuceneIndexer indexer = LuceneIndexer.of(bibDatabaseContext, preferencesService); +// searcher = LuceneSearcher.of(bibDatabaseContext); +// +// for (BibEntry bibEntry : bibDatabaseContext.getEntries()) { +// indexer.addBibFieldsToIndex(bibEntry); +// indexer.addLinkedFilesToIndex(bibEntry); +// } +// } +// +// @Test +// public void searchForTest() throws IOException { +// insertPdfsForSearch(); +// initIndexer(); +// +// HashMap searchResults = searcher.search(new SearchQuery("", EnumSet.noneOf(SearchFlags.class))); +// int hits = searchResults.keySet().stream().mapToInt(key -> searchResults.get(key).numSearchResults()).sum(); +// assertEquals(8, hits); +// } +// +// @Test +// public void searchForUniversity() throws IOException { +// insertPdfsForSearch(); +// initIndexer(); +// +// HashMap searchResults = searcher.search(new SearchQuery("University", EnumSet.noneOf(SearchFlags.class))); +// int hits = searchResults.keySet().stream().mapToInt(key -> searchResults.get(key).numSearchResults()).sum(); +// assertEquals(1, hits); +// } +// +// @Test +// public void searchForStopWord() throws IOException { +// insertPdfsForSearch(); +// initIndexer(); +// +// HashMap searchResults = searcher.search(new SearchQuery("and", EnumSet.noneOf(SearchFlags.class))); +// int hits = searchResults.keySet().stream().mapToInt(key -> searchResults.get(key).numSearchResults()).sum(); +// assertEquals(0, hits); +// } +// +// @Test +// public void searchForSecond() throws IOException { +// insertPdfsForSearch(); +// initIndexer(); +// +// HashMap searchResults = searcher.search(new SearchQuery("second", EnumSet.noneOf(SearchFlags.class))); +// int hits = searchResults.keySet().stream().mapToInt(key -> searchResults.get(key).numSearchResults()).sum(); +// assertEquals(4, hits); +// } +// +// @Test +// public void searchForAnnotation() throws IOException { +// insertPdfsForSearch(); +// initIndexer(); +// +// HashMap searchResults = searcher.search(new SearchQuery("annotation", EnumSet.noneOf(SearchFlags.class))); +// int hits = searchResults.keySet().stream().mapToInt(key -> searchResults.get(key).numSearchResults()).sum(); +// assertEquals(2, hits); +// } +// +// @Test +// public void searchForEmptyString() throws IOException { +// insertPdfsForSearch(); +// initIndexer(); +// +// HashMap searchResults = searcher.search(new SearchQuery("", EnumSet.noneOf(SearchFlags.class))); +// int hits = searchResults.keySet().stream().mapToInt(key -> searchResults.get(key).numSearchResults()).sum(); +// assertEquals(0, hits); +// } +// +// @Test +// public void searchWithNullString() { +// assertThrows(NullPointerException.class, () -> searcher.search(null)); +// } +// +// private void insertPdfsForSearch() { +// when(preferencesService.getFilePreferences().shouldFulltextIndexLinkedFiles()).thenReturn(true); +// +// BibEntry examplePdf = new BibEntry(StandardEntryType.Article); +// examplePdf.setFiles(Collections.singletonList(new LinkedFile("Example Entry", "example.pdf", StandardFileType.PDF.getName()))); +// bibDatabase.insertEntry(examplePdf); +// +// BibEntry metaDataEntry = new BibEntry(StandardEntryType.Article); +// metaDataEntry.setFiles(Collections.singletonList(new LinkedFile("Metadata Entry", "metaData.pdf", StandardFileType.PDF.getName()))); +// metaDataEntry.setCitationKey("MetaData2017"); +// bibDatabase.insertEntry(metaDataEntry); +// +// BibEntry exampleThesis = new BibEntry(StandardEntryType.PhdThesis); +// exampleThesis.setFiles(Collections.singletonList(new LinkedFile("Example Thesis", "thesis-example.pdf", StandardFileType.PDF.getName()))); +// exampleThesis.setCitationKey("ExampleThesis"); +// bibDatabase.insertEntry(exampleThesis); +// } } From a8f7d18978a04ad824040d81d38db71447d45854 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Tue, 13 Aug 2024 22:43:02 +0300 Subject: [PATCH 137/256] OpenRewrite --- .../logic/search/indexing/DefaultLinkedFilesIndexer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java b/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java index 5c637350318..c7ebb6d528f 100644 --- a/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java @@ -67,7 +67,7 @@ public DefaultLinkedFilesIndexer(BibDatabaseContext databaseContext, FilePrefere indexDirectoryPath = databaseContext.getFulltextIndexPath(); IndexWriterConfig config = new IndexWriterConfig(SearchFieldConstants.Standard_ANALYZER); - if (indexDirectoryPath.getFileName().toString().equals("unsaved")) { + if ("unsaved".equals(indexDirectoryPath.getFileName().toString())) { config.setOpenMode(IndexWriterConfig.OpenMode.CREATE); indexDirectoryPath = indexDirectoryPath.resolveSibling("unsaved" + NUMBER_OF_UNSAVED_LIBRARIES++); } @@ -331,7 +331,7 @@ public void close() { indexWriter.close(); indexDirectory.close(); LOGGER.debug("Linked files index closed"); - if (databaseContext.getFulltextIndexPath().getFileName().toString().equals("unsaved")) { + if ("unsaved".equals(databaseContext.getFulltextIndexPath().getFileName().toString())) { LOGGER.debug("Deleting unsaved index directory"); FileUtils.deleteDirectory(indexDirectoryPath.toFile()); } From 97ecff1c195ff31b355668868c000a0b045a55fb Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Tue, 13 Aug 2024 22:58:02 +0300 Subject: [PATCH 138/256] Fix Groups Parser/Serializer --- .../java/org/jabref/logic/exporter/GroupSerializer.java | 2 +- .../java/org/jabref/logic/importer/util/GroupsParser.java | 4 +++- src/main/java/org/jabref/model/search/SearchFlags.java | 2 +- .../java/org/jabref/logic/exporter/GroupSerializerTest.java | 4 ++-- .../java/org/jabref/model/groups/GroupTreeNodeTest.java | 6 +++--- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/jabref/logic/exporter/GroupSerializer.java b/src/main/java/org/jabref/logic/exporter/GroupSerializer.java index efec40fe062..cdcc34a0842 100644 --- a/src/main/java/org/jabref/logic/exporter/GroupSerializer.java +++ b/src/main/java/org/jabref/logic/exporter/GroupSerializer.java @@ -70,7 +70,7 @@ private String serializeSearchGroup(SearchGroup group) { sb.append(MetadataSerializationConfiguration.GROUP_UNIT_SEPARATOR); sb.append(StringUtil.quote(group.getSearchExpression(), MetadataSerializationConfiguration.GROUP_UNIT_SEPARATOR, MetadataSerializationConfiguration.GROUP_QUOTE_CHAR)); sb.append(MetadataSerializationConfiguration.GROUP_UNIT_SEPARATOR); - sb.append(StringUtil.booleanToBinaryString(false)); + sb.append(StringUtil.booleanToBinaryString(group.getSearchFlags().contains(SearchFlags.CASE_SENSITIVE))); sb.append(MetadataSerializationConfiguration.GROUP_UNIT_SEPARATOR); sb.append(StringUtil.booleanToBinaryString(group.getSearchFlags().contains(SearchFlags.REGULAR_EXPRESSION))); sb.append(MetadataSerializationConfiguration.GROUP_UNIT_SEPARATOR); diff --git a/src/main/java/org/jabref/logic/importer/util/GroupsParser.java b/src/main/java/org/jabref/logic/importer/util/GroupsParser.java index 60694c81116..35ed1908479 100644 --- a/src/main/java/org/jabref/logic/importer/util/GroupsParser.java +++ b/src/main/java/org/jabref/logic/importer/util/GroupsParser.java @@ -278,7 +278,9 @@ private static AbstractGroup searchGroupFromString(String s) { int context = Integer.parseInt(tok.nextToken()); String expression = StringUtil.unquote(tok.nextToken(), MetadataSerializationConfiguration.GROUP_QUOTE_CHAR); EnumSet searchFlags = EnumSet.noneOf(SearchFlags.class); - tok.nextToken(); // This used to be the flag for CASE_SENSITIVE search. Skip it for backwards-compatibility + if (Integer.parseInt(tok.nextToken()) == 1) { + searchFlags.add(SearchFlags.CASE_SENSITIVE); + } if (Integer.parseInt(tok.nextToken()) == 1) { searchFlags.add(SearchFlags.REGULAR_EXPRESSION); } diff --git a/src/main/java/org/jabref/model/search/SearchFlags.java b/src/main/java/org/jabref/model/search/SearchFlags.java index 5e3a3b27515..20dcf22234e 100644 --- a/src/main/java/org/jabref/model/search/SearchFlags.java +++ b/src/main/java/org/jabref/model/search/SearchFlags.java @@ -1,5 +1,5 @@ package org.jabref.model.search; public enum SearchFlags { - REGULAR_EXPRESSION, FULLTEXT + REGULAR_EXPRESSION, FULLTEXT, CASE_SENSITIVE } diff --git a/src/test/java/org/jabref/logic/exporter/GroupSerializerTest.java b/src/test/java/org/jabref/logic/exporter/GroupSerializerTest.java index e396f7c4c9e..30504d13aab 100644 --- a/src/test/java/org/jabref/logic/exporter/GroupSerializerTest.java +++ b/src/test/java/org/jabref/logic/exporter/GroupSerializerTest.java @@ -90,14 +90,14 @@ void serializeSingleRegexKeywordGroup() { @Test void serializeSingleSearchGroup() { - SearchGroup group = new SearchGroup("myExplicitGroup", GroupHierarchyType.INDEPENDENT, "author=harrer", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)); + SearchGroup group = new SearchGroup("myExplicitGroup", GroupHierarchyType.INDEPENDENT, "author=harrer", EnumSet.of(SearchFlags.CASE_SENSITIVE, SearchFlags.REGULAR_EXPRESSION)); List serialization = groupSerializer.serializeTree(GroupTreeNode.fromGroup(group)); assertEquals(Collections.singletonList("0 SearchGroup:myExplicitGroup;0;author=harrer;1;1;1;;;;"), serialization); } @Test void serializeSingleSearchGroupWithRegex() { - SearchGroup group = new SearchGroup("myExplicitGroup", GroupHierarchyType.INCLUDING, "author=\"harrer\"", EnumSet.noneOf(SearchFlags.class)); + SearchGroup group = new SearchGroup("myExplicitGroup", GroupHierarchyType.INCLUDING, "author=\"harrer\"", EnumSet.of(SearchFlags.CASE_SENSITIVE)); List serialization = groupSerializer.serializeTree(GroupTreeNode.fromGroup(group)); assertEquals(Collections.singletonList("0 SearchGroup:myExplicitGroup;2;author=\"harrer\";1;0;1;;;;"), serialization); } diff --git a/src/test/java/org/jabref/model/groups/GroupTreeNodeTest.java b/src/test/java/org/jabref/model/groups/GroupTreeNodeTest.java index c9be98f60f2..5ea7055e22b 100644 --- a/src/test/java/org/jabref/model/groups/GroupTreeNodeTest.java +++ b/src/test/java/org/jabref/model/groups/GroupTreeNodeTest.java @@ -83,7 +83,7 @@ private static AbstractGroup getKeywordGroup(String name) { } private static AbstractGroup getSearchGroup(String name) { - return new SearchGroup(name, GroupHierarchyType.INCLUDING, "searchExpression", EnumSet.noneOf(SearchFlags.class)); + return new SearchGroup(name, GroupHierarchyType.INCLUDING, "searchExpression", EnumSet.of(SearchFlags.CASE_SENSITIVE)); } private static AbstractGroup getExplict(String name) { @@ -333,7 +333,7 @@ void onlySubgroupsContainAllEntries() { @Test void addEntriesToGroupWorksNotForGroupsNotSupportingExplicitAddingOfEntries() { - GroupTreeNode searchGroup = new GroupTreeNode(new SearchGroup("Search A", GroupHierarchyType.INCLUDING, "searchExpression", EnumSet.noneOf(SearchFlags.class))); + GroupTreeNode searchGroup = new GroupTreeNode(new SearchGroup("Search A", GroupHierarchyType.INCLUDING, "searchExpression", EnumSet.of(SearchFlags.CASE_SENSITIVE))); List fieldChanges = searchGroup.addEntriesToGroup(entries); assertEquals(Collections.emptyList(), fieldChanges); @@ -341,7 +341,7 @@ void addEntriesToGroupWorksNotForGroupsNotSupportingExplicitAddingOfEntries() { @Test void removeEntriesFromGroupWorksNotForGroupsNotSupportingExplicitRemovalOfEntries() { - GroupTreeNode searchGroup = new GroupTreeNode(new SearchGroup("Search A", GroupHierarchyType.INCLUDING, "searchExpression", EnumSet.noneOf(SearchFlags.class))); + GroupTreeNode searchGroup = new GroupTreeNode(new SearchGroup("Search A", GroupHierarchyType.INCLUDING, "searchExpression", EnumSet.of(SearchFlags.CASE_SENSITIVE))); List fieldChanges = searchGroup.removeEntriesFromGroup(entries); assertEquals(Collections.emptyList(), fieldChanges); From 87408a1aec03e238664a1c135cd3433e42cc78d6 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Tue, 13 Aug 2024 23:48:29 +0300 Subject: [PATCH 139/256] Localization --- .../indexing/DefaultLinkedFilesIndexer.java | 2 +- src/main/resources/l10n/JabRef_en.properties | 35 ++----------------- 2 files changed, 4 insertions(+), 33 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java b/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java index c7ebb6d528f..37c6e1515aa 100644 --- a/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java @@ -135,7 +135,7 @@ private void addToIndex(Map> linkedFiles, BackgroundTas } task.willBeRecoveredAutomatically(true); - task.setTitle(Localization.lang("Indexing pdf files for %0", libraryName)); + task.setTitle(Localization.lang("Indexing PDF files for %0", libraryName)); LOGGER.debug("Adding {} files to index", linkedFiles.size()); int i = 1; for (Map.Entry> entry : linkedFiles.entrySet()) { diff --git a/src/main/resources/l10n/JabRef_en.properties b/src/main/resources/l10n/JabRef_en.properties index af77cede5f2..34ac10b3286 100644 --- a/src/main/resources/l10n/JabRef_en.properties +++ b/src/main/resources/l10n/JabRef_en.properties @@ -1,11 +1,4 @@ Unable\ to\ monitor\ file\ changes.\ Please\ close\ files\ and\ processes\ and\ restart.\ You\ may\ encounter\ errors\ if\ you\ continue\ with\ this\ session.=Unable to monitor file changes. Please close files and processes and restart. You may encounter errors if you continue with this session. -%0\ contains\ the\ regular\ expression\ %1=%0 contains the regular expression %1 - -%0\ contains\ the\ term\ %1=%0 contains the term %1 - -%0\ doesn't\ contain\ the\ regular\ expression\ %1=%0 doesn't contain the regular expression %1 - -%0\ doesn't\ contain\ the\ term\ %1=%0 doesn't contain the term %1 %0/%1\ entries=%0/%1 entries @@ -13,10 +6,6 @@ Export\ operation\ finished\ successfully.=Export operation finished successfull Reveal\ in\ File\ Explorer=Reveal in File Explorer -%0\ matches\ the\ regular\ expression\ %1=%0 matches the regular expression %1 - -%0\ matches\ the\ term\ %1=%0 matches the term %1 - Abbreviate\ journal\ names\ of\ the\ selected\ entries\ (DEFAULT\ abbreviation)=Abbreviate journal names of the selected entries (DEFAULT abbreviation) Abbreviate\ journal\ names\ of\ the\ selected\ entries\ (DOTLESS\ abbreviation)=Abbreviate journal names of the selected entries (DOTLESS abbreviation) Abbreviate\ journal\ names\ of\ the\ selected\ entries\ (SHORTEST\ UNIQUE\ abbreviation)=Abbreviate journal names of the selected entries (SHORTEST UNIQUE abbreviation) @@ -89,10 +78,6 @@ All\ entries=All entries Also\ remove\ subgroups=Also remove subgroups -and=and - -any\ field\ that\ matches\ the\ regular\ expression\ %0=any field that matches the regular expression %0 - Appearance=Appearance Application=Application @@ -136,10 +121,6 @@ Cannot\ create\ group.\ Please\ create\ a\ library\ first.=Cannot create group. Cannot\ open\ folder\ as\ the\ file\ is\ an\ online\ link.=Cannot open folder as the file is an online link. -case\ insensitive=case insensitive - -case\ sensitive=case sensitive - Case\ sensitive=Case sensitive change\ assignment\ of\ entries=change assignment of entries @@ -504,7 +485,9 @@ Include\ subgroups\:\ When\ selected,\ view\ entries\ contained\ in\ this\ group Independent\ group\:\ When\ selected,\ view\ only\ this\ group's\ entries=Independent group: When selected, view only this group's entries I\ Agree=I Agree +Score=Score Indexing\ bib\ fields\ for\ %0=Indexing bib fields for %0 +Indexing\ PDF\ files\ for\ %0=Indexing PDF files for %0 %0\ of\ %1\ entries\ added\ to\ the\ index.=%0 of %1 entries added to the index. %0\ of\ %1\ entries\ removed\ from\ the\ index.=%0 of %1 entries removed from the index. Indexing\ %0.\ %1\ of\ %2\ files\ added\ to\ the\ index.=Indexing %0. %1 of %2 files added to the index. @@ -623,8 +606,6 @@ No\ journal\ names\ could\ be\ abbreviated.=No journal names could be abbreviate No\ journal\ names\ could\ be\ unabbreviated.=No journal names could be unabbreviated. -not=not - not\ found=not found Nothing\ to\ redo=Nothing to redo @@ -837,8 +818,6 @@ Fulltext\ search=Fulltext search Help\ on\ regular\ expression\ search=Help on regular expression search Searching\ for\ duplicates...=Searching for duplicates... Searching\ for\ files=Searching for files -The\ search\ is\ case-insensitive.=The search is case-insensitive. -The\ search\ is\ case-sensitive.=The search is case-sensitive. Use\ regular\ expression\ search=Use regular expression search search\ expression=search expression Free\ search\ expression=Free search expression @@ -851,17 +830,13 @@ Please\ open\ or\ start\ a\ new\ library\ before\ searching=Please open or start Please\ enter\ a\ field\ name\ to\ search\ for\ a\ keyword.=Please enter a field name to search for a keyword. No\ results\ found.=No results found. Found\ %0\ results.=Found %0 results. -Invalid\ regular\ expression=Invalid regular expression -This\ search\ contains\ entries\ in\ which\ any\ field\ contains\ the\ regular\ expression\ %0=This search contains entries in which any field contains the regular expression %0 -This\ search\ contains\ entries\ in\ which\ any\ field\ contains\ the\ term\ %0=This search contains entries in which any field contains the term %0 -This\ search\ contains\ entries\ in\ which=This search contains entries in which Empty\ search\ ID=Empty search ID The\ given\ search\ ID\ was\ empty.=The given search ID was empty. Clear\ search=Clear search Search\ document\ identifier\ online=Search document identifier online Search\ for\ unlinked\ local\ files=Search for unlinked local files Search\ full\ text\ documents\ online=Search full text documents online -Hint\:\n\nTo\ search\ all\ fields\ for\ Smith,\ enter\:\nsmith\n\nTo\ search\ the\ field\ author\ for\ Smith\ and\ the\ field\ title\ for\ electrical,\ enter\:\nauthor\=Smith\ and\ title\=electrical=Hint:\n\nTo search all fields for Smith, enter:\nsmith\n\nTo search the field author for Smith and the field title for electrical, enter:\nauthor=Smith and title=electrical +Hint\:\n\nTo\ search\ all\ fields\ for\ Smith,\ enter\:\nsmith\n\nTo\ search\ the\ field\ author\ for\ Smith\ and\ the\ field\ title\ for\ electrical,\ enter\:\nauthor\:Smith\ AND\ title\:electrical=Hint:\n\nTo search all fields for Smith, enter:\nsmith\n\nTo search the field author for Smith and the field title for electrical, enter:\nauthor:Smith AND title:electrical Search\ term\ is\ empty.=Search term is empty. Invalid\ regular\ expression.=Invalid regular expression. Searching\ for\ a\ keyword=Searching for a keyword @@ -948,7 +923,6 @@ New\ BibTeX\ sublibrary=New BibTeX sublibrary Switches\ between\ full\ and\ abbreviated\ journal\ name\ if\ the\ journal\ name\ is\ known.=Switches between full and abbreviated journal name if the journal name is known. -the\ field\ %0=the field %0 The\ group\ "%0"\ already\ contains\ the\ selection.=The group "%0" already contains the selection. The\ output\ option\ depends\ on\ a\ valid\ import\ option.=The output option depends on a valid import option. @@ -2147,8 +2121,6 @@ Hierarchical\ keyword\ delimiter=Hierarchical keyword delimiter Escape\ ampersands=Escape ampersands Escape\ dollar\ sign=Escape dollar sign -Hint\:\n\nTo\ search\ all\ fields\ for\ Smith,\ enter\:\nsmith\n\nTo\ search\ the\ field\ author\ for\ Smith\ and\ the\ field\ title\ for\ electrical,\ enter\:\nauthor\:Smith\ AND\ title\:electrical=Hint:\n\nTo search all fields for Smith, enter:\nsmith\n\nTo search the field author for Smith and the field title for electrical, enter:\nauthor:Smith AND title:electrical - Copied\ '%0'\ to\ clipboard.=Copied '%0' to clipboard. This\ operation\ requires\ an\ open\ library.=This operation requires an open library. @@ -2556,7 +2528,6 @@ Left\ Entry=Left Entry Merge\ %0=Merge %0 Right\ Entry=Right Entry Unmerge\ %0=Unmerge %0 -plain\ text=plain text The\ %0s\ are\ the\ same.\ However,\ the\ order\ of\ field\ content\ differs=The %0s are the same. However, the order of field content differs From b5a277437b92a68a54bb020f1eeb81feb859947a Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Thu, 15 Aug 2024 22:53:23 +0300 Subject: [PATCH 140/256] Search groups --- .../gui/maintable/MainTableDataModel.java | 20 +++++++++++++++++++ .../jabref/logic/search/LuceneManager.java | 16 +++++++++------ .../indexing/DefaultLinkedFilesIndexer.java | 4 ---- .../search/retrieval/LuceneSearcher.java | 7 +++++++ .../org/jabref/model/groups/SearchGroup.java | 17 +++++++++------- 5 files changed, 47 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index bbf3b5092d5..832c6f2b407 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -26,10 +26,12 @@ import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.groups.GroupTreeNode; +import org.jabref.model.groups.SearchGroup; import org.jabref.model.search.SearchFieldConstants; import org.jabref.model.search.SearchQuery; import org.jabref.model.search.SearchResults; import org.jabref.model.search.envent.IndexAddedOrUpdatedEvent; +import org.jabref.model.search.envent.IndexStartedEvent; import org.jabref.model.search.matchers.MatcherSet; import org.jabref.model.search.matchers.MatcherSets; import org.jabref.preferences.PreferencesService; @@ -59,6 +61,7 @@ public class MainTableDataModel { private final Subscription groupViewModeSubscription; private final LuceneIndexListener indexUpdatedListener; private final OptionalObjectProperty searchQueryProperty; + private final ListProperty selectedGroupsProperty; private Optional groupsMatcher; public MainTableDataModel(BibDatabaseContext context, @@ -77,6 +80,7 @@ public MainTableDataModel(BibDatabaseContext context, this.luceneManager = luceneManager; this.bibDatabaseContext = context; this.groupsMatcher = createGroupMatcher(selectedGroupsProperty.get(), groupsPreferences); + this.selectedGroupsProperty = selectedGroupsProperty; this.searchQueryProperty = searchQueryProperty; this.indexUpdatedListener = new LuceneIndexListener(); if (luceneManager != null) { @@ -226,5 +230,21 @@ public void listen(IndexAddedOrUpdatedEvent indexAddedOrUpdatedEvent) { }).onSuccess(index -> FilteredListProxy.refilterListReflection(entriesFiltered, index, index + 1)).executeWith(taskExecutor); }); } + + @Subscribe + public void listen(IndexStartedEvent indexStartedEvent) { + bibDatabaseContext.getMetaData().getGroups().ifPresent(this::setLuceneManagerForSearchGroups); + updateSearchMatches(searchQueryProperty.get()); + updateGroupMatches(selectedGroupsProperty); + } + + private void setLuceneManagerForSearchGroups(GroupTreeNode root) { + for (GroupTreeNode node : root.getChildren()) { + if (node.getGroup() instanceof SearchGroup) { + ((SearchGroup) node.getGroup()).setLuceneManager(luceneManager); + } + setLuceneManagerForSearchGroups(node); + } + } } } diff --git a/src/main/java/org/jabref/logic/search/LuceneManager.java b/src/main/java/org/jabref/logic/search/LuceneManager.java index ef90d488e23..ea3019b6c9b 100644 --- a/src/main/java/org/jabref/logic/search/LuceneManager.java +++ b/src/main/java/org/jabref/logic/search/LuceneManager.java @@ -238,6 +238,14 @@ public AutoCloseable blockLinkedFileIndexer() { return () -> isLinkedFilesIndexerBlocked.set(false); } + public void registerListener(Object listener) { + this.eventBus.register(listener); + } + + public void unregisterListener(Object listener) { + eventBus.unregister(listener); + } + public IndexSearcher getIndexSearcher(SearchQuery query) { if (query.getSearchFlags().contains(SearchFlags.FULLTEXT) && shouldIndexLinkedFiles.get()) { try { @@ -256,11 +264,7 @@ public SearchResults search(SearchQuery query) { return luceneSearcher.search(query, getIndexSearcher(query)); } - public void registerListener(Object listener) { - this.eventBus.register(listener); - } - - public void unregisterListener(Object listener) { - eventBus.unregister(listener); + public boolean isMatched(BibEntry entry, SearchQuery query) { + return luceneSearcher.isMatched(entry, query, getIndexSearcher(query)); } } diff --git a/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java b/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java index 37c6e1515aa..d83b23efaf5 100644 --- a/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java @@ -291,10 +291,6 @@ private Pair getLinkedFileInfo(LinkedFile linkedFile) { public IndexSearcher getIndexSearcher() { LOGGER.debug("Getting index searcher for linked files index"); try { - if (indexSearcher != null) { - LOGGER.debug("Releasing index searcher for linked files index"); - searcherManager.release(indexSearcher); - } searcherManager.maybeRefresh(); indexSearcher = searcherManager.acquire(); } catch (IOException e) { diff --git a/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java b/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java index 8383b2f9b4b..a74e55e12fe 100644 --- a/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java +++ b/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java @@ -38,6 +38,13 @@ public LuceneSearcher(BibDatabaseContext databaseContext) { this.databaseContext = databaseContext; } + public boolean isMatched(BibEntry entry, SearchQuery searchQuery, IndexSearcher indexSearcher) { + String query = "+" + SearchFieldConstants.ENTRY_ID + " +" + searchQuery.getSearchExpression(); + SearchQuery newSearchQuery = new SearchQuery(query, searchQuery.getSearchFlags()); + SearchResults searchResults = search(searchQuery, indexSearcher); + return searchResults.getSearchScoreForEntry(entry) > 0; + } + /** * Search for results matching a query in the Lucene search index * diff --git a/src/main/java/org/jabref/model/groups/SearchGroup.java b/src/main/java/org/jabref/model/groups/SearchGroup.java index 67b0ede3a5c..9911feaaf0c 100644 --- a/src/main/java/org/jabref/model/groups/SearchGroup.java +++ b/src/main/java/org/jabref/model/groups/SearchGroup.java @@ -2,8 +2,8 @@ import java.util.EnumSet; import java.util.Objects; -import java.util.Set; +import org.jabref.logic.search.LuceneManager; import org.jabref.model.entry.BibEntry; import org.jabref.model.search.GroupSearchQuery; import org.jabref.model.search.SearchFlags; @@ -19,7 +19,7 @@ public class SearchGroup extends AbstractGroup { private static final Logger LOGGER = LoggerFactory.getLogger(SearchGroup.class); private final GroupSearchQuery query; - private Set matches = Set.of(); + private LuceneManager luceneManager; public SearchGroup(String name, GroupHierarchyType context, String searchExpression, EnumSet searchFlags) { super(name, context); @@ -34,14 +34,14 @@ public GroupSearchQuery getQuery() { return query; } - public void setMatches(Set matches) { - this.matches = matches; - } - public EnumSet getSearchFlags() { return query.getSearchFlags(); } + public void setLuceneManager(LuceneManager luceneManager) { + this.luceneManager = luceneManager; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -58,7 +58,10 @@ public boolean equals(Object o) { @Override public boolean contains(BibEntry entry) { - return matches.contains(entry); + if (luceneManager == null) { + return false; + } + return luceneManager.isMatched(entry, query); } @Override From 3939064c994fa299fd48c8376685f670f9c01538 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Wed, 21 Aug 2024 00:52:35 +0300 Subject: [PATCH 141/256] Release `IndexSearcher` after completing search task --- src/main/java/org/jabref/gui/LibraryTab.java | 2 +- .../gui/maintable/MainTableDataModel.java | 9 +- .../jabref/logic/search/LuceneManager.java | 90 ++++++------------ .../search/indexing/BibFieldsIndexer.java | 31 +++---- .../indexing/DefaultLinkedFilesIndexer.java | 22 ++--- .../indexing/ReadOnlyLinkedFilesIndexer.java | 15 +-- .../search/retrieval/LuceneSearcher.java | 91 +++++++++++++------ .../jabref/model/search/LuceneIndexer.java | 4 +- 8 files changed, 121 insertions(+), 143 deletions(-) diff --git a/src/main/java/org/jabref/gui/LibraryTab.java b/src/main/java/org/jabref/gui/LibraryTab.java index edce8fe9eed..d4722500c92 100644 --- a/src/main/java/org/jabref/gui/LibraryTab.java +++ b/src/main/java/org/jabref/gui/LibraryTab.java @@ -290,7 +290,7 @@ private void onDatabaseLoadingSucceed(ParserResult result) { } public void setLuceneManager() { - luceneManager = new LuceneManager(bibDatabaseContext, taskExecutor, preferencesService); + luceneManager = new LuceneManager(bibDatabaseContext, taskExecutor, preferencesService.getFilePreferences()); luceneManager.updateOnStart(); } diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index 832c6f2b407..db39e5cc395 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -210,6 +210,7 @@ public void listen(IndexAddedOrUpdatedEvent indexAddedOrUpdatedEvent) { if (index >= 0) { BibEntryTableViewModel viewModel = entriesViewModel.get(index); boolean isFloatingMode = searchPreferences.getSearchDisplayMode() == SearchDisplayMode.FLOAT; + boolean isMatched = true; if (searchQueryProperty.get().isPresent()) { SearchQuery searchQuery = searchQueryProperty.get().get(); String newSearchExpression = "+" + SearchFieldConstants.ENTRY_ID + ":" + entry.getId() + " +" + searchQuery.getSearchExpression(); @@ -218,12 +219,12 @@ public void listen(IndexAddedOrUpdatedEvent indexAddedOrUpdatedEvent) { viewModel.searchScoreProperty().set(results.getSearchScoreForEntry(entry)); viewModel.hasFullTextResultsProperty().set(results.hasFulltextResults(entry)); - updateEntrySearchMatch(viewModel, viewModel.searchScoreProperty().get() > 0, isFloatingMode); + isMatched = viewModel.searchScoreProperty().get() > 0; } else { viewModel.searchScoreProperty().set(0); viewModel.hasFullTextResultsProperty().set(false); - updateEntrySearchMatch(viewModel, true, isFloatingMode); } + updateEntrySearchMatch(viewModel, isMatched, isFloatingMode); updateEntryGroupMatch(viewModel, groupsMatcher, groupsPreferences.getGroupViewMode().contains(GroupViewMode.INVERT), !groupsPreferences.getGroupViewMode().contains(GroupViewMode.FILTER)); } return index; @@ -240,8 +241,8 @@ public void listen(IndexStartedEvent indexStartedEvent) { private void setLuceneManagerForSearchGroups(GroupTreeNode root) { for (GroupTreeNode node : root.getChildren()) { - if (node.getGroup() instanceof SearchGroup) { - ((SearchGroup) node.getGroup()).setLuceneManager(luceneManager); + if (node.getGroup() instanceof SearchGroup searchGroup) { + searchGroup.setLuceneManager(luceneManager); } setLuceneManagerForSearchGroups(node); } diff --git a/src/main/java/org/jabref/logic/search/LuceneManager.java b/src/main/java/org/jabref/logic/search/LuceneManager.java index ea3019b6c9b..2a1a3fadb62 100644 --- a/src/main/java/org/jabref/logic/search/LuceneManager.java +++ b/src/main/java/org/jabref/logic/search/LuceneManager.java @@ -16,73 +16,52 @@ import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.search.LuceneIndexer; -import org.jabref.model.search.SearchFlags; import org.jabref.model.search.SearchQuery; import org.jabref.model.search.SearchResults; import org.jabref.model.search.envent.IndexAddedOrUpdatedEvent; import org.jabref.model.search.envent.IndexRemovedEvent; import org.jabref.model.search.envent.IndexStartedEvent; -import org.jabref.preferences.PreferencesService; +import org.jabref.preferences.FilePreferences; import com.google.common.eventbus.EventBus; -import org.apache.lucene.index.MultiReader; -import org.apache.lucene.search.IndexSearcher; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class LuceneManager { - private static final Logger LOGGER = LoggerFactory.getLogger(LuceneManager.class); + private final EventBus eventBus = new EventBus(); - private final BibDatabaseContext databaseContext; private final TaskExecutor taskExecutor; - private final PreferencesService preferences; private final BooleanProperty shouldIndexLinkedFiles; - private final BooleanProperty isLinkedFilesIndexerBlocked; + private final BooleanProperty isLinkedFilesIndexerBlocked = new SimpleBooleanProperty(false);; private final ChangeListener preferencesListener; private final LuceneSearcher luceneSearcher; - private LuceneIndexer linkedFilesIndexer; - private LuceneIndexer bibFieldsIndexer; + private final LuceneIndexer bibFieldsIndexer; + private final LuceneIndexer linkedFilesIndexer; - public LuceneManager(BibDatabaseContext databaseContext, TaskExecutor executor, PreferencesService preferences) { - this.databaseContext = databaseContext; + public LuceneManager(BibDatabaseContext databaseContext, TaskExecutor executor, FilePreferences preferences) { this.taskExecutor = executor; - this.preferences = preferences; - this.isLinkedFilesIndexerBlocked = new SimpleBooleanProperty(false); - this.shouldIndexLinkedFiles = preferences.getFilePreferences().fulltextIndexLinkedFilesProperty(); + + this.shouldIndexLinkedFiles = preferences.fulltextIndexLinkedFilesProperty(); this.preferencesListener = (observable, oldValue, newValue) -> bindToPreferences(newValue); this.shouldIndexLinkedFiles.addListener(preferencesListener); - this.luceneSearcher = new LuceneSearcher(databaseContext); - - initializeIndexers(); - } - private void initializeIndexers() { - try { - bibFieldsIndexer = new BibFieldsIndexer(databaseContext); - } catch (IOException e) { - LOGGER.error("Error initializing bib fields index", e); - } - initializeLinkedFilesIndexer(); - if (!shouldIndexLinkedFiles.get() && linkedFilesIndexer != null) { - linkedFilesIndexer.removeAllFromIndex(); - linkedFilesIndexer.close(); - linkedFilesIndexer = null; - } - } + this.bibFieldsIndexer = new BibFieldsIndexer(databaseContext); - private void initializeLinkedFilesIndexer() { + LuceneIndexer indexer; try { - linkedFilesIndexer = new DefaultLinkedFilesIndexer(databaseContext, preferences.getFilePreferences()); + indexer = new DefaultLinkedFilesIndexer(databaseContext, preferences); } catch (IOException e) { LOGGER.debug("Error initializing linked files index - using read only index"); - linkedFilesIndexer = new ReadOnlyLinkedFilesIndexer(databaseContext); + indexer = new ReadOnlyLinkedFilesIndexer(databaseContext); } + linkedFilesIndexer = indexer; + + this.luceneSearcher = new LuceneSearcher(databaseContext, bibFieldsIndexer, linkedFilesIndexer); } private void bindToPreferences(boolean newValue) { if (newValue) { - initializeLinkedFilesIndexer(); new BackgroundTask<>() { @Override protected Object call() { @@ -92,8 +71,6 @@ protected Object call() { }.showToUser(true).executeWith(taskExecutor); } else { linkedFilesIndexer.removeAllFromIndex(); - linkedFilesIndexer.close(); - linkedFilesIndexer = null; } } @@ -108,7 +85,7 @@ protected Object call() { .onFinished(() -> this.eventBus.post(new IndexStartedEvent())) .executeWith(taskExecutor); - if (shouldIndexLinkedFiles.get() && linkedFilesIndexer != null) { + if (shouldIndexLinkedFiles.get()) { new BackgroundTask<>() { @Override protected Object call() { @@ -129,7 +106,7 @@ protected Object call() { }.onFinished(() -> this.eventBus.post(new IndexAddedOrUpdatedEvent(entries))) .showToUser(true).executeWith(taskExecutor); - if (shouldIndexLinkedFiles.get() && !isLinkedFilesIndexerBlocked.get() && linkedFilesIndexer != null) { + if (shouldIndexLinkedFiles.get() && !isLinkedFilesIndexerBlocked.get()) { new BackgroundTask<>() { @Override protected Object call() { @@ -150,7 +127,7 @@ protected Object call() { }.onFinished(() -> this.eventBus.post(new IndexRemovedEvent())) .showToUser(true).executeWith(taskExecutor); - if (shouldIndexLinkedFiles.get() && linkedFilesIndexer != null) { + if (shouldIndexLinkedFiles.get()) { new BackgroundTask<>() { @Override protected Object call() { @@ -171,7 +148,7 @@ protected Object call() { }.onFinished(() -> this.eventBus.post(new IndexAddedOrUpdatedEvent(List.of(entry)))) .showToUser(true).executeWith(taskExecutor); - if (isLinkedFile && shouldIndexLinkedFiles.get() && !isLinkedFilesIndexerBlocked.get() && linkedFilesIndexer != null) { + if (isLinkedFile && shouldIndexLinkedFiles.get() && !isLinkedFilesIndexerBlocked.get()) { new BackgroundTask<>() { @Override protected Object call() { @@ -192,7 +169,7 @@ protected Object call() { }.onFinished(() -> this.eventBus.post(new IndexAddedOrUpdatedEvent(List.of(entry)))) .showToUser(true).executeWith(taskExecutor); - if (shouldIndexLinkedFiles.get() && !isLinkedFilesIndexerBlocked.get() && linkedFilesIndexer != null) { + if (shouldIndexLinkedFiles.get() && !isLinkedFilesIndexerBlocked.get()) { new BackgroundTask<>() { @Override protected Object call() { @@ -213,7 +190,7 @@ protected Object call() { }.onFinished(() -> this.eventBus.post(new IndexStartedEvent())) .showToUser(true).executeWith(taskExecutor); - if (shouldIndexLinkedFiles.get() && linkedFilesIndexer != null) { + if (shouldIndexLinkedFiles.get()) { new BackgroundTask<>() { @Override protected Object call() { @@ -227,9 +204,7 @@ protected Object call() { public void close() { bibFieldsIndexer.close(); shouldIndexLinkedFiles.removeListener(preferencesListener); - if (linkedFilesIndexer != null) { - linkedFilesIndexer.close(); - } + linkedFilesIndexer.close(); } public AutoCloseable blockLinkedFileIndexer() { @@ -246,25 +221,14 @@ public void unregisterListener(Object listener) { eventBus.unregister(listener); } - public IndexSearcher getIndexSearcher(SearchQuery query) { - if (query.getSearchFlags().contains(SearchFlags.FULLTEXT) && shouldIndexLinkedFiles.get()) { - try { - MultiReader reader = new MultiReader(bibFieldsIndexer.getIndexSearcher().getIndexReader(), linkedFilesIndexer.getIndexSearcher().getIndexReader()); - LOGGER.debug("Using index searcher for bib fields and linked files"); - return new IndexSearcher(reader); - } catch (IOException e) { - LOGGER.error("Error getting index searcher", e); - } - } - LOGGER.debug("Using index searcher for bib fields only"); - return bibFieldsIndexer.getIndexSearcher(); - } - public SearchResults search(SearchQuery query) { - return luceneSearcher.search(query, getIndexSearcher(query)); + if (query.isValid()) { + return luceneSearcher.search(query.getParsedQuery(), query.getSearchFlags()); + } + return new SearchResults(); } public boolean isMatched(BibEntry entry, SearchQuery query) { - return luceneSearcher.isMatched(entry, query, getIndexSearcher(query)); + return luceneSearcher.isMatched(entry, query); } } diff --git a/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java b/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java index 57b7e3edd87..8fa8be9c517 100644 --- a/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java @@ -19,7 +19,6 @@ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.Term; -import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.SearcherManager; import org.apache.lucene.store.ByteBuffersDirectory; import org.apache.lucene.store.Directory; @@ -33,19 +32,22 @@ public class BibFieldsIndexer implements LuceneIndexer { private final BibDatabaseContext databaseContext; private final String libraryName; private final Directory indexDirectory; - private final IndexWriter indexWriter; - private final SearcherManager searcherManager; - private IndexSearcher indexSearcher; + private IndexWriter indexWriter; + private SearcherManager searcherManager; - public BibFieldsIndexer(BibDatabaseContext databaseContext) throws IOException { + public BibFieldsIndexer(BibDatabaseContext databaseContext) { this.databaseContext = databaseContext; this.libraryName = databaseContext.getDatabasePath().map(path -> path.getFileName().toString()).orElseGet(() -> "unsaved"); IndexWriterConfig config = new IndexWriterConfig(SearchFieldConstants.NGram_Analyzer_For_INDEXING); this.indexDirectory = new ByteBuffersDirectory(); - this.indexWriter = new IndexWriter(indexDirectory, config); - this.searcherManager = new SearcherManager(indexWriter, null); + try { + this.indexWriter = new IndexWriter(indexDirectory, config); + this.searcherManager = new SearcherManager(indexWriter, null); + } catch (IOException e) { + LOGGER.error("Error initializing bib fields index", e); + } } @Override @@ -147,19 +149,8 @@ public void rebuildIndex(BackgroundTask task) { } @Override - public IndexSearcher getIndexSearcher() { - LOGGER.debug("Getting index searcher for bib fields index"); - try { - if (indexSearcher != null) { - LOGGER.debug("Releasing bib fields index searcher"); - searcherManager.release(indexSearcher); - } - searcherManager.maybeRefresh(); - indexSearcher = searcherManager.acquire(); - } catch (IOException e) { - LOGGER.error("Error refreshing searcher for bib fields index", e); - } - return indexSearcher; + public SearcherManager getSearcherManager() { + return searcherManager; } @Override diff --git a/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java b/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java index d83b23efaf5..98bc712ce7f 100644 --- a/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java @@ -57,7 +57,6 @@ public class DefaultLinkedFilesIndexer implements LuceneIndexer { private final SearcherManager searcherManager; private Path indexDirectoryPath; private Map indexedFiles; - private IndexSearcher indexSearcher; public DefaultLinkedFilesIndexer(BibDatabaseContext databaseContext, FilePreferences filePreferences) throws IOException { this.databaseContext = databaseContext; @@ -238,7 +237,8 @@ private Map getLinkedFilesFromIndex() { Map linkedFiles = new HashMap<>(); try { TermQuery query = new TermQuery(new Term(SearchFieldConstants.PAGE_NUMBER.toString(), "1")); - IndexSearcher searcher = getIndexSearcher(); + searcherManager.maybeRefresh(); + IndexSearcher searcher = searcherManager.acquire(); StoredFields storedFields = searcher.storedFields(); TopDocs allDocs = searcher.search(query, Integer.MAX_VALUE); for (ScoreDoc scoreDoc : allDocs.scoreDocs) { @@ -249,6 +249,7 @@ private Map getLinkedFilesFromIndex() { linkedFiles.put(pathField.stringValue(), Long.valueOf(modifiedField.stringValue())); } } + searcherManager.release(searcher); } catch (IOException e) { LOGGER.error("Error getting linked files from index", e); } @@ -287,18 +288,6 @@ private Pair getLinkedFileInfo(LinkedFile linkedFile) { } } - @Override - public IndexSearcher getIndexSearcher() { - LOGGER.debug("Getting index searcher for linked files index"); - try { - searcherManager.maybeRefresh(); - indexSearcher = searcherManager.acquire(); - } catch (IOException e) { - LOGGER.error("Error refreshing searcher", e); - } - return indexSearcher; - } - private void optimizeIndex() { LOGGER.debug("Optimizing index"); if (indexWriter.hasDeletions()) { @@ -317,6 +306,11 @@ private void optimizeIndex() { } } + @Override + public SearcherManager getSearcherManager() { + return searcherManager; + } + @Override public void close() { HeadlessExecutorService.INSTANCE.execute(() -> { diff --git a/src/main/java/org/jabref/logic/search/indexing/ReadOnlyLinkedFilesIndexer.java b/src/main/java/org/jabref/logic/search/indexing/ReadOnlyLinkedFilesIndexer.java index 19e63e7ab16..8c4e9200330 100644 --- a/src/main/java/org/jabref/logic/search/indexing/ReadOnlyLinkedFilesIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/ReadOnlyLinkedFilesIndexer.java @@ -9,7 +9,6 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.search.LuceneIndexer; -import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.SearcherManager; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; @@ -20,7 +19,6 @@ public class ReadOnlyLinkedFilesIndexer implements LuceneIndexer { private static final Logger LOGGER = LoggerFactory.getLogger(ReadOnlyLinkedFilesIndexer.class); private Directory indexDirectory; private SearcherManager searcherManager; - private IndexSearcher indexSearcher; public ReadOnlyLinkedFilesIndexer(BibDatabaseContext databaseContext) { try { @@ -56,17 +54,8 @@ public void rebuildIndex(BackgroundTask task) { } @Override - public IndexSearcher getIndexSearcher() { - try { - if (indexSearcher != null) { - searcherManager.release(indexSearcher); - } - searcherManager.maybeRefresh(); - indexSearcher = searcherManager.acquire(); - } catch (IOException e) { - LOGGER.error("Error refreshing searcher", e); - } - return indexSearcher; + public SearcherManager getSearcherManager() { + return searcherManager; } @Override diff --git a/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java b/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java index a74e55e12fe..72c150a9da2 100644 --- a/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java +++ b/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java @@ -2,24 +2,33 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.LinkedFile; +import org.jabref.model.search.LuceneIndexer; import org.jabref.model.search.SearchFieldConstants; +import org.jabref.model.search.SearchFlags; import org.jabref.model.search.SearchQuery; import org.jabref.model.search.SearchResult; import org.jabref.model.search.SearchResults; import org.apache.lucene.document.Document; +import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.StoredFields; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.SearcherManager; +import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.highlight.Highlighter; import org.apache.lucene.search.highlight.QueryScorer; @@ -29,44 +38,65 @@ public final class LuceneSearcher { private static final Logger LOGGER = LoggerFactory.getLogger(LuceneSearcher.class); + private final BibDatabaseContext databaseContext; - private SearchQuery searchQuery; - private TopDocs topDocs; - private StoredFields storedFields; + private final SearcherManager bibFieldsSearcherManager; + private final SearcherManager linkedFilesSearcherManager; - public LuceneSearcher(BibDatabaseContext databaseContext) { + public LuceneSearcher(BibDatabaseContext databaseContext, LuceneIndexer bibFieldsIndexer, LuceneIndexer linkedFilesIndexer) { + this.bibFieldsSearcherManager = bibFieldsIndexer.getSearcherManager(); + this.linkedFilesSearcherManager = linkedFilesIndexer.getSearcherManager(); this.databaseContext = databaseContext; } - public boolean isMatched(BibEntry entry, SearchQuery searchQuery, IndexSearcher indexSearcher) { - String query = "+" + SearchFieldConstants.ENTRY_ID + " +" + searchQuery.getSearchExpression(); - SearchQuery newSearchQuery = new SearchQuery(query, searchQuery.getSearchFlags()); - SearchResults searchResults = search(searchQuery, indexSearcher); - return searchResults.getSearchScoreForEntry(entry) > 0; + public boolean isMatched(BibEntry entry, SearchQuery searchQuery) { + BooleanQuery booleanQuery = createBooleanQuery(entry, searchQuery); + SearchResults results = search(booleanQuery, searchQuery.getSearchFlags()); + return results.getSearchScoreForEntry(entry) > 0; + } + + private BooleanQuery createBooleanQuery(BibEntry entry, SearchQuery searchQuery) { + Query query = searchQuery.getParsedQuery(); + TermQuery termQuery = new TermQuery(new Term(SearchFieldConstants.ENTRY_ID.toString(), entry.getId())); + return new BooleanQuery.Builder() + .add(query, BooleanClause.Occur.MUST) + .add(termQuery, BooleanClause.Occur.MUST) + .build(); } - /** - * Search for results matching a query in the Lucene search index - * - * @param searchQuery query to search for - * @param indexSearcher the index searcher to use - * @return a result map of all entries that have matches in any fields - */ - public SearchResults search(SearchQuery searchQuery, IndexSearcher indexSearcher) { - this.searchQuery = Objects.requireNonNull(searchQuery); + public SearchResults search(Query searchQuery, EnumSet searchFlags) { + LOGGER.debug("Searching for entries matching query: {}", searchQuery); try { - LOGGER.debug("Searching for entries matching query: {}", searchQuery.getParsedQuery()); - topDocs = indexSearcher.search(searchQuery.getParsedQuery(), Integer.MAX_VALUE); - storedFields = indexSearcher.storedFields(); - LOGGER.debug("Found {} matches", topDocs.totalHits.value); - return getSearchResults(); + if (searchFlags.contains(SearchFlags.FULLTEXT)) { + IndexSearcher bibFieldsIndexSearcher = getIndexedSearcher(bibFieldsSearcherManager); + IndexSearcher linkedFilesIndexSearcher = getIndexedSearcher(linkedFilesSearcherManager); + MultiReader multiReader = new MultiReader(bibFieldsIndexSearcher.getIndexReader(), linkedFilesIndexSearcher.getIndexReader()); + IndexSearcher indexSearcher = new IndexSearcher(multiReader); + + SearchResults searchResults = search(indexSearcher, searchQuery); + releaseIndexSearcher(bibFieldsSearcherManager, bibFieldsIndexSearcher); + releaseIndexSearcher(linkedFilesSearcherManager, linkedFilesIndexSearcher); + return searchResults; + } else { + IndexSearcher indexSearcher = getIndexedSearcher(bibFieldsSearcherManager); + SearchResults searchResults = search(indexSearcher, searchQuery); + releaseIndexSearcher(bibFieldsSearcherManager, indexSearcher); + return searchResults; + } } catch (IOException | IndexSearcher.TooManyClauses e) { LOGGER.error("Error while searching", e); } return new SearchResults(); } - private SearchResults getSearchResults() throws IOException { + private SearchResults search(IndexSearcher indexSearcher, Query searchQuery) throws IOException { + TopDocs topDocs = indexSearcher.search(searchQuery, Integer.MAX_VALUE); + StoredFields storedFields = indexSearcher.storedFields(); + LOGGER.debug("Found {} matches", topDocs.totalHits.value); + return getSearchResults(topDocs, storedFields, searchQuery); + } + + private SearchResults getSearchResults(TopDocs topDocs, StoredFields storedFields, Query searchQuery) throws IOException { SearchResults searchResults = new SearchResults(); Map entriesMap = new HashMap<>(); Map> linkedFilesMap = new HashMap<>(); @@ -79,7 +109,7 @@ private SearchResults getSearchResults() throws IOException { } long startTime = System.currentTimeMillis(); - Highlighter highlighter = new Highlighter(new SimpleHTMLFormatter("", ""), new QueryScorer(searchQuery.getParsedQuery())); + Highlighter highlighter = new Highlighter(new SimpleHTMLFormatter("", ""), new QueryScorer(searchQuery)); for (ScoreDoc scoreDoc : topDocs.scoreDocs) { float score = scoreDoc.score; Document document = storedFields.document(scoreDoc.doc); @@ -107,4 +137,13 @@ private SearchResults getSearchResults() throws IOException { private static String getFieldContents(Document document, SearchFieldConstants field) { return Optional.ofNullable(document.get(field.toString())).orElse(""); } + + private static IndexSearcher getIndexedSearcher(SearcherManager searcherManager) throws IOException { + searcherManager.maybeRefresh(); + return searcherManager.acquire(); + } + + private static void releaseIndexSearcher(SearcherManager searcherManager, IndexSearcher indexSearcher) throws IOException { + searcherManager.release(indexSearcher); + } } diff --git a/src/main/java/org/jabref/model/search/LuceneIndexer.java b/src/main/java/org/jabref/model/search/LuceneIndexer.java index f8ed054a2c9..1f0dfbd6edb 100644 --- a/src/main/java/org/jabref/model/search/LuceneIndexer.java +++ b/src/main/java/org/jabref/model/search/LuceneIndexer.java @@ -5,7 +5,7 @@ import org.jabref.gui.util.BackgroundTask; import org.jabref.model.entry.BibEntry; -import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.SearcherManager; public interface LuceneIndexer { void updateOnStart(BackgroundTask task); @@ -20,7 +20,7 @@ public interface LuceneIndexer { void rebuildIndex(BackgroundTask task); - IndexSearcher getIndexSearcher(); + SearcherManager getSearcherManager(); void close(); } From 8c8becc06c3b12b448cffcf9579e12f5fb80e0c5 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Wed, 21 Aug 2024 01:06:54 +0300 Subject: [PATCH 142/256] Checkstyle --- src/main/java/org/jabref/logic/search/LuceneManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/jabref/logic/search/LuceneManager.java b/src/main/java/org/jabref/logic/search/LuceneManager.java index 2a1a3fadb62..464aa2a4c76 100644 --- a/src/main/java/org/jabref/logic/search/LuceneManager.java +++ b/src/main/java/org/jabref/logic/search/LuceneManager.java @@ -33,7 +33,7 @@ public class LuceneManager { private final EventBus eventBus = new EventBus(); private final TaskExecutor taskExecutor; private final BooleanProperty shouldIndexLinkedFiles; - private final BooleanProperty isLinkedFilesIndexerBlocked = new SimpleBooleanProperty(false);; + private final BooleanProperty isLinkedFilesIndexerBlocked = new SimpleBooleanProperty(false); private final ChangeListener preferencesListener; private final LuceneSearcher luceneSearcher; private final LuceneIndexer bibFieldsIndexer; From 363f46b9a1139d3b73c675c14419493c35aa14bb Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Sun, 25 Aug 2024 20:02:33 +0300 Subject: [PATCH 143/256] Correct typo --- .../java/org/jabref/gui/maintable/MainTableDataModel.java | 4 ++-- src/main/java/org/jabref/logic/search/LuceneManager.java | 6 +++--- .../org/jabref/model/search/envent/IndexRemovedEvent.java | 4 ---- .../org/jabref/model/search/envent/IndexStartedEvent.java | 4 ---- .../org/jabref/model/search/envent/IndexUpdatedEvent.java | 4 ---- .../search/{envent => event}/IndexAddedOrUpdatedEvent.java | 2 +- .../org/jabref/model/search/event/IndexRemovedEvent.java | 4 ++++ .../org/jabref/model/search/event/IndexStartedEvent.java | 4 ++++ .../org/jabref/model/search/event/IndexUpdatedEvent.java | 4 ++++ 9 files changed, 18 insertions(+), 18 deletions(-) delete mode 100644 src/main/java/org/jabref/model/search/envent/IndexRemovedEvent.java delete mode 100644 src/main/java/org/jabref/model/search/envent/IndexStartedEvent.java delete mode 100644 src/main/java/org/jabref/model/search/envent/IndexUpdatedEvent.java rename src/main/java/org/jabref/model/search/{envent => event}/IndexAddedOrUpdatedEvent.java (77%) create mode 100644 src/main/java/org/jabref/model/search/event/IndexRemovedEvent.java create mode 100644 src/main/java/org/jabref/model/search/event/IndexStartedEvent.java create mode 100644 src/main/java/org/jabref/model/search/event/IndexUpdatedEvent.java diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index db39e5cc395..c556922b32d 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -30,8 +30,8 @@ import org.jabref.model.search.SearchFieldConstants; import org.jabref.model.search.SearchQuery; import org.jabref.model.search.SearchResults; -import org.jabref.model.search.envent.IndexAddedOrUpdatedEvent; -import org.jabref.model.search.envent.IndexStartedEvent; +import org.jabref.model.search.event.IndexAddedOrUpdatedEvent; +import org.jabref.model.search.event.IndexStartedEvent; import org.jabref.model.search.matchers.MatcherSet; import org.jabref.model.search.matchers.MatcherSets; import org.jabref.preferences.PreferencesService; diff --git a/src/main/java/org/jabref/logic/search/LuceneManager.java b/src/main/java/org/jabref/logic/search/LuceneManager.java index 464aa2a4c76..e9acc94348c 100644 --- a/src/main/java/org/jabref/logic/search/LuceneManager.java +++ b/src/main/java/org/jabref/logic/search/LuceneManager.java @@ -18,9 +18,9 @@ import org.jabref.model.search.LuceneIndexer; import org.jabref.model.search.SearchQuery; import org.jabref.model.search.SearchResults; -import org.jabref.model.search.envent.IndexAddedOrUpdatedEvent; -import org.jabref.model.search.envent.IndexRemovedEvent; -import org.jabref.model.search.envent.IndexStartedEvent; +import org.jabref.model.search.event.IndexAddedOrUpdatedEvent; +import org.jabref.model.search.event.IndexRemovedEvent; +import org.jabref.model.search.event.IndexStartedEvent; import org.jabref.preferences.FilePreferences; import com.google.common.eventbus.EventBus; diff --git a/src/main/java/org/jabref/model/search/envent/IndexRemovedEvent.java b/src/main/java/org/jabref/model/search/envent/IndexRemovedEvent.java deleted file mode 100644 index dd00c496c0d..00000000000 --- a/src/main/java/org/jabref/model/search/envent/IndexRemovedEvent.java +++ /dev/null @@ -1,4 +0,0 @@ -package org.jabref.model.search.envent; - -public class IndexRemovedEvent { -} diff --git a/src/main/java/org/jabref/model/search/envent/IndexStartedEvent.java b/src/main/java/org/jabref/model/search/envent/IndexStartedEvent.java deleted file mode 100644 index 0bd7f738167..00000000000 --- a/src/main/java/org/jabref/model/search/envent/IndexStartedEvent.java +++ /dev/null @@ -1,4 +0,0 @@ -package org.jabref.model.search.envent; - -public class IndexStartedEvent { -} diff --git a/src/main/java/org/jabref/model/search/envent/IndexUpdatedEvent.java b/src/main/java/org/jabref/model/search/envent/IndexUpdatedEvent.java deleted file mode 100644 index e43c8677011..00000000000 --- a/src/main/java/org/jabref/model/search/envent/IndexUpdatedEvent.java +++ /dev/null @@ -1,4 +0,0 @@ -package org.jabref.model.search.envent; - -public class IndexUpdatedEvent { -} diff --git a/src/main/java/org/jabref/model/search/envent/IndexAddedOrUpdatedEvent.java b/src/main/java/org/jabref/model/search/event/IndexAddedOrUpdatedEvent.java similarity index 77% rename from src/main/java/org/jabref/model/search/envent/IndexAddedOrUpdatedEvent.java rename to src/main/java/org/jabref/model/search/event/IndexAddedOrUpdatedEvent.java index d192dcd643d..280d9baa0e0 100644 --- a/src/main/java/org/jabref/model/search/envent/IndexAddedOrUpdatedEvent.java +++ b/src/main/java/org/jabref/model/search/event/IndexAddedOrUpdatedEvent.java @@ -1,4 +1,4 @@ -package org.jabref.model.search.envent; +package org.jabref.model.search.event; import java.util.List; diff --git a/src/main/java/org/jabref/model/search/event/IndexRemovedEvent.java b/src/main/java/org/jabref/model/search/event/IndexRemovedEvent.java new file mode 100644 index 00000000000..7fff2f223ba --- /dev/null +++ b/src/main/java/org/jabref/model/search/event/IndexRemovedEvent.java @@ -0,0 +1,4 @@ +package org.jabref.model.search.event; + +record IndexRemovedEvent() { +} diff --git a/src/main/java/org/jabref/model/search/event/IndexStartedEvent.java b/src/main/java/org/jabref/model/search/event/IndexStartedEvent.java new file mode 100644 index 00000000000..d1794191446 --- /dev/null +++ b/src/main/java/org/jabref/model/search/event/IndexStartedEvent.java @@ -0,0 +1,4 @@ +package org.jabref.model.search.event; + +record IndexStartedEvent() { +} diff --git a/src/main/java/org/jabref/model/search/event/IndexUpdatedEvent.java b/src/main/java/org/jabref/model/search/event/IndexUpdatedEvent.java new file mode 100644 index 00000000000..6572aace6ea --- /dev/null +++ b/src/main/java/org/jabref/model/search/event/IndexUpdatedEvent.java @@ -0,0 +1,4 @@ +package org.jabref.model.search.event; + +record IndexUpdatedEvent() { +} From 78ab566490964ae2adc3b3133b22ff5873c4c8d5 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Sun, 25 Aug 2024 20:08:07 +0300 Subject: [PATCH 144/256] Remove GroupSearchQuery --- src/main/java/org/jabref/model/groups/SearchGroup.java | 8 ++++---- .../java/org/jabref/model/search/GroupSearchQuery.java | 10 ---------- .../jabref/model/search/event/IndexRemovedEvent.java | 2 +- .../jabref/model/search/event/IndexStartedEvent.java | 2 +- .../jabref/model/search/event/IndexUpdatedEvent.java | 2 +- 5 files changed, 7 insertions(+), 17 deletions(-) delete mode 100644 src/main/java/org/jabref/model/search/GroupSearchQuery.java diff --git a/src/main/java/org/jabref/model/groups/SearchGroup.java b/src/main/java/org/jabref/model/groups/SearchGroup.java index 9911feaaf0c..e97d44916ea 100644 --- a/src/main/java/org/jabref/model/groups/SearchGroup.java +++ b/src/main/java/org/jabref/model/groups/SearchGroup.java @@ -5,8 +5,8 @@ import org.jabref.logic.search.LuceneManager; import org.jabref.model.entry.BibEntry; -import org.jabref.model.search.GroupSearchQuery; import org.jabref.model.search.SearchFlags; +import org.jabref.model.search.SearchQuery; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -18,19 +18,19 @@ public class SearchGroup extends AbstractGroup { private static final Logger LOGGER = LoggerFactory.getLogger(SearchGroup.class); - private final GroupSearchQuery query; + private final SearchQuery query; private LuceneManager luceneManager; public SearchGroup(String name, GroupHierarchyType context, String searchExpression, EnumSet searchFlags) { super(name, context); - this.query = new GroupSearchQuery(searchExpression, searchFlags); + this.query = new SearchQuery(searchExpression, searchFlags); } public String getSearchExpression() { return query.getSearchExpression(); } - public GroupSearchQuery getQuery() { + public SearchQuery getQuery() { return query; } diff --git a/src/main/java/org/jabref/model/search/GroupSearchQuery.java b/src/main/java/org/jabref/model/search/GroupSearchQuery.java deleted file mode 100644 index a7813697412..00000000000 --- a/src/main/java/org/jabref/model/search/GroupSearchQuery.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.jabref.model.search; - -import java.util.EnumSet; - -public class GroupSearchQuery extends SearchQuery { - - public GroupSearchQuery(String query, EnumSet searchFlags) { - super(query, searchFlags); - } -} diff --git a/src/main/java/org/jabref/model/search/event/IndexRemovedEvent.java b/src/main/java/org/jabref/model/search/event/IndexRemovedEvent.java index 7fff2f223ba..72b5a404f44 100644 --- a/src/main/java/org/jabref/model/search/event/IndexRemovedEvent.java +++ b/src/main/java/org/jabref/model/search/event/IndexRemovedEvent.java @@ -1,4 +1,4 @@ package org.jabref.model.search.event; -record IndexRemovedEvent() { +public record IndexRemovedEvent() { } diff --git a/src/main/java/org/jabref/model/search/event/IndexStartedEvent.java b/src/main/java/org/jabref/model/search/event/IndexStartedEvent.java index d1794191446..8bb268ab73c 100644 --- a/src/main/java/org/jabref/model/search/event/IndexStartedEvent.java +++ b/src/main/java/org/jabref/model/search/event/IndexStartedEvent.java @@ -1,4 +1,4 @@ package org.jabref.model.search.event; -record IndexStartedEvent() { +public record IndexStartedEvent() { } diff --git a/src/main/java/org/jabref/model/search/event/IndexUpdatedEvent.java b/src/main/java/org/jabref/model/search/event/IndexUpdatedEvent.java index 6572aace6ea..b0ec923c1c1 100644 --- a/src/main/java/org/jabref/model/search/event/IndexUpdatedEvent.java +++ b/src/main/java/org/jabref/model/search/event/IndexUpdatedEvent.java @@ -1,4 +1,4 @@ package org.jabref.model.search.event; -record IndexUpdatedEvent() { +public record IndexUpdatedEvent() { } From 210a824001be957f9083b17147c2f738a9960d52 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Mon, 26 Aug 2024 01:04:34 +0300 Subject: [PATCH 145/256] Remove EventBus from LuceneManager and use BibDatabase eventBus --- .../gui/maintable/MainTableDataModel.java | 8 ++---- .../jabref/logic/search/LuceneManager.java | 25 ++++++------------- .../jabref/model/database/BibDatabase.java | 4 +++ 3 files changed, 14 insertions(+), 23 deletions(-) diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index c556922b32d..e7a5a425210 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -83,9 +83,7 @@ public MainTableDataModel(BibDatabaseContext context, this.selectedGroupsProperty = selectedGroupsProperty; this.searchQueryProperty = searchQueryProperty; this.indexUpdatedListener = new LuceneIndexListener(); - if (luceneManager != null) { - this.luceneManager.registerListener(indexUpdatedListener); - } + this.bibDatabaseContext.getDatabase().registerListener(indexUpdatedListener); resetFieldFormatter(); @@ -188,9 +186,7 @@ public void unbind() { searchDisplayModeSubscription.unsubscribe(); selectedGroupsSubscription.unsubscribe(); groupViewModeSubscription.unsubscribe(); - if (luceneManager != null) { - luceneManager.unregisterListener(indexUpdatedListener); - } + bibDatabaseContext.getDatabase().unregisterListener(indexUpdatedListener); } public SortedList getEntriesFilteredAndSorted() { diff --git a/src/main/java/org/jabref/logic/search/LuceneManager.java b/src/main/java/org/jabref/logic/search/LuceneManager.java index e9acc94348c..21ce36b23f2 100644 --- a/src/main/java/org/jabref/logic/search/LuceneManager.java +++ b/src/main/java/org/jabref/logic/search/LuceneManager.java @@ -23,15 +23,14 @@ import org.jabref.model.search.event.IndexStartedEvent; import org.jabref.preferences.FilePreferences; -import com.google.common.eventbus.EventBus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class LuceneManager { private static final Logger LOGGER = LoggerFactory.getLogger(LuceneManager.class); - private final EventBus eventBus = new EventBus(); private final TaskExecutor taskExecutor; + private final BibDatabaseContext databaseContext; private final BooleanProperty shouldIndexLinkedFiles; private final BooleanProperty isLinkedFilesIndexerBlocked = new SimpleBooleanProperty(false); private final ChangeListener preferencesListener; @@ -41,7 +40,7 @@ public class LuceneManager { public LuceneManager(BibDatabaseContext databaseContext, TaskExecutor executor, FilePreferences preferences) { this.taskExecutor = executor; - + this.databaseContext = databaseContext; this.shouldIndexLinkedFiles = preferences.fulltextIndexLinkedFilesProperty(); this.preferencesListener = (observable, oldValue, newValue) -> bindToPreferences(newValue); this.shouldIndexLinkedFiles.addListener(preferencesListener); @@ -82,7 +81,7 @@ protected Object call() { return null; } }.showToUser(true) - .onFinished(() -> this.eventBus.post(new IndexStartedEvent())) + .onFinished(() -> this.databaseContext.getDatabase().postEvent(new IndexStartedEvent())) .executeWith(taskExecutor); if (shouldIndexLinkedFiles.get()) { @@ -103,7 +102,7 @@ protected Object call() { bibFieldsIndexer.addToIndex(entries, this); return null; } - }.onFinished(() -> this.eventBus.post(new IndexAddedOrUpdatedEvent(entries))) + }.onFinished(() -> this.databaseContext.getDatabase().postEvent(new IndexAddedOrUpdatedEvent(entries))) .showToUser(true).executeWith(taskExecutor); if (shouldIndexLinkedFiles.get() && !isLinkedFilesIndexerBlocked.get()) { @@ -124,7 +123,7 @@ protected Object call() { bibFieldsIndexer.removeFromIndex(entries, this); return null; } - }.onFinished(() -> this.eventBus.post(new IndexRemovedEvent())) + }.onFinished(() -> this.databaseContext.getDatabase().postEvent(new IndexRemovedEvent())) .showToUser(true).executeWith(taskExecutor); if (shouldIndexLinkedFiles.get()) { @@ -145,7 +144,7 @@ protected Object call() { bibFieldsIndexer.updateEntry(entry, oldValue, newValue, this); return null; } - }.onFinished(() -> this.eventBus.post(new IndexAddedOrUpdatedEvent(List.of(entry)))) + }.onFinished(() -> this.databaseContext.getDatabase().postEvent(new IndexAddedOrUpdatedEvent(List.of(entry)))) .showToUser(true).executeWith(taskExecutor); if (isLinkedFile && shouldIndexLinkedFiles.get() && !isLinkedFilesIndexerBlocked.get()) { @@ -166,7 +165,7 @@ protected Object call() { bibFieldsIndexer.updateEntry(entry, "", "", this); return null; } - }.onFinished(() -> this.eventBus.post(new IndexAddedOrUpdatedEvent(List.of(entry)))) + }.onFinished(() -> this.databaseContext.getDatabase().postEvent(new IndexAddedOrUpdatedEvent(List.of(entry)))) .showToUser(true).executeWith(taskExecutor); if (shouldIndexLinkedFiles.get() && !isLinkedFilesIndexerBlocked.get()) { @@ -187,7 +186,7 @@ protected Object call() { bibFieldsIndexer.rebuildIndex(this); return null; } - }.onFinished(() -> this.eventBus.post(new IndexStartedEvent())) + }.onFinished(() -> this.databaseContext.getDatabase().postEvent(new IndexStartedEvent())) .showToUser(true).executeWith(taskExecutor); if (shouldIndexLinkedFiles.get()) { @@ -213,14 +212,6 @@ public AutoCloseable blockLinkedFileIndexer() { return () -> isLinkedFilesIndexerBlocked.set(false); } - public void registerListener(Object listener) { - this.eventBus.register(listener); - } - - public void unregisterListener(Object listener) { - eventBus.unregister(listener); - } - public SearchResults search(SearchQuery query) { if (query.isValid()) { return luceneSearcher.search(query.getParsedQuery(), query.getSearchFlags()); diff --git a/src/main/java/org/jabref/model/database/BibDatabase.java b/src/main/java/org/jabref/model/database/BibDatabase.java index b28f6cc6398..3ec1adb20d3 100644 --- a/src/main/java/org/jabref/model/database/BibDatabase.java +++ b/src/main/java/org/jabref/model/database/BibDatabase.java @@ -550,6 +550,10 @@ public void registerListener(Object listener) { this.eventBus.register(listener); } + public void postEvent(Object event) { + this.eventBus.post(event); + } + /** * Unregisters an listener object. * From 193d53307f850111a83f22b1b70f07ba74c1983a Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Mon, 26 Aug 2024 01:16:06 +0300 Subject: [PATCH 146/256] Fix number of matched entries in groups --- .../jabref/gui/groups/GroupNodeViewModel.java | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java b/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java index 75ce49baf99..43dc4805a94 100644 --- a/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java +++ b/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java @@ -14,6 +14,7 @@ import javafx.collections.FXCollections; import javafx.collections.ListChangeListener; import javafx.collections.ObservableList; +import javafx.collections.ObservableMap; import javafx.scene.input.Dragboard; import javafx.scene.paint.Color; @@ -58,15 +59,17 @@ public class GroupNodeViewModel { private final BibDatabaseContext databaseContext; private final StateManager stateManager; private final GroupTreeNode groupNode; - private final ObservableList matchedEntries = FXCollections.observableArrayList(); + private final ObservableMap matchedEntries = FXCollections.observableHashMap(); private final SimpleBooleanProperty hasChildren; private final SimpleBooleanProperty expandedProperty = new SimpleBooleanProperty(); private final BooleanBinding anySelectedEntriesMatched; private final BooleanBinding allSelectedEntriesMatched; private final TaskExecutor taskExecutor; private final CustomLocalDragboard localDragBoard; - private final ObservableList entriesList; private final PreferencesService preferencesService; + @SuppressWarnings("FieldCanBeLocal") + private final ObservableList entriesList; + @SuppressWarnings("FieldCanBeLocal") private final InvalidationListener onInvalidatedGroup = listener -> refreshGroup(); public GroupNodeViewModel(BibDatabaseContext databaseContext, StateManager stateManager, TaskExecutor taskExecutor, GroupTreeNode groupNode, CustomLocalDragboard localDragBoard, PreferencesService preferencesService) { @@ -236,22 +239,18 @@ private void onDatabaseChanged(ListChangeListener.Change cha } else if (change.wasUpdated()) { for (BibEntry changedEntry : change.getList().subList(change.getFrom(), change.getTo())) { if (groupNode.matches(changedEntry)) { - if (!matchedEntries.contains(changedEntry)) { - matchedEntries.add(changedEntry); - } + matchedEntries.put(System.identityHashCode(changedEntry), changedEntry); } else { - matchedEntries.remove(changedEntry); + matchedEntries.remove(System.identityHashCode(changedEntry)); } } } else { for (BibEntry removedEntry : change.getRemoved()) { - matchedEntries.remove(removedEntry); + matchedEntries.remove(System.identityHashCode(removedEntry)); } for (BibEntry addedEntry : change.getAddedSubList()) { if (groupNode.matches(addedEntry)) { - if (!matchedEntries.contains(addedEntry)) { - matchedEntries.add(addedEntry); - } + matchedEntries.put(System.identityHashCode(addedEntry), addedEntry); } } } @@ -278,7 +277,7 @@ private void updateMatchedEntries() { .wrap(() -> groupNode.findMatches(databaseContext.getDatabase())) .onSuccess(entries -> { matchedEntries.clear(); - matchedEntries.addAll(entries); + entries.forEach(entry -> matchedEntries.put(System.identityHashCode(entry), entry)); }) .executeWith(taskExecutor); } From 4ca8afb89bf5c9ae29c8b0238534ea5a91942a7f Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Mon, 26 Aug 2024 01:27:58 +0300 Subject: [PATCH 147/256] Fix search groups --- .../jabref/gui/groups/GroupNodeViewModel.java | 24 ++++++++--- .../gui/groups/GroupTreeNodeViewModel.java | 37 +++++++--------- .../org/jabref/gui/groups/GroupTreeView.java | 14 ++----- .../gui/maintable/MainTableDataModel.java | 42 +++++++++++++++++-- .../jabref/logic/search/LuceneManager.java | 2 +- .../org/jabref/model/groups/SearchGroup.java | 31 ++++++++++---- .../jabref/model/search/SearchResults.java | 5 ++- .../model/search/event/IndexRemovedEvent.java | 6 ++- .../model/search/event/IndexUpdatedEvent.java | 4 -- 9 files changed, 106 insertions(+), 59 deletions(-) delete mode 100644 src/main/java/org/jabref/model/search/event/IndexUpdatedEvent.java diff --git a/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java b/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java index 43dc4805a94..acac38400a8 100644 --- a/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java +++ b/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java @@ -59,7 +59,7 @@ public class GroupNodeViewModel { private final BibDatabaseContext databaseContext; private final StateManager stateManager; private final GroupTreeNode groupNode; - private final ObservableMap matchedEntries = FXCollections.observableHashMap(); + private final ObservableMap matchedEntries; private final SimpleBooleanProperty hasChildren; private final SimpleBooleanProperty expandedProperty = new SimpleBooleanProperty(); private final BooleanBinding anySelectedEntriesMatched; @@ -94,6 +94,13 @@ public GroupNodeViewModel(BibDatabaseContext databaseContext, StateManager state if (groupNode.getGroup() instanceof TexGroup) { databaseContext.getMetaData().groupsBinding().addListener(new WeakInvalidationListener(onInvalidatedGroup)); } + + if (groupNode.getGroup() instanceof SearchGroup searchGroup) { + matchedEntries = searchGroup.getMatchedEntries(); + } else { + matchedEntries = FXCollections.observableHashMap(); + } + hasChildren = new SimpleBooleanProperty(); hasChildren.bind(Bindings.isNotEmpty(children)); EasyBind.subscribe(preferencesService.getGroupsPreferences().displayGroupCountProperty(), shouldDisplay -> updateMatchedEntries()); @@ -231,8 +238,13 @@ public GroupTreeNode getGroupNode() { /** * Gets invoked if an entry in the current database changes. + * + * @implNote Search groups are updated in {@link org.jabref.gui.maintable.MainTableDataModel.LuceneIndexListener#updateSearchGroupsMatches(org.jabref.model.entry.BibEntry, org.jabref.model.groups.GroupTreeNode)}. */ private void onDatabaseChanged(ListChangeListener.Change change) { + if (groupNode.getGroup() instanceof SearchGroup) { + return; + } while (change.next()) { if (change.wasPermutated()) { // Nothing to do, as permutation doesn't change matched entries @@ -388,7 +400,7 @@ public boolean canAddEntriesIn() { return true; } else if (group instanceof LastNameGroup || group instanceof RegexKeywordGroup) { return groupNode.getParent() - .map(parent -> parent.getGroup()) + .map(GroupTreeNode::getGroup) .map(groupParent -> groupParent instanceof AutomaticKeywordGroup || groupParent instanceof AutomaticPersonsGroup) .orElse(false); } else if (group instanceof KeywordGroup) { @@ -416,7 +428,7 @@ public boolean canBeDragged() { } else if (group instanceof KeywordGroup) { // KeywordGroup is parent of LastNameGroup, RegexKeywordGroup and WordKeywordGroup return groupNode.getParent() - .map(parent -> parent.getGroup()) + .map(GroupTreeNode::getGroup) .map(groupParent -> !(groupParent instanceof AutomaticKeywordGroup || groupParent instanceof AutomaticPersonsGroup)) .orElse(false); } else if (group instanceof SearchGroup) { @@ -441,7 +453,7 @@ public boolean canAddGroupsIn() { } else if (group instanceof KeywordGroup) { // KeywordGroup is parent of LastNameGroup, RegexKeywordGroup and WordKeywordGroup return groupNode.getParent() - .map(parent -> parent.getGroup()) + .map(GroupTreeNode::getGroup) .map(groupParent -> !(groupParent instanceof AutomaticKeywordGroup || groupParent instanceof AutomaticPersonsGroup)) .orElse(false); } else if (group instanceof SearchGroup) { @@ -466,7 +478,7 @@ public boolean canRemove() { } else if (group instanceof KeywordGroup) { // KeywordGroup is parent of LastNameGroup, RegexKeywordGroup and WordKeywordGroup return groupNode.getParent() - .map(parent -> parent.getGroup()) + .map(GroupTreeNode::getGroup) .map(groupParent -> !(groupParent instanceof AutomaticKeywordGroup || groupParent instanceof AutomaticPersonsGroup)) .orElse(false); } else if (group instanceof SearchGroup) { @@ -491,7 +503,7 @@ public boolean isEditable() { } else if (group instanceof KeywordGroup) { // KeywordGroup is parent of LastNameGroup, RegexKeywordGroup and WordKeywordGroup return groupNode.getParent() - .map(parent -> parent.getGroup()) + .map(GroupTreeNode::getGroup) .map(groupParent -> !(groupParent instanceof AutomaticKeywordGroup || groupParent instanceof AutomaticPersonsGroup)) .orElse(false); } else if (group instanceof SearchGroup) { diff --git a/src/main/java/org/jabref/gui/groups/GroupTreeNodeViewModel.java b/src/main/java/org/jabref/gui/groups/GroupTreeNodeViewModel.java index 4383ec85ebf..38f9c532988 100644 --- a/src/main/java/org/jabref/gui/groups/GroupTreeNodeViewModel.java +++ b/src/main/java/org/jabref/gui/groups/GroupTreeNodeViewModel.java @@ -18,13 +18,7 @@ import org.jabref.model.groups.KeywordGroup; import org.jabref.model.groups.SearchGroup; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - public class GroupTreeNodeViewModel { - - private static final Logger LOGGER = LoggerFactory.getLogger(GroupTreeNodeViewModel.class); - private final GroupTreeNode node; public GroupTreeNodeViewModel(GroupTreeNode node) { @@ -33,10 +27,7 @@ public GroupTreeNodeViewModel(GroupTreeNode node) { @Override public String toString() { - final StringBuilder sb = new StringBuilder("GroupTreeNodeViewModel{"); - sb.append("node=").append(node); - sb.append('}'); - return sb.toString(); + return "GroupTreeNodeViewModel{" + "node=" + node + '}'; } public GroupTreeNode getNode() { @@ -59,15 +50,17 @@ public String getDescription() { AbstractGroup group = node.getGroup(); String shortDescription = ""; boolean showDynamic = true; - if (group instanceof ExplicitGroup explicitGroup) { - shortDescription = GroupDescriptions.getShortDescriptionExplicitGroup(explicitGroup); - } else if (group instanceof KeywordGroup keywordGroup) { - shortDescription = GroupDescriptions.getShortDescriptionKeywordGroup(keywordGroup, showDynamic); - } else if (group instanceof SearchGroup searchGroup) { - shortDescription = GroupDescriptions.getShortDescription(searchGroup, showDynamic); - } else { - shortDescription = GroupDescriptions.getShortDescriptionAllEntriesGroup(); - } + shortDescription = switch (group) { + case ExplicitGroup explicitGroup -> + GroupDescriptions.getShortDescriptionExplicitGroup(explicitGroup); + case KeywordGroup keywordGroup -> + GroupDescriptions.getShortDescriptionKeywordGroup(keywordGroup, showDynamic); + case SearchGroup searchGroup -> + GroupDescriptions.getShortDescription(searchGroup, showDynamic); + case null, + default -> + GroupDescriptions.getShortDescriptionAllEntriesGroup(); + }; return "" + shortDescription + ""; } @@ -112,12 +105,12 @@ public boolean canBeEdited() { } public boolean canMoveUp() { - return (getNode().getPreviousSibling() != null) + return (getNode().getPreviousSibling().isPresent()) && !(getNode().getGroup() instanceof AllEntriesGroup); } public boolean canMoveDown() { - return (getNode().getNextSibling() != null) + return (getNode().getNextSibling().isPresent()) && !(getNode().getGroup() instanceof AllEntriesGroup); } @@ -128,7 +121,7 @@ public boolean canMoveLeft() { } public boolean canMoveRight() { - return (getNode().getPreviousSibling() != null) + return (getNode().getPreviousSibling().isPresent()) && !(getNode().getGroup() instanceof AllEntriesGroup); } diff --git a/src/main/java/org/jabref/gui/groups/GroupTreeView.java b/src/main/java/org/jabref/gui/groups/GroupTreeView.java index ef0084fe4f7..0b50b7d6d20 100644 --- a/src/main/java/org/jabref/gui/groups/GroupTreeView.java +++ b/src/main/java/org/jabref/gui/groups/GroupTreeView.java @@ -74,30 +74,24 @@ public class GroupTreeView extends BorderPane { private static final Logger LOGGER = LoggerFactory.getLogger(GroupTreeView.class); - private static final PseudoClass PSEUDOCLASS_ANYSELECTED = PseudoClass.getPseudoClass("any-selected"); private static final PseudoClass PSEUDOCLASS_ALLSELECTED = PseudoClass.getPseudoClass("all-selected"); private static final PseudoClass PSEUDOCLASS_ROOTELEMENT = PseudoClass.getPseudoClass("root"); private static final PseudoClass PSEUDOCLASS_SUBELEMENT = PseudoClass.getPseudoClass("sub"); // > 1 deep - private static final double SCROLL_SPEED_UP = 3.0; + private final StateManager stateManager; + private final DialogService dialogService; + private final TaskExecutor taskExecutor; + private final PreferencesService preferencesService; private TreeTableView groupTree; private TreeTableColumn mainColumn; private TreeTableColumn numberColumn; private TreeTableColumn expansionNodeColumn; private CustomTextField searchField; - - private final StateManager stateManager; - private final DialogService dialogService; - private final TaskExecutor taskExecutor; - private final PreferencesService preferencesService; - private GroupTreeViewModel viewModel; private CustomLocalDragboard localDragboard; - private DragExpansionHandler dragExpansionHandler; - private Timer scrollTimer; private double scrollVelocity = 0; private double scrollableAreaHeight; diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index e7a5a425210..b72d367c911 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -31,6 +31,7 @@ import org.jabref.model.search.SearchQuery; import org.jabref.model.search.SearchResults; import org.jabref.model.search.event.IndexAddedOrUpdatedEvent; +import org.jabref.model.search.event.IndexRemovedEvent; import org.jabref.model.search.event.IndexStartedEvent; import org.jabref.model.search.matchers.MatcherSet; import org.jabref.model.search.matchers.MatcherSets; @@ -220,6 +221,9 @@ public void listen(IndexAddedOrUpdatedEvent indexAddedOrUpdatedEvent) { viewModel.searchScoreProperty().set(0); viewModel.hasFullTextResultsProperty().set(false); } + + bibDatabaseContext.getMetaData().getGroups().ifPresent(root -> updateSearchGroupsMatches(entry, root)); + updateEntrySearchMatch(viewModel, isMatched, isFloatingMode); updateEntryGroupMatch(viewModel, groupsMatcher, groupsPreferences.getGroupViewMode().contains(GroupViewMode.INVERT), !groupsPreferences.getGroupViewMode().contains(GroupViewMode.FILTER)); } @@ -230,17 +234,47 @@ public void listen(IndexAddedOrUpdatedEvent indexAddedOrUpdatedEvent) { @Subscribe public void listen(IndexStartedEvent indexStartedEvent) { - bibDatabaseContext.getMetaData().getGroups().ifPresent(this::setLuceneManagerForSearchGroups); + bibDatabaseContext.getMetaData().getGroups().ifPresent(this::setSearchGroupsMatches); updateSearchMatches(searchQueryProperty.get()); updateGroupMatches(selectedGroupsProperty); } - private void setLuceneManagerForSearchGroups(GroupTreeNode root) { + private void setSearchGroupsMatches(GroupTreeNode root) { + for (GroupTreeNode node : root.getChildren()) { + if (node.getGroup() instanceof SearchGroup searchGroup) { + SearchQuery query = searchGroup.getQuery(); + SearchResults searchResults = luceneManager.search(query); + searchGroup.setMatchedEntries(searchResults.getMatchedEntries()); + } + setSearchGroupsMatches(node); + } + } + + private void updateSearchGroupsMatches(BibEntry entry, GroupTreeNode root) { + for (GroupTreeNode node : root.getChildren()) { + if (node.getGroup() instanceof SearchGroup searchGroup) { + searchGroup.updateEntry(entry, luceneManager.isMatched(entry, searchGroup.getQuery())); + searchGroup.updateEntry(entry, node.getSearchMatcher().isMatch(entry)); + } + updateSearchGroupsMatches(entry, node); + } + } + + @Subscribe + public void listen(IndexRemovedEvent indexRemovedEvent) { + BackgroundTask.wrap(() -> { + for (BibEntry entry : indexRemovedEvent.removedEntries()) { + bibDatabaseContext.getMetaData().getGroups().ifPresent(root -> removeSearchGroupsMatches(entry, root)); + } + }).executeWith(taskExecutor); + } + + private void removeSearchGroupsMatches(BibEntry entry, GroupTreeNode root) { for (GroupTreeNode node : root.getChildren()) { if (node.getGroup() instanceof SearchGroup searchGroup) { - searchGroup.setLuceneManager(luceneManager); + searchGroup.updateEntry(entry, false); } - setLuceneManagerForSearchGroups(node); + removeSearchGroupsMatches(entry, node); } } } diff --git a/src/main/java/org/jabref/logic/search/LuceneManager.java b/src/main/java/org/jabref/logic/search/LuceneManager.java index 21ce36b23f2..31297183bb3 100644 --- a/src/main/java/org/jabref/logic/search/LuceneManager.java +++ b/src/main/java/org/jabref/logic/search/LuceneManager.java @@ -123,7 +123,7 @@ protected Object call() { bibFieldsIndexer.removeFromIndex(entries, this); return null; } - }.onFinished(() -> this.databaseContext.getDatabase().postEvent(new IndexRemovedEvent())) + }.onFinished(() -> this.databaseContext.getDatabase().postEvent(new IndexRemovedEvent(entries))) .showToUser(true).executeWith(taskExecutor); if (shouldIndexLinkedFiles.get()) { diff --git a/src/main/java/org/jabref/model/groups/SearchGroup.java b/src/main/java/org/jabref/model/groups/SearchGroup.java index e97d44916ea..34fdcebb0f3 100644 --- a/src/main/java/org/jabref/model/groups/SearchGroup.java +++ b/src/main/java/org/jabref/model/groups/SearchGroup.java @@ -1,9 +1,12 @@ package org.jabref.model.groups; +import java.util.Collection; import java.util.EnumSet; import java.util.Objects; -import org.jabref.logic.search.LuceneManager; +import javafx.collections.FXCollections; +import javafx.collections.ObservableMap; + import org.jabref.model.entry.BibEntry; import org.jabref.model.search.SearchFlags; import org.jabref.model.search.SearchQuery; @@ -18,8 +21,8 @@ public class SearchGroup extends AbstractGroup { private static final Logger LOGGER = LoggerFactory.getLogger(SearchGroup.class); + private final ObservableMap matchedEntries = FXCollections.observableHashMap(); private final SearchQuery query; - private LuceneManager luceneManager; public SearchGroup(String name, GroupHierarchyType context, String searchExpression, EnumSet searchFlags) { super(name, context); @@ -38,10 +41,6 @@ public EnumSet getSearchFlags() { return query.getSearchFlags(); } - public void setLuceneManager(LuceneManager luceneManager) { - this.luceneManager = luceneManager; - } - @Override public boolean equals(Object o) { if (this == o) { @@ -58,10 +57,24 @@ public boolean equals(Object o) { @Override public boolean contains(BibEntry entry) { - if (luceneManager == null) { - return false; + return matchedEntries.containsKey(System.identityHashCode(entry)); + } + + public void setMatchedEntries(Collection entries) { + matchedEntries.clear(); + entries.forEach(entry -> matchedEntries.put(System.identityHashCode(entry), entry)); + } + + public void updateEntry(BibEntry entry, boolean matched) { + if (matched) { + matchedEntries.put(System.identityHashCode(entry), entry); + } else { + matchedEntries.remove(System.identityHashCode(entry)); } - return luceneManager.isMatched(entry, query); + } + + public ObservableMap getMatchedEntries() { + return matchedEntries; } @Override diff --git a/src/main/java/org/jabref/model/search/SearchResults.java b/src/main/java/org/jabref/model/search/SearchResults.java index 283a863e9d7..4b6c9221dc5 100644 --- a/src/main/java/org/jabref/model/search/SearchResults.java +++ b/src/main/java/org/jabref/model/search/SearchResults.java @@ -5,6 +5,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import org.jabref.model.entry.BibEntry; @@ -46,8 +47,8 @@ public Map> getFileSearchResultsForEntry(BibEntry ent return results; } - public Map> getAllSearchResults() { - return searchResults; + public Set getMatchedEntries() { + return searchResults.keySet(); } public int getNumberOfResults() { diff --git a/src/main/java/org/jabref/model/search/event/IndexRemovedEvent.java b/src/main/java/org/jabref/model/search/event/IndexRemovedEvent.java index 72b5a404f44..7e69986c45b 100644 --- a/src/main/java/org/jabref/model/search/event/IndexRemovedEvent.java +++ b/src/main/java/org/jabref/model/search/event/IndexRemovedEvent.java @@ -1,4 +1,8 @@ package org.jabref.model.search.event; -public record IndexRemovedEvent() { +import java.util.List; + +import org.jabref.model.entry.BibEntry; + +public record IndexRemovedEvent(List removedEntries) { } diff --git a/src/main/java/org/jabref/model/search/event/IndexUpdatedEvent.java b/src/main/java/org/jabref/model/search/event/IndexUpdatedEvent.java deleted file mode 100644 index b0ec923c1c1..00000000000 --- a/src/main/java/org/jabref/model/search/event/IndexUpdatedEvent.java +++ /dev/null @@ -1,4 +0,0 @@ -package org.jabref.model.search.event; - -public record IndexUpdatedEvent() { -} From 26a49d4c7cfd71cc654324738e997b8f28488216 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Mon, 26 Aug 2024 01:43:39 +0300 Subject: [PATCH 148/256] Localization --- src/main/java/org/jabref/gui/search/GlobalSearchBar.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java index eb8c7c37d01..f0c7720e5cc 100644 --- a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java +++ b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java @@ -132,7 +132,7 @@ public GlobalSearchBar(LibraryTabContainer tabContainer, return Localization.lang("Search failed: illegal search expression"); } else if (invalid) { searchField.pseudoClassStateChanged(ILLEGAL_SEARCH, true); - return Localization.lang("Invalid regular expression"); + return Localization.lang("Invalid regular expression."); } else if (searchQuery.isEmpty()) { searchField.pseudoClassStateChanged(ILLEGAL_SEARCH, false); return ""; From d26929e44ab49c47dd0cf800f352740e70c9f282 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Mon, 26 Aug 2024 04:02:41 +0300 Subject: [PATCH 149/256] Remove bib fields highlighter --- .../logic/search/retrieval/LuceneSearcher.java | 2 +- .../java/org/jabref/model/search/SearchResult.java | 14 ++------------ 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java b/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java index 72c150a9da2..933acd1a359 100644 --- a/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java +++ b/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java @@ -127,7 +127,7 @@ private SearchResults getSearchResults(TopDocs topDocs, StoredFields storedField } } else { String entryId = getFieldContents(document, SearchFieldConstants.ENTRY_ID); - searchResults.addSearchResult(entriesMap.get(entryId), new SearchResult(score, highlighter)); + searchResults.addSearchResult(entriesMap.get(entryId), new SearchResult(score)); } } LOGGER.debug("Mapping search results took {} ms", System.currentTimeMillis() - startTime); diff --git a/src/main/java/org/jabref/model/search/SearchResult.java b/src/main/java/org/jabref/model/search/SearchResult.java index 72e36b6d029..c2a76c8fc63 100644 --- a/src/main/java/org/jabref/model/search/SearchResult.java +++ b/src/main/java/org/jabref/model/search/SearchResult.java @@ -4,8 +4,6 @@ import java.util.Arrays; import java.util.List; -import org.jabref.model.entry.BibEntry; - import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.search.highlight.Highlighter; import org.apache.lucene.search.highlight.InvalidTokenOffsetsException; @@ -22,7 +20,6 @@ public final class SearchResult { private final Highlighter highlighter; private List contentResultStringsHtml; private List annotationsResultStringsHtml; - private List bibEntryStringsHtml; private SearchResult(float luceneScore, boolean hasFulltextResults, @@ -40,8 +37,8 @@ private SearchResult(float luceneScore, this.highlighter = highlighter; } - public SearchResult(float luceneScore, Highlighter highlighter) { - this(luceneScore, false, "", "", "", -1, highlighter); + public SearchResult(float luceneScore) { + this(luceneScore, false, "", "", "", -1, null); } public SearchResult(float luceneScore, String path, String pageContent, String annotation, int pageNumber, Highlighter highlighter) { @@ -62,13 +59,6 @@ public List getAnnotationsResultStringsHtml() { return annotationsResultStringsHtml; } - public List getBibEntryStringsHtml(BibEntry bibEntry) { - if (bibEntryStringsHtml == null) { - bibEntryStringsHtml = getHighlighterFragments(highlighter, SearchFieldConstants.ENTRY_ID, bibEntry.getParsedSerialization()); - } - return bibEntryStringsHtml; - } - public float getLuceneScore() { return luceneScore; } From e4c9d915bd6221e09345920b2087a0d90f5c9dad Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Mon, 26 Aug 2024 05:36:29 +0300 Subject: [PATCH 150/256] Pass LuceneManager to search groups --- src/main/java/org/jabref/gui/LibraryTab.java | 1 + .../java/org/jabref/gui/StateManager.java | 10 ++++ .../jabref/gui/groups/GroupDialogView.java | 4 +- .../gui/groups/GroupDialogViewModel.java | 9 ++- .../jabref/gui/groups/GroupNodeViewModel.java | 59 ++++++++++++++++--- .../gui/maintable/MainTableDataModel.java | 47 +-------------- .../jabref/logic/search/LuceneManager.java | 2 + .../search/retrieval/LuceneSearcher.java | 2 +- .../model/database/BibDatabaseContext.java | 3 +- .../org/jabref/model/groups/SearchGroup.java | 42 ++++++------- .../event/IndexAddedOrUpdatedEvent.java | 2 +- .../model/search/event/IndexClosedEvent.java | 4 ++ .../model/search/event/IndexRemovedEvent.java | 2 +- 13 files changed, 101 insertions(+), 86 deletions(-) create mode 100644 src/main/java/org/jabref/model/search/event/IndexClosedEvent.java diff --git a/src/main/java/org/jabref/gui/LibraryTab.java b/src/main/java/org/jabref/gui/LibraryTab.java index d4722500c92..f8e81270cab 100644 --- a/src/main/java/org/jabref/gui/LibraryTab.java +++ b/src/main/java/org/jabref/gui/LibraryTab.java @@ -291,6 +291,7 @@ private void onDatabaseLoadingSucceed(ParserResult result) { public void setLuceneManager() { luceneManager = new LuceneManager(bibDatabaseContext, taskExecutor, preferencesService.getFilePreferences()); + stateManager.setLuceneManager(bibDatabaseContext, luceneManager); luceneManager.updateOnStart(); } diff --git a/src/main/java/org/jabref/gui/StateManager.java b/src/main/java/org/jabref/gui/StateManager.java index d2fa41388e8..10868ffe91b 100644 --- a/src/main/java/org/jabref/gui/StateManager.java +++ b/src/main/java/org/jabref/gui/StateManager.java @@ -25,6 +25,7 @@ import org.jabref.gui.util.CustomLocalDragboard; import org.jabref.gui.util.DialogWindowState; import org.jabref.gui.util.OptionalObjectProperty; +import org.jabref.logic.search.LuceneManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.groups.GroupTreeNode; @@ -57,6 +58,7 @@ public class StateManager { private final OptionalObjectProperty activeTab = OptionalObjectProperty.empty(); private final ObservableList selectedEntries = FXCollections.observableArrayList(); private final ObservableMap> selectedGroups = FXCollections.observableHashMap(); + private final ObservableMap luceneManagers = FXCollections.observableHashMap(); private final OptionalObjectProperty activeSearchQuery = OptionalObjectProperty.empty(); private final OptionalObjectProperty activeGlobalSearchQuery = OptionalObjectProperty.empty(); private final IntegerProperty searchResultSize = new SimpleIntegerProperty(0); @@ -121,6 +123,14 @@ public void clearSelectedGroups(BibDatabaseContext context) { selectedGroups.computeIfAbsent(context.getUid(), k -> FXCollections.observableArrayList()).clear(); } + public void setLuceneManager(BibDatabaseContext database, LuceneManager luceneManager) { + luceneManagers.put(database.getUid(), luceneManager); + } + + public Optional getLuceneManager(BibDatabaseContext database) { + return Optional.of(luceneManagers.get(database.getUid())); + } + public Optional getActiveDatabase() { return activeDatabase.get(); } diff --git a/src/main/java/org/jabref/gui/groups/GroupDialogView.java b/src/main/java/org/jabref/gui/groups/GroupDialogView.java index 03017fa06c6..1dbf4df21b4 100644 --- a/src/main/java/org/jabref/gui/groups/GroupDialogView.java +++ b/src/main/java/org/jabref/gui/groups/GroupDialogView.java @@ -30,6 +30,7 @@ import javafx.scene.paint.Color; import org.jabref.gui.DialogService; +import org.jabref.gui.StateManager; import org.jabref.gui.actions.ActionFactory; import org.jabref.gui.actions.StandardActions; import org.jabref.gui.help.HelpAction; @@ -113,6 +114,7 @@ public class GroupDialogView extends BaseDialog { @Inject private FileUpdateMonitor fileUpdateMonitor; @Inject private DialogService dialogService; @Inject private PreferencesService preferencesService; + @Inject private StateManager stateManager; public GroupDialogView(BibDatabaseContext currentDatabase, @Nullable GroupTreeNode parentNode, @@ -170,7 +172,7 @@ public GroupDialogView(BibDatabaseContext currentDatabase, @FXML public void initialize() { - viewModel = new GroupDialogViewModel(dialogService, currentDatabase, preferencesService, editedGroup, parentNode, fileUpdateMonitor); + viewModel = new GroupDialogViewModel(dialogService, currentDatabase, preferencesService, editedGroup, parentNode, fileUpdateMonitor, stateManager); setResultConverter(viewModel::resultConverter); diff --git a/src/main/java/org/jabref/gui/groups/GroupDialogViewModel.java b/src/main/java/org/jabref/gui/groups/GroupDialogViewModel.java index 71554fc4720..64e8a531a11 100644 --- a/src/main/java/org/jabref/gui/groups/GroupDialogViewModel.java +++ b/src/main/java/org/jabref/gui/groups/GroupDialogViewModel.java @@ -24,6 +24,7 @@ import javafx.scene.paint.Color; import org.jabref.gui.DialogService; +import org.jabref.gui.StateManager; import org.jabref.gui.icon.IconTheme; import org.jabref.gui.util.FileDialogConfiguration; import org.jabref.logic.auxparser.DefaultAuxParser; @@ -111,19 +112,22 @@ public class GroupDialogViewModel { private final AbstractGroup editedGroup; private final GroupTreeNode parentNode; private final FileUpdateMonitor fileUpdateMonitor; + private final StateManager stateManager; public GroupDialogViewModel(DialogService dialogService, BibDatabaseContext currentDatabase, PreferencesService preferencesService, @Nullable AbstractGroup editedGroup, @Nullable GroupTreeNode parentNode, - FileUpdateMonitor fileUpdateMonitor) { + FileUpdateMonitor fileUpdateMonitor, + StateManager stateManager) { this.dialogService = dialogService; this.preferencesService = preferencesService; this.currentDatabase = currentDatabase; this.editedGroup = editedGroup; this.parentNode = parentNode; this.fileUpdateMonitor = fileUpdateMonitor; + this.stateManager = stateManager; setupValidation(); setValues(); @@ -332,7 +336,8 @@ public AbstractGroup resultConverter(ButtonType button) { groupName, groupHierarchySelectedProperty.getValue(), searchGroupSearchTermProperty.getValue().trim(), - searchFlagsProperty.getValue()); + searchFlagsProperty.getValue(), + stateManager.getLuceneManager(currentDatabase).orElse(null)); } else if (typeAutoProperty.getValue()) { if (autoGroupKeywordsOptionProperty.getValue()) { // Set default value for delimiters: ',' for base and '>' for hierarchical diff --git a/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java b/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java index acac38400a8..716facdafea 100644 --- a/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java +++ b/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java @@ -45,9 +45,14 @@ import org.jabref.model.groups.RegexKeywordGroup; import org.jabref.model.groups.SearchGroup; import org.jabref.model.groups.TexGroup; +import org.jabref.model.search.event.IndexAddedOrUpdatedEvent; +import org.jabref.model.search.event.IndexClosedEvent; +import org.jabref.model.search.event.IndexRemovedEvent; +import org.jabref.model.search.event.IndexStartedEvent; import org.jabref.model.strings.StringUtil; import org.jabref.preferences.PreferencesService; +import com.google.common.eventbus.Subscribe; import com.tobiasdiez.easybind.EasyBind; import com.tobiasdiez.easybind.EasyObservableList; @@ -59,7 +64,7 @@ public class GroupNodeViewModel { private final BibDatabaseContext databaseContext; private final StateManager stateManager; private final GroupTreeNode groupNode; - private final ObservableMap matchedEntries; + private final ObservableMap matchedEntries = FXCollections.observableHashMap(); private final SimpleBooleanProperty hasChildren; private final SimpleBooleanProperty expandedProperty = new SimpleBooleanProperty(); private final BooleanBinding anySelectedEntriesMatched; @@ -95,12 +100,6 @@ public GroupNodeViewModel(BibDatabaseContext databaseContext, StateManager state databaseContext.getMetaData().groupsBinding().addListener(new WeakInvalidationListener(onInvalidatedGroup)); } - if (groupNode.getGroup() instanceof SearchGroup searchGroup) { - matchedEntries = searchGroup.getMatchedEntries(); - } else { - matchedEntries = FXCollections.observableHashMap(); - } - hasChildren = new SimpleBooleanProperty(); hasChildren.bind(Bindings.isNotEmpty(children)); EasyBind.subscribe(preferencesService.getGroupsPreferences().displayGroupCountProperty(), shouldDisplay -> updateMatchedEntries()); @@ -116,6 +115,8 @@ public GroupNodeViewModel(BibDatabaseContext databaseContext, StateManager state anySelectedEntriesMatched = selectedEntriesMatchStatus.anyMatch(matched -> matched); // 'all' returns 'true' for empty streams, so this has to be checked explicitly allSelectedEntriesMatched = selectedEntriesMatchStatus.isEmptyBinding().not().and(selectedEntriesMatchStatus.allMatch(matched -> matched)); + + this.databaseContext.getDatabase().registerListener(new LuceneIndexListener()); } public GroupNodeViewModel(BibDatabaseContext databaseContext, StateManager stateManager, TaskExecutor taskExecutor, AbstractGroup group, CustomLocalDragboard localDragboard, PreferencesService preferencesService) { @@ -239,7 +240,7 @@ public GroupTreeNode getGroupNode() { /** * Gets invoked if an entry in the current database changes. * - * @implNote Search groups are updated in {@link org.jabref.gui.maintable.MainTableDataModel.LuceneIndexListener#updateSearchGroupsMatches(org.jabref.model.entry.BibEntry, org.jabref.model.groups.GroupTreeNode)}. + * @implNote Search groups are updated in {@link LuceneIndexListener}. */ private void onDatabaseChanged(ListChangeListener.Change change) { if (groupNode.getGroup() instanceof SearchGroup) { @@ -518,4 +519,46 @@ public boolean isEditable() { throw new UnsupportedOperationException("isEditable method not yet implemented in group: " + group.getClass().getName()); } } + + class LuceneIndexListener { + @Subscribe + public void listen(IndexStartedEvent event) { + if (groupNode.getGroup() instanceof SearchGroup group) { + group.setLuceneManager(stateManager.getLuceneManager(databaseContext).get()); + refreshGroup(); + } + } + + @Subscribe + public void listen(IndexAddedOrUpdatedEvent event) { + if (groupNode.getGroup() instanceof SearchGroup) { + BackgroundTask.wrap(() -> { + for (BibEntry entry : event.entries()) { + if (groupNode.matches(entry)) { + matchedEntries.put(System.identityHashCode(entry), entry); + } else { + matchedEntries.remove(System.identityHashCode(entry)); + } + } + }).executeWith(taskExecutor); + } + } + + @Subscribe + public void listen(IndexRemovedEvent event) { + if (groupNode.getGroup() instanceof SearchGroup) { + for (BibEntry entry : event.entries()) { + matchedEntries.remove(System.identityHashCode(entry)); + } + } + } + + @Subscribe + public void listen(IndexClosedEvent event) { + if (groupNode.getGroup() instanceof SearchGroup group) { + group.setLuceneManager(null); + databaseContext.getDatabase().unregisterListener(this); + } + } + } } diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index b72d367c911..93ad5ef65e8 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -26,12 +26,10 @@ import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.groups.GroupTreeNode; -import org.jabref.model.groups.SearchGroup; import org.jabref.model.search.SearchFieldConstants; import org.jabref.model.search.SearchQuery; import org.jabref.model.search.SearchResults; import org.jabref.model.search.event.IndexAddedOrUpdatedEvent; -import org.jabref.model.search.event.IndexRemovedEvent; import org.jabref.model.search.event.IndexStartedEvent; import org.jabref.model.search.matchers.MatcherSet; import org.jabref.model.search.matchers.MatcherSets; @@ -201,7 +199,7 @@ public void resetFieldFormatter() { class LuceneIndexListener { @Subscribe public void listen(IndexAddedOrUpdatedEvent indexAddedOrUpdatedEvent) { - indexAddedOrUpdatedEvent.addedEntries().forEach(entry -> { + indexAddedOrUpdatedEvent.entries().forEach(entry -> { BackgroundTask.wrap(() -> { int index = allEntries.indexOf(entry); if (index >= 0) { @@ -222,8 +220,6 @@ public void listen(IndexAddedOrUpdatedEvent indexAddedOrUpdatedEvent) { viewModel.hasFullTextResultsProperty().set(false); } - bibDatabaseContext.getMetaData().getGroups().ifPresent(root -> updateSearchGroupsMatches(entry, root)); - updateEntrySearchMatch(viewModel, isMatched, isFloatingMode); updateEntryGroupMatch(viewModel, groupsMatcher, groupsPreferences.getGroupViewMode().contains(GroupViewMode.INVERT), !groupsPreferences.getGroupViewMode().contains(GroupViewMode.FILTER)); } @@ -234,48 +230,7 @@ public void listen(IndexAddedOrUpdatedEvent indexAddedOrUpdatedEvent) { @Subscribe public void listen(IndexStartedEvent indexStartedEvent) { - bibDatabaseContext.getMetaData().getGroups().ifPresent(this::setSearchGroupsMatches); updateSearchMatches(searchQueryProperty.get()); - updateGroupMatches(selectedGroupsProperty); - } - - private void setSearchGroupsMatches(GroupTreeNode root) { - for (GroupTreeNode node : root.getChildren()) { - if (node.getGroup() instanceof SearchGroup searchGroup) { - SearchQuery query = searchGroup.getQuery(); - SearchResults searchResults = luceneManager.search(query); - searchGroup.setMatchedEntries(searchResults.getMatchedEntries()); - } - setSearchGroupsMatches(node); - } - } - - private void updateSearchGroupsMatches(BibEntry entry, GroupTreeNode root) { - for (GroupTreeNode node : root.getChildren()) { - if (node.getGroup() instanceof SearchGroup searchGroup) { - searchGroup.updateEntry(entry, luceneManager.isMatched(entry, searchGroup.getQuery())); - searchGroup.updateEntry(entry, node.getSearchMatcher().isMatch(entry)); - } - updateSearchGroupsMatches(entry, node); - } - } - - @Subscribe - public void listen(IndexRemovedEvent indexRemovedEvent) { - BackgroundTask.wrap(() -> { - for (BibEntry entry : indexRemovedEvent.removedEntries()) { - bibDatabaseContext.getMetaData().getGroups().ifPresent(root -> removeSearchGroupsMatches(entry, root)); - } - }).executeWith(taskExecutor); - } - - private void removeSearchGroupsMatches(BibEntry entry, GroupTreeNode root) { - for (GroupTreeNode node : root.getChildren()) { - if (node.getGroup() instanceof SearchGroup searchGroup) { - searchGroup.updateEntry(entry, false); - } - removeSearchGroupsMatches(entry, node); - } } } } diff --git a/src/main/java/org/jabref/logic/search/LuceneManager.java b/src/main/java/org/jabref/logic/search/LuceneManager.java index 31297183bb3..55ee8af275d 100644 --- a/src/main/java/org/jabref/logic/search/LuceneManager.java +++ b/src/main/java/org/jabref/logic/search/LuceneManager.java @@ -19,6 +19,7 @@ import org.jabref.model.search.SearchQuery; import org.jabref.model.search.SearchResults; import org.jabref.model.search.event.IndexAddedOrUpdatedEvent; +import org.jabref.model.search.event.IndexClosedEvent; import org.jabref.model.search.event.IndexRemovedEvent; import org.jabref.model.search.event.IndexStartedEvent; import org.jabref.preferences.FilePreferences; @@ -204,6 +205,7 @@ public void close() { bibFieldsIndexer.close(); shouldIndexLinkedFiles.removeListener(preferencesListener); linkedFilesIndexer.close(); + databaseContext.getDatabase().postEvent(new IndexClosedEvent()); } public AutoCloseable blockLinkedFileIndexer() { diff --git a/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java b/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java index 933acd1a359..84e74b97b2d 100644 --- a/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java +++ b/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java @@ -139,7 +139,7 @@ private static String getFieldContents(Document document, SearchFieldConstants f } private static IndexSearcher getIndexedSearcher(SearcherManager searcherManager) throws IOException { - searcherManager.maybeRefresh(); + searcherManager.maybeRefreshBlocking(); return searcherManager.acquire(); } diff --git a/src/main/java/org/jabref/model/database/BibDatabaseContext.java b/src/main/java/org/jabref/model/database/BibDatabaseContext.java index 2f072bfb53a..611aa4855fb 100644 --- a/src/main/java/org/jabref/model/database/BibDatabaseContext.java +++ b/src/main/java/org/jabref/model/database/BibDatabaseContext.java @@ -10,7 +10,6 @@ import java.util.UUID; import org.jabref.architecture.AllowedToUseLogic; -import org.jabref.gui.LibraryTab; import org.jabref.gui.desktop.JabRefDesktop; import org.jabref.logic.crawler.Crawler; import org.jabref.logic.crawler.StudyRepository; @@ -40,7 +39,7 @@ @AllowedToUseLogic("because it needs access to shared database features") public class BibDatabaseContext { - private static final Logger LOGGER = LoggerFactory.getLogger(LibraryTab.class); + private static final Logger LOGGER = LoggerFactory.getLogger(BibDatabaseContext.class); private final BibDatabase database; private MetaData metaData; diff --git a/src/main/java/org/jabref/model/groups/SearchGroup.java b/src/main/java/org/jabref/model/groups/SearchGroup.java index 34fdcebb0f3..8d1cc8ee712 100644 --- a/src/main/java/org/jabref/model/groups/SearchGroup.java +++ b/src/main/java/org/jabref/model/groups/SearchGroup.java @@ -1,12 +1,10 @@ package org.jabref.model.groups; -import java.util.Collection; import java.util.EnumSet; import java.util.Objects; -import javafx.collections.FXCollections; -import javafx.collections.ObservableMap; - +import org.jabref.architecture.AllowedToUseLogic; +import org.jabref.logic.search.LuceneManager; import org.jabref.model.entry.BibEntry; import org.jabref.model.search.SearchFlags; import org.jabref.model.search.SearchQuery; @@ -18,15 +16,21 @@ * This group matches entries by a complex search pattern, which might include conditions about the values of * multiple fields. */ +@AllowedToUseLogic("because it needs access to lucene manager") public class SearchGroup extends AbstractGroup { private static final Logger LOGGER = LoggerFactory.getLogger(SearchGroup.class); - private final ObservableMap matchedEntries = FXCollections.observableHashMap(); private final SearchQuery query; + private LuceneManager luceneManager; - public SearchGroup(String name, GroupHierarchyType context, String searchExpression, EnumSet searchFlags) { + public SearchGroup(String name, GroupHierarchyType context, String searchExpression, EnumSet searchFlags, LuceneManager luceneManager) { super(name, context); this.query = new SearchQuery(searchExpression, searchFlags); + this.luceneManager = luceneManager; + } + + public SearchGroup(String name, GroupHierarchyType context, String searchExpression, EnumSet searchFlags) { + this(name, context, searchExpression, searchFlags, null); } public String getSearchExpression() { @@ -41,6 +45,10 @@ public EnumSet getSearchFlags() { return query.getSearchFlags(); } + public void setLuceneManager(LuceneManager luceneManager) { + this.luceneManager = luceneManager; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -57,30 +65,16 @@ public boolean equals(Object o) { @Override public boolean contains(BibEntry entry) { - return matchedEntries.containsKey(System.identityHashCode(entry)); - } - - public void setMatchedEntries(Collection entries) { - matchedEntries.clear(); - entries.forEach(entry -> matchedEntries.put(System.identityHashCode(entry), entry)); - } - - public void updateEntry(BibEntry entry, boolean matched) { - if (matched) { - matchedEntries.put(System.identityHashCode(entry), entry); - } else { - matchedEntries.remove(System.identityHashCode(entry)); + if (luceneManager == null) { + return false; } - } - - public ObservableMap getMatchedEntries() { - return matchedEntries; + return luceneManager.isMatched(entry, query); } @Override public AbstractGroup deepCopy() { try { - return new SearchGroup(getName(), getHierarchicalContext(), getSearchExpression(), getSearchFlags()); + return new SearchGroup(getName(), getHierarchicalContext(), getSearchExpression(), getSearchFlags(), luceneManager); } catch (Throwable t) { // this should never happen, because the constructor obviously // succeeded in creating _this_ instance! diff --git a/src/main/java/org/jabref/model/search/event/IndexAddedOrUpdatedEvent.java b/src/main/java/org/jabref/model/search/event/IndexAddedOrUpdatedEvent.java index 280d9baa0e0..ed6f432ce25 100644 --- a/src/main/java/org/jabref/model/search/event/IndexAddedOrUpdatedEvent.java +++ b/src/main/java/org/jabref/model/search/event/IndexAddedOrUpdatedEvent.java @@ -4,5 +4,5 @@ import org.jabref.model.entry.BibEntry; -public record IndexAddedOrUpdatedEvent(List addedEntries) { +public record IndexAddedOrUpdatedEvent(List entries) { } diff --git a/src/main/java/org/jabref/model/search/event/IndexClosedEvent.java b/src/main/java/org/jabref/model/search/event/IndexClosedEvent.java new file mode 100644 index 00000000000..a79475c80dd --- /dev/null +++ b/src/main/java/org/jabref/model/search/event/IndexClosedEvent.java @@ -0,0 +1,4 @@ +package org.jabref.model.search.event; + +public record IndexClosedEvent() { +} diff --git a/src/main/java/org/jabref/model/search/event/IndexRemovedEvent.java b/src/main/java/org/jabref/model/search/event/IndexRemovedEvent.java index 7e69986c45b..a89cad1e2bd 100644 --- a/src/main/java/org/jabref/model/search/event/IndexRemovedEvent.java +++ b/src/main/java/org/jabref/model/search/event/IndexRemovedEvent.java @@ -4,5 +4,5 @@ import org.jabref.model.entry.BibEntry; -public record IndexRemovedEvent(List removedEntries) { +public record IndexRemovedEvent(List entries) { } From 3400c922456210f359d98ebce5a5519e67e61474 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Mon, 26 Aug 2024 07:00:32 +0300 Subject: [PATCH 151/256] Fix performance issues by caching matched entries --- .../java/org/jabref/gui/StateManager.java | 2 +- .../jabref/gui/groups/GroupNodeViewModel.java | 23 +++++++++----- .../org/jabref/model/groups/SearchGroup.java | 31 ++++++++++++++++--- 3 files changed, 44 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/jabref/gui/StateManager.java b/src/main/java/org/jabref/gui/StateManager.java index 10868ffe91b..0883bfb5d15 100644 --- a/src/main/java/org/jabref/gui/StateManager.java +++ b/src/main/java/org/jabref/gui/StateManager.java @@ -128,7 +128,7 @@ public void setLuceneManager(BibDatabaseContext database, LuceneManager luceneMa } public Optional getLuceneManager(BibDatabaseContext database) { - return Optional.of(luceneManagers.get(database.getUid())); + return Optional.ofNullable(luceneManagers.get(database.getUid())); } public Optional getActiveDatabase() { diff --git a/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java b/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java index 716facdafea..b3804e52985 100644 --- a/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java +++ b/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java @@ -98,6 +98,10 @@ public GroupNodeViewModel(BibDatabaseContext databaseContext, StateManager state } if (groupNode.getGroup() instanceof TexGroup) { databaseContext.getMetaData().groupsBinding().addListener(new WeakInvalidationListener(onInvalidatedGroup)); + } else if (groupNode.getGroup() instanceof SearchGroup searchGroup) { + stateManager.getLuceneManager(databaseContext).ifPresent(searchGroup::setLuceneManager); + searchGroup.updateMatches(); + refreshGroup(); } hasChildren = new SimpleBooleanProperty(); @@ -523,17 +527,19 @@ public boolean isEditable() { class LuceneIndexListener { @Subscribe public void listen(IndexStartedEvent event) { - if (groupNode.getGroup() instanceof SearchGroup group) { - group.setLuceneManager(stateManager.getLuceneManager(databaseContext).get()); + if (groupNode.getGroup() instanceof SearchGroup searchGroup) { + stateManager.getLuceneManager(databaseContext).ifPresent(searchGroup::setLuceneManager); + searchGroup.updateMatches(); refreshGroup(); } } @Subscribe public void listen(IndexAddedOrUpdatedEvent event) { - if (groupNode.getGroup() instanceof SearchGroup) { + if (groupNode.getGroup() instanceof SearchGroup searchGroup) { BackgroundTask.wrap(() -> { for (BibEntry entry : event.entries()) { + searchGroup.updateMatches(entry); if (groupNode.matches(entry)) { matchedEntries.put(System.identityHashCode(entry), entry); } else { @@ -546,10 +552,13 @@ public void listen(IndexAddedOrUpdatedEvent event) { @Subscribe public void listen(IndexRemovedEvent event) { - if (groupNode.getGroup() instanceof SearchGroup) { - for (BibEntry entry : event.entries()) { - matchedEntries.remove(System.identityHashCode(entry)); - } + if (groupNode.getGroup() instanceof SearchGroup searchGroup) { + BackgroundTask.wrap(() -> { + for (BibEntry entry : event.entries()) { + searchGroup.updateMatches(entry); + matchedEntries.remove(System.identityHashCode(entry)); + } + }).executeWith(taskExecutor); } } diff --git a/src/main/java/org/jabref/model/groups/SearchGroup.java b/src/main/java/org/jabref/model/groups/SearchGroup.java index 8d1cc8ee712..0b6f6cadf75 100644 --- a/src/main/java/org/jabref/model/groups/SearchGroup.java +++ b/src/main/java/org/jabref/model/groups/SearchGroup.java @@ -3,6 +3,9 @@ import java.util.EnumSet; import java.util.Objects; +import javafx.collections.FXCollections; +import javafx.collections.ObservableMap; + import org.jabref.architecture.AllowedToUseLogic; import org.jabref.logic.search.LuceneManager; import org.jabref.model.entry.BibEntry; @@ -20,13 +23,16 @@ public class SearchGroup extends AbstractGroup { private static final Logger LOGGER = LoggerFactory.getLogger(SearchGroup.class); + private final SearchQuery query; + private final ObservableMap matchedEntries = FXCollections.observableHashMap(); private LuceneManager luceneManager; public SearchGroup(String name, GroupHierarchyType context, String searchExpression, EnumSet searchFlags, LuceneManager luceneManager) { super(name, context); this.query = new SearchQuery(searchExpression, searchFlags); this.luceneManager = luceneManager; + updateMatches(); } public SearchGroup(String name, GroupHierarchyType context, String searchExpression, EnumSet searchFlags) { @@ -49,6 +55,26 @@ public void setLuceneManager(LuceneManager luceneManager) { this.luceneManager = luceneManager; } + public void updateMatches() { + if (luceneManager == null) { + return; + } + matchedEntries.clear(); + // TODO: Search should be done in a background thread + luceneManager.search(query).getMatchedEntries().forEach(entry -> matchedEntries.put(System.identityHashCode(entry), entry)); + } + + public void updateMatches(BibEntry entry) { + if (luceneManager == null) { + return; + } + if (luceneManager.isMatched(entry, query)) { + matchedEntries.put(System.identityHashCode(entry), entry); + } else { + matchedEntries.remove(System.identityHashCode(entry)); + } + } + @Override public boolean equals(Object o) { if (this == o) { @@ -65,10 +91,7 @@ public boolean equals(Object o) { @Override public boolean contains(BibEntry entry) { - if (luceneManager == null) { - return false; - } - return luceneManager.isMatched(entry, query); + return matchedEntries.containsKey(System.identityHashCode(entry)); } @Override From e4fe911ca0b42c86deac4aebd28469e16c82d176 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Mon, 26 Aug 2024 07:02:56 +0300 Subject: [PATCH 152/256] Update GroupDialogViewModelTest.java --- .../org/jabref/gui/groups/GroupDialogViewModelTest.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/test/java/org/jabref/gui/groups/GroupDialogViewModelTest.java b/src/test/java/org/jabref/gui/groups/GroupDialogViewModelTest.java index 1c47527d0f8..71c44d39e48 100644 --- a/src/test/java/org/jabref/gui/groups/GroupDialogViewModelTest.java +++ b/src/test/java/org/jabref/gui/groups/GroupDialogViewModelTest.java @@ -5,6 +5,7 @@ import java.util.Optional; import org.jabref.gui.DialogService; +import org.jabref.gui.StateManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.groups.AbstractGroup; import org.jabref.model.groups.GroupHierarchyType; @@ -31,6 +32,7 @@ class GroupDialogViewModelTest { private Path temporaryFolder; private BibDatabaseContext bibDatabaseContext; private final MetaData metaData = mock(MetaData.class); + private final StateManager stateManager = mock(StateManager.class); private final GroupsPreferences groupsPreferences = mock(GroupsPreferences.class); private final DialogService dialogService = mock(DialogService.class); private final AbstractGroup group = mock(AbstractGroup.class); @@ -51,7 +53,7 @@ void setUp(@TempDir Path temporaryFolder) { bibDatabaseContext.setMetaData(metaData); - viewModel = new GroupDialogViewModel(dialogService, bibDatabaseContext, preferencesService, group, null, new DummyFileUpdateMonitor()); + viewModel = new GroupDialogViewModel(dialogService, bibDatabaseContext, preferencesService, group, null, new DummyFileUpdateMonitor(), stateManager); } @Test @@ -88,7 +90,7 @@ void validateExistingRelativePath() throws Exception { void hierarchicalContextFromGroup() throws Exception { GroupHierarchyType groupHierarchyType = GroupHierarchyType.INCLUDING; when(group.getHierarchicalContext()).thenReturn(groupHierarchyType); - viewModel = new GroupDialogViewModel(dialogService, bibDatabaseContext, preferencesService, group, null, new DummyFileUpdateMonitor()); + viewModel = new GroupDialogViewModel(dialogService, bibDatabaseContext, preferencesService, group, null, new DummyFileUpdateMonitor(), stateManager); assertEquals(groupHierarchyType, viewModel.groupHierarchySelectedProperty().getValue()); } @@ -97,7 +99,7 @@ void hierarchicalContextFromGroup() throws Exception { void defaultHierarchicalContext() throws Exception { GroupHierarchyType defaultHierarchicalContext = GroupHierarchyType.REFINING; when(preferencesService.getGroupsPreferences().getDefaultHierarchicalContext()).thenReturn(defaultHierarchicalContext); - viewModel = new GroupDialogViewModel(dialogService, bibDatabaseContext, preferencesService, null, null, new DummyFileUpdateMonitor()); + viewModel = new GroupDialogViewModel(dialogService, bibDatabaseContext, preferencesService, null, null, new DummyFileUpdateMonitor(), stateManager); assertEquals(defaultHierarchicalContext, viewModel.groupHierarchySelectedProperty().getValue()); } From 707063a7d53c14d531ae3592634f34b7e2f06379 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Mon, 26 Aug 2024 07:35:47 +0300 Subject: [PATCH 153/256] Update main table matches --- .../jabref/gui/groups/GroupNodeViewModel.java | 4 +++- .../gui/maintable/MainTableDataModel.java | 17 ++++++++++++++++- .../groups/event/SearchGroupUpdatedEvent.java | 6 ++++++ 3 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 src/main/java/org/jabref/model/groups/event/SearchGroupUpdatedEvent.java diff --git a/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java b/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java index b3804e52985..fe4adbe4813 100644 --- a/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java +++ b/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java @@ -45,6 +45,7 @@ import org.jabref.model.groups.RegexKeywordGroup; import org.jabref.model.groups.SearchGroup; import org.jabref.model.groups.TexGroup; +import org.jabref.model.groups.event.SearchGroupUpdatedEvent; import org.jabref.model.search.event.IndexAddedOrUpdatedEvent; import org.jabref.model.search.event.IndexClosedEvent; import org.jabref.model.search.event.IndexRemovedEvent; @@ -531,6 +532,7 @@ public void listen(IndexStartedEvent event) { stateManager.getLuceneManager(databaseContext).ifPresent(searchGroup::setLuceneManager); searchGroup.updateMatches(); refreshGroup(); + databaseContext.getDatabase().postEvent(new SearchGroupUpdatedEvent(groupNode)); } } @@ -546,7 +548,7 @@ public void listen(IndexAddedOrUpdatedEvent event) { matchedEntries.remove(System.identityHashCode(entry)); } } - }).executeWith(taskExecutor); + }).onFinished(() -> databaseContext.getDatabase().postEvent(new SearchGroupUpdatedEvent(groupNode))).executeWith(taskExecutor); } } diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index 93ad5ef65e8..95c0fccf3fd 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -26,6 +26,7 @@ import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.groups.GroupTreeNode; +import org.jabref.model.groups.event.SearchGroupUpdatedEvent; import org.jabref.model.search.SearchFieldConstants; import org.jabref.model.search.SearchQuery; import org.jabref.model.search.SearchResults; @@ -59,6 +60,7 @@ public class MainTableDataModel { private final Subscription selectedGroupsSubscription; private final Subscription groupViewModeSubscription; private final LuceneIndexListener indexUpdatedListener; + private final SearchGroupsListener searchGroupsListener; private final OptionalObjectProperty searchQueryProperty; private final ListProperty selectedGroupsProperty; private Optional groupsMatcher; @@ -82,8 +84,10 @@ public MainTableDataModel(BibDatabaseContext context, this.selectedGroupsProperty = selectedGroupsProperty; this.searchQueryProperty = searchQueryProperty; this.indexUpdatedListener = new LuceneIndexListener(); - this.bibDatabaseContext.getDatabase().registerListener(indexUpdatedListener); + this.searchGroupsListener = new SearchGroupsListener(); + this.bibDatabaseContext.getDatabase().registerListener(indexUpdatedListener); + this.bibDatabaseContext.getDatabase().registerListener(searchGroupsListener); resetFieldFormatter(); allEntries = BindingsHelper.forUI(context.getDatabase().getEntries()); @@ -185,7 +189,9 @@ public void unbind() { searchDisplayModeSubscription.unsubscribe(); selectedGroupsSubscription.unsubscribe(); groupViewModeSubscription.unsubscribe(); + bibDatabaseContext.getDatabase().unregisterListener(indexUpdatedListener); + bibDatabaseContext.getDatabase().unregisterListener(searchGroupsListener); } public SortedList getEntriesFilteredAndSorted() { @@ -233,4 +239,13 @@ public void listen(IndexStartedEvent indexStartedEvent) { updateSearchMatches(searchQueryProperty.get()); } } + + class SearchGroupsListener { + @Subscribe + public void listen(SearchGroupUpdatedEvent event) { + if (selectedGroupsProperty.get().contains(event.group())) { + updateGroupMatches(selectedGroupsProperty.get()); + } + } + } } diff --git a/src/main/java/org/jabref/model/groups/event/SearchGroupUpdatedEvent.java b/src/main/java/org/jabref/model/groups/event/SearchGroupUpdatedEvent.java new file mode 100644 index 00000000000..56df3d4c609 --- /dev/null +++ b/src/main/java/org/jabref/model/groups/event/SearchGroupUpdatedEvent.java @@ -0,0 +1,6 @@ +package org.jabref.model.groups.event; + +import org.jabref.model.groups.GroupTreeNode; + +public record SearchGroupUpdatedEvent(GroupTreeNode group) { +} From 63cba32334f4ee4a98c27ac1d3f0b2c7b218e7f1 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Mon, 26 Aug 2024 10:28:47 +0300 Subject: [PATCH 154/256] Fix groups icon --- src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java b/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java index fe4adbe4813..c2eefb61009 100644 --- a/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java +++ b/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java @@ -103,6 +103,7 @@ public GroupNodeViewModel(BibDatabaseContext databaseContext, StateManager state stateManager.getLuceneManager(databaseContext).ifPresent(searchGroup::setLuceneManager); searchGroup.updateMatches(); refreshGroup(); + databaseContext.getMetaData().groupsBinding().invalidate(); } hasChildren = new SimpleBooleanProperty(); @@ -533,6 +534,7 @@ public void listen(IndexStartedEvent event) { searchGroup.updateMatches(); refreshGroup(); databaseContext.getDatabase().postEvent(new SearchGroupUpdatedEvent(groupNode)); + databaseContext.getMetaData().groupsBinding().invalidate(); } } @@ -548,6 +550,7 @@ public void listen(IndexAddedOrUpdatedEvent event) { matchedEntries.remove(System.identityHashCode(entry)); } } + databaseContext.getMetaData().groupsBinding().invalidate(); }).onFinished(() -> databaseContext.getDatabase().postEvent(new SearchGroupUpdatedEvent(groupNode))).executeWith(taskExecutor); } } @@ -560,6 +563,7 @@ public void listen(IndexRemovedEvent event) { searchGroup.updateMatches(entry); matchedEntries.remove(System.identityHashCode(entry)); } + databaseContext.getMetaData().groupsBinding().invalidate(); }).executeWith(taskExecutor); } } From fd531a6fdca7450e332d6f3899a0425ce6ec0658 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Mon, 26 Aug 2024 10:47:33 +0300 Subject: [PATCH 155/256] Restore Search.g4 and GrammarBasedSearchRule --- build.gradle | 13 + src/main/antlr4/org/jabref/search/Search.g4 | 52 ++++ .../search/rules/GrammarBasedSearchRule.java | 259 ++++++++++++++++++ .../jabref/model/search/rules/SearchRule.java | 10 + .../model/search/rules/SearchRules.java | 44 +++ 5 files changed, 378 insertions(+) create mode 100644 src/main/antlr4/org/jabref/search/Search.g4 create mode 100644 src/main/java/org/jabref/model/search/rules/GrammarBasedSearchRule.java create mode 100644 src/main/java/org/jabref/model/search/rules/SearchRule.java create mode 100644 src/main/java/org/jabref/model/search/rules/SearchRules.java diff --git a/build.gradle b/build.gradle index 7cbd403b8f8..8c71cadb6f6 100644 --- a/build.gradle +++ b/build.gradle @@ -415,6 +415,7 @@ processResources { tasks.register('generateSource') { dependsOn("generateBstGrammarSource", + "generateSearchGrammarSource", "generateCitaviSource") group = 'JabRef' description 'Generates all necessary (Java) source files.' @@ -432,6 +433,18 @@ tasks.register("generateBstGrammarSource", JavaExec) { args = ["-o", "src-gen/main/java/org/jabref/logic/bst/", "-visitor", "-no-listener", "-package", "org.jabref.logic.bst", "$projectDir/src/main/antlr4/org/jabref/bst/Bst.g4"] } +tasks.register("generateSearchGrammarSource", JavaExec) { + group = 'JabRef' + description = "Generates java files for Search.g antlr4." + classpath = configurations.antlr4 + mainClass = "org.antlr.v4.Tool" + javaLauncher.set(javaToolchains.launcherFor(java.toolchain)) + + inputs.dir("src/main/antlr4/org/jabref/search/") + outputs.dir("src-gen/main/java/org/jabref/search/") + args = ["-o","src-gen/main/java/org/jabref/search" , "-visitor", "-no-listener", "-package", "org.jabref.search", "$projectDir/src/main/antlr4/org/jabref/search/Search.g4"] +} + tasks.register("generateJournalListMV", JavaExec) { group = "JabRef" description = "Converts the comma-separated journal abbreviation file to a H2 MVStore" diff --git a/src/main/antlr4/org/jabref/search/Search.g4 b/src/main/antlr4/org/jabref/search/Search.g4 new file mode 100644 index 00000000000..e2ee78b6c9d --- /dev/null +++ b/src/main/antlr4/org/jabref/search/Search.g4 @@ -0,0 +1,52 @@ +/** + * This is the antlr v4 grammar for defining search expressions. + * + * These search expressions are used for searching the bibtex library. They are heavily used for search groups. + */ +grammar Search; + +WS: [ \t] -> skip; // whitespace is ignored/skipped + +LPAREN:'('; +RPAREN:')'; + +EQUAL:'='; // semantically the same as CONTAINS +EEQUAL:'=='; // semantically the same as MATCHES +NEQUAL:'!='; + +AND:[aA][nN][dD]; // 'and' case insensitive +OR:[oO][rR]; // 'or' case insensitive +CONTAINS:[cC][oO][nN][tT][aA][iI][nN][sS]; // 'contains' case insensitive +MATCHES:[mM][aA][tT][cC][hH][eE][sS]; // 'matches' case insensitive +NOT:[nN][oO][tT]; // 'not' case insensitive + +STRING:QUOTE (~'"')* QUOTE; +QUOTE:'"'; + +FIELDTYPE:LETTER+; +// fragments are not accessible from the code, they are only for describing the grammar better +fragment LETTER : ~[ \t"()=!]; + + +start: + expression EOF; + +// labels are used to refer to parts of the rules in the generated code later on +// label=actualThingy +expression: + LPAREN expression RPAREN #parenExpression // example: (author=miller) + | NOT expression #unaryExpression // example: not author = miller + | left=expression operator=AND right=expression #binaryExpression // example: author = miller and title = test + | left=expression operator=OR right=expression #binaryExpression // example: author = miller or title = test + | comparison #atomExpression + ; + +comparison: + left=name operator=(CONTAINS | MATCHES | EQUAL | EEQUAL | NEQUAL) right=name // example: author != miller + | right=name // example: miller (search all fields) + ; + +name: + STRING // example: "miller" + | FIELDTYPE // example: author + ; diff --git a/src/main/java/org/jabref/model/search/rules/GrammarBasedSearchRule.java b/src/main/java/org/jabref/model/search/rules/GrammarBasedSearchRule.java new file mode 100644 index 00000000000..e062ff48532 --- /dev/null +++ b/src/main/java/org/jabref/model/search/rules/GrammarBasedSearchRule.java @@ -0,0 +1,259 @@ +package org.jabref.model.search.rules; + +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.Predicate; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import org.jabref.architecture.AllowedToUseLogic; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.Keyword; +import org.jabref.model.entry.field.Field; +import org.jabref.model.entry.field.InternalField; +import org.jabref.model.search.SearchFlags; +import org.jabref.model.search.SearchResult; +import org.jabref.model.strings.StringUtil; +import org.jabref.search.SearchBaseVisitor; +import org.jabref.search.SearchLexer; +import org.jabref.search.SearchParser; + +import org.antlr.v4.runtime.ANTLRInputStream; +import org.antlr.v4.runtime.BailErrorStrategy; +import org.antlr.v4.runtime.BaseErrorListener; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.RecognitionException; +import org.antlr.v4.runtime.Recognizer; +import org.antlr.v4.runtime.misc.ParseCancellationException; +import org.antlr.v4.runtime.tree.ParseTree; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The search query must be specified in an expression that is acceptable by the Search.g4 grammar. + *

+ * This class implements the "Advanced Search Mode" described in the help + */ +@AllowedToUseLogic("Because access to the lucene index is needed") +public class GrammarBasedSearchRule implements SearchRule { + + private static final Logger LOGGER = LoggerFactory.getLogger(GrammarBasedSearchRule.class); + + private final EnumSet searchFlags; + + private ParseTree tree; + private String query; + private List searchResults = new ArrayList<>(); + + public static class ThrowingErrorListener extends BaseErrorListener { + + public static final ThrowingErrorListener INSTANCE = new ThrowingErrorListener(); + + @Override + public void syntaxError(Recognizer recognizer, Object offendingSymbol, + int line, int charPositionInLine, String msg, RecognitionException e) + throws ParseCancellationException { + throw new ParseCancellationException("line " + line + ":" + charPositionInLine + " " + msg); + } + } + + public GrammarBasedSearchRule(EnumSet searchFlags) throws RecognitionException { + this.searchFlags = searchFlags; + } + + public static boolean isValid(EnumSet searchFlags, String query) { + return new GrammarBasedSearchRule(searchFlags).validateSearchStrings(query); + } + + public ParseTree getTree() { + return this.tree; + } + + public String getQuery() { + return this.query; + } + + private void init(String query) throws ParseCancellationException { + if (Objects.equals(this.query, query)) { + return; + } + + SearchLexer lexer = new SearchLexer(new ANTLRInputStream(query)); + lexer.removeErrorListeners(); // no infos on file system + lexer.addErrorListener(ThrowingErrorListener.INSTANCE); + SearchParser parser = new SearchParser(new CommonTokenStream(lexer)); + parser.removeErrorListeners(); // no infos on file system + parser.addErrorListener(ThrowingErrorListener.INSTANCE); + parser.setErrorHandler(new BailErrorStrategy()); // ParseCancellationException on parse errors + tree = parser.start(); + this.query = query; + } + + @Override + public boolean applyRule(String query, BibEntry bibEntry) { + try { + return new BibtexSearchVisitor(searchFlags, bibEntry).visit(tree); + } catch (Exception e) { + LOGGER.info("Search failed", e); + return false; + } + } + + @Override + public boolean validateSearchStrings(String query) { + try { + init(query); + return true; + } catch (ParseCancellationException e) { + LOGGER.debug("Search query invalid", e); + return false; + } + } + + public EnumSet getSearchFlags() { + return searchFlags; + } + + public enum ComparisonOperator { + EXACT, CONTAINS, DOES_NOT_CONTAIN; + + public static ComparisonOperator build(String value) { + if ("CONTAINS".equalsIgnoreCase(value) || "=".equals(value)) { + return CONTAINS; + } else if ("MATCHES".equalsIgnoreCase(value) || "==".equals(value)) { + return EXACT; + } else { + return DOES_NOT_CONTAIN; + } + } + } + + public static class Comparator { + + private final ComparisonOperator operator; + private final Pattern fieldPattern; + private final Pattern valuePattern; + + public Comparator(String field, String value, ComparisonOperator operator, EnumSet searchFlags) { + this.operator = operator; + + int option = searchFlags.contains(SearchFlags.CASE_SENSITIVE) ? 0 : Pattern.CASE_INSENSITIVE; + this.fieldPattern = Pattern.compile(searchFlags.contains(SearchFlags.REGULAR_EXPRESSION) ? StringUtil.stripAccents(field) : "\\Q" + StringUtil.stripAccents(field) + "\\E", option); + this.valuePattern = Pattern.compile(searchFlags.contains(SearchFlags.REGULAR_EXPRESSION) ? StringUtil.stripAccents(value) : "\\Q" + StringUtil.stripAccents(value) + "\\E", option); + } + + public boolean compare(BibEntry entry) { + // special case for searching for entrytype=phdthesis + if (fieldPattern.matcher(InternalField.TYPE_HEADER.getName()).matches()) { + return matchFieldValue(entry.getType().getName()); + } + + // special case for searching a single keyword + if (fieldPattern.matcher("anykeyword").matches()) { + return entry.getKeywords(',').stream().map(Keyword::toString).anyMatch(this::matchFieldValue); + } + + // specification of fieldsKeys to search is done in the search expression itself + Set fieldsKeys = entry.getFields(); + + // special case for searching allfields=cat and title=dog + if (!fieldPattern.matcher("anyfield").matches()) { + // Filter out the requested fields + fieldsKeys = fieldsKeys.stream().filter(matchFieldKey()).collect(Collectors.toSet()); + } + + for (Field field : fieldsKeys) { + Optional fieldValue = entry.getFieldLatexFree(field); + if (fieldValue.isPresent()) { + if (matchFieldValue(StringUtil.stripAccents(fieldValue.get()))) { + return true; + } + } + } + + // special case of asdf!=whatever and entry does not contain asdf + return fieldsKeys.isEmpty() && (operator == ComparisonOperator.DOES_NOT_CONTAIN); + } + + private Predicate matchFieldKey() { + return field -> fieldPattern.matcher(field.getName()).matches(); + } + + public boolean matchFieldValue(String content) { + Matcher matcher = valuePattern.matcher(content); + if (operator == ComparisonOperator.CONTAINS) { + return matcher.find(); + } else if (operator == ComparisonOperator.EXACT) { + return matcher.matches(); + } else if (operator == ComparisonOperator.DOES_NOT_CONTAIN) { + return !matcher.find(); + } else { + throw new IllegalStateException("MUST NOT HAPPEN"); + } + } + } + + /** + * Search results in boolean. It may be later on converted to an int. + */ + static class BibtexSearchVisitor extends SearchBaseVisitor { + + private final EnumSet searchFlags; + + private final BibEntry entry; + + public BibtexSearchVisitor(EnumSet searchFlags, BibEntry bibEntry) { + this.searchFlags = searchFlags; + this.entry = bibEntry; + } + + public boolean comparison(String field, ComparisonOperator operator, String value) { + return new Comparator(field, value, operator, searchFlags).compare(entry); + } + + @Override + public Boolean visitStart(SearchParser.StartContext ctx) { + return visit(ctx.expression()); + } + + @Override + public Boolean visitComparison(SearchParser.ComparisonContext context) { + // remove possible enclosing " symbols + String right = context.right.getText(); + if (right.startsWith("\"") && right.endsWith("\"")) { + right = right.substring(1, right.length() - 1); + } + + Optional fieldDescriptor = Optional.ofNullable(context.left); + if (fieldDescriptor.isPresent()) { + return comparison(fieldDescriptor.get().getText(), ComparisonOperator.build(context.operator.getText()), right); + } else { + return SearchRules.getSearchRule(searchFlags).applyRule(right, entry); + } + } + + @Override + public Boolean visitUnaryExpression(SearchParser.UnaryExpressionContext ctx) { + return !visit(ctx.expression()); // negate + } + + @Override + public Boolean visitParenExpression(SearchParser.ParenExpressionContext ctx) { + return visit(ctx.expression()); // ignore parenthesis + } + + @Override + public Boolean visitBinaryExpression(SearchParser.BinaryExpressionContext ctx) { + if ("AND".equalsIgnoreCase(ctx.operator.getText())) { + return visit(ctx.left) && visit(ctx.right); // and + } else { + return visit(ctx.left) || visit(ctx.right); // or + } + } + } +} diff --git a/src/main/java/org/jabref/model/search/rules/SearchRule.java b/src/main/java/org/jabref/model/search/rules/SearchRule.java new file mode 100644 index 00000000000..ffc4dced699 --- /dev/null +++ b/src/main/java/org/jabref/model/search/rules/SearchRule.java @@ -0,0 +1,10 @@ +package org.jabref.model.search.rules; + +import org.jabref.model.entry.BibEntry; + +public interface SearchRule { + + boolean applyRule(String query, BibEntry bibEntry); + + boolean validateSearchStrings(String query); +} diff --git a/src/main/java/org/jabref/model/search/rules/SearchRules.java b/src/main/java/org/jabref/model/search/rules/SearchRules.java new file mode 100644 index 00000000000..2ab7c3f01a4 --- /dev/null +++ b/src/main/java/org/jabref/model/search/rules/SearchRules.java @@ -0,0 +1,44 @@ +package org.jabref.model.search.rules; + +import java.util.EnumSet; +import java.util.regex.Pattern; + +import org.jabref.model.search.SearchFlags; + +/** + * This is a factory to instantiate the matching SearchRule implementation matching a given query + */ +public class SearchRules { + + private static final Pattern SIMPLE_EXPRESSION = Pattern.compile("[^\\p{Punct}]*"); + + private SearchRules() { + } + + /** + * Returns the appropriate search rule that fits best to the given parameter. + */ + public static SearchRule getSearchRuleByQuery(String query, EnumSet searchFlags) { + // this searches specified fields if specified, + // and all fields otherwise + SearchRule searchExpression = new GrammarBasedSearchRule(searchFlags); + if (searchExpression.validateSearchStrings(query)) { + return searchExpression; + } else { + return getSearchRule(searchFlags); + } + } + + private static boolean isSimpleQuery(String query) { + return SIMPLE_EXPRESSION.matcher(query).matches(); + } + + static SearchRule getSearchRule(EnumSet searchFlags) { + if (searchFlags.contains(SearchFlags.REGULAR_EXPRESSION)) { + // return new RegexBasedSearchRule(searchFlags); + } else { + // return new ContainsBasedSearchRule(searchFlags); + } + return new GrammarBasedSearchRule(searchFlags); + } +} From fcc8c9ca1d2ec5bd81c70302a23326fd7e3aecfe Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Mon, 26 Aug 2024 14:47:47 +0200 Subject: [PATCH 156/256] First version of search group migration Co-authored-by: Loay Ghreeb <52158423+LoayGhreeb@users.noreply.github.com> --- .../importer/actions/OpenDatabaseAction.java | 2 + .../actions/SearchGroupsMigrationAction.java | 104 +++++++++++++++ .../actions/SearchToLuceneVisitor.java | 126 ++++++++++++++++++ .../logic/exporter/MetaDataSerializer.java | 3 + .../logic/importer/util/MetaDataParser.java | 4 + .../org/jabref/model/groups/SearchGroup.java | 16 ++- .../org/jabref/model/metadata/MetaData.java | 13 ++ src/main/resources/tinylog.properties | 3 + .../actions/SearchToLuceneVisitorTest.java | 44 ++++++ .../logic/exporter/GroupSerializerTest.java | 3 + 10 files changed, 317 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java create mode 100644 src/main/java/org/jabref/gui/importer/actions/SearchToLuceneVisitor.java create mode 100644 src/test/java/org/jabref/gui/importer/actions/SearchToLuceneVisitorTest.java diff --git a/src/main/java/org/jabref/gui/importer/actions/OpenDatabaseAction.java b/src/main/java/org/jabref/gui/importer/actions/OpenDatabaseAction.java index f94a8595eec..d278051a25a 100644 --- a/src/main/java/org/jabref/gui/importer/actions/OpenDatabaseAction.java +++ b/src/main/java/org/jabref/gui/importer/actions/OpenDatabaseAction.java @@ -55,6 +55,8 @@ public class OpenDatabaseAction extends SimpleCommand { new MergeReviewIntoCommentAction(), // Check for new custom entry types loaded from the BIB file: new CheckForNewEntryTypesAction(), + // Migrate search groups from Search.g4 to Lucene syntax + new SearchGroupsMigrationAction(), // AI chat history links BibEntry with citation key. When citation key is changed, chat history should be transferred from old citation key to new citation key new ListenForCitationKeyChangeForAiAction()); diff --git a/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java b/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java new file mode 100644 index 00000000000..7e87f29ecf2 --- /dev/null +++ b/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java @@ -0,0 +1,104 @@ +package org.jabref.gui.importer.actions; + +import java.util.Optional; + +import org.jabref.gui.DialogService; +import org.jabref.logic.importer.ParserResult; +import org.jabref.model.groups.GroupTreeNode; +import org.jabref.model.groups.SearchGroup; +import org.jabref.model.search.SearchFieldConstants; +import org.jabref.model.search.SearchFlags; +import org.jabref.model.search.rules.GrammarBasedSearchRule; +import org.jabref.preferences.PreferencesService; +import org.jabref.search.SearchLexer; +import org.jabref.search.SearchParser; + +import com.google.common.annotations.VisibleForTesting; +import org.antlr.v4.runtime.ANTLRInputStream; +import org.antlr.v4.runtime.BailErrorStrategy; +import org.antlr.v4.runtime.CommonTokenStream; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; +import org.apache.lucene.queryparser.flexible.standard.parser.EscapeQuerySyntaxImpl; + +/** + * This action checks whether the syntax for SearchGroups is the new one. + * If not we ask the user whether to migrate. + */ +public class SearchGroupsMigrationAction implements GUIPostOpenAction { + + @Override + public boolean isActionNecessary(ParserResult parserResult, PreferencesService preferencesService) { + if (parserResult.getMetaData().getGroupSearchSyntaxVersion().isPresent()) { + // Currently the presence of any version is enough to know that no migration is necessary + return false; + } + + Optional groups = parserResult.getMetaData().getGroups(); + if (groups.isEmpty()) { + return false; + } + + return groupOrSubgroupIsSearchGrooup(groups.get()); + } + + private boolean groupOrSubgroupIsSearchGrooup(GroupTreeNode groupTreeNode) { + if (groupTreeNode.getGroup() instanceof SearchGroup) { + return true; + } + for (GroupTreeNode child : groupTreeNode.getChildren()) { + if (groupOrSubgroupIsSearchGrooup(child)) { + return true; + } + } + return false; + } + + @Override + public void performAction(ParserResult parserResult, DialogService dialogService, PreferencesService preferencesService) { + // TODO: Localization + if (!dialogService.showConfirmationDialogAndWait("Search groups migration of " + parserResult.getPath().map(path -> path.toString()).orElse(""), + "The search groups syntax is outdated. Do you want to migrate to the new syntax?", + "Migrate", "Cancel")) { + return; + } + + parserResult.getMetaData().getGroups().ifPresent(this::migrateGroups); + + parserResult.getMetaData().setGroupSearchSyntaxVersion(SearchGroup.VERSION_6_0_ALPHA); + } + + private void migrateGroups(GroupTreeNode node) { + if (node.getGroup() instanceof SearchGroup searchGroup) { + String luceneSearchExpression = migrateToLuceneSyntax(searchGroup.getSearchExpression(), searchGroup.getSearchFlags().contains(SearchFlags.REGULAR_EXPRESSION)); + searchGroup.setSearchExpression(luceneSearchExpression); + } + for (GroupTreeNode child : node.getChildren()) { + migrateGroups(child); + } + } + + @VisibleForTesting + static String migrateToLuceneSyntax(String searchExpression, boolean isRegularExpression) { + SearchParser.StartContext context = getStartContext(searchExpression); + SearchToLuceneVisitor searchToLuceneVisitor = new SearchToLuceneVisitor(isRegularExpression); + QueryNode luceneQueryNode = searchToLuceneVisitor.visit(context); + String result = luceneQueryNode.toQueryString(new EscapeQuerySyntaxImpl()).toString(); + if (!searchToLuceneVisitor.isNegation()) { + // Remove "all:" prefix for some cleaner search queries + // There is no UnfieldedQueryNode in Lucene, only FieldQueryNode + return result.replace(SearchFieldConstants.DEFAULT_FIELD + ":", ""); + } + return result; + } + + private static SearchParser.StartContext getStartContext(String searchExpression) { + SearchLexer lexer = new SearchLexer(new ANTLRInputStream(searchExpression)); + lexer.removeErrorListeners(); // no infos on file system + lexer.addErrorListener(GrammarBasedSearchRule.ThrowingErrorListener.INSTANCE); + SearchParser parser = new SearchParser(new CommonTokenStream(lexer)); + parser.removeErrorListeners(); // no infos on file system + parser.addErrorListener(GrammarBasedSearchRule.ThrowingErrorListener.INSTANCE); + parser.setErrorHandler(new BailErrorStrategy()); // ParseCancellationException on parse errors + return parser.start(); + } +} diff --git a/src/main/java/org/jabref/gui/importer/actions/SearchToLuceneVisitor.java b/src/main/java/org/jabref/gui/importer/actions/SearchToLuceneVisitor.java new file mode 100644 index 00000000000..995d7eb6337 --- /dev/null +++ b/src/main/java/org/jabref/gui/importer/actions/SearchToLuceneVisitor.java @@ -0,0 +1,126 @@ +package org.jabref.gui.importer.actions; + +import java.util.List; +import java.util.Optional; + +import org.jabref.model.search.SearchFieldConstants; +import org.jabref.search.SearchBaseVisitor; +import org.jabref.search.SearchParser; + +import org.apache.lucene.queryparser.flexible.core.nodes.AndQueryNode; +import org.apache.lucene.queryparser.flexible.core.nodes.FieldQueryNode; +import org.apache.lucene.queryparser.flexible.core.nodes.GroupQueryNode; +import org.apache.lucene.queryparser.flexible.core.nodes.ModifierQueryNode; +import org.apache.lucene.queryparser.flexible.core.nodes.OrQueryNode; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; +import org.apache.lucene.queryparser.flexible.standard.nodes.RegexpQueryNode; + +/** + * Converts to a Lucene index with the assumption that the ngram analyzer is used. + */ +public class SearchToLuceneVisitor extends SearchBaseVisitor { + + private final boolean isRegularExpression; + + private boolean isNegation = false; + + public SearchToLuceneVisitor(boolean isRegularExpression) { + this.isRegularExpression = isRegularExpression; + } + + @Override + public QueryNode visitStart(SearchParser.StartContext ctx) { + QueryNode result = visit(ctx.expression()); + + // If user searches for a single negation, Lucene also (!) interprets it as filter on the entities matched by the other terms + // We need to add a "filter" to match all entities + // See https://github.com/LoayGhreeb/lucene-mwe/issues/1 for more details + if (result instanceof ModifierQueryNode modifierQueryNode) { + if (modifierQueryNode.getModifier() == ModifierQueryNode.Modifier.MOD_NOT) { + isNegation = true; + return new AndQueryNode(List.of(new FieldQueryNode(SearchFieldConstants.DEFAULT_FIELD.toString(), "*", 0, 0), modifierQueryNode)); + } + } + + // User might search for NOT this AND NOT that - we also need to convert properly + if (result instanceof AndQueryNode andQueryNode) { + if (andQueryNode.getChildren().stream().allMatch(child -> child instanceof ModifierQueryNode modifierQueryNode && modifierQueryNode.getModifier() == ModifierQueryNode.Modifier.MOD_NOT)) { + isNegation = true; + return new AndQueryNode(List.of(new FieldQueryNode(SearchFieldConstants.DEFAULT_FIELD.toString(), "*", 0, 0), result)); + } + } + + return result; + } + + @Override + public QueryNode visitUnaryExpression(SearchParser.UnaryExpressionContext ctx) { + return new ModifierQueryNode(visit(ctx.expression()), ModifierQueryNode.Modifier.MOD_NOT); + } + + @Override + public QueryNode visitParenExpression(SearchParser.ParenExpressionContext ctx) { + return new GroupQueryNode(visit(ctx.expression())); + } + + @Override + public QueryNode visitBinaryExpression(SearchParser.BinaryExpressionContext ctx) { + if ("AND".equalsIgnoreCase(ctx.operator.getText())) { + return new AndQueryNode(List.of(visit(ctx.left), visit(ctx.right))); + } else { + return new OrQueryNode(List.of(visit(ctx.left), visit(ctx.right))); + } + } + + @Override + public QueryNode visitComparison(SearchParser.ComparisonContext context) { + // The comparison is a leaf node in the tree + + // remove possible enclosing " symbols + String right = context.right.getText(); + if (right.startsWith("\"") && right.endsWith("\"")) { + right = right.substring(1, right.length() - 1); + } + + Optional fieldDescriptor = Optional.ofNullable(context.left); + int startIndex = context.getStart().getStartIndex(); + int stopIndex = context.getStop().getStopIndex(); + if (fieldDescriptor.isPresent()) { + String field = fieldDescriptor.get().getText(); + + // Direct comparison does not work + // context.CONTAINS() and others are null if absent (thus, we cannot check for getText()) + if (context.CONTAINS() != null || + context.MATCHES() != null || + context.EQUAL() != null || + context.EEQUAL() != null) { // exact match + return getFieldQueryNode(field, right, startIndex, stopIndex); + } + + assert (context.NEQUAL() != null); + return new ModifierQueryNode(getFieldQueryNode(field, right, startIndex, stopIndex), ModifierQueryNode.Modifier.MOD_NOT); + } else { + return getFieldQueryNode(SearchFieldConstants.DEFAULT_FIELD.toString(), right, startIndex, stopIndex); + } + } + + /** + * A search query can be either a regular expression or a normal query. + * In Lucene, this is represented by a RegexpQueryNode or a FieldQueryNode. + * They are created in this class accordingly. + */ + private QueryNode getFieldQueryNode(String field, String term, int startIndex, int stopIndex) { + if (isRegularExpression) { + // Lucene does a sanity check on the positions, thus we provide other fake positions + return new RegexpQueryNode(field, term, 0, term.length() - 1); + } + return new FieldQueryNode(field, term, startIndex, stopIndex); + } + + /** + * Returns whether the search query is a negation (and was patched to be a filter). + */ + public boolean isNegation() { + return this.isNegation; + } +} diff --git a/src/main/java/org/jabref/logic/exporter/MetaDataSerializer.java b/src/main/java/org/jabref/logic/exporter/MetaDataSerializer.java index 87120af65fc..9c0e2dfe95c 100644 --- a/src/main/java/org/jabref/logic/exporter/MetaDataSerializer.java +++ b/src/main/java/org/jabref/logic/exporter/MetaDataSerializer.java @@ -73,6 +73,9 @@ public static Map getSerializedStringMap(MetaData metaData, metaData.getGroups().filter(root -> root.getNumberOfChildren() > 0).ifPresent( root -> serializedMetaData.put(MetaData.GROUPSTREE, serializeGroups(root))); + metaData.getGroupSearchSyntaxVersion().ifPresent( + version -> serializedMetaData.put(MetaData.GROUPS_SEARCH_SYNTAX_VERSION, version.toString())); + // finally add all unknown meta data items to the serialization map Map> unknownMetaData = metaData.getUnknownMetaData(); for (Map.Entry> entry : unknownMetaData.entrySet()) { diff --git a/src/main/java/org/jabref/logic/importer/util/MetaDataParser.java b/src/main/java/org/jabref/logic/importer/util/MetaDataParser.java index f427e6a6276..70a17d7d5f5 100644 --- a/src/main/java/org/jabref/logic/importer/util/MetaDataParser.java +++ b/src/main/java/org/jabref/logic/importer/util/MetaDataParser.java @@ -15,6 +15,7 @@ import org.jabref.logic.citationkeypattern.CitationKeyPattern; import org.jabref.logic.cleanup.FieldFormatterCleanups; import org.jabref.logic.importer.ParseException; +import org.jabref.logic.util.Version; import org.jabref.model.database.BibDatabaseMode; import org.jabref.model.entry.BibEntryType; import org.jabref.model.entry.BibEntryTypeBuilder; @@ -129,6 +130,9 @@ public MetaData parse(MetaData metaData, Map data, Character key metaData.setSaveOrder(SaveOrder.parse(values)); } else if (entry.getKey().equals(MetaData.GROUPSTREE) || entry.getKey().equals(MetaData.GROUPSTREE_LEGACY)) { metaData.setGroups(GroupsParser.importGroups(values, keywordSeparator, fileMonitor, metaData)); + } else if (entry.getKey().equals(MetaData.GROUPS_SEARCH_SYNTAX_VERSION)) { + Version version = Version.parse(getSingleItem(values)); + metaData.setGroupSearchSyntaxVersion(version); } else if (entry.getKey().equals(MetaData.VERSION_DB_STRUCT)) { metaData.setVersionDBStructure(getSingleItem(values)); } else { diff --git a/src/main/java/org/jabref/model/groups/SearchGroup.java b/src/main/java/org/jabref/model/groups/SearchGroup.java index 0b6f6cadf75..7cc98dda875 100644 --- a/src/main/java/org/jabref/model/groups/SearchGroup.java +++ b/src/main/java/org/jabref/model/groups/SearchGroup.java @@ -8,6 +8,7 @@ import org.jabref.architecture.AllowedToUseLogic; import org.jabref.logic.search.LuceneManager; +import org.jabref.logic.util.Version; import org.jabref.model.entry.BibEntry; import org.jabref.model.search.SearchFlags; import org.jabref.model.search.SearchQuery; @@ -22,10 +23,14 @@ @AllowedToUseLogic("because it needs access to lucene manager") public class SearchGroup extends AbstractGroup { + // We cannot have this constant in Version java becuase of recursion errors + // Thus, we keep it here, because it is (currently) used only in the context of groups. + public static final Version VERSION_6_0_ALPHA = Version.parse("6.0-alpha"); + private static final Logger LOGGER = LoggerFactory.getLogger(SearchGroup.class); - private final SearchQuery query; private final ObservableMap matchedEntries = FXCollections.observableHashMap(); + private SearchQuery query; private LuceneManager luceneManager; public SearchGroup(String name, GroupHierarchyType context, String searchExpression, EnumSet searchFlags, LuceneManager luceneManager) { @@ -43,6 +48,15 @@ public String getSearchExpression() { return query.getSearchExpression(); } + /** + * Used by {@link org.jabref.gui.importer.actions.SearchGroupsMigrationAction} to update the search expression. + * Do not use otherwise. + */ + public void setSearchExpression(String searchExpression) { + LOGGER.debug("Setting search expression {}", searchExpression); + this.query = new SearchQuery(searchExpression, query.getSearchFlags()); + } + public SearchQuery getQuery() { return query; } diff --git a/src/main/java/org/jabref/model/metadata/MetaData.java b/src/main/java/org/jabref/model/metadata/MetaData.java index 941dc667426..9c05df7cdaf 100644 --- a/src/main/java/org/jabref/model/metadata/MetaData.java +++ b/src/main/java/org/jabref/model/metadata/MetaData.java @@ -19,6 +19,7 @@ import org.jabref.logic.citationkeypattern.DatabaseCitationKeyPatterns; import org.jabref.logic.citationkeypattern.GlobalCitationKeyPatterns; import org.jabref.logic.cleanup.FieldFormatterCleanups; +import org.jabref.logic.util.Version; import org.jabref.model.database.BibDatabaseMode; import org.jabref.model.database.event.ChangePropagation; import org.jabref.model.entry.field.Field; @@ -46,6 +47,7 @@ public class MetaData { public static final String VERSION_DB_STRUCT = "VersionDBStructure"; public static final String GROUPSTREE = "grouping"; public static final String GROUPSTREE_LEGACY = "groupstree"; + public static final String GROUPS_SEARCH_SYNTAX_VERSION = "groups-search-syntax-version"; public static final String FILE_DIRECTORY = "fileDirectory"; public static final String FILE_DIRECTORY_LATEX = "fileDirectoryLatex"; public static final String PROTECTED_FLAG_META = "protectedFlag"; @@ -62,8 +64,11 @@ public class MetaData { private final Map citeKeyPatterns = new HashMap<>(); // private final Map userFileDirectory = new HashMap<>(); // private final Map laTexFileDirectory = new HashMap<>(); // + private final ObjectProperty groupsRoot = new SimpleObjectProperty<>(null); private final OptionalBinding groupsRootBinding = new OptionalWrapper<>(groupsRoot); + private Optional groupSearchSyntaxVersion = Optional.empty(); + private Charset encoding; private SaveOrder saveOrder; private String defaultCiteKeyPattern; @@ -113,6 +118,14 @@ public void setGroups(GroupTreeNode root) { postChange(); } + public void setGroupSearchSyntaxVersion(Version version) { + groupSearchSyntaxVersion = Optional.of(version); + } + + public Optional getGroupSearchSyntaxVersion() { + return this.groupSearchSyntaxVersion; + } + /** * @return the stored label patterns */ diff --git a/src/main/resources/tinylog.properties b/src/main/resources/tinylog.properties index 8edda96f0d6..898616c193f 100644 --- a/src/main/resources/tinylog.properties +++ b/src/main/resources/tinylog.properties @@ -24,3 +24,6 @@ level@org.jabref.http.server.Server = debug #level@org.jabref.logic.ai.FileEmbeddingsManager = trace #level@org.jabref.logic.ai.impl.embeddings.LowLevelIngestor = trace #level@org.jabref.logic.ai.chat.AiChatLogic = trace + +level@org.jabref.model.groups.SearchGroup = debug + diff --git a/src/test/java/org/jabref/gui/importer/actions/SearchToLuceneVisitorTest.java b/src/test/java/org/jabref/gui/importer/actions/SearchToLuceneVisitorTest.java new file mode 100644 index 00000000000..7bcc48bd8aa --- /dev/null +++ b/src/test/java/org/jabref/gui/importer/actions/SearchToLuceneVisitorTest.java @@ -0,0 +1,44 @@ +package org.jabref.gui.importer.actions; + +import java.util.stream.Stream; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class SearchToLuceneVisitorTest { + + public static Stream transformationNormal() { + return Stream.of( + Arguments.of("all:* AND -title:chocolate", "title != chocolate"), + Arguments.of("chocolate", "chocolate"), + Arguments.of("title:chocolate OR author:chocolate", "title = chocolate or author = chocolate"), + Arguments.of("title:chocolate AND author:chocolate", "title = \"chocolate\" AND author = \"chocolate\""), + Arguments.of("title:chocolate OR author:chocolate", "title == chocolate or author == chocolate"), + Arguments.of("title:image\\ processing AND author:smith", "title = \"image processing\" AND author= smith") + ); + } + + @ParameterizedTest + @MethodSource + void transformationNormal(String expected, String query) { + String result = SearchGroupsMigrationAction.migrateToLuceneSyntax(query, false); + assertEquals(expected, result); + } + + public static Stream transformationRegularExpression() { + return Stream.of( + Arguments.of("all:* AND -groups:/./", "groups != .+"), + Arguments.of("all:* AND ( -groups:/./ AND -readstatus:/./ )", "groups != .+ and readstatus != .+") + ); + } + + @ParameterizedTest + @MethodSource + void transformationRegularExpression(String expected, String query) { + String result = SearchGroupsMigrationAction.migrateToLuceneSyntax(query, true); + assertEquals(expected, result); + } +} diff --git a/src/test/java/org/jabref/logic/exporter/GroupSerializerTest.java b/src/test/java/org/jabref/logic/exporter/GroupSerializerTest.java index 30504d13aab..64346f80347 100644 --- a/src/test/java/org/jabref/logic/exporter/GroupSerializerTest.java +++ b/src/test/java/org/jabref/logic/exporter/GroupSerializerTest.java @@ -32,6 +32,9 @@ import static org.junit.jupiter.api.Assertions.assertEquals; +/** + * Loading of groups is tested at {@link org.jabref.logic.importer.util.GroupsParserTest}. + */ class GroupSerializerTest { private GroupSerializer groupSerializer; From 660e3541c508f6565591234c09888047cb27cc4c Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Mon, 26 Aug 2024 20:46:06 +0300 Subject: [PATCH 157/256] Add groups field to the index --- .../java/org/jabref/gui/groups/GroupNodeViewModel.java | 1 - .../jabref/logic/search/indexing/BibFieldsIndexer.java | 9 +-------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java b/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java index c2eefb61009..90853e2e503 100644 --- a/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java +++ b/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java @@ -563,7 +563,6 @@ public void listen(IndexRemovedEvent event) { searchGroup.updateMatches(entry); matchedEntries.remove(System.identityHashCode(entry)); } - databaseContext.getMetaData().groupsBinding().invalidate(); }).executeWith(taskExecutor); } } diff --git a/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java b/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java index 8fa8be9c517..5a94d152959 100644 --- a/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java @@ -25,8 +25,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.jabref.model.entry.field.StandardField.GROUPS; - public class BibFieldsIndexer implements LuceneIndexer { private static final Logger LOGGER = LoggerFactory.getLogger(BibFieldsIndexer.class); private final BibDatabaseContext databaseContext; @@ -84,12 +82,7 @@ private void addToIndex(BibEntry bibEntry) { StringBuilder allFields = new StringBuilder(bibEntry.getType().getName()); for (Map.Entry mapEntry : bibEntry.getFieldMap().entrySet()) { - Field field = mapEntry.getKey(); - if (field == GROUPS) { - // https://github.com/JabRef/jabref/issues/7996 - continue; - } - document.add(new TextField(field.getName(), mapEntry.getValue(), storeDisabled)); + document.add(new TextField(mapEntry.getKey().getName(), mapEntry.getValue(), storeDisabled)); allFields.append('\n').append(mapEntry.getValue()); } document.add(new TextField(SearchFieldConstants.DEFAULT_FIELD.toString(), allFields.toString(), storeDisabled)); From c6c906598eeb4f4c09f72d8d4a91a7bcaa1a2756 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Mon, 26 Aug 2024 20:50:44 +0300 Subject: [PATCH 158/256] Remove search rules --- .../search/rules/GrammarBasedSearchRule.java | 259 ------------------ .../jabref/model/search/rules/SearchRule.java | 10 - .../model/search/rules/SearchRules.java | 44 --- 3 files changed, 313 deletions(-) delete mode 100644 src/main/java/org/jabref/model/search/rules/GrammarBasedSearchRule.java delete mode 100644 src/main/java/org/jabref/model/search/rules/SearchRule.java delete mode 100644 src/main/java/org/jabref/model/search/rules/SearchRules.java diff --git a/src/main/java/org/jabref/model/search/rules/GrammarBasedSearchRule.java b/src/main/java/org/jabref/model/search/rules/GrammarBasedSearchRule.java deleted file mode 100644 index e062ff48532..00000000000 --- a/src/main/java/org/jabref/model/search/rules/GrammarBasedSearchRule.java +++ /dev/null @@ -1,259 +0,0 @@ -package org.jabref.model.search.rules; - -import java.util.ArrayList; -import java.util.EnumSet; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.function.Predicate; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -import org.jabref.architecture.AllowedToUseLogic; -import org.jabref.model.entry.BibEntry; -import org.jabref.model.entry.Keyword; -import org.jabref.model.entry.field.Field; -import org.jabref.model.entry.field.InternalField; -import org.jabref.model.search.SearchFlags; -import org.jabref.model.search.SearchResult; -import org.jabref.model.strings.StringUtil; -import org.jabref.search.SearchBaseVisitor; -import org.jabref.search.SearchLexer; -import org.jabref.search.SearchParser; - -import org.antlr.v4.runtime.ANTLRInputStream; -import org.antlr.v4.runtime.BailErrorStrategy; -import org.antlr.v4.runtime.BaseErrorListener; -import org.antlr.v4.runtime.CommonTokenStream; -import org.antlr.v4.runtime.RecognitionException; -import org.antlr.v4.runtime.Recognizer; -import org.antlr.v4.runtime.misc.ParseCancellationException; -import org.antlr.v4.runtime.tree.ParseTree; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * The search query must be specified in an expression that is acceptable by the Search.g4 grammar. - *

- * This class implements the "Advanced Search Mode" described in the help - */ -@AllowedToUseLogic("Because access to the lucene index is needed") -public class GrammarBasedSearchRule implements SearchRule { - - private static final Logger LOGGER = LoggerFactory.getLogger(GrammarBasedSearchRule.class); - - private final EnumSet searchFlags; - - private ParseTree tree; - private String query; - private List searchResults = new ArrayList<>(); - - public static class ThrowingErrorListener extends BaseErrorListener { - - public static final ThrowingErrorListener INSTANCE = new ThrowingErrorListener(); - - @Override - public void syntaxError(Recognizer recognizer, Object offendingSymbol, - int line, int charPositionInLine, String msg, RecognitionException e) - throws ParseCancellationException { - throw new ParseCancellationException("line " + line + ":" + charPositionInLine + " " + msg); - } - } - - public GrammarBasedSearchRule(EnumSet searchFlags) throws RecognitionException { - this.searchFlags = searchFlags; - } - - public static boolean isValid(EnumSet searchFlags, String query) { - return new GrammarBasedSearchRule(searchFlags).validateSearchStrings(query); - } - - public ParseTree getTree() { - return this.tree; - } - - public String getQuery() { - return this.query; - } - - private void init(String query) throws ParseCancellationException { - if (Objects.equals(this.query, query)) { - return; - } - - SearchLexer lexer = new SearchLexer(new ANTLRInputStream(query)); - lexer.removeErrorListeners(); // no infos on file system - lexer.addErrorListener(ThrowingErrorListener.INSTANCE); - SearchParser parser = new SearchParser(new CommonTokenStream(lexer)); - parser.removeErrorListeners(); // no infos on file system - parser.addErrorListener(ThrowingErrorListener.INSTANCE); - parser.setErrorHandler(new BailErrorStrategy()); // ParseCancellationException on parse errors - tree = parser.start(); - this.query = query; - } - - @Override - public boolean applyRule(String query, BibEntry bibEntry) { - try { - return new BibtexSearchVisitor(searchFlags, bibEntry).visit(tree); - } catch (Exception e) { - LOGGER.info("Search failed", e); - return false; - } - } - - @Override - public boolean validateSearchStrings(String query) { - try { - init(query); - return true; - } catch (ParseCancellationException e) { - LOGGER.debug("Search query invalid", e); - return false; - } - } - - public EnumSet getSearchFlags() { - return searchFlags; - } - - public enum ComparisonOperator { - EXACT, CONTAINS, DOES_NOT_CONTAIN; - - public static ComparisonOperator build(String value) { - if ("CONTAINS".equalsIgnoreCase(value) || "=".equals(value)) { - return CONTAINS; - } else if ("MATCHES".equalsIgnoreCase(value) || "==".equals(value)) { - return EXACT; - } else { - return DOES_NOT_CONTAIN; - } - } - } - - public static class Comparator { - - private final ComparisonOperator operator; - private final Pattern fieldPattern; - private final Pattern valuePattern; - - public Comparator(String field, String value, ComparisonOperator operator, EnumSet searchFlags) { - this.operator = operator; - - int option = searchFlags.contains(SearchFlags.CASE_SENSITIVE) ? 0 : Pattern.CASE_INSENSITIVE; - this.fieldPattern = Pattern.compile(searchFlags.contains(SearchFlags.REGULAR_EXPRESSION) ? StringUtil.stripAccents(field) : "\\Q" + StringUtil.stripAccents(field) + "\\E", option); - this.valuePattern = Pattern.compile(searchFlags.contains(SearchFlags.REGULAR_EXPRESSION) ? StringUtil.stripAccents(value) : "\\Q" + StringUtil.stripAccents(value) + "\\E", option); - } - - public boolean compare(BibEntry entry) { - // special case for searching for entrytype=phdthesis - if (fieldPattern.matcher(InternalField.TYPE_HEADER.getName()).matches()) { - return matchFieldValue(entry.getType().getName()); - } - - // special case for searching a single keyword - if (fieldPattern.matcher("anykeyword").matches()) { - return entry.getKeywords(',').stream().map(Keyword::toString).anyMatch(this::matchFieldValue); - } - - // specification of fieldsKeys to search is done in the search expression itself - Set fieldsKeys = entry.getFields(); - - // special case for searching allfields=cat and title=dog - if (!fieldPattern.matcher("anyfield").matches()) { - // Filter out the requested fields - fieldsKeys = fieldsKeys.stream().filter(matchFieldKey()).collect(Collectors.toSet()); - } - - for (Field field : fieldsKeys) { - Optional fieldValue = entry.getFieldLatexFree(field); - if (fieldValue.isPresent()) { - if (matchFieldValue(StringUtil.stripAccents(fieldValue.get()))) { - return true; - } - } - } - - // special case of asdf!=whatever and entry does not contain asdf - return fieldsKeys.isEmpty() && (operator == ComparisonOperator.DOES_NOT_CONTAIN); - } - - private Predicate matchFieldKey() { - return field -> fieldPattern.matcher(field.getName()).matches(); - } - - public boolean matchFieldValue(String content) { - Matcher matcher = valuePattern.matcher(content); - if (operator == ComparisonOperator.CONTAINS) { - return matcher.find(); - } else if (operator == ComparisonOperator.EXACT) { - return matcher.matches(); - } else if (operator == ComparisonOperator.DOES_NOT_CONTAIN) { - return !matcher.find(); - } else { - throw new IllegalStateException("MUST NOT HAPPEN"); - } - } - } - - /** - * Search results in boolean. It may be later on converted to an int. - */ - static class BibtexSearchVisitor extends SearchBaseVisitor { - - private final EnumSet searchFlags; - - private final BibEntry entry; - - public BibtexSearchVisitor(EnumSet searchFlags, BibEntry bibEntry) { - this.searchFlags = searchFlags; - this.entry = bibEntry; - } - - public boolean comparison(String field, ComparisonOperator operator, String value) { - return new Comparator(field, value, operator, searchFlags).compare(entry); - } - - @Override - public Boolean visitStart(SearchParser.StartContext ctx) { - return visit(ctx.expression()); - } - - @Override - public Boolean visitComparison(SearchParser.ComparisonContext context) { - // remove possible enclosing " symbols - String right = context.right.getText(); - if (right.startsWith("\"") && right.endsWith("\"")) { - right = right.substring(1, right.length() - 1); - } - - Optional fieldDescriptor = Optional.ofNullable(context.left); - if (fieldDescriptor.isPresent()) { - return comparison(fieldDescriptor.get().getText(), ComparisonOperator.build(context.operator.getText()), right); - } else { - return SearchRules.getSearchRule(searchFlags).applyRule(right, entry); - } - } - - @Override - public Boolean visitUnaryExpression(SearchParser.UnaryExpressionContext ctx) { - return !visit(ctx.expression()); // negate - } - - @Override - public Boolean visitParenExpression(SearchParser.ParenExpressionContext ctx) { - return visit(ctx.expression()); // ignore parenthesis - } - - @Override - public Boolean visitBinaryExpression(SearchParser.BinaryExpressionContext ctx) { - if ("AND".equalsIgnoreCase(ctx.operator.getText())) { - return visit(ctx.left) && visit(ctx.right); // and - } else { - return visit(ctx.left) || visit(ctx.right); // or - } - } - } -} diff --git a/src/main/java/org/jabref/model/search/rules/SearchRule.java b/src/main/java/org/jabref/model/search/rules/SearchRule.java deleted file mode 100644 index ffc4dced699..00000000000 --- a/src/main/java/org/jabref/model/search/rules/SearchRule.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.jabref.model.search.rules; - -import org.jabref.model.entry.BibEntry; - -public interface SearchRule { - - boolean applyRule(String query, BibEntry bibEntry); - - boolean validateSearchStrings(String query); -} diff --git a/src/main/java/org/jabref/model/search/rules/SearchRules.java b/src/main/java/org/jabref/model/search/rules/SearchRules.java deleted file mode 100644 index 2ab7c3f01a4..00000000000 --- a/src/main/java/org/jabref/model/search/rules/SearchRules.java +++ /dev/null @@ -1,44 +0,0 @@ -package org.jabref.model.search.rules; - -import java.util.EnumSet; -import java.util.regex.Pattern; - -import org.jabref.model.search.SearchFlags; - -/** - * This is a factory to instantiate the matching SearchRule implementation matching a given query - */ -public class SearchRules { - - private static final Pattern SIMPLE_EXPRESSION = Pattern.compile("[^\\p{Punct}]*"); - - private SearchRules() { - } - - /** - * Returns the appropriate search rule that fits best to the given parameter. - */ - public static SearchRule getSearchRuleByQuery(String query, EnumSet searchFlags) { - // this searches specified fields if specified, - // and all fields otherwise - SearchRule searchExpression = new GrammarBasedSearchRule(searchFlags); - if (searchExpression.validateSearchStrings(query)) { - return searchExpression; - } else { - return getSearchRule(searchFlags); - } - } - - private static boolean isSimpleQuery(String query) { - return SIMPLE_EXPRESSION.matcher(query).matches(); - } - - static SearchRule getSearchRule(EnumSet searchFlags) { - if (searchFlags.contains(SearchFlags.REGULAR_EXPRESSION)) { - // return new RegexBasedSearchRule(searchFlags); - } else { - // return new ContainsBasedSearchRule(searchFlags); - } - return new GrammarBasedSearchRule(searchFlags); - } -} From 05aab8dde155d209eb08c9344febcd4b88c3347e Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Mon, 26 Aug 2024 20:57:18 +0300 Subject: [PATCH 159/256] Localization --- .../actions/SearchGroupsMigrationAction.java | 25 ++++++++----------- .../model/search/ThrowingErrorListener.java | 18 +++++++++++++ src/main/resources/l10n/JabRef_en.properties | 3 +++ 3 files changed, 32 insertions(+), 14 deletions(-) create mode 100644 src/main/java/org/jabref/model/search/ThrowingErrorListener.java diff --git a/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java b/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java index 7e87f29ecf2..f5077035159 100644 --- a/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java +++ b/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java @@ -1,14 +1,16 @@ package org.jabref.gui.importer.actions; +import java.nio.file.Path; import java.util.Optional; import org.jabref.gui.DialogService; import org.jabref.logic.importer.ParserResult; +import org.jabref.logic.l10n.Localization; import org.jabref.model.groups.GroupTreeNode; import org.jabref.model.groups.SearchGroup; import org.jabref.model.search.SearchFieldConstants; import org.jabref.model.search.SearchFlags; -import org.jabref.model.search.rules.GrammarBasedSearchRule; +import org.jabref.model.search.ThrowingErrorListener; import org.jabref.preferences.PreferencesService; import org.jabref.search.SearchLexer; import org.jabref.search.SearchParser; @@ -34,19 +36,15 @@ public boolean isActionNecessary(ParserResult parserResult, PreferencesService p } Optional groups = parserResult.getMetaData().getGroups(); - if (groups.isEmpty()) { - return false; - } - - return groupOrSubgroupIsSearchGrooup(groups.get()); + return groups.filter(this::groupOrSubgroupIsSearchGroup).isPresent(); } - private boolean groupOrSubgroupIsSearchGrooup(GroupTreeNode groupTreeNode) { + private boolean groupOrSubgroupIsSearchGroup(GroupTreeNode groupTreeNode) { if (groupTreeNode.getGroup() instanceof SearchGroup) { return true; } for (GroupTreeNode child : groupTreeNode.getChildren()) { - if (groupOrSubgroupIsSearchGrooup(child)) { + if (groupOrSubgroupIsSearchGroup(child)) { return true; } } @@ -55,10 +53,9 @@ private boolean groupOrSubgroupIsSearchGrooup(GroupTreeNode groupTreeNode) { @Override public void performAction(ParserResult parserResult, DialogService dialogService, PreferencesService preferencesService) { - // TODO: Localization - if (!dialogService.showConfirmationDialogAndWait("Search groups migration of " + parserResult.getPath().map(path -> path.toString()).orElse(""), - "The search groups syntax is outdated. Do you want to migrate to the new syntax?", - "Migrate", "Cancel")) { + if (!dialogService.showConfirmationDialogAndWait(Localization.lang("Search groups migration of %0", parserResult.getPath().map(Path::toString).orElse("")), + Localization.lang("The search groups syntax is outdated. Do you want to migrate to the new syntax?"), + Localization.lang("Migrate"), Localization.lang("Cancel"))) { return; } @@ -94,10 +91,10 @@ static String migrateToLuceneSyntax(String searchExpression, boolean isRegularEx private static SearchParser.StartContext getStartContext(String searchExpression) { SearchLexer lexer = new SearchLexer(new ANTLRInputStream(searchExpression)); lexer.removeErrorListeners(); // no infos on file system - lexer.addErrorListener(GrammarBasedSearchRule.ThrowingErrorListener.INSTANCE); + lexer.addErrorListener(ThrowingErrorListener.INSTANCE); SearchParser parser = new SearchParser(new CommonTokenStream(lexer)); parser.removeErrorListeners(); // no infos on file system - parser.addErrorListener(GrammarBasedSearchRule.ThrowingErrorListener.INSTANCE); + parser.addErrorListener(ThrowingErrorListener.INSTANCE); parser.setErrorHandler(new BailErrorStrategy()); // ParseCancellationException on parse errors return parser.start(); } diff --git a/src/main/java/org/jabref/model/search/ThrowingErrorListener.java b/src/main/java/org/jabref/model/search/ThrowingErrorListener.java new file mode 100644 index 00000000000..d51c2b24474 --- /dev/null +++ b/src/main/java/org/jabref/model/search/ThrowingErrorListener.java @@ -0,0 +1,18 @@ +package org.jabref.model.search; + +import org.antlr.v4.runtime.BaseErrorListener; +import org.antlr.v4.runtime.RecognitionException; +import org.antlr.v4.runtime.Recognizer; +import org.antlr.v4.runtime.misc.ParseCancellationException; + +public class ThrowingErrorListener extends BaseErrorListener { + + public static final ThrowingErrorListener INSTANCE = new ThrowingErrorListener(); + + @Override + public void syntaxError(Recognizer recognizer, Object offendingSymbol, + int line, int charPositionInLine, String msg, RecognitionException e) + throws ParseCancellationException { + throw new ParseCancellationException("line " + line + ":" + charPositionInLine + " " + msg); + } +} diff --git a/src/main/resources/l10n/JabRef_en.properties b/src/main/resources/l10n/JabRef_en.properties index 7c90ef3db7d..2551f90bb05 100644 --- a/src/main/resources/l10n/JabRef_en.properties +++ b/src/main/resources/l10n/JabRef_en.properties @@ -269,6 +269,9 @@ Dynamically\ group\ entries\ by\ searching\ a\ field\ for\ a\ keyword=Dynamicall Each\ line\ must\ be\ of\ the\ following\ form\:\ \'tab\:field1;field2;...;fieldN\'.=Each line must be of the following form\: 'tab\:field1;field2;...;fieldN'. +Search\ groups\ migration\ of\ %0=Search groups migration of %0 +The\ search\ groups\ syntax\ is\ outdated.\ Do\ you\ want\ to\ migrate\ to\ the\ new\ syntax?= The search groups syntax is outdated. Do you want to migrate to the new syntax? +Migrate=Migrate Edit=Edit Edit\ file\ type=Edit file type From c19d01d15be1255b20e096bae1ab52a9983680f5 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Tue, 27 Aug 2024 09:31:11 +0300 Subject: [PATCH 160/256] Add test cases --- .../org/jabref/model/groups/SearchGroup.java | 3 +-- .../org/jabref/model/metadata/MetaData.java | 1 + .../actions/SearchToLuceneVisitorTest.java | 23 +++++++++++++------ 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/jabref/model/groups/SearchGroup.java b/src/main/java/org/jabref/model/groups/SearchGroup.java index 7cc98dda875..3efc67a0a0d 100644 --- a/src/main/java/org/jabref/model/groups/SearchGroup.java +++ b/src/main/java/org/jabref/model/groups/SearchGroup.java @@ -23,10 +23,9 @@ @AllowedToUseLogic("because it needs access to lucene manager") public class SearchGroup extends AbstractGroup { - // We cannot have this constant in Version java becuase of recursion errors + // We cannot have this constant in Version java because of recursion errors // Thus, we keep it here, because it is (currently) used only in the context of groups. public static final Version VERSION_6_0_ALPHA = Version.parse("6.0-alpha"); - private static final Logger LOGGER = LoggerFactory.getLogger(SearchGroup.class); private final ObservableMap matchedEntries = FXCollections.observableHashMap(); diff --git a/src/main/java/org/jabref/model/metadata/MetaData.java b/src/main/java/org/jabref/model/metadata/MetaData.java index 9c05df7cdaf..8f7c4a1dcd2 100644 --- a/src/main/java/org/jabref/model/metadata/MetaData.java +++ b/src/main/java/org/jabref/model/metadata/MetaData.java @@ -120,6 +120,7 @@ public void setGroups(GroupTreeNode root) { public void setGroupSearchSyntaxVersion(Version version) { groupSearchSyntaxVersion = Optional.of(version); + postChange(); } public Optional getGroupSearchSyntaxVersion() { diff --git a/src/test/java/org/jabref/gui/importer/actions/SearchToLuceneVisitorTest.java b/src/test/java/org/jabref/gui/importer/actions/SearchToLuceneVisitorTest.java index 7bcc48bd8aa..adbbc6c3678 100644 --- a/src/test/java/org/jabref/gui/importer/actions/SearchToLuceneVisitorTest.java +++ b/src/test/java/org/jabref/gui/importer/actions/SearchToLuceneVisitorTest.java @@ -12,12 +12,21 @@ class SearchToLuceneVisitorTest { public static Stream transformationNormal() { return Stream.of( - Arguments.of("all:* AND -title:chocolate", "title != chocolate"), Arguments.of("chocolate", "chocolate"), - Arguments.of("title:chocolate OR author:chocolate", "title = chocolate or author = chocolate"), - Arguments.of("title:chocolate AND author:chocolate", "title = \"chocolate\" AND author = \"chocolate\""), - Arguments.of("title:chocolate OR author:chocolate", "title == chocolate or author == chocolate"), - Arguments.of("title:image\\ processing AND author:smith", "title = \"image processing\" AND author= smith") + Arguments.of("title:chocolate", "title=chocolate"), + Arguments.of("title:chocolate OR author:smith", "title = chocolate or author = smith"), + Arguments.of("title:chocolate AND author:smith", "title = \"chocolate\" AND author = \"smith\""), + Arguments.of("title:chocolate AND author:smith", "title contains \"chocolate\" AND author matches \"smith\""), + Arguments.of("( title:chocolate ) OR ( author:smith )", "(title == chocolate) or (author == smith)"), + Arguments.of("( title:chocolate OR author:smith ) AND ( year:2024 )", "(title contains chocolate or author matches smith) AND (year = 2024)"), + Arguments.of("video AND year:1932", "video and year == 1932"), + Arguments.of("title:neighbou?r", "title =neighbou?r"), + Arguments.of("abstract:model\\{1,2\\}ing", "abstract = model{1,2}ing"), + Arguments.of("all:* AND -title:chocolate", "title != chocolate"), + Arguments.of("all:* AND -title:chocolate", "not title contains chocolate"), + Arguments.of("title:\"image processing\" OR keywords:\"image processing\"", "title|keywords = \"image processing\""), + Arguments.of("( author:miller OR title:\"image processing\" OR keywords:\"image processing\" ) AND NOT author:brown AND NOT author:blue", "(author = miller or title|keywords = \"image processing\") and not author = brown and != author = blue"), + Arguments.of("title:\"image processing\" AND author:smith", "title = \"image processing\" AND author= smith") ); } @@ -30,8 +39,8 @@ void transformationNormal(String expected, String query) { public static Stream transformationRegularExpression() { return Stream.of( - Arguments.of("all:* AND -groups:/./", "groups != .+"), - Arguments.of("all:* AND ( -groups:/./ AND -readstatus:/./ )", "groups != .+ and readstatus != .+") + Arguments.of("all:* AND -groups:/.+/", "groups != .+"), + Arguments.of("(all:* AND -groups:/.+/) AND (all:* AND -readstatus:/.+/)", "groups != .+ and readstatus != .+") ); } From 67c6c4d131013758abcc17c91605b803eb91ffcd Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Tue, 27 Aug 2024 08:49:48 +0200 Subject: [PATCH 161/256] Fix names Co-authored-by: Loay Ghreeb <52158423+LoayGhreeb@users.noreply.github.com> --- .../actions/SearchGroupsMigrationAction.java | 39 +------------------ .../migrations/SearchToLuceneMigration.java | 38 ++++++++++++++++++ .../SearchToLuceneVisitor.java | 2 +- .../SearchToLuceneMigrationTest.java} | 8 ++-- 4 files changed, 45 insertions(+), 42 deletions(-) create mode 100644 src/main/java/org/jabref/migrations/SearchToLuceneMigration.java rename src/main/java/org/jabref/{gui/importer/actions => migrations}/SearchToLuceneVisitor.java (99%) rename src/test/java/org/jabref/{gui/importer/actions/SearchToLuceneVisitorTest.java => migrations/SearchToLuceneMigrationTest.java} (91%) diff --git a/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java b/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java index f5077035159..95e7501c0b9 100644 --- a/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java +++ b/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java @@ -6,21 +6,11 @@ import org.jabref.gui.DialogService; import org.jabref.logic.importer.ParserResult; import org.jabref.logic.l10n.Localization; +import org.jabref.migrations.SearchToLuceneMigration; import org.jabref.model.groups.GroupTreeNode; import org.jabref.model.groups.SearchGroup; -import org.jabref.model.search.SearchFieldConstants; import org.jabref.model.search.SearchFlags; -import org.jabref.model.search.ThrowingErrorListener; import org.jabref.preferences.PreferencesService; -import org.jabref.search.SearchLexer; -import org.jabref.search.SearchParser; - -import com.google.common.annotations.VisibleForTesting; -import org.antlr.v4.runtime.ANTLRInputStream; -import org.antlr.v4.runtime.BailErrorStrategy; -import org.antlr.v4.runtime.CommonTokenStream; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; -import org.apache.lucene.queryparser.flexible.standard.parser.EscapeQuerySyntaxImpl; /** * This action checks whether the syntax for SearchGroups is the new one. @@ -66,36 +56,11 @@ public void performAction(ParserResult parserResult, DialogService dialogService private void migrateGroups(GroupTreeNode node) { if (node.getGroup() instanceof SearchGroup searchGroup) { - String luceneSearchExpression = migrateToLuceneSyntax(searchGroup.getSearchExpression(), searchGroup.getSearchFlags().contains(SearchFlags.REGULAR_EXPRESSION)); + String luceneSearchExpression = SearchToLuceneMigration.migrateToLuceneSyntax(searchGroup.getSearchExpression(), searchGroup.getSearchFlags().contains(SearchFlags.REGULAR_EXPRESSION)); searchGroup.setSearchExpression(luceneSearchExpression); } for (GroupTreeNode child : node.getChildren()) { migrateGroups(child); } } - - @VisibleForTesting - static String migrateToLuceneSyntax(String searchExpression, boolean isRegularExpression) { - SearchParser.StartContext context = getStartContext(searchExpression); - SearchToLuceneVisitor searchToLuceneVisitor = new SearchToLuceneVisitor(isRegularExpression); - QueryNode luceneQueryNode = searchToLuceneVisitor.visit(context); - String result = luceneQueryNode.toQueryString(new EscapeQuerySyntaxImpl()).toString(); - if (!searchToLuceneVisitor.isNegation()) { - // Remove "all:" prefix for some cleaner search queries - // There is no UnfieldedQueryNode in Lucene, only FieldQueryNode - return result.replace(SearchFieldConstants.DEFAULT_FIELD + ":", ""); - } - return result; - } - - private static SearchParser.StartContext getStartContext(String searchExpression) { - SearchLexer lexer = new SearchLexer(new ANTLRInputStream(searchExpression)); - lexer.removeErrorListeners(); // no infos on file system - lexer.addErrorListener(ThrowingErrorListener.INSTANCE); - SearchParser parser = new SearchParser(new CommonTokenStream(lexer)); - parser.removeErrorListeners(); // no infos on file system - parser.addErrorListener(ThrowingErrorListener.INSTANCE); - parser.setErrorHandler(new BailErrorStrategy()); // ParseCancellationException on parse errors - return parser.start(); - } } diff --git a/src/main/java/org/jabref/migrations/SearchToLuceneMigration.java b/src/main/java/org/jabref/migrations/SearchToLuceneMigration.java new file mode 100644 index 00000000000..ef1326063d7 --- /dev/null +++ b/src/main/java/org/jabref/migrations/SearchToLuceneMigration.java @@ -0,0 +1,38 @@ +package org.jabref.migrations; + +import org.jabref.model.search.SearchFieldConstants; +import org.jabref.model.search.ThrowingErrorListener; +import org.jabref.search.SearchLexer; +import org.jabref.search.SearchParser; + +import org.antlr.v4.runtime.ANTLRInputStream; +import org.antlr.v4.runtime.BailErrorStrategy; +import org.antlr.v4.runtime.CommonTokenStream; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; +import org.apache.lucene.queryparser.flexible.standard.parser.EscapeQuerySyntaxImpl; + +public class SearchToLuceneMigration { + public static String migrateToLuceneSyntax(String searchExpression, boolean isRegularExpression) { + SearchParser.StartContext context = getStartContext(searchExpression); + SearchToLuceneVisitor searchToLuceneVisitor = new SearchToLuceneVisitor(isRegularExpression); + QueryNode luceneQueryNode = searchToLuceneVisitor.visit(context); + String result = luceneQueryNode.toQueryString(new EscapeQuerySyntaxImpl()).toString(); + if (!searchToLuceneVisitor.isNegation()) { + // Remove "all:" prefix for some cleaner search queries + // There is no UnfieldedQueryNode in Lucene, only FieldQueryNode + return result.replace(SearchFieldConstants.DEFAULT_FIELD + ":", ""); + } + return result; + } + + private static SearchParser.StartContext getStartContext(String searchExpression) { + SearchLexer lexer = new SearchLexer(new ANTLRInputStream(searchExpression)); + lexer.removeErrorListeners(); // no infos on file system + lexer.addErrorListener(ThrowingErrorListener.INSTANCE); + SearchParser parser = new SearchParser(new CommonTokenStream(lexer)); + parser.removeErrorListeners(); // no infos on file system + parser.addErrorListener(ThrowingErrorListener.INSTANCE); + parser.setErrorHandler(new BailErrorStrategy()); // ParseCancellationException on parse errors + return parser.start(); + } +} diff --git a/src/main/java/org/jabref/gui/importer/actions/SearchToLuceneVisitor.java b/src/main/java/org/jabref/migrations/SearchToLuceneVisitor.java similarity index 99% rename from src/main/java/org/jabref/gui/importer/actions/SearchToLuceneVisitor.java rename to src/main/java/org/jabref/migrations/SearchToLuceneVisitor.java index 995d7eb6337..396a6bdc96c 100644 --- a/src/main/java/org/jabref/gui/importer/actions/SearchToLuceneVisitor.java +++ b/src/main/java/org/jabref/migrations/SearchToLuceneVisitor.java @@ -1,4 +1,4 @@ -package org.jabref.gui.importer.actions; +package org.jabref.migrations; import java.util.List; import java.util.Optional; diff --git a/src/test/java/org/jabref/gui/importer/actions/SearchToLuceneVisitorTest.java b/src/test/java/org/jabref/migrations/SearchToLuceneMigrationTest.java similarity index 91% rename from src/test/java/org/jabref/gui/importer/actions/SearchToLuceneVisitorTest.java rename to src/test/java/org/jabref/migrations/SearchToLuceneMigrationTest.java index adbbc6c3678..646ee38d0b8 100644 --- a/src/test/java/org/jabref/gui/importer/actions/SearchToLuceneVisitorTest.java +++ b/src/test/java/org/jabref/migrations/SearchToLuceneMigrationTest.java @@ -1,4 +1,4 @@ -package org.jabref.gui.importer.actions; +package org.jabref.migrations; import java.util.stream.Stream; @@ -8,7 +8,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; -class SearchToLuceneVisitorTest { +class SearchToLuceneMigrationTest { public static Stream transformationNormal() { return Stream.of( @@ -33,7 +33,7 @@ public static Stream transformationNormal() { @ParameterizedTest @MethodSource void transformationNormal(String expected, String query) { - String result = SearchGroupsMigrationAction.migrateToLuceneSyntax(query, false); + String result = SearchToLuceneMigration.migrateToLuceneSyntax(query, false); assertEquals(expected, result); } @@ -47,7 +47,7 @@ public static Stream transformationRegularExpression() { @ParameterizedTest @MethodSource void transformationRegularExpression(String expected, String query) { - String result = SearchGroupsMigrationAction.migrateToLuceneSyntax(query, true); + String result = SearchToLuceneMigration.migrateToLuceneSyntax(query, true); assertEquals(expected, result); } } From ba660d413b873f3d76235ebeb8178a343d74f405 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Tue, 27 Aug 2024 09:22:35 +0200 Subject: [PATCH 162/256] Add some more functionality Co-authored-by: Loay Ghreeb <52158423+LoayGhreeb@users.noreply.github.com> --- .../migrations/SearchToLuceneVisitor.java | 27 +++++++++++++++++-- .../SearchToLuceneMigrationTest.java | 20 +++++++++++--- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/jabref/migrations/SearchToLuceneVisitor.java b/src/main/java/org/jabref/migrations/SearchToLuceneVisitor.java index 396a6bdc96c..56ad9e743f9 100644 --- a/src/main/java/org/jabref/migrations/SearchToLuceneVisitor.java +++ b/src/main/java/org/jabref/migrations/SearchToLuceneVisitor.java @@ -3,6 +3,8 @@ import java.util.List; import java.util.Optional; +import org.jabref.model.entry.field.InternalField; +import org.jabref.model.entry.field.StandardField; import org.jabref.model.search.SearchFieldConstants; import org.jabref.search.SearchBaseVisitor; import org.jabref.search.SearchParser; @@ -14,12 +16,18 @@ import org.apache.lucene.queryparser.flexible.core.nodes.OrQueryNode; import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.apache.lucene.queryparser.flexible.standard.nodes.RegexpQueryNode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Converts to a Lucene index with the assumption that the ngram analyzer is used. + * + * Tests are located in {@link org.jabref.migrations.SearchToLuceneMigrationTest}. */ public class SearchToLuceneVisitor extends SearchBaseVisitor { + private static final Logger LOGGER = LoggerFactory.getLogger(SearchToLuceneVisitor.class); + private final boolean isRegularExpression; private boolean isNegation = false; @@ -46,7 +54,12 @@ public QueryNode visitStart(SearchParser.StartContext ctx) { if (result instanceof AndQueryNode andQueryNode) { if (andQueryNode.getChildren().stream().allMatch(child -> child instanceof ModifierQueryNode modifierQueryNode && modifierQueryNode.getModifier() == ModifierQueryNode.Modifier.MOD_NOT)) { isNegation = true; - return new AndQueryNode(List.of(new FieldQueryNode(SearchFieldConstants.DEFAULT_FIELD.toString(), "*", 0, 0), result)); + List children = andQueryNode.getChildren().stream() + // prepend "all:* AND" to each child + .map(child -> new AndQueryNode(List.of(new FieldQueryNode(SearchFieldConstants.DEFAULT_FIELD.toString(), "*", 0, 0), child))) + .map(child -> (QueryNode) child) + .toList(); + return new AndQueryNode(children); } } @@ -94,6 +107,9 @@ public QueryNode visitComparison(SearchParser.ComparisonContext context) { context.MATCHES() != null || context.EQUAL() != null || context.EEQUAL() != null) { // exact match + if (LOGGER.isDebugEnabled() && context.EEQUAL() != null) { + LOGGER.warn("Exact match is currently supported by Lucene, using contains instead. Term: {}", context.getText()); + } return getFieldQueryNode(field, right, startIndex, stopIndex); } @@ -110,9 +126,16 @@ public QueryNode visitComparison(SearchParser.ComparisonContext context) { * They are created in this class accordingly. */ private QueryNode getFieldQueryNode(String field, String term, int startIndex, int stopIndex) { + field = switch (field) { + case "anyfield" -> SearchFieldConstants.DEFAULT_FIELD.toString(); + case "anykeyword" -> StandardField.KEYWORDS.getName(); + case "key" -> InternalField.KEY_FIELD.getName(); + default -> field; + }; + if (isRegularExpression) { // Lucene does a sanity check on the positions, thus we provide other fake positions - return new RegexpQueryNode(field, term, 0, term.length() - 1); + return new RegexpQueryNode(field, term, 0, term.length()); } return new FieldQueryNode(field, term, startIndex, stopIndex); } diff --git a/src/test/java/org/jabref/migrations/SearchToLuceneMigrationTest.java b/src/test/java/org/jabref/migrations/SearchToLuceneMigrationTest.java index 646ee38d0b8..59eaec810ba 100644 --- a/src/test/java/org/jabref/migrations/SearchToLuceneMigrationTest.java +++ b/src/test/java/org/jabref/migrations/SearchToLuceneMigrationTest.java @@ -13,6 +13,7 @@ class SearchToLuceneMigrationTest { public static Stream transformationNormal() { return Stream.of( Arguments.of("chocolate", "chocolate"), + Arguments.of("title:chocolate", "title=chocolate"), Arguments.of("title:chocolate OR author:smith", "title = chocolate or author = smith"), Arguments.of("title:chocolate AND author:smith", "title = \"chocolate\" AND author = \"smith\""), @@ -24,9 +25,20 @@ public static Stream transformationNormal() { Arguments.of("abstract:model\\{1,2\\}ing", "abstract = model{1,2}ing"), Arguments.of("all:* AND -title:chocolate", "title != chocolate"), Arguments.of("all:* AND -title:chocolate", "not title contains chocolate"), - Arguments.of("title:\"image processing\" OR keywords:\"image processing\"", "title|keywords = \"image processing\""), - Arguments.of("( author:miller OR title:\"image processing\" OR keywords:\"image processing\" ) AND NOT author:brown AND NOT author:blue", "(author = miller or title|keywords = \"image processing\") and not author = brown and != author = blue"), - Arguments.of("title:\"image processing\" AND author:smith", "title = \"image processing\" AND author= smith") + + // not converted, because not working in JabRef 5.x + // Arguments.of("title:\"image processing\" OR keywords:\"image processing\"", "title|keywords = \"image processing\""), + + // not converted, because wrong syntax for JabRef 5.x + // Arguments.of("( author:miller OR title:\"image processing\" OR keywords:\"image processing\" ) AND NOT author:brown AND NOT author:blue", "(author = miller or title|keywords = \"image processing\") and not author = brown and != author = blue"), + + // String with a space + Arguments.of("title:image\\ processing AND author:smith", "title = \"image processing\" AND author= smith"), + + // We renamed fields + Arguments.of("all:somecontent", "anyfield = somecontent"), + Arguments.of("keywords:somekeyword", "anykeyword = somekeyword"), + Arguments.of("citationkey:somebibtexkey", "key = somebibtexkey") ); } @@ -40,7 +52,7 @@ void transformationNormal(String expected, String query) { public static Stream transformationRegularExpression() { return Stream.of( Arguments.of("all:* AND -groups:/.+/", "groups != .+"), - Arguments.of("(all:* AND -groups:/.+/) AND (all:* AND -readstatus:/.+/)", "groups != .+ and readstatus != .+") + Arguments.of("( all:* AND -groups:/.+/ ) AND ( all:* AND -readstatus:/.+/ )", "groups != .+ and readstatus != .+") ); } From e46e0a23d7b9526bd449cbecfb189ae6dbc40a28 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Tue, 27 Aug 2024 09:23:58 +0200 Subject: [PATCH 163/256] Always add "all" prefix Co-authored-by: Loay Ghreeb <52158423+LoayGhreeb@users.noreply.github.com> --- .../org/jabref/migrations/SearchToLuceneMigration.java | 9 +-------- .../jabref/migrations/SearchToLuceneMigrationTest.java | 4 ++-- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/jabref/migrations/SearchToLuceneMigration.java b/src/main/java/org/jabref/migrations/SearchToLuceneMigration.java index ef1326063d7..4a0d2db6297 100644 --- a/src/main/java/org/jabref/migrations/SearchToLuceneMigration.java +++ b/src/main/java/org/jabref/migrations/SearchToLuceneMigration.java @@ -1,6 +1,5 @@ package org.jabref.migrations; -import org.jabref.model.search.SearchFieldConstants; import org.jabref.model.search.ThrowingErrorListener; import org.jabref.search.SearchLexer; import org.jabref.search.SearchParser; @@ -16,13 +15,7 @@ public static String migrateToLuceneSyntax(String searchExpression, boolean isRe SearchParser.StartContext context = getStartContext(searchExpression); SearchToLuceneVisitor searchToLuceneVisitor = new SearchToLuceneVisitor(isRegularExpression); QueryNode luceneQueryNode = searchToLuceneVisitor.visit(context); - String result = luceneQueryNode.toQueryString(new EscapeQuerySyntaxImpl()).toString(); - if (!searchToLuceneVisitor.isNegation()) { - // Remove "all:" prefix for some cleaner search queries - // There is no UnfieldedQueryNode in Lucene, only FieldQueryNode - return result.replace(SearchFieldConstants.DEFAULT_FIELD + ":", ""); - } - return result; + return luceneQueryNode.toQueryString(new EscapeQuerySyntaxImpl()).toString(); } private static SearchParser.StartContext getStartContext(String searchExpression) { diff --git a/src/test/java/org/jabref/migrations/SearchToLuceneMigrationTest.java b/src/test/java/org/jabref/migrations/SearchToLuceneMigrationTest.java index 59eaec810ba..ea1a67789a6 100644 --- a/src/test/java/org/jabref/migrations/SearchToLuceneMigrationTest.java +++ b/src/test/java/org/jabref/migrations/SearchToLuceneMigrationTest.java @@ -12,7 +12,7 @@ class SearchToLuceneMigrationTest { public static Stream transformationNormal() { return Stream.of( - Arguments.of("chocolate", "chocolate"), + Arguments.of("all:chocolate", "chocolate"), Arguments.of("title:chocolate", "title=chocolate"), Arguments.of("title:chocolate OR author:smith", "title = chocolate or author = smith"), @@ -20,7 +20,7 @@ public static Stream transformationNormal() { Arguments.of("title:chocolate AND author:smith", "title contains \"chocolate\" AND author matches \"smith\""), Arguments.of("( title:chocolate ) OR ( author:smith )", "(title == chocolate) or (author == smith)"), Arguments.of("( title:chocolate OR author:smith ) AND ( year:2024 )", "(title contains chocolate or author matches smith) AND (year = 2024)"), - Arguments.of("video AND year:1932", "video and year == 1932"), + Arguments.of("all:video AND year:1932", "video and year == 1932"), Arguments.of("title:neighbou?r", "title =neighbou?r"), Arguments.of("abstract:model\\{1,2\\}ing", "abstract = model{1,2}ing"), Arguments.of("all:* AND -title:chocolate", "title != chocolate"), From 699e9216929846508e3d7522d8c4060b0d1a541d Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Tue, 27 Aug 2024 09:31:42 +0200 Subject: [PATCH 164/256] Add comment for alternative implementation Co-authored-by: Loay Ghreeb <52158423+LoayGhreeb@users.noreply.github.com> --- .../java/org/jabref/migrations/SearchToLuceneMigrationTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/org/jabref/migrations/SearchToLuceneMigrationTest.java b/src/test/java/org/jabref/migrations/SearchToLuceneMigrationTest.java index ea1a67789a6..ca8255e2c1b 100644 --- a/src/test/java/org/jabref/migrations/SearchToLuceneMigrationTest.java +++ b/src/test/java/org/jabref/migrations/SearchToLuceneMigrationTest.java @@ -12,6 +12,7 @@ class SearchToLuceneMigrationTest { public static Stream transformationNormal() { return Stream.of( + // If "all:" should not be added, see e46e0a23d7b9526bd449cbecfb189ae6dbc40a28 for a fix Arguments.of("all:chocolate", "chocolate"), Arguments.of("title:chocolate", "title=chocolate"), From f671f088b4fdb5b4cc2b13d0b5984bffedba7037 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Tue, 27 Aug 2024 09:32:02 +0200 Subject: [PATCH 165/256] Mark library tab changed after migration Co-authored-by: Loay Ghreeb <52158423+LoayGhreeb@users.noreply.github.com> --- src/main/java/org/jabref/gui/LibraryTab.java | 8 +++++--- .../jabref/gui/frame/JabRefFrameViewModel.java | 18 +++++++++++++----- .../importer/actions/OpenDatabaseAction.java | 9 +++++---- 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/jabref/gui/LibraryTab.java b/src/main/java/org/jabref/gui/LibraryTab.java index f8e81270cab..a9cc84258da 100644 --- a/src/main/java/org/jabref/gui/LibraryTab.java +++ b/src/main/java/org/jabref/gui/LibraryTab.java @@ -279,10 +279,12 @@ private void onDatabaseLoadingStarted() { } private void onDatabaseLoadingSucceed(ParserResult result) { - BibDatabaseContext context = result.getDatabaseContext(); - OpenDatabaseAction.performPostOpenActions(result, dialogService, preferencesService); + boolean migrationPerformed = OpenDatabaseAction.performPostOpenActions(result, dialogService, preferencesService); + if (migrationPerformed) { + this.markBaseChanged(); + } - setDatabaseContext(context); + setDatabaseContext(result.getDatabaseContext()); LOGGER.trace("loading.set(false);"); loading.set(false); diff --git a/src/main/java/org/jabref/gui/frame/JabRefFrameViewModel.java b/src/main/java/org/jabref/gui/frame/JabRefFrameViewModel.java index 502f3d53447..4557ac623f6 100644 --- a/src/main/java/org/jabref/gui/frame/JabRefFrameViewModel.java +++ b/src/main/java/org/jabref/gui/frame/JabRefFrameViewModel.java @@ -251,10 +251,7 @@ private void openDatabases(List parserResults) { for (ParserResult parserResult : parserResults) { if (parserResult.hasWarnings()) { ParserResultWarningDialog.showParserResultWarningDialog(parserResult, dialogService); - tabContainer.getLibraryTabs().stream() - .filter(tab -> parserResult.getDatabase().equals(tab.getDatabase())) - .findAny() - .ifPresent(tabContainer::showLibraryTab); + getLibraryTab(parserResult).ifPresent(tabContainer::showLibraryTab); } } @@ -263,11 +260,22 @@ private void openDatabases(List parserResults) { // if we found new entry types that can be imported, or checking // if the database contents should be modified due to new features // in this version of JabRef. - parserResults.forEach(pr -> OpenDatabaseAction.performPostOpenActions(pr, dialogService, preferences)); + parserResults.forEach(pr -> { + boolean migrationPerformed = OpenDatabaseAction.performPostOpenActions(pr, dialogService, preferences); + if (migrationPerformed) { + getLibraryTab(pr).ifPresent(LibraryTab::markBaseChanged); + } + }); LOGGER.debug("Finished adding panels"); } + private Optional getLibraryTab(ParserResult parserResult) { + return tabContainer.getLibraryTabs().stream() + .filter(tab -> parserResult.getDatabase().equals(tab.getDatabase())) + .findAny(); + } + /** * Should be called when a user asks JabRef at the command line * i) to import a file or diff --git a/src/main/java/org/jabref/gui/importer/actions/OpenDatabaseAction.java b/src/main/java/org/jabref/gui/importer/actions/OpenDatabaseAction.java index d278051a25a..916633b211a 100644 --- a/src/main/java/org/jabref/gui/importer/actions/OpenDatabaseAction.java +++ b/src/main/java/org/jabref/gui/importer/actions/OpenDatabaseAction.java @@ -94,16 +94,17 @@ public OpenDatabaseAction(LibraryTabContainer tabContainer, } /** - * Go through the list of post open actions, and perform those that need to be performed. - * - * @param result The result of the BIB file parse operation. + * @return true if any post open action was performed. False otherwise. */ - public static void performPostOpenActions(ParserResult result, DialogService dialogService, PreferencesService preferencesService) { + public static boolean performPostOpenActions(ParserResult result, DialogService dialogService, PreferencesService preferencesService) { + boolean anyActionPerformed = false; for (GUIPostOpenAction action : OpenDatabaseAction.POST_OPEN_ACTIONS) { if (action.isActionNecessary(result, preferencesService)) { action.performAction(result, dialogService, preferencesService); + anyActionPerformed = true; } } + return anyActionPerformed; } @Override From 3e6a179cc0241c0b5e8f01692a83b0030dcc21c5 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Tue, 27 Aug 2024 09:36:44 +0200 Subject: [PATCH 166/256] Add another test for regular expression Co-authored-by: Loay Ghreeb <52158423+LoayGhreeb@users.noreply.github.com> --- .../org/jabref/migrations/SearchToLuceneMigrationTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/jabref/migrations/SearchToLuceneMigrationTest.java b/src/test/java/org/jabref/migrations/SearchToLuceneMigrationTest.java index ca8255e2c1b..4cefc0b5c08 100644 --- a/src/test/java/org/jabref/migrations/SearchToLuceneMigrationTest.java +++ b/src/test/java/org/jabref/migrations/SearchToLuceneMigrationTest.java @@ -53,7 +53,8 @@ void transformationNormal(String expected, String query) { public static Stream transformationRegularExpression() { return Stream.of( Arguments.of("all:* AND -groups:/.+/", "groups != .+"), - Arguments.of("( all:* AND -groups:/.+/ ) AND ( all:* AND -readstatus:/.+/ )", "groups != .+ and readstatus != .+") + Arguments.of("( all:* AND -groups:/.+/ ) AND ( all:* AND -readstatus:/.+/ )", "groups != .+ and readstatus != .+"), + Arguments.of("author:/(John|Doe).+(John|Doe)/", "author = \"(John|Doe).+(John|Doe)\"") ); } From 2018ceaff890a8625001c8e8f041d3d554ecd9c6 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Tue, 27 Aug 2024 09:51:31 +0200 Subject: [PATCH 167/256] Small fixes --- .../java/org/jabref/gui/importer/actions/GUIPostOpenAction.java | 2 -- .../gui/importer/actions/SearchGroupsMigrationAction.java | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/org/jabref/gui/importer/actions/GUIPostOpenAction.java b/src/main/java/org/jabref/gui/importer/actions/GUIPostOpenAction.java index 3b392a1bc5a..0e8fa96fd3b 100644 --- a/src/main/java/org/jabref/gui/importer/actions/GUIPostOpenAction.java +++ b/src/main/java/org/jabref/gui/importer/actions/GUIPostOpenAction.java @@ -31,8 +31,6 @@ public interface GUIPostOpenAction { * Note: if several such methods need to be called sequentially, it is * important that all implementations of this method do not return * until the operation is finished. - * - * @param pr The result of the BIB parse operation. */ void performAction(ParserResult pr, DialogService dialogService, PreferencesService preferencesService); } diff --git a/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java b/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java index 95e7501c0b9..1f4de228f28 100644 --- a/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java +++ b/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java @@ -45,7 +45,7 @@ private boolean groupOrSubgroupIsSearchGroup(GroupTreeNode groupTreeNode) { public void performAction(ParserResult parserResult, DialogService dialogService, PreferencesService preferencesService) { if (!dialogService.showConfirmationDialogAndWait(Localization.lang("Search groups migration of %0", parserResult.getPath().map(Path::toString).orElse("")), Localization.lang("The search groups syntax is outdated. Do you want to migrate to the new syntax?"), - Localization.lang("Migrate"), Localization.lang("Cancel"))) { + Localization.lang("Migrate"), Localization.lang("Keep as is"))) { return; } From 04105a870134960314e932c5a94b2bffb9b15861 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Tue, 27 Aug 2024 09:55:37 +0200 Subject: [PATCH 168/256] Fix markBaseChanged --- src/main/java/org/jabref/gui/LibraryTab.java | 4 ++-- .../java/org/jabref/gui/frame/JabRefFrameViewModel.java | 4 ++-- .../jabref/gui/importer/actions/OpenDatabaseAction.java | 8 +------- .../gui/importer/actions/SearchGroupsMigrationAction.java | 2 +- src/main/java/org/jabref/logic/importer/ParserResult.java | 2 +- .../migrations/MergeReviewIntoCommentMigration.java | 6 +++++- 6 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/jabref/gui/LibraryTab.java b/src/main/java/org/jabref/gui/LibraryTab.java index a9cc84258da..2545add20d4 100644 --- a/src/main/java/org/jabref/gui/LibraryTab.java +++ b/src/main/java/org/jabref/gui/LibraryTab.java @@ -279,8 +279,8 @@ private void onDatabaseLoadingStarted() { } private void onDatabaseLoadingSucceed(ParserResult result) { - boolean migrationPerformed = OpenDatabaseAction.performPostOpenActions(result, dialogService, preferencesService); - if (migrationPerformed) { + OpenDatabaseAction.performPostOpenActions(result, dialogService, preferencesService); + if (result.getChangedOnMigration()) { this.markBaseChanged(); } diff --git a/src/main/java/org/jabref/gui/frame/JabRefFrameViewModel.java b/src/main/java/org/jabref/gui/frame/JabRefFrameViewModel.java index 4557ac623f6..924a8baa675 100644 --- a/src/main/java/org/jabref/gui/frame/JabRefFrameViewModel.java +++ b/src/main/java/org/jabref/gui/frame/JabRefFrameViewModel.java @@ -261,8 +261,8 @@ private void openDatabases(List parserResults) { // if the database contents should be modified due to new features // in this version of JabRef. parserResults.forEach(pr -> { - boolean migrationPerformed = OpenDatabaseAction.performPostOpenActions(pr, dialogService, preferences); - if (migrationPerformed) { + OpenDatabaseAction.performPostOpenActions(pr, dialogService, preferences); + if (pr.getChangedOnMigration()) { getLibraryTab(pr).ifPresent(LibraryTab::markBaseChanged); } }); diff --git a/src/main/java/org/jabref/gui/importer/actions/OpenDatabaseAction.java b/src/main/java/org/jabref/gui/importer/actions/OpenDatabaseAction.java index 916633b211a..c01e63c1570 100644 --- a/src/main/java/org/jabref/gui/importer/actions/OpenDatabaseAction.java +++ b/src/main/java/org/jabref/gui/importer/actions/OpenDatabaseAction.java @@ -93,18 +93,12 @@ public OpenDatabaseAction(LibraryTabContainer tabContainer, this.taskExecutor = taskExecutor; } - /** - * @return true if any post open action was performed. False otherwise. - */ - public static boolean performPostOpenActions(ParserResult result, DialogService dialogService, PreferencesService preferencesService) { - boolean anyActionPerformed = false; + public static void performPostOpenActions(ParserResult result, DialogService dialogService, PreferencesService preferencesService) { for (GUIPostOpenAction action : OpenDatabaseAction.POST_OPEN_ACTIONS) { if (action.isActionNecessary(result, preferencesService)) { action.performAction(result, dialogService, preferencesService); - anyActionPerformed = true; } } - return anyActionPerformed; } @Override diff --git a/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java b/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java index 1f4de228f28..ed720a8f41b 100644 --- a/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java +++ b/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java @@ -50,8 +50,8 @@ public void performAction(ParserResult parserResult, DialogService dialogService } parserResult.getMetaData().getGroups().ifPresent(this::migrateGroups); - parserResult.getMetaData().setGroupSearchSyntaxVersion(SearchGroup.VERSION_6_0_ALPHA); + parserResult.setChangedOnMigration(true); } private void migrateGroups(GroupTreeNode node) { diff --git a/src/main/java/org/jabref/logic/importer/ParserResult.java b/src/main/java/org/jabref/logic/importer/ParserResult.java index c5f61ef3513..60dd5af278a 100644 --- a/src/main/java/org/jabref/logic/importer/ParserResult.java +++ b/src/main/java/org/jabref/logic/importer/ParserResult.java @@ -153,7 +153,7 @@ public boolean isEmpty() { this.getMetaData().isEmpty(); } - public boolean wasChangedOnMigration() { + public boolean getChangedOnMigration() { return changedOnMigration; } diff --git a/src/main/java/org/jabref/migrations/MergeReviewIntoCommentMigration.java b/src/main/java/org/jabref/migrations/MergeReviewIntoCommentMigration.java index f7d86408070..7b08a38e52e 100644 --- a/src/main/java/org/jabref/migrations/MergeReviewIntoCommentMigration.java +++ b/src/main/java/org/jabref/migrations/MergeReviewIntoCommentMigration.java @@ -26,6 +26,10 @@ public void performMigration(ParserResult parserResult) { */ List entries = Objects.requireNonNull(parserResult).getDatabase().getEntries(); + if (!entries.isEmpty()) { + parserResult.setChangedOnMigration(true); + } + entries.stream() .filter(MergeReviewIntoCommentMigration::hasReviewField) .filter(entry -> !MergeReviewIntoCommentMigration.hasCommentField(entry)) @@ -64,7 +68,7 @@ private String mergeCommentFieldIfPresent(BibEntry entry, String review) { private void migrate(BibEntry entry, ParserResult parserResult) { if (hasReviewField(entry)) { updateFields(entry, mergeCommentFieldIfPresent(entry, entry.getField(StandardField.REVIEW).get())); - parserResult.wasChangedOnMigration(); + parserResult.setChangedOnMigration(true); } } From cc3c002dcdb50b6f03e5ae7120a0e01a6a749195 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Wed, 28 Aug 2024 04:04:17 +0300 Subject: [PATCH 169/256] Fix adding new entries did not update MatchCategory --- .../gui/maintable/MainTableDataModel.java | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index 95c0fccf3fd..6d150ee892d 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -207,7 +207,17 @@ class LuceneIndexListener { public void listen(IndexAddedOrUpdatedEvent indexAddedOrUpdatedEvent) { indexAddedOrUpdatedEvent.entries().forEach(entry -> { BackgroundTask.wrap(() -> { - int index = allEntries.indexOf(entry); + // Find the index of the entry in the list. + // The indexOf() method is not used because it relies on the equals(), + // which can return the wrong index if some entries are equal but different instances. + // For example, two different instances of an empty entry would be treated as equal by .equals(). + int index = -1; + for (int i = 0; i < allEntries.size(); i++) { + if (allEntries.get(i) == entry) { + index = i; + break; + } + } if (index >= 0) { BibEntryTableViewModel viewModel = entriesViewModel.get(index); boolean isFloatingMode = searchPreferences.getSearchDisplayMode() == SearchDisplayMode.FLOAT; @@ -230,7 +240,11 @@ public void listen(IndexAddedOrUpdatedEvent indexAddedOrUpdatedEvent) { updateEntryGroupMatch(viewModel, groupsMatcher, groupsPreferences.getGroupViewMode().contains(GroupViewMode.INVERT), !groupsPreferences.getGroupViewMode().contains(GroupViewMode.FILTER)); } return index; - }).onSuccess(index -> FilteredListProxy.refilterListReflection(entriesFiltered, index, index + 1)).executeWith(taskExecutor); + }).onSuccess(index -> { + if (index >= 0) { + FilteredListProxy.refilterListReflection(entriesFiltered, index, index + 1); + } + }).executeWith(taskExecutor); }); } From 909afe490adacfd57ad3339757ab5121aa344890 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Wed, 28 Aug 2024 07:33:54 +0300 Subject: [PATCH 170/256] Fix searching for Non-ASCII characters --- .../search/indexing/BibFieldsIndexer.java | 8 +++- .../org/jabref/model/search/SearchQuery.java | 10 ++--- .../logic/search/LuceneQueryParserTest.java | 38 +++++++++++++++++++ 3 files changed, 49 insertions(+), 7 deletions(-) create mode 100644 src/test/java/org/jabref/logic/search/LuceneQueryParserTest.java diff --git a/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java b/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java index 5a94d152959..f3d76248c7f 100644 --- a/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java @@ -5,7 +5,9 @@ import java.util.Map; import org.jabref.gui.util.BackgroundTask; +import org.jabref.logic.cleanup.Formatter; import org.jabref.logic.l10n.Localization; +import org.jabref.logic.layout.format.LatexToUnicodeFormatter; import org.jabref.logic.util.HeadlessExecutorService; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; @@ -27,6 +29,7 @@ public class BibFieldsIndexer implements LuceneIndexer { private static final Logger LOGGER = LoggerFactory.getLogger(BibFieldsIndexer.class); + private static final Formatter FORMATTER = new LatexToUnicodeFormatter(); private final BibDatabaseContext databaseContext; private final String libraryName; private final Directory indexDirectory; @@ -82,8 +85,9 @@ private void addToIndex(BibEntry bibEntry) { StringBuilder allFields = new StringBuilder(bibEntry.getType().getName()); for (Map.Entry mapEntry : bibEntry.getFieldMap().entrySet()) { - document.add(new TextField(mapEntry.getKey().getName(), mapEntry.getValue(), storeDisabled)); - allFields.append('\n').append(mapEntry.getValue()); + String value = FORMATTER.format(mapEntry.getValue()); + document.add(new TextField(mapEntry.getKey().getName(), value, storeDisabled)); + allFields.append('\n').append(value); } document.add(new TextField(SearchFieldConstants.DEFAULT_FIELD.toString(), allFields.toString(), storeDisabled)); indexWriter.addDocument(document); diff --git a/src/main/java/org/jabref/model/search/SearchQuery.java b/src/main/java/org/jabref/model/search/SearchQuery.java index 4ef1f6a0d46..674c502d3df 100644 --- a/src/main/java/org/jabref/model/search/SearchQuery.java +++ b/src/main/java/org/jabref/model/search/SearchQuery.java @@ -12,16 +12,17 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import org.jabref.logic.cleanup.Formatter; +import org.jabref.logic.layout.format.LatexToUnicodeFormatter; + import org.apache.lucene.queryparser.classic.MultiFieldQueryParser; import org.apache.lucene.queryparser.classic.ParseException; import org.apache.lucene.search.Query; import org.apache.lucene.search.highlight.QueryTermExtractor; import org.apache.lucene.search.highlight.WeightedTerm; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class SearchQuery { - + private static final Formatter FORMATTER = new LatexToUnicodeFormatter(); /** * The mode of escaping special characters in regular expressions */ @@ -59,7 +60,6 @@ String format(String regex) { abstract String format(String regex); } - private final static Logger LOGGER = LoggerFactory.getLogger(SearchQuery.class); protected final String query; protected Query parsedQuery; protected String parseError; @@ -87,7 +87,7 @@ public SearchQuery(String query, EnumSet searchFlags) { queryParser.setAllowLeadingWildcard(true); try { - parsedQuery = queryParser.parse(query); + parsedQuery = queryParser.parse(FORMATTER.format(query)); parseError = null; } catch (ParseException e) { parsedQuery = null; diff --git a/src/test/java/org/jabref/logic/search/LuceneQueryParserTest.java b/src/test/java/org/jabref/logic/search/LuceneQueryParserTest.java new file mode 100644 index 00000000000..9a471e305bc --- /dev/null +++ b/src/test/java/org/jabref/logic/search/LuceneQueryParserTest.java @@ -0,0 +1,38 @@ +package org.jabref.logic.search; + +import java.util.EnumSet; +import java.util.stream.Stream; + +import org.jabref.model.search.SearchFlags; +import org.jabref.model.search.SearchQuery; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class LuceneQueryParserTest { + + public static Stream searchQuires() { + return Stream.of( + // unicode + Arguments.of("preissinger", "preißinger"), + Arguments.of("jesus", "jesús"), + Arguments.of("breitenbucher", "breitenbücher"), + + // latex + Arguments.of("preissinger", "prei{\\ss}inger"), + Arguments.of("jesus", "jes{\\'{u}}s"), + Arguments.of("breitenbucher", "breitenb{\\\"{u}}cher") + ); + } + + @ParameterizedTest + @MethodSource + void searchQuires(String expected, String query) { + expected = "(all:" + expected + ")^4.0"; + SearchQuery searchQuery = new SearchQuery(query, EnumSet.noneOf(SearchFlags.class)); + assertEquals(expected, searchQuery.getParsedQuery().toString()); + } +} From bfa13565fd3affa44b4b539de15c8c79b5a9542c Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Wed, 28 Aug 2024 09:28:48 +0300 Subject: [PATCH 171/256] Fix escaping special characters Use WhitespaceTokenizer instead of StandardTokenizer https://stackoverflow.com/a/6119584/21694752 --- .../indexing/DefaultLinkedFilesIndexer.java | 2 +- .../migrations/SearchToLuceneVisitor.java | 13 +------- .../jabref/model/search/NGramAnalyzer.java | 4 +-- .../model/search/SearchFieldConstants.java | 2 +- .../org/jabref/model/search/SearchQuery.java | 2 +- .../org/jabref/model/search/SearchResult.java | 2 +- ...dAnalyzer.java => WhitespaceAnalyzer.java} | 8 ++--- .../logic/search/LuceneQueryParserTest.java | 33 +++++++++++-------- .../SearchToLuceneMigrationTest.java | 2 ++ 9 files changed, 33 insertions(+), 35 deletions(-) rename src/main/java/org/jabref/model/search/{StandardAnalyzer.java => WhitespaceAnalyzer.java} (77%) diff --git a/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java b/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java index 98bc712ce7f..9d4c6ad5b2e 100644 --- a/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java @@ -65,7 +65,7 @@ public DefaultLinkedFilesIndexer(BibDatabaseContext databaseContext, FilePrefere this.indexedFiles = new ConcurrentHashMap<>(); indexDirectoryPath = databaseContext.getFulltextIndexPath(); - IndexWriterConfig config = new IndexWriterConfig(SearchFieldConstants.Standard_ANALYZER); + IndexWriterConfig config = new IndexWriterConfig(SearchFieldConstants.Whitespace_ANALYZER); if ("unsaved".equals(indexDirectoryPath.getFileName().toString())) { config.setOpenMode(IndexWriterConfig.OpenMode.CREATE); indexDirectoryPath = indexDirectoryPath.resolveSibling("unsaved" + NUMBER_OF_UNSAVED_LIBRARIES++); diff --git a/src/main/java/org/jabref/migrations/SearchToLuceneVisitor.java b/src/main/java/org/jabref/migrations/SearchToLuceneVisitor.java index 56ad9e743f9..5ddaa260d8b 100644 --- a/src/main/java/org/jabref/migrations/SearchToLuceneVisitor.java +++ b/src/main/java/org/jabref/migrations/SearchToLuceneVisitor.java @@ -30,8 +30,6 @@ public class SearchToLuceneVisitor extends SearchBaseVisitor { private final boolean isRegularExpression; - private boolean isNegation = false; - public SearchToLuceneVisitor(boolean isRegularExpression) { this.isRegularExpression = isRegularExpression; } @@ -45,7 +43,6 @@ public QueryNode visitStart(SearchParser.StartContext ctx) { // See https://github.com/LoayGhreeb/lucene-mwe/issues/1 for more details if (result instanceof ModifierQueryNode modifierQueryNode) { if (modifierQueryNode.getModifier() == ModifierQueryNode.Modifier.MOD_NOT) { - isNegation = true; return new AndQueryNode(List.of(new FieldQueryNode(SearchFieldConstants.DEFAULT_FIELD.toString(), "*", 0, 0), modifierQueryNode)); } } @@ -53,7 +50,6 @@ public QueryNode visitStart(SearchParser.StartContext ctx) { // User might search for NOT this AND NOT that - we also need to convert properly if (result instanceof AndQueryNode andQueryNode) { if (andQueryNode.getChildren().stream().allMatch(child -> child instanceof ModifierQueryNode modifierQueryNode && modifierQueryNode.getModifier() == ModifierQueryNode.Modifier.MOD_NOT)) { - isNegation = true; List children = andQueryNode.getChildren().stream() // prepend "all:* AND" to each child .map(child -> new AndQueryNode(List.of(new FieldQueryNode(SearchFieldConstants.DEFAULT_FIELD.toString(), "*", 0, 0), child))) @@ -108,7 +104,7 @@ public QueryNode visitComparison(SearchParser.ComparisonContext context) { context.EQUAL() != null || context.EEQUAL() != null) { // exact match if (LOGGER.isDebugEnabled() && context.EEQUAL() != null) { - LOGGER.warn("Exact match is currently supported by Lucene, using contains instead. Term: {}", context.getText()); + LOGGER.warn("Exact match is currently not supported by Lucene, using contains instead. Term: {}", context.getText()); } return getFieldQueryNode(field, right, startIndex, stopIndex); } @@ -139,11 +135,4 @@ private QueryNode getFieldQueryNode(String field, String term, int startIndex, i } return new FieldQueryNode(field, term, startIndex, stopIndex); } - - /** - * Returns whether the search query is a negation (and was patched to be a filter). - */ - public boolean isNegation() { - return this.isNegation; - } } diff --git a/src/main/java/org/jabref/model/search/NGramAnalyzer.java b/src/main/java/org/jabref/model/search/NGramAnalyzer.java index d226b887d4b..206e1568619 100644 --- a/src/main/java/org/jabref/model/search/NGramAnalyzer.java +++ b/src/main/java/org/jabref/model/search/NGramAnalyzer.java @@ -6,9 +6,9 @@ import org.apache.lucene.analysis.StopFilter; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.analysis.core.WhitespaceTokenizer; import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter; import org.apache.lucene.analysis.ngram.NGramTokenFilter; -import org.apache.lucene.analysis.standard.StandardTokenizer; public class NGramAnalyzer extends Analyzer { private final int minGram; @@ -23,7 +23,7 @@ public NGramAnalyzer(int minGram, int maxGram, CharArraySet stopWords) { @Override protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer source = new StandardTokenizer(); + Tokenizer source = new WhitespaceTokenizer(); TokenStream result = new LowerCaseFilter(source); result = new StopFilter(result, stopWords); result = new ASCIIFoldingFilter(result); diff --git a/src/main/java/org/jabref/model/search/SearchFieldConstants.java b/src/main/java/org/jabref/model/search/SearchFieldConstants.java index 77b5f7cb957..ae37feb9475 100644 --- a/src/main/java/org/jabref/model/search/SearchFieldConstants.java +++ b/src/main/java/org/jabref/model/search/SearchFieldConstants.java @@ -17,7 +17,7 @@ public enum SearchFieldConstants { PAGE_NUMBER("pageNumber"), MODIFIED("modified"); - public static final Analyzer Standard_ANALYZER = new StandardAnalyzer(EnglishAnalyzer.ENGLISH_STOP_WORDS_SET); + public static final Analyzer Whitespace_ANALYZER = new WhitespaceAnalyzer(EnglishAnalyzer.ENGLISH_STOP_WORDS_SET); public static final Analyzer NGram_Analyzer_For_INDEXING = new NGramAnalyzer(1, Integer.MAX_VALUE, EnglishAnalyzer.ENGLISH_STOP_WORDS_SET); public static final List PDF_FIELDS = List.of(PATH.toString(), CONTENT.toString(), ANNOTATIONS.toString()); private final String field; diff --git a/src/main/java/org/jabref/model/search/SearchQuery.java b/src/main/java/org/jabref/model/search/SearchQuery.java index 674c502d3df..2ea0840de2c 100644 --- a/src/main/java/org/jabref/model/search/SearchQuery.java +++ b/src/main/java/org/jabref/model/search/SearchQuery.java @@ -83,7 +83,7 @@ public SearchQuery(String query, EnumSet searchFlags) { query = '/' + query + '/'; } - MultiFieldQueryParser queryParser = new MultiFieldQueryParser(fieldsToSearchArray, SearchFieldConstants.Standard_ANALYZER, boosts); + MultiFieldQueryParser queryParser = new MultiFieldQueryParser(fieldsToSearchArray, SearchFieldConstants.Whitespace_ANALYZER, boosts); queryParser.setAllowLeadingWildcard(true); try { diff --git a/src/main/java/org/jabref/model/search/SearchResult.java b/src/main/java/org/jabref/model/search/SearchResult.java index c2a76c8fc63..98c0bac5410 100644 --- a/src/main/java/org/jabref/model/search/SearchResult.java +++ b/src/main/java/org/jabref/model/search/SearchResult.java @@ -76,7 +76,7 @@ public int getPageNumber() { } private static List getHighlighterFragments(Highlighter highlighter, SearchFieldConstants field, String content) { - try (TokenStream contentStream = SearchFieldConstants.Standard_ANALYZER.tokenStream(field.toString(), content)) { + try (TokenStream contentStream = SearchFieldConstants.Whitespace_ANALYZER.tokenStream(field.toString(), content)) { TextFragment[] frags = highlighter.getBestTextFragments(contentStream, content, true, 10); return Arrays.stream(frags).map(TextFragment::toString).toList(); } catch (IOException | InvalidTokenOffsetsException e) { diff --git a/src/main/java/org/jabref/model/search/StandardAnalyzer.java b/src/main/java/org/jabref/model/search/WhitespaceAnalyzer.java similarity index 77% rename from src/main/java/org/jabref/model/search/StandardAnalyzer.java rename to src/main/java/org/jabref/model/search/WhitespaceAnalyzer.java index cf50b518ced..c25b1852680 100644 --- a/src/main/java/org/jabref/model/search/StandardAnalyzer.java +++ b/src/main/java/org/jabref/model/search/WhitespaceAnalyzer.java @@ -6,18 +6,18 @@ import org.apache.lucene.analysis.StopFilter; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.analysis.core.WhitespaceTokenizer; import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter; -import org.apache.lucene.analysis.standard.StandardTokenizer; -public class StandardAnalyzer extends Analyzer { +public class WhitespaceAnalyzer extends Analyzer { private final CharArraySet stopWords; - public StandardAnalyzer(CharArraySet stopWords) { + public WhitespaceAnalyzer(CharArraySet stopWords) { this.stopWords = stopWords; } @Override protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer source = new StandardTokenizer(); + Tokenizer source = new WhitespaceTokenizer(); TokenStream result = new LowerCaseFilter(source); result = new StopFilter(result, stopWords); result = new ASCIIFoldingFilter(result); diff --git a/src/test/java/org/jabref/logic/search/LuceneQueryParserTest.java b/src/test/java/org/jabref/logic/search/LuceneQueryParserTest.java index 9a471e305bc..7580d9f9899 100644 --- a/src/test/java/org/jabref/logic/search/LuceneQueryParserTest.java +++ b/src/test/java/org/jabref/logic/search/LuceneQueryParserTest.java @@ -1,11 +1,14 @@ package org.jabref.logic.search; -import java.util.EnumSet; import java.util.stream.Stream; -import org.jabref.model.search.SearchFlags; -import org.jabref.model.search.SearchQuery; +import org.jabref.logic.cleanup.Formatter; +import org.jabref.logic.layout.format.LatexToUnicodeFormatter; +import org.jabref.model.search.SearchFieldConstants; +import org.apache.lucene.analysis.core.WhitespaceAnalyzer; +import org.apache.lucene.queryparser.classic.ParseException; +import org.apache.lucene.queryparser.classic.QueryParser; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -13,26 +16,30 @@ import static org.junit.jupiter.api.Assertions.assertEquals; public class LuceneQueryParserTest { + private static final Formatter FORMATTER = new LatexToUnicodeFormatter(); public static Stream searchQuires() { return Stream.of( // unicode - Arguments.of("preissinger", "preißinger"), - Arguments.of("jesus", "jesús"), - Arguments.of("breitenbucher", "breitenbücher"), + Arguments.of("all:preissinger", "preißinger"), + Arguments.of("all:jesus", "jesús"), + Arguments.of("all:breitenbucher", "breitenbücher"), // latex - Arguments.of("preissinger", "prei{\\ss}inger"), - Arguments.of("jesus", "jes{\\'{u}}s"), - Arguments.of("breitenbucher", "breitenb{\\\"{u}}cher") + Arguments.of("all:preissinger", "prei{\\ss}inger"), + Arguments.of("all:jesus", "jes{\\'{u}}s"), + Arguments.of("all:breitenbucher", "breitenb{\\\"{u}}cher"), + + Arguments.of("groups:/exclude", "groups:\\/exclude") ); } @ParameterizedTest @MethodSource - void searchQuires(String expected, String query) { - expected = "(all:" + expected + ")^4.0"; - SearchQuery searchQuery = new SearchQuery(query, EnumSet.noneOf(SearchFlags.class)); - assertEquals(expected, searchQuery.getParsedQuery().toString()); + void searchQuires(String expected, String query) throws ParseException { + QueryParser parser = new QueryParser(SearchFieldConstants.DEFAULT_FIELD.toString(), new WhitespaceAnalyzer()); + query = FORMATTER.format(query); + String result = parser.parse(query).toString(); + assertEquals(expected, result); } } diff --git a/src/test/java/org/jabref/migrations/SearchToLuceneMigrationTest.java b/src/test/java/org/jabref/migrations/SearchToLuceneMigrationTest.java index 4cefc0b5c08..6b6a2732e1c 100644 --- a/src/test/java/org/jabref/migrations/SearchToLuceneMigrationTest.java +++ b/src/test/java/org/jabref/migrations/SearchToLuceneMigrationTest.java @@ -17,6 +17,7 @@ public static Stream transformationNormal() { Arguments.of("title:chocolate", "title=chocolate"), Arguments.of("title:chocolate OR author:smith", "title = chocolate or author = smith"), + Arguments.of("groups:\\/exclude", "groups= /exclude"), Arguments.of("title:chocolate AND author:smith", "title = \"chocolate\" AND author = \"smith\""), Arguments.of("title:chocolate AND author:smith", "title contains \"chocolate\" AND author matches \"smith\""), Arguments.of("( title:chocolate ) OR ( author:smith )", "(title == chocolate) or (author == smith)"), @@ -26,6 +27,7 @@ public static Stream transformationNormal() { Arguments.of("abstract:model\\{1,2\\}ing", "abstract = model{1,2}ing"), Arguments.of("all:* AND -title:chocolate", "title != chocolate"), Arguments.of("all:* AND -title:chocolate", "not title contains chocolate"), + Arguments.of("groups=:\\:paywall AND -file=\"\" AND -groups=\\/exclude", "groups=:paywall and file!=\"\" and groups!=/exclude"), // not converted, because not working in JabRef 5.x // Arguments.of("title:\"image processing\" OR keywords:\"image processing\"", "title|keywords = \"image processing\""), From ec5bfbbbe1bcced8868f87cc26de3a00a33bb498 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Wed, 28 Aug 2024 10:14:59 +0300 Subject: [PATCH 172/256] Fix tests Co-Authored-By: Oliver Kopp --- .../migrations/SearchToLuceneVisitor.java | 20 ++++++++++++++----- .../jabref/model/search/NGramAnalyzer.java | 2 ++ .../logic/search/LuceneQueryParserTest.java | 3 +-- .../SearchToLuceneMigrationTest.java | 3 ++- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/jabref/migrations/SearchToLuceneVisitor.java b/src/main/java/org/jabref/migrations/SearchToLuceneVisitor.java index 5ddaa260d8b..6e3c75bfcf6 100644 --- a/src/main/java/org/jabref/migrations/SearchToLuceneVisitor.java +++ b/src/main/java/org/jabref/migrations/SearchToLuceneVisitor.java @@ -106,13 +106,23 @@ public QueryNode visitComparison(SearchParser.ComparisonContext context) { if (LOGGER.isDebugEnabled() && context.EEQUAL() != null) { LOGGER.warn("Exact match is currently not supported by Lucene, using contains instead. Term: {}", context.getText()); } - return getFieldQueryNode(field, right, startIndex, stopIndex); + return getFieldQueryNode(field, right, startIndex, stopIndex, false); } assert (context.NEQUAL() != null); - return new ModifierQueryNode(getFieldQueryNode(field, right, startIndex, stopIndex), ModifierQueryNode.Modifier.MOD_NOT); + + // Treating of "wrong" query field != "". This did not work in v5.x, but should work in v6.x + boolean forceRegex; + if (right.isEmpty()) { + forceRegex = true; + right = ".+"; + } else { + forceRegex = false; + } + + return new ModifierQueryNode(getFieldQueryNode(field, right, startIndex, stopIndex, forceRegex), ModifierQueryNode.Modifier.MOD_NOT); } else { - return getFieldQueryNode(SearchFieldConstants.DEFAULT_FIELD.toString(), right, startIndex, stopIndex); + return getFieldQueryNode(SearchFieldConstants.DEFAULT_FIELD.toString(), right, startIndex, stopIndex, false); } } @@ -121,7 +131,7 @@ public QueryNode visitComparison(SearchParser.ComparisonContext context) { * In Lucene, this is represented by a RegexpQueryNode or a FieldQueryNode. * They are created in this class accordingly. */ - private QueryNode getFieldQueryNode(String field, String term, int startIndex, int stopIndex) { + private QueryNode getFieldQueryNode(String field, String term, int startIndex, int stopIndex, boolean forceRegex) { field = switch (field) { case "anyfield" -> SearchFieldConstants.DEFAULT_FIELD.toString(); case "anykeyword" -> StandardField.KEYWORDS.getName(); @@ -129,7 +139,7 @@ private QueryNode getFieldQueryNode(String field, String term, int startIndex, i default -> field; }; - if (isRegularExpression) { + if (isRegularExpression || forceRegex) { // Lucene does a sanity check on the positions, thus we provide other fake positions return new RegexpQueryNode(field, term, 0, term.length()); } diff --git a/src/main/java/org/jabref/model/search/NGramAnalyzer.java b/src/main/java/org/jabref/model/search/NGramAnalyzer.java index 206e1568619..df84d568002 100644 --- a/src/main/java/org/jabref/model/search/NGramAnalyzer.java +++ b/src/main/java/org/jabref/model/search/NGramAnalyzer.java @@ -9,6 +9,7 @@ import org.apache.lucene.analysis.core.WhitespaceTokenizer; import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter; import org.apache.lucene.analysis.ngram.NGramTokenFilter; +import org.apache.lucene.analysis.shingle.ShingleFilter; public class NGramAnalyzer extends Analyzer { private final int minGram; @@ -27,6 +28,7 @@ protected TokenStreamComponents createComponents(String fieldName) { TokenStream result = new LowerCaseFilter(source); result = new StopFilter(result, stopWords); result = new ASCIIFoldingFilter(result); + result = new ShingleFilter(result); result = new NGramTokenFilter(result, minGram, maxGram, true); return new TokenStreamComponents(source, result); } diff --git a/src/test/java/org/jabref/logic/search/LuceneQueryParserTest.java b/src/test/java/org/jabref/logic/search/LuceneQueryParserTest.java index 7580d9f9899..f4b4eb08ad9 100644 --- a/src/test/java/org/jabref/logic/search/LuceneQueryParserTest.java +++ b/src/test/java/org/jabref/logic/search/LuceneQueryParserTest.java @@ -6,7 +6,6 @@ import org.jabref.logic.layout.format.LatexToUnicodeFormatter; import org.jabref.model.search.SearchFieldConstants; -import org.apache.lucene.analysis.core.WhitespaceAnalyzer; import org.apache.lucene.queryparser.classic.ParseException; import org.apache.lucene.queryparser.classic.QueryParser; import org.junit.jupiter.params.ParameterizedTest; @@ -37,7 +36,7 @@ public static Stream searchQuires() { @ParameterizedTest @MethodSource void searchQuires(String expected, String query) throws ParseException { - QueryParser parser = new QueryParser(SearchFieldConstants.DEFAULT_FIELD.toString(), new WhitespaceAnalyzer()); + QueryParser parser = new QueryParser(SearchFieldConstants.DEFAULT_FIELD.toString(), SearchFieldConstants.Whitespace_ANALYZER); query = FORMATTER.format(query); String result = parser.parse(query).toString(); assertEquals(expected, result); diff --git a/src/test/java/org/jabref/migrations/SearchToLuceneMigrationTest.java b/src/test/java/org/jabref/migrations/SearchToLuceneMigrationTest.java index 6b6a2732e1c..8fea29c57c9 100644 --- a/src/test/java/org/jabref/migrations/SearchToLuceneMigrationTest.java +++ b/src/test/java/org/jabref/migrations/SearchToLuceneMigrationTest.java @@ -27,7 +27,8 @@ public static Stream transformationNormal() { Arguments.of("abstract:model\\{1,2\\}ing", "abstract = model{1,2}ing"), Arguments.of("all:* AND -title:chocolate", "title != chocolate"), Arguments.of("all:* AND -title:chocolate", "not title contains chocolate"), - Arguments.of("groups=:\\:paywall AND -file=\"\" AND -groups=\\/exclude", "groups=:paywall and file!=\"\" and groups!=/exclude"), + // https://github.com/JabRef/jabref/issues/11654#issuecomment-2313178736 + Arguments.of("( groups:\\:paywall AND -file:/.+/ ) AND -groups:\\/exclude", "groups=:paywall and file!=\"\" and groups!=/exclude"), // not converted, because not working in JabRef 5.x // Arguments.of("title:\"image processing\" OR keywords:\"image processing\"", "title|keywords = \"image processing\""), From 9efbf5580512d51c97ecb8d85b150cf3b70eaf9d Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Wed, 28 Aug 2024 09:47:06 +0200 Subject: [PATCH 173/256] Add first draft of LatexToUnicodeFoldingFilter Co-authored-by: Loay Ghreeb <52158423+LoayGhreeb@users.noreply.github.com> --- .../search/LatexToUnicodeFoldingFilter.java | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 src/main/java/org/jabref/model/search/LatexToUnicodeFoldingFilter.java diff --git a/src/main/java/org/jabref/model/search/LatexToUnicodeFoldingFilter.java b/src/main/java/org/jabref/model/search/LatexToUnicodeFoldingFilter.java new file mode 100644 index 00000000000..2c80165549e --- /dev/null +++ b/src/main/java/org/jabref/model/search/LatexToUnicodeFoldingFilter.java @@ -0,0 +1,83 @@ +package org.jabref.model.search; + +import java.io.IOException; +import java.util.Arrays; + +import org.jabref.logic.cleanup.Formatter; +import org.jabref.logic.layout.format.LatexToUnicodeFormatter; + +import org.apache.lucene.analysis.TokenFilter; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; + +/** + * @implNote Implementation based on {@link org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter} + */ +public class LatexToUnicodeFoldingFilter extends TokenFilter { + private static final Formatter FORMATTER = new LatexToUnicodeFormatter(); + + private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); + private final PositionIncrementAttribute posIncAttr = + addAttribute(PositionIncrementAttribute.class); + + private State state; + + public LatexToUnicodeFoldingFilter(TokenStream input) { + super(input); + } + + private record FoldingResult(char[] output, int length) { + } + + @Override + public boolean incrementToken() throws IOException { + if (state != null) { + restoreState(state); + posIncAttr.setPositionIncrement(0); + state = null; + return true; + } + if (input.incrementToken()) { + final char[] buffer = termAtt.buffer(); + final int length = termAtt.length(); + FoldingResult foldingResult = foldToUnicode(buffer, length); + termAtt.copyBuffer(foldingResult.output, 0, foldingResult.length); + return true; + } else { + return false; + } + } + + @Override + public void reset() throws IOException { + super.reset(); + state = null; + } + + /** + * @param input The string to fold + * @param length The number of characters in the input string + * @return + */ + public FoldingResult foldToUnicode(char[] input, int length) { + FoldingResult result = foldToUnicode(input, 0, length); + if (result.length != length) { + state = captureState(); + } + return result; + } + + /** + * @param input The characters to fold + * @param inputPos Index of the first character to fold + * @param length The number of characters to fold + */ + public static FoldingResult foldToUnicode(char[] input, int inputPos, int length) { + char[] subArray = Arrays.copyOfRange(input, inputPos, inputPos + length); + String s = new String(subArray); + String result = FORMATTER.format(s); + + return new FoldingResult(result.toCharArray(), result.length()); + } +} From e95674a290c0f508f29773ec534c9704028e5a39 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Wed, 28 Aug 2024 13:01:34 +0300 Subject: [PATCH 174/256] Fix LatexToUnicodeFoldingFilter Co-Authored-By: Oliver Kopp --- .../search/indexing/DefaultLinkedFilesIndexer.java | 2 +- .../{WhitespaceAnalyzer.java => JabRefAnalyzer.java} | 7 ++++--- .../model/search/LatexToUnicodeFoldingFilter.java | 12 ++++++++---- .../jabref/model/search/SearchFieldConstants.java | 2 +- .../java/org/jabref/model/search/SearchQuery.java | 2 +- .../java/org/jabref/model/search/SearchResult.java | 2 +- .../jabref/logic/search/LuceneQueryParserTest.java | 12 ++++-------- src/test/resources/tinylog-test.properties | 1 + 8 files changed, 21 insertions(+), 19 deletions(-) rename src/main/java/org/jabref/model/search/{WhitespaceAnalyzer.java => JabRefAnalyzer.java} (79%) diff --git a/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java b/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java index 9d4c6ad5b2e..80e6045051e 100644 --- a/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java @@ -65,7 +65,7 @@ public DefaultLinkedFilesIndexer(BibDatabaseContext databaseContext, FilePrefere this.indexedFiles = new ConcurrentHashMap<>(); indexDirectoryPath = databaseContext.getFulltextIndexPath(); - IndexWriterConfig config = new IndexWriterConfig(SearchFieldConstants.Whitespace_ANALYZER); + IndexWriterConfig config = new IndexWriterConfig(SearchFieldConstants.JABREF_ANALYZER); if ("unsaved".equals(indexDirectoryPath.getFileName().toString())) { config.setOpenMode(IndexWriterConfig.OpenMode.CREATE); indexDirectoryPath = indexDirectoryPath.resolveSibling("unsaved" + NUMBER_OF_UNSAVED_LIBRARIES++); diff --git a/src/main/java/org/jabref/model/search/WhitespaceAnalyzer.java b/src/main/java/org/jabref/model/search/JabRefAnalyzer.java similarity index 79% rename from src/main/java/org/jabref/model/search/WhitespaceAnalyzer.java rename to src/main/java/org/jabref/model/search/JabRefAnalyzer.java index c25b1852680..317827a3c2d 100644 --- a/src/main/java/org/jabref/model/search/WhitespaceAnalyzer.java +++ b/src/main/java/org/jabref/model/search/JabRefAnalyzer.java @@ -9,18 +9,19 @@ import org.apache.lucene.analysis.core.WhitespaceTokenizer; import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter; -public class WhitespaceAnalyzer extends Analyzer { +public class JabRefAnalyzer extends Analyzer { private final CharArraySet stopWords; - public WhitespaceAnalyzer(CharArraySet stopWords) { + public JabRefAnalyzer(CharArraySet stopWords) { this.stopWords = stopWords; } @Override protected TokenStreamComponents createComponents(String fieldName) { Tokenizer source = new WhitespaceTokenizer(); - TokenStream result = new LowerCaseFilter(source); + TokenStream result = new LatexToUnicodeFoldingFilter(source); result = new StopFilter(result, stopWords); result = new ASCIIFoldingFilter(result); + result = new LowerCaseFilter(result); return new TokenStreamComponents(source, result); } } diff --git a/src/main/java/org/jabref/model/search/LatexToUnicodeFoldingFilter.java b/src/main/java/org/jabref/model/search/LatexToUnicodeFoldingFilter.java index 2c80165549e..f2a1da5313a 100644 --- a/src/main/java/org/jabref/model/search/LatexToUnicodeFoldingFilter.java +++ b/src/main/java/org/jabref/model/search/LatexToUnicodeFoldingFilter.java @@ -10,11 +10,14 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * @implNote Implementation based on {@link org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter} */ -public class LatexToUnicodeFoldingFilter extends TokenFilter { +public final class LatexToUnicodeFoldingFilter extends TokenFilter { + private static final Logger LOGGER = LoggerFactory.getLogger(LatexToUnicodeFoldingFilter.class); private static final Formatter FORMATTER = new LatexToUnicodeFormatter(); private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); @@ -58,12 +61,13 @@ public void reset() throws IOException { /** * @param input The string to fold * @param length The number of characters in the input string - * @return */ public FoldingResult foldToUnicode(char[] input, int length) { FoldingResult result = foldToUnicode(input, 0, length); if (result.length != length) { - state = captureState(); + // ASCIIFoldingFilter does "state = captureState();" + // We do not do anything since the index also contains clean LaTeX only. + // If we capture the state, the result is Synonym(LaTeX, Unicode) } return result; } @@ -77,7 +81,7 @@ public static FoldingResult foldToUnicode(char[] input, int inputPos, int length char[] subArray = Arrays.copyOfRange(input, inputPos, inputPos + length); String s = new String(subArray); String result = FORMATTER.format(s); - + LOGGER.debug("Folding {} to {}", s, result); return new FoldingResult(result.toCharArray(), result.length()); } } diff --git a/src/main/java/org/jabref/model/search/SearchFieldConstants.java b/src/main/java/org/jabref/model/search/SearchFieldConstants.java index ae37feb9475..f704cd082c0 100644 --- a/src/main/java/org/jabref/model/search/SearchFieldConstants.java +++ b/src/main/java/org/jabref/model/search/SearchFieldConstants.java @@ -17,7 +17,7 @@ public enum SearchFieldConstants { PAGE_NUMBER("pageNumber"), MODIFIED("modified"); - public static final Analyzer Whitespace_ANALYZER = new WhitespaceAnalyzer(EnglishAnalyzer.ENGLISH_STOP_WORDS_SET); + public static final Analyzer JABREF_ANALYZER = new JabRefAnalyzer(EnglishAnalyzer.ENGLISH_STOP_WORDS_SET); public static final Analyzer NGram_Analyzer_For_INDEXING = new NGramAnalyzer(1, Integer.MAX_VALUE, EnglishAnalyzer.ENGLISH_STOP_WORDS_SET); public static final List PDF_FIELDS = List.of(PATH.toString(), CONTENT.toString(), ANNOTATIONS.toString()); private final String field; diff --git a/src/main/java/org/jabref/model/search/SearchQuery.java b/src/main/java/org/jabref/model/search/SearchQuery.java index 2ea0840de2c..36363ceee48 100644 --- a/src/main/java/org/jabref/model/search/SearchQuery.java +++ b/src/main/java/org/jabref/model/search/SearchQuery.java @@ -83,7 +83,7 @@ public SearchQuery(String query, EnumSet searchFlags) { query = '/' + query + '/'; } - MultiFieldQueryParser queryParser = new MultiFieldQueryParser(fieldsToSearchArray, SearchFieldConstants.Whitespace_ANALYZER, boosts); + MultiFieldQueryParser queryParser = new MultiFieldQueryParser(fieldsToSearchArray, SearchFieldConstants.JABREF_ANALYZER, boosts); queryParser.setAllowLeadingWildcard(true); try { diff --git a/src/main/java/org/jabref/model/search/SearchResult.java b/src/main/java/org/jabref/model/search/SearchResult.java index 98c0bac5410..67cc6d9eb4d 100644 --- a/src/main/java/org/jabref/model/search/SearchResult.java +++ b/src/main/java/org/jabref/model/search/SearchResult.java @@ -76,7 +76,7 @@ public int getPageNumber() { } private static List getHighlighterFragments(Highlighter highlighter, SearchFieldConstants field, String content) { - try (TokenStream contentStream = SearchFieldConstants.Whitespace_ANALYZER.tokenStream(field.toString(), content)) { + try (TokenStream contentStream = SearchFieldConstants.JABREF_ANALYZER.tokenStream(field.toString(), content)) { TextFragment[] frags = highlighter.getBestTextFragments(contentStream, content, true, 10); return Arrays.stream(frags).map(TextFragment::toString).toList(); } catch (IOException | InvalidTokenOffsetsException e) { diff --git a/src/test/java/org/jabref/logic/search/LuceneQueryParserTest.java b/src/test/java/org/jabref/logic/search/LuceneQueryParserTest.java index f4b4eb08ad9..99cb1222e1d 100644 --- a/src/test/java/org/jabref/logic/search/LuceneQueryParserTest.java +++ b/src/test/java/org/jabref/logic/search/LuceneQueryParserTest.java @@ -2,8 +2,6 @@ import java.util.stream.Stream; -import org.jabref.logic.cleanup.Formatter; -import org.jabref.logic.layout.format.LatexToUnicodeFormatter; import org.jabref.model.search.SearchFieldConstants; import org.apache.lucene.queryparser.classic.ParseException; @@ -15,7 +13,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; public class LuceneQueryParserTest { - private static final Formatter FORMATTER = new LatexToUnicodeFormatter(); public static Stream searchQuires() { return Stream.of( @@ -25,9 +22,9 @@ public static Stream searchQuires() { Arguments.of("all:breitenbucher", "breitenbücher"), // latex - Arguments.of("all:preissinger", "prei{\\ss}inger"), - Arguments.of("all:jesus", "jes{\\'{u}}s"), - Arguments.of("all:breitenbucher", "breitenb{\\\"{u}}cher"), + Arguments.of("all:preissinger", "\"prei{\\\\ss}inger\""), + Arguments.of("all:jesus", "\"jes{\\\\'{u}}s\""), + Arguments.of("all:breitenbucher", "\"breitenb{\\\\\\\"{u}}cher\""), Arguments.of("groups:/exclude", "groups:\\/exclude") ); @@ -36,8 +33,7 @@ public static Stream searchQuires() { @ParameterizedTest @MethodSource void searchQuires(String expected, String query) throws ParseException { - QueryParser parser = new QueryParser(SearchFieldConstants.DEFAULT_FIELD.toString(), SearchFieldConstants.Whitespace_ANALYZER); - query = FORMATTER.format(query); + QueryParser parser = new QueryParser(SearchFieldConstants.DEFAULT_FIELD.toString(), SearchFieldConstants.JABREF_ANALYZER); String result = parser.parse(query).toString(); assertEquals(expected, result); } diff --git a/src/test/resources/tinylog-test.properties b/src/test/resources/tinylog-test.properties index 640a69a1005..ed6c4288b2a 100644 --- a/src/test/resources/tinylog-test.properties +++ b/src/test/resources/tinylog-test.properties @@ -4,3 +4,4 @@ writer = console #level@org.jabref.model.entry.BibEntry = debug #level@org.jabref.logic.importer.fetcher.ResearchGate = trace #level@org.jabref.logic.importer.fetcher.DoiFetcher = trace +level@org.jabref.model.search.LatexToUnicodeFoldingFilter= debug\ From 02c8ca0aa94dcad97d432b52709e2b29a20af5f5 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Wed, 28 Aug 2024 13:08:24 +0300 Subject: [PATCH 175/256] Remove LatexToUnicode from SearchQuery --- .../jabref/logic/search/indexing/BibFieldsIndexer.java | 10 +++------- .../java/org/jabref/model/search/NGramAnalyzer.java | 5 ++--- .../org/jabref/model/search/SearchFieldConstants.java | 2 +- src/main/java/org/jabref/model/search/SearchQuery.java | 6 +----- 4 files changed, 7 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java b/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java index f3d76248c7f..aed7b64e0b0 100644 --- a/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java @@ -5,9 +5,7 @@ import java.util.Map; import org.jabref.gui.util.BackgroundTask; -import org.jabref.logic.cleanup.Formatter; import org.jabref.logic.l10n.Localization; -import org.jabref.logic.layout.format.LatexToUnicodeFormatter; import org.jabref.logic.util.HeadlessExecutorService; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; @@ -29,7 +27,6 @@ public class BibFieldsIndexer implements LuceneIndexer { private static final Logger LOGGER = LoggerFactory.getLogger(BibFieldsIndexer.class); - private static final Formatter FORMATTER = new LatexToUnicodeFormatter(); private final BibDatabaseContext databaseContext; private final String libraryName; private final Directory indexDirectory; @@ -40,7 +37,7 @@ public BibFieldsIndexer(BibDatabaseContext databaseContext) { this.databaseContext = databaseContext; this.libraryName = databaseContext.getDatabasePath().map(path -> path.getFileName().toString()).orElseGet(() -> "unsaved"); - IndexWriterConfig config = new IndexWriterConfig(SearchFieldConstants.NGram_Analyzer_For_INDEXING); + IndexWriterConfig config = new IndexWriterConfig(SearchFieldConstants.NGRAM_ANALYZER); this.indexDirectory = new ByteBuffersDirectory(); try { @@ -85,9 +82,8 @@ private void addToIndex(BibEntry bibEntry) { StringBuilder allFields = new StringBuilder(bibEntry.getType().getName()); for (Map.Entry mapEntry : bibEntry.getFieldMap().entrySet()) { - String value = FORMATTER.format(mapEntry.getValue()); - document.add(new TextField(mapEntry.getKey().getName(), value, storeDisabled)); - allFields.append('\n').append(value); + document.add(new TextField(mapEntry.getKey().getName(), mapEntry.getValue(), storeDisabled)); + allFields.append('\n').append(mapEntry.getValue()); } document.add(new TextField(SearchFieldConstants.DEFAULT_FIELD.toString(), allFields.toString(), storeDisabled)); indexWriter.addDocument(document); diff --git a/src/main/java/org/jabref/model/search/NGramAnalyzer.java b/src/main/java/org/jabref/model/search/NGramAnalyzer.java index df84d568002..bdae3cecafc 100644 --- a/src/main/java/org/jabref/model/search/NGramAnalyzer.java +++ b/src/main/java/org/jabref/model/search/NGramAnalyzer.java @@ -9,7 +9,6 @@ import org.apache.lucene.analysis.core.WhitespaceTokenizer; import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter; import org.apache.lucene.analysis.ngram.NGramTokenFilter; -import org.apache.lucene.analysis.shingle.ShingleFilter; public class NGramAnalyzer extends Analyzer { private final int minGram; @@ -25,10 +24,10 @@ public NGramAnalyzer(int minGram, int maxGram, CharArraySet stopWords) { @Override protected TokenStreamComponents createComponents(String fieldName) { Tokenizer source = new WhitespaceTokenizer(); - TokenStream result = new LowerCaseFilter(source); + TokenStream result = new LatexToUnicodeFoldingFilter(source); result = new StopFilter(result, stopWords); result = new ASCIIFoldingFilter(result); - result = new ShingleFilter(result); + result = new LowerCaseFilter(result); result = new NGramTokenFilter(result, minGram, maxGram, true); return new TokenStreamComponents(source, result); } diff --git a/src/main/java/org/jabref/model/search/SearchFieldConstants.java b/src/main/java/org/jabref/model/search/SearchFieldConstants.java index f704cd082c0..94781a3179c 100644 --- a/src/main/java/org/jabref/model/search/SearchFieldConstants.java +++ b/src/main/java/org/jabref/model/search/SearchFieldConstants.java @@ -18,7 +18,7 @@ public enum SearchFieldConstants { MODIFIED("modified"); public static final Analyzer JABREF_ANALYZER = new JabRefAnalyzer(EnglishAnalyzer.ENGLISH_STOP_WORDS_SET); - public static final Analyzer NGram_Analyzer_For_INDEXING = new NGramAnalyzer(1, Integer.MAX_VALUE, EnglishAnalyzer.ENGLISH_STOP_WORDS_SET); + public static final Analyzer NGRAM_ANALYZER = new NGramAnalyzer(1, Integer.MAX_VALUE, EnglishAnalyzer.ENGLISH_STOP_WORDS_SET); public static final List PDF_FIELDS = List.of(PATH.toString(), CONTENT.toString(), ANNOTATIONS.toString()); private final String field; diff --git a/src/main/java/org/jabref/model/search/SearchQuery.java b/src/main/java/org/jabref/model/search/SearchQuery.java index 36363ceee48..0fa1c22dbaf 100644 --- a/src/main/java/org/jabref/model/search/SearchQuery.java +++ b/src/main/java/org/jabref/model/search/SearchQuery.java @@ -12,9 +12,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import org.jabref.logic.cleanup.Formatter; -import org.jabref.logic.layout.format.LatexToUnicodeFormatter; - import org.apache.lucene.queryparser.classic.MultiFieldQueryParser; import org.apache.lucene.queryparser.classic.ParseException; import org.apache.lucene.search.Query; @@ -22,7 +19,6 @@ import org.apache.lucene.search.highlight.WeightedTerm; public class SearchQuery { - private static final Formatter FORMATTER = new LatexToUnicodeFormatter(); /** * The mode of escaping special characters in regular expressions */ @@ -87,7 +83,7 @@ public SearchQuery(String query, EnumSet searchFlags) { queryParser.setAllowLeadingWildcard(true); try { - parsedQuery = queryParser.parse(FORMATTER.format(query)); + parsedQuery = queryParser.parse(query); parseError = null; } catch (ParseException e) { parsedQuery = null; From 7b95f8dc7d565728a4a6e5d871d6705010581ba9 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Wed, 28 Aug 2024 13:20:13 +0300 Subject: [PATCH 176/256] Localization --- src/main/resources/l10n/JabRef_en.properties | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/resources/l10n/JabRef_en.properties b/src/main/resources/l10n/JabRef_en.properties index 2551f90bb05..b9054753ef3 100644 --- a/src/main/resources/l10n/JabRef_en.properties +++ b/src/main/resources/l10n/JabRef_en.properties @@ -272,6 +272,7 @@ Each\ line\ must\ be\ of\ the\ following\ form\:\ \'tab\:field1;field2;...;field Search\ groups\ migration\ of\ %0=Search groups migration of %0 The\ search\ groups\ syntax\ is\ outdated.\ Do\ you\ want\ to\ migrate\ to\ the\ new\ syntax?= The search groups syntax is outdated. Do you want to migrate to the new syntax? Migrate=Migrate +Keep\ as\ is=Keep as is Edit=Edit Edit\ file\ type=Edit file type From a0ff7fb4d6eae7de23be23c3e1082b2be176cce0 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Wed, 28 Aug 2024 13:20:17 +0300 Subject: [PATCH 177/256] AllowedToUseLogic --- .../org/jabref/model/search/LatexToUnicodeFoldingFilter.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/jabref/model/search/LatexToUnicodeFoldingFilter.java b/src/main/java/org/jabref/model/search/LatexToUnicodeFoldingFilter.java index f2a1da5313a..dcb4de0d3fe 100644 --- a/src/main/java/org/jabref/model/search/LatexToUnicodeFoldingFilter.java +++ b/src/main/java/org/jabref/model/search/LatexToUnicodeFoldingFilter.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.util.Arrays; +import org.jabref.architecture.AllowedToUseLogic; import org.jabref.logic.cleanup.Formatter; import org.jabref.logic.layout.format.LatexToUnicodeFormatter; @@ -16,6 +17,7 @@ /** * @implNote Implementation based on {@link org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter} */ +@AllowedToUseLogic("because it needs access to the LaTeXToUnicodeFormatter") public final class LatexToUnicodeFoldingFilter extends TokenFilter { private static final Logger LOGGER = LoggerFactory.getLogger(LatexToUnicodeFoldingFilter.class); private static final Formatter FORMATTER = new LatexToUnicodeFormatter(); From b55ab317f46296473fab161567a62fe9876ec1f7 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Wed, 28 Aug 2024 12:27:53 +0200 Subject: [PATCH 178/256] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b022706225..0fdf486f01e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv ### Added +- We added probable search hits instead of exact matches. Sorting by hit score can be done by the new score table column. [#11326](https://github.com/JabRef/jabref/pull/11326) - We added an AI-based chat for entries with linked PDF files. [#11430](https://github.com/JabRef/jabref/pull/11430) - We added an AI-based summarization possibility for entries with linked PDF files. [#11430](https://github.com/JabRef/jabref/pull/11430) - We added support for selecting and using CSL Styles in JabRef's OpenOffice/LibreOffice integration for inserting bibliographic and in-text citations into a document. [#2146](https://github.com/JabRef/jabref/issues/2146), [#8893](https://github.com/JabRef/jabref/issues/8893) @@ -28,7 +29,6 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv ### Changed -- The search in the library now displays probable search hits instead of exact matches. Sorting by hit score can be done by the new score table column. [#11326](https://github.com/JabRef/jabref/pull/11326) - When a communication error with an [online service](https://docs.jabref.org/collect/import-using-online-bibliographic-database) occurs, JabRef displays the HTTP error. [#11223](https://github.com/JabRef/jabref/issues/11223) - The Pubmed/Medline Plain importer now imports the PMID field as well [#11488](https://github.com/JabRef/jabref/issues/11488) - The 'Check for updates' menu bar button is now always enabled. [#11485](https://github.com/JabRef/jabref/pull/11485) From 0b7284da5d63776a9fa571f718f64184292401da Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Wed, 28 Aug 2024 20:38:06 +0200 Subject: [PATCH 179/256] Use sentence case for search result heading --- .../entryeditor/fileannotationtab/FulltextSearchResultsTab.java | 2 +- src/main/resources/l10n/JabRef_en.properties | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java b/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java index 32b93921f44..61930b38169 100644 --- a/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java @@ -110,7 +110,7 @@ protected void bindToEntry(BibEntry entry) { content.getChildren().addAll(new Text(System.lineSeparator()), lineSeparator(0.8), createPageLink(linkedFile, searchResult.getPageNumber())); } if (!searchResult.getAnnotationsResultStringsHtml().isEmpty()) { - Text annotationsText = new Text(System.lineSeparator() + Localization.lang("Found matches in Annotations:") + System.lineSeparator() + System.lineSeparator()); + Text annotationsText = new Text(System.lineSeparator() + Localization.lang("Found matches in annotations:") + System.lineSeparator() + System.lineSeparator()); annotationsText.setStyle("-fx-font-style: italic;"); content.getChildren().add(annotationsText); } diff --git a/src/main/resources/l10n/JabRef_en.properties b/src/main/resources/l10n/JabRef_en.properties index b9054753ef3..cfd2891e3d6 100644 --- a/src/main/resources/l10n/JabRef_en.properties +++ b/src/main/resources/l10n/JabRef_en.properties @@ -2457,7 +2457,7 @@ Rebuild\ fulltext\ search\ index\ for\ current\ library?=Rebuild fulltext search Rebuilding\ fulltext\ search\ index...=Rebuilding fulltext search index... Found\ match\ in\ %0=Found match in %0 On\ page\ %0=On page %0 -Found\ matches\ in\ Annotations\:=Found matches in Annotations: +Found\ matches\ in\ annotations\:=Found matches in annotations: Fetcher\ cannot\ be\ tested\!=Fetcher cannot be tested! Fetcher\ unknown\!=Fetcher unknown! From 37c8325c16b084281401123b966ef3642fc08493 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Wed, 28 Aug 2024 20:44:07 +0200 Subject: [PATCH 180/256] Add CHANGELOG for change in JabRefFrameViewModel --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0fdf486f01e..4f53e6d98a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,7 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv - We fixed an issue with query transformers (JStor and others). [#11643](https://github.com/JabRef/jabref/pull/11643) - We fixed an issue where a new unsaved library was not marked with an asterisk. [#11519](https://github.com/JabRef/jabref/pull/11519) - We fixed an issue where JabRef starts without window decorations. [#11440](https://github.com/JabRef/jabref/pull/11440) +- We fixed an issue when the library was not marked changed after a migration. [#11542](https://github.com/JabRef/jabref/pull/11542) - We fixed an issue where the entry preview highlight was not working when searching before opening the entry editor. [#11659](https://github.com/JabRef/jabref/pull/11659) - We fixed an issue where text in Dark mode inside "Citation information" was not readable. [#11512](https://github.com/JabRef/jabref/issues/11512) - We fixed an issue where the selection of an entry in the table lost after searching for a group. [#3176](https://github.com/JabRef/jabref/issues/3176) From 8faad3334630c0c57b69e1c4493d42e63dfc991d Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Wed, 28 Aug 2024 20:49:09 +0200 Subject: [PATCH 181/256] Add more changes to CHANGELOG.md --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f53e6d98a2..3a4464021b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv ### Added - We added probable search hits instead of exact matches. Sorting by hit score can be done by the new score table column. [#11326](https://github.com/JabRef/jabref/pull/11326) +- We added support finding LaTeX-encoded special characters based on plain Unicode and vice versa. [#11542](https://github.com/JabRef/jabref/pull/11542) - We added an AI-based chat for entries with linked PDF files. [#11430](https://github.com/JabRef/jabref/pull/11430) - We added an AI-based summarization possibility for entries with linked PDF files. [#11430](https://github.com/JabRef/jabref/pull/11430) - We added support for selecting and using CSL Styles in JabRef's OpenOffice/LibreOffice integration for inserting bibliographic and in-text citations into a document. [#2146](https://github.com/JabRef/jabref/issues/2146), [#8893](https://github.com/JabRef/jabref/issues/8893) @@ -29,6 +30,7 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv ### Changed +- The search syntax is changed to [Apache Lucene syntax](https://docs.jabref.org/collect/import-using-online-bibliographic-database#search-syntax) (also to be similar to the online search syntax). [#11542](https://github.com/JabRef/jabref/pull/11542/) - When a communication error with an [online service](https://docs.jabref.org/collect/import-using-online-bibliographic-database) occurs, JabRef displays the HTTP error. [#11223](https://github.com/JabRef/jabref/issues/11223) - The Pubmed/Medline Plain importer now imports the PMID field as well [#11488](https://github.com/JabRef/jabref/issues/11488) - The 'Check for updates' menu bar button is now always enabled. [#11485](https://github.com/JabRef/jabref/pull/11485) @@ -56,6 +58,7 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv ### Removed +- We removed support for case-insentive and exact search. [#11542](https://github.com/JabRef/jabref/pull/11542) - We removed support for importing using the SilverPlatterImporter (`Record INSPEC`). [#11576](https://github.com/JabRef/jabref/pull/11576) From cd71f11069b822dd51170f112e633c0cd4621dbe Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Thu, 29 Aug 2024 00:12:53 +0200 Subject: [PATCH 182/256] Add ADR-0038 --- ...038-use-identityhashcode-for-bibentries.md | 30 +++++++++++++++++++ .../duplicationFinder/DuplicateSearch.java | 2 ++ .../jabref/gui/groups/GroupNodeViewModel.java | 8 +++++ .../org/jabref/model/groups/SearchGroup.java | 4 +++ 4 files changed, 44 insertions(+) create mode 100644 docs/decisions/0038-use-identityhashcode-for-bibentries.md diff --git a/docs/decisions/0038-use-identityhashcode-for-bibentries.md b/docs/decisions/0038-use-identityhashcode-for-bibentries.md new file mode 100644 index 00000000000..d5c9fb25062 --- /dev/null +++ b/docs/decisions/0038-use-identityhashcode-for-bibentries.md @@ -0,0 +1,30 @@ +--- +title: Use System.identityHashCode for BibEntry at indexing +nav_order: 38 +parent: Decision Records +--- + + +# Use `System.identityHashCode` for BibEntries at Indexing + +## Context and Problem Statement + +The `BibEntry` class has `equals` and `hashCode` implemented on the content of the bib entry. +Thus, if two bib entries have the same type, the same fields, and the same content, they are equal. + +This, however, is not useful in the UI, where equal entries are not the same entries. + +## Decision Drivers + +* Simple code +* Not changing much other JabRef code +* Working Lucene + +## Considered Options + +* Use `System.identityHashCode` for indexing `BibEntry` +* Rewrite `BibEntry` logic + +## Decision Outcome + +Chosen option: "Use `System.identityHashCode` for indexing `BibEntry`", because is the "natural" thing to ensure distinction between two instances of a `BibEntry` object - regardless of equality. diff --git a/src/main/java/org/jabref/gui/duplicationFinder/DuplicateSearch.java b/src/main/java/org/jabref/gui/duplicationFinder/DuplicateSearch.java index 789c5d33f18..de084e65aff 100644 --- a/src/main/java/org/jabref/gui/duplicationFinder/DuplicateSearch.java +++ b/src/main/java/org/jabref/gui/duplicationFinder/DuplicateSearch.java @@ -233,6 +233,7 @@ public synchronized List getToAdd() { } public synchronized void remove(BibEntry entry) { + // ADR-0038 toRemove.put(System.identityHashCode(entry), entry); duplicates++; } @@ -250,6 +251,7 @@ public synchronized void replace(BibEntry entry, BibEntry replacement) { } public synchronized boolean isToRemove(BibEntry entry) { + // ADR-0038 return toRemove.containsKey(System.identityHashCode(entry)); } diff --git a/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java b/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java index 90853e2e503..7458aa0b455 100644 --- a/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java +++ b/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java @@ -258,17 +258,21 @@ private void onDatabaseChanged(ListChangeListener.Change cha } else if (change.wasUpdated()) { for (BibEntry changedEntry : change.getList().subList(change.getFrom(), change.getTo())) { if (groupNode.matches(changedEntry)) { + // ADR-0038 matchedEntries.put(System.identityHashCode(changedEntry), changedEntry); } else { + // ADR-0038 matchedEntries.remove(System.identityHashCode(changedEntry)); } } } else { for (BibEntry removedEntry : change.getRemoved()) { + // ADR-0038 matchedEntries.remove(System.identityHashCode(removedEntry)); } for (BibEntry addedEntry : change.getAddedSubList()) { if (groupNode.matches(addedEntry)) { + // ADR-0038 matchedEntries.put(System.identityHashCode(addedEntry), addedEntry); } } @@ -296,6 +300,7 @@ private void updateMatchedEntries() { .wrap(() -> groupNode.findMatches(databaseContext.getDatabase())) .onSuccess(entries -> { matchedEntries.clear(); + // ADR-0038 entries.forEach(entry -> matchedEntries.put(System.identityHashCode(entry), entry)); }) .executeWith(taskExecutor); @@ -545,8 +550,10 @@ public void listen(IndexAddedOrUpdatedEvent event) { for (BibEntry entry : event.entries()) { searchGroup.updateMatches(entry); if (groupNode.matches(entry)) { + // ADR-0038 matchedEntries.put(System.identityHashCode(entry), entry); } else { + // ADR-0038 matchedEntries.remove(System.identityHashCode(entry)); } } @@ -561,6 +568,7 @@ public void listen(IndexRemovedEvent event) { BackgroundTask.wrap(() -> { for (BibEntry entry : event.entries()) { searchGroup.updateMatches(entry); + // ADR-0038 matchedEntries.remove(System.identityHashCode(entry)); } }).executeWith(taskExecutor); diff --git a/src/main/java/org/jabref/model/groups/SearchGroup.java b/src/main/java/org/jabref/model/groups/SearchGroup.java index 3efc67a0a0d..eaf0e9412de 100644 --- a/src/main/java/org/jabref/model/groups/SearchGroup.java +++ b/src/main/java/org/jabref/model/groups/SearchGroup.java @@ -74,6 +74,7 @@ public void updateMatches() { } matchedEntries.clear(); // TODO: Search should be done in a background thread + // ADR-0038 luceneManager.search(query).getMatchedEntries().forEach(entry -> matchedEntries.put(System.identityHashCode(entry), entry)); } @@ -82,8 +83,10 @@ public void updateMatches(BibEntry entry) { return; } if (luceneManager.isMatched(entry, query)) { + // ADR-0038 matchedEntries.put(System.identityHashCode(entry), entry); } else { + // ADR-0038 matchedEntries.remove(System.identityHashCode(entry)); } } @@ -104,6 +107,7 @@ public boolean equals(Object o) { @Override public boolean contains(BibEntry entry) { + // ADR-0038 return matchedEntries.containsKey(System.identityHashCode(entry)); } From beb38811838b7b345956004c838051b7dfbb2a80 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Thu, 29 Aug 2024 00:16:04 +0200 Subject: [PATCH 183/256] Rename "SCORE" to "MATCH_SCORE" --- .../java/org/jabref/gui/maintable/MainTableColumnFactory.java | 4 ++-- .../java/org/jabref/gui/maintable/MainTableColumnModel.java | 2 +- .../org/jabref/gui/preferences/table/TableTabViewModel.java | 2 +- src/main/java/org/jabref/gui/search/SearchResultsTable.java | 2 +- src/main/resources/l10n/JabRef_en.properties | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/jabref/gui/maintable/MainTableColumnFactory.java b/src/main/java/org/jabref/gui/maintable/MainTableColumnFactory.java index 6966b261c08..2fe8228ad30 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableColumnFactory.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableColumnFactory.java @@ -87,7 +87,7 @@ public MainTableColumnFactory(BibDatabaseContext database, public TableColumn createColumn(MainTableColumnModel column) { TableColumn returnColumn = null; switch (column.getType()) { - case SCORE: + case MATCH_SCORE: returnColumn = createScoreColumn(column); break; case INDEX: @@ -167,7 +167,7 @@ private TableColumn createScoreColumn(MainTableC TableColumn column = new MainTableColumn<>(columnModel); Node header = new Text(Localization.lang("Score")); header.getStyleClass().add("mainTable-header"); - Tooltip.install(header, new Tooltip(MainTableColumnModel.Type.SCORE.getDisplayName())); + Tooltip.install(header, new Tooltip(MainTableColumnModel.Type.MATCH_SCORE.getDisplayName())); column.setGraphic(header); column.setStyle("-fx-alignment: CENTER-RIGHT;"); column.setCellValueFactory(cellData -> cellData.getValue().searchScoreProperty()); diff --git a/src/main/java/org/jabref/gui/maintable/MainTableColumnModel.java b/src/main/java/org/jabref/gui/maintable/MainTableColumnModel.java index ee5baccc1f2..0c67b1610c5 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableColumnModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableColumnModel.java @@ -36,7 +36,7 @@ public class MainTableColumnModel { private static final Logger LOGGER = LoggerFactory.getLogger(MainTableColumnModel.class); public enum Type { MATCH_CATEGORY("match_category"), // Not localized, because this column is always hidden - SCORE("search_score", Localization.lang("Search score")), + MATCH_SCORE("match_score", Localization.lang("Match score")), INDEX("index", Localization.lang("Index")), EXTRAFILE("extrafile", Localization.lang("File type")), FILES("files", Localization.lang("Linked files")), diff --git a/src/main/java/org/jabref/gui/preferences/table/TableTabViewModel.java b/src/main/java/org/jabref/gui/preferences/table/TableTabViewModel.java index 2d7b8c7191a..459c2ab6769 100644 --- a/src/main/java/org/jabref/gui/preferences/table/TableTabViewModel.java +++ b/src/main/java/org/jabref/gui/preferences/table/TableTabViewModel.java @@ -123,7 +123,7 @@ public void setValues() { availableColumnsProperty.clear(); availableColumnsProperty.addAll( - new MainTableColumnModel(MainTableColumnModel.Type.SCORE), + new MainTableColumnModel(MainTableColumnModel.Type.MATCH_SCORE), new MainTableColumnModel(MainTableColumnModel.Type.INDEX), new MainTableColumnModel(MainTableColumnModel.Type.LINKED_IDENTIFIER), new MainTableColumnModel(MainTableColumnModel.Type.GROUPS), diff --git a/src/main/java/org/jabref/gui/search/SearchResultsTable.java b/src/main/java/org/jabref/gui/search/SearchResultsTable.java index 2bb9ac86b96..6b6e2778f8d 100644 --- a/src/main/java/org/jabref/gui/search/SearchResultsTable.java +++ b/src/main/java/org/jabref/gui/search/SearchResultsTable.java @@ -55,7 +55,7 @@ public SearchResultsTable(SearchResultsTableDataModel model, TableColumn scoreColumn = this.getColumns().stream() .filter(c -> c instanceof MainTableColumn) .map(c -> ((MainTableColumn) c)) - .filter(c -> c.getModel().getType() == MainTableColumnModel.Type.SCORE) + .filter(c -> c.getModel().getType() == MainTableColumnModel.Type.MATCH_SCORE) .findFirst().orElse(null); this.getSortOrder().clear(); diff --git a/src/main/resources/l10n/JabRef_en.properties b/src/main/resources/l10n/JabRef_en.properties index cfd2891e3d6..8338ee0ecb3 100644 --- a/src/main/resources/l10n/JabRef_en.properties +++ b/src/main/resources/l10n/JabRef_en.properties @@ -808,7 +808,7 @@ Character\ encoding\ '%0'\ is\ not\ supported.=Character encoding '%0' is not su Filter\ search\ results=Filter search results Filter\ by\ groups=Filter by groups Invert\ groups=Invert groups -Search\ score=Search score +Match\ score=Match score Scroll\ to\ previous\ match\ category=Scroll to previous match category Scroll\ to\ next\ match\ category=Scroll to next match category Search=Search From 59a5e54f5226237bbdc861011236086de93eb8f2 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Thu, 29 Aug 2024 00:17:21 +0200 Subject: [PATCH 184/256] Add link to ADR-0038 --- src/main/java/org/jabref/gui/maintable/MainTableDataModel.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index 6d150ee892d..371caed6736 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -211,6 +211,7 @@ public void listen(IndexAddedOrUpdatedEvent indexAddedOrUpdatedEvent) { // The indexOf() method is not used because it relies on the equals(), // which can return the wrong index if some entries are equal but different instances. // For example, two different instances of an empty entry would be treated as equal by .equals(). + // See also ADR-0038 int index = -1; for (int i = 0; i < allEntries.size(); i++) { if (allEntries.get(i) == entry) { From bc2853208ad483b1dbd3b014a28c1796a61a93c3 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Thu, 29 Aug 2024 00:27:31 +0200 Subject: [PATCH 185/256] Add another CHANGELOG.md entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a4464021b0..f2099151978 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv - We added probable search hits instead of exact matches. Sorting by hit score can be done by the new score table column. [#11326](https://github.com/JabRef/jabref/pull/11326) - We added support finding LaTeX-encoded special characters based on plain Unicode and vice versa. [#11542](https://github.com/JabRef/jabref/pull/11542) +- When a search hits a file, the file icon of that entry is changed accordingly. [#11542](https://github.com/JabRef/jabref/pull/11542) - We added an AI-based chat for entries with linked PDF files. [#11430](https://github.com/JabRef/jabref/pull/11430) - We added an AI-based summarization possibility for entries with linked PDF files. [#11430](https://github.com/JabRef/jabref/pull/11430) - We added support for selecting and using CSL Styles in JabRef's OpenOffice/LibreOffice integration for inserting bibliographic and in-text citations into a document. [#2146](https://github.com/JabRef/jabref/issues/2146), [#8893](https://github.com/JabRef/jabref/issues/8893) From 85f4a4bdc08b90dd0cc9e29d5655c013d1940571 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Thu, 29 Aug 2024 00:32:17 +0200 Subject: [PATCH 186/256] Add CHANGELOG.md entry --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2099151978..2e6353ba68c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,7 +61,7 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv - We removed support for case-insentive and exact search. [#11542](https://github.com/JabRef/jabref/pull/11542) - We removed support for importing using the SilverPlatterImporter (`Record INSPEC`). [#11576](https://github.com/JabRef/jabref/pull/11576) - +- We removed the descroption of search strings. [#11565](https://github.com/JabRef/jabref/pull/11565) From d22ef7b2a9c5b6bec43bad0c01a6cacc2ba205bb Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Thu, 29 Aug 2024 00:44:58 +0200 Subject: [PATCH 187/256] Revert change of filename --- .../java/org/jabref/model/database/BibDatabaseContext.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jabref/model/database/BibDatabaseContext.java b/src/main/java/org/jabref/model/database/BibDatabaseContext.java index 611aa4855fb..b412533a11b 100644 --- a/src/main/java/org/jabref/model/database/BibDatabaseContext.java +++ b/src/main/java/org/jabref/model/database/BibDatabaseContext.java @@ -262,7 +262,8 @@ public Path getFulltextIndexPath() { if (getDatabasePath().isPresent()) { Path databasePath = getDatabasePath().get(); - String fileName = BackupFileUtil.getUniqueFilePrefix(databasePath) + "-" + databasePath.getFileName(); + // Eventually, this leads to filenames as "40daf3b0--fuu.bib--2022-09-04--01.36.25.bib" --> "--" is used as separator between "groups" + String fileName = BackupFileUtil.getUniqueFilePrefix(databasePath) + "--" + databasePath.getFileName(); indexPath = appData.resolve(fileName); LOGGER.debug("Index path for {} is {}", getDatabasePath().get(), indexPath); return indexPath; From 9ca8416357b000a81c015206fe34754b648a1f9c Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Thu, 29 Aug 2024 00:47:56 +0200 Subject: [PATCH 188/256] Add JavaDoc comment --- src/main/java/org/jabref/model/search/JabRefAnalyzer.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/org/jabref/model/search/JabRefAnalyzer.java b/src/main/java/org/jabref/model/search/JabRefAnalyzer.java index 317827a3c2d..055025ac737 100644 --- a/src/main/java/org/jabref/model/search/JabRefAnalyzer.java +++ b/src/main/java/org/jabref/model/search/JabRefAnalyzer.java @@ -9,6 +9,10 @@ import org.apache.lucene.analysis.core.WhitespaceTokenizer; import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter; +/** + * Lucene analyzer respecting the special "features" of JabRef. + * Especially, LaTeX-encoded text. + */ public class JabRefAnalyzer extends Analyzer { private final CharArraySet stopWords; public JabRefAnalyzer(CharArraySet stopWords) { From 4ebb256bb2106e0db4cc002939f9ea73e33abb11 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Thu, 29 Aug 2024 00:55:49 +0200 Subject: [PATCH 189/256] Trying to find better names --- .../org/jabref/logic/search/indexing/BibFieldsIndexer.java | 2 +- .../logic/search/indexing/DefaultLinkedFilesIndexer.java | 2 +- .../search/{JabRefAnalyzer.java => LatexAwareAnalyzer.java} | 4 ++-- .../{NGramAnalyzer.java => LatexAwareNGramAnalyzer.java} | 4 ++-- .../java/org/jabref/model/search/SearchFieldConstants.java | 4 ++-- src/main/java/org/jabref/model/search/SearchQuery.java | 2 +- src/main/java/org/jabref/model/search/SearchResult.java | 2 +- .../java/org/jabref/logic/search/LuceneQueryParserTest.java | 2 +- 8 files changed, 11 insertions(+), 11 deletions(-) rename src/main/java/org/jabref/model/search/{JabRefAnalyzer.java => LatexAwareAnalyzer.java} (90%) rename src/main/java/org/jabref/model/search/{NGramAnalyzer.java => LatexAwareNGramAnalyzer.java} (89%) diff --git a/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java b/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java index aed7b64e0b0..5277fba3f19 100644 --- a/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java @@ -37,7 +37,7 @@ public BibFieldsIndexer(BibDatabaseContext databaseContext) { this.databaseContext = databaseContext; this.libraryName = databaseContext.getDatabasePath().map(path -> path.getFileName().toString()).orElseGet(() -> "unsaved"); - IndexWriterConfig config = new IndexWriterConfig(SearchFieldConstants.NGRAM_ANALYZER); + IndexWriterConfig config = new IndexWriterConfig(SearchFieldConstants.LATEX_AWARE_NGRAM_ANALYZER); this.indexDirectory = new ByteBuffersDirectory(); try { diff --git a/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java b/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java index 80e6045051e..b53f602628f 100644 --- a/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java @@ -65,7 +65,7 @@ public DefaultLinkedFilesIndexer(BibDatabaseContext databaseContext, FilePrefere this.indexedFiles = new ConcurrentHashMap<>(); indexDirectoryPath = databaseContext.getFulltextIndexPath(); - IndexWriterConfig config = new IndexWriterConfig(SearchFieldConstants.JABREF_ANALYZER); + IndexWriterConfig config = new IndexWriterConfig(SearchFieldConstants.LATEX_AWARE_ANALYZER); if ("unsaved".equals(indexDirectoryPath.getFileName().toString())) { config.setOpenMode(IndexWriterConfig.OpenMode.CREATE); indexDirectoryPath = indexDirectoryPath.resolveSibling("unsaved" + NUMBER_OF_UNSAVED_LIBRARIES++); diff --git a/src/main/java/org/jabref/model/search/JabRefAnalyzer.java b/src/main/java/org/jabref/model/search/LatexAwareAnalyzer.java similarity index 90% rename from src/main/java/org/jabref/model/search/JabRefAnalyzer.java rename to src/main/java/org/jabref/model/search/LatexAwareAnalyzer.java index 055025ac737..2d306b89036 100644 --- a/src/main/java/org/jabref/model/search/JabRefAnalyzer.java +++ b/src/main/java/org/jabref/model/search/LatexAwareAnalyzer.java @@ -13,9 +13,9 @@ * Lucene analyzer respecting the special "features" of JabRef. * Especially, LaTeX-encoded text. */ -public class JabRefAnalyzer extends Analyzer { +public class LatexAwareAnalyzer extends Analyzer { private final CharArraySet stopWords; - public JabRefAnalyzer(CharArraySet stopWords) { + public LatexAwareAnalyzer(CharArraySet stopWords) { this.stopWords = stopWords; } diff --git a/src/main/java/org/jabref/model/search/NGramAnalyzer.java b/src/main/java/org/jabref/model/search/LatexAwareNGramAnalyzer.java similarity index 89% rename from src/main/java/org/jabref/model/search/NGramAnalyzer.java rename to src/main/java/org/jabref/model/search/LatexAwareNGramAnalyzer.java index bdae3cecafc..c4db64b6aca 100644 --- a/src/main/java/org/jabref/model/search/NGramAnalyzer.java +++ b/src/main/java/org/jabref/model/search/LatexAwareNGramAnalyzer.java @@ -10,12 +10,12 @@ import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter; import org.apache.lucene.analysis.ngram.NGramTokenFilter; -public class NGramAnalyzer extends Analyzer { +public class LatexAwareNGramAnalyzer extends Analyzer { private final int minGram; private final int maxGram; private final CharArraySet stopWords; - public NGramAnalyzer(int minGram, int maxGram, CharArraySet stopWords) { + public LatexAwareNGramAnalyzer(int minGram, int maxGram, CharArraySet stopWords) { this.minGram = minGram; this.maxGram = maxGram; this.stopWords = stopWords; diff --git a/src/main/java/org/jabref/model/search/SearchFieldConstants.java b/src/main/java/org/jabref/model/search/SearchFieldConstants.java index 94781a3179c..cb9f3b21d7f 100644 --- a/src/main/java/org/jabref/model/search/SearchFieldConstants.java +++ b/src/main/java/org/jabref/model/search/SearchFieldConstants.java @@ -17,8 +17,8 @@ public enum SearchFieldConstants { PAGE_NUMBER("pageNumber"), MODIFIED("modified"); - public static final Analyzer JABREF_ANALYZER = new JabRefAnalyzer(EnglishAnalyzer.ENGLISH_STOP_WORDS_SET); - public static final Analyzer NGRAM_ANALYZER = new NGramAnalyzer(1, Integer.MAX_VALUE, EnglishAnalyzer.ENGLISH_STOP_WORDS_SET); + public static final Analyzer LATEX_AWARE_ANALYZER = new LatexAwareAnalyzer(EnglishAnalyzer.ENGLISH_STOP_WORDS_SET); + public static final Analyzer LATEX_AWARE_NGRAM_ANALYZER = new LatexAwareNGramAnalyzer(1, Integer.MAX_VALUE, EnglishAnalyzer.ENGLISH_STOP_WORDS_SET); public static final List PDF_FIELDS = List.of(PATH.toString(), CONTENT.toString(), ANNOTATIONS.toString()); private final String field; diff --git a/src/main/java/org/jabref/model/search/SearchQuery.java b/src/main/java/org/jabref/model/search/SearchQuery.java index 0fa1c22dbaf..04e1846d36e 100644 --- a/src/main/java/org/jabref/model/search/SearchQuery.java +++ b/src/main/java/org/jabref/model/search/SearchQuery.java @@ -79,7 +79,7 @@ public SearchQuery(String query, EnumSet searchFlags) { query = '/' + query + '/'; } - MultiFieldQueryParser queryParser = new MultiFieldQueryParser(fieldsToSearchArray, SearchFieldConstants.JABREF_ANALYZER, boosts); + MultiFieldQueryParser queryParser = new MultiFieldQueryParser(fieldsToSearchArray, SearchFieldConstants.LATEX_AWARE_ANALYZER, boosts); queryParser.setAllowLeadingWildcard(true); try { diff --git a/src/main/java/org/jabref/model/search/SearchResult.java b/src/main/java/org/jabref/model/search/SearchResult.java index 67cc6d9eb4d..5789b48aead 100644 --- a/src/main/java/org/jabref/model/search/SearchResult.java +++ b/src/main/java/org/jabref/model/search/SearchResult.java @@ -76,7 +76,7 @@ public int getPageNumber() { } private static List getHighlighterFragments(Highlighter highlighter, SearchFieldConstants field, String content) { - try (TokenStream contentStream = SearchFieldConstants.JABREF_ANALYZER.tokenStream(field.toString(), content)) { + try (TokenStream contentStream = SearchFieldConstants.LATEX_AWARE_ANALYZER.tokenStream(field.toString(), content)) { TextFragment[] frags = highlighter.getBestTextFragments(contentStream, content, true, 10); return Arrays.stream(frags).map(TextFragment::toString).toList(); } catch (IOException | InvalidTokenOffsetsException e) { diff --git a/src/test/java/org/jabref/logic/search/LuceneQueryParserTest.java b/src/test/java/org/jabref/logic/search/LuceneQueryParserTest.java index 99cb1222e1d..0ce6a3bda13 100644 --- a/src/test/java/org/jabref/logic/search/LuceneQueryParserTest.java +++ b/src/test/java/org/jabref/logic/search/LuceneQueryParserTest.java @@ -33,7 +33,7 @@ public static Stream searchQuires() { @ParameterizedTest @MethodSource void searchQuires(String expected, String query) throws ParseException { - QueryParser parser = new QueryParser(SearchFieldConstants.DEFAULT_FIELD.toString(), SearchFieldConstants.JABREF_ANALYZER); + QueryParser parser = new QueryParser(SearchFieldConstants.DEFAULT_FIELD.toString(), SearchFieldConstants.LATEX_AWARE_ANALYZER); String result = parser.parse(query).toString(); assertEquals(expected, result); } From e64242d5fa2e3723f915f60b5bedc4200894c65a Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Thu, 29 Aug 2024 00:58:51 +0200 Subject: [PATCH 190/256] Discard changes to src/main/resources/tinylog.properties --- src/main/resources/tinylog.properties | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/main/resources/tinylog.properties b/src/main/resources/tinylog.properties index 898616c193f..a1168dd1506 100644 --- a/src/main/resources/tinylog.properties +++ b/src/main/resources/tinylog.properties @@ -8,9 +8,8 @@ writerConsole.format = {date} [{thread}] {class}.{method}()\n{level}: {message}\ exception = strip: jdk.internal #level@org.jabref.model.entry.BibEntry = debug -level@org.jabref.gui.maintable = debug -level@org.jabref.logic.search = debug -level@org.jabref.model.search = debug +#level@org.jabref.gui.maintable.PersistenceVisualStateTable = debug + level@org.jabref.http.server.Server = debug #level@org.jabref.gui.JabRefGUI = debug @@ -24,6 +23,3 @@ level@org.jabref.http.server.Server = debug #level@org.jabref.logic.ai.FileEmbeddingsManager = trace #level@org.jabref.logic.ai.impl.embeddings.LowLevelIngestor = trace #level@org.jabref.logic.ai.chat.AiChatLogic = trace - -level@org.jabref.model.groups.SearchGroup = debug - From f6b3d94c689e7ef5ded23de54b04a6f84fff7e7e Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Thu, 29 Aug 2024 01:02:00 +0200 Subject: [PATCH 191/256] Remove commented out code --- .../search/DatabaseSearcherWithBibFilesTest.java | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java b/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java index 4caef90eaf0..da3674f96ca 100644 --- a/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java +++ b/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java @@ -25,7 +25,6 @@ import org.jabref.preferences.PreferencesService; import com.airhacks.afterburner.injection.Injector; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -98,19 +97,9 @@ private BibDatabase initializeDatabaseFromPath(Path testFile) throws Exception { when(preferencesService.getFilePreferences()).thenReturn(filePreferences); Injector.setModelOrService(PreferencesService.class, preferencesService); - // pdfIndexer = PdfIndexerManager.getIndexer(context, filePreferences); - // Alternative - For debugging with Luke (part of the Apache Lucene distribution) - // pdfIndexer = PdfIndexer.of(context, Path.of("C:\\temp\\index"), filePreferences); - - // pdfIndexer.rebuildIndex(); return database; } - @AfterEach - void tearDown() throws Exception { - // pdfIndexer.close(); - } - private static Stream searchLibrary() { return Stream.of( // empty library From 7d2834ba9f5039dc3fa2f15739a8049d07e332ae Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Thu, 29 Aug 2024 01:04:28 +0200 Subject: [PATCH 192/256] Remove obsolete testing class --- .../org/jabref/logic/search/LuceneTest.java | 52 ------------------- 1 file changed, 52 deletions(-) delete mode 100644 src/test/java/org/jabref/logic/search/LuceneTest.java diff --git a/src/test/java/org/jabref/logic/search/LuceneTest.java b/src/test/java/org/jabref/logic/search/LuceneTest.java deleted file mode 100644 index 1e006c4e718..00000000000 --- a/src/test/java/org/jabref/logic/search/LuceneTest.java +++ /dev/null @@ -1,52 +0,0 @@ -package org.jabref.logic.search; - -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.standard.StandardAnalyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.TextField; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.queryparser.classic.QueryParser; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.store.ByteBuffersDirectory; -import org.apache.lucene.store.Directory; - -public class LuceneTest { - public static void main(String[] args) throws Exception { - // Setup the analyzer - Analyzer analyzer = new StandardAnalyzer(); - - // Store the index in memory - Directory directory = new ByteBuffersDirectory(); - - // Configure index writer - IndexWriterConfig config = new IndexWriterConfig(analyzer); - IndexWriter indexWriter = new IndexWriter(directory, config); - - // Index sample data - String[] texts = {"running", "runner", "ran", "trial", "trials"}; - for (String text : texts) { - Document document = new Document(); - document.add(new TextField("content", text, Field.Store.YES)); - indexWriter.addDocument(document); - } - indexWriter.close(); - - search("trials", directory, analyzer); - } - - public static void search(String queryString, Directory directory, Analyzer analyzer) throws Exception { - Query query = new QueryParser("content", analyzer).parse(queryString); - IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(directory)); - ScoreDoc[] hits = searcher.search(query, 10).scoreDocs; - - for (ScoreDoc scoreDoc : hits) { - Document doc = searcher.doc(scoreDoc.doc); -// System.out.println(doc.get("content")); - } - } -} From 033226cb1e55d14304e01be2a4ccc535ddd26485 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Thu, 29 Aug 2024 01:06:32 +0200 Subject: [PATCH 193/256] Remove obsolete test --- .../jabref/logic/search/SearchQueryTest.java | 272 ------------------ 1 file changed, 272 deletions(-) delete mode 100644 src/test/java/org/jabref/logic/search/SearchQueryTest.java diff --git a/src/test/java/org/jabref/logic/search/SearchQueryTest.java b/src/test/java/org/jabref/logic/search/SearchQueryTest.java deleted file mode 100644 index 8d58dbb02ad..00000000000 --- a/src/test/java/org/jabref/logic/search/SearchQueryTest.java +++ /dev/null @@ -1,272 +0,0 @@ -package org.jabref.logic.search; - -import java.io.IOException; -import java.nio.file.Path; -import java.util.EnumSet; - -import org.jabref.logic.search.retrieval.LuceneSearcher; -import org.jabref.model.database.BibDatabase; -import org.jabref.model.database.BibDatabaseContext; -import org.jabref.model.search.SearchFlags; -import org.jabref.model.search.SearchQuery; -import org.jabref.preferences.PreferencesService; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -import static org.junit.jupiter.api.Assertions.assertNull; - -public class SearchQueryTest { - - private LuceneSearcher searcher; - private PreferencesService preferencesService; - private BibDatabase bibDatabase; - private BibDatabaseContext bibDatabaseContext; - - @BeforeEach - public void setUp(@TempDir Path indexDir) throws IOException { -// preferencesService = mock(PreferencesService.class); -// when(preferencesService.getFilePreferences()).thenReturn(mock(FilePreferences.class)); -// -// bibDatabase = new BibDatabase(); -// bibDatabaseContext = mock(BibDatabaseContext.class); -// when(bibDatabaseContext.getFileDirectories(Mockito.any())).thenReturn(Collections.singletonList(Path.of("src/test/resources/pdfs"))); -// when(bibDatabaseContext.getFulltextIndexPath()).thenReturn(indexDir); -// when(bibDatabaseContext.getDatabase()).thenReturn(bibDatabase); -// when(bibDatabaseContext.getEntries()).thenReturn(bibDatabase.getEntries()); -// -// LuceneIndexer indexer = LuceneIndexer.of(bibDatabaseContext, preferencesService); -// indexer.createIndex(); -// -// searcher = LuceneSearcher.of(bibDatabaseContext); - } - - @Test - public void nullWhenQueryBlank() { - assertNull(new SearchQuery("", EnumSet.noneOf(SearchFlags.class)).getParsedQuery()); - } - -// @Test -// public void testToString() { -// assertEquals("\"asdf\" (case sensitive, regular expression) [CASE_SENSITIVE, REGULAR_EXPRESSION]", new SearchQuery("asdf", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)).toString()); -// assertEquals("\"asdf\" (case insensitive, plain text) []", new SearchQuery("asdf", EnumSet.noneOf(SearchFlags.class)).toString()); -// } -// -// @Test -// public void isContainsBasedSearch() { -// assertTrue(new SearchQuery("asdf", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE)).isContainsBasedSearch()); -// assertTrue(new SearchQuery("asdf", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)).isContainsBasedSearch()); -// assertFalse(new SearchQuery("author=asdf", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE)).isContainsBasedSearch()); -// } -// -// @Test -// public void isGrammarBasedSearch() { -// assertFalse(new SearchQuery("asdf", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE)).isGrammarBasedSearch()); -// assertFalse(new SearchQuery("asdf", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)).isGrammarBasedSearch()); -// assertTrue(new SearchQuery("author=asdf", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE)).isGrammarBasedSearch()); -// } -// -// @Test -// public void grammarSearch() { -// BibEntry entry = new BibEntry(); -// entry.addKeyword("one two", ','); -// SearchQuery searchQuery = new SearchQuery("keywords=\"one two\"", EnumSet.noneOf(SearchFlags.class)); -// assertTrue(searchQuery.isMatch(entry)); -// } -// -// @Test -// public void grammarSearchFullEntryLastCharMissing() { -// BibEntry entry = new BibEntry(); -// entry.setField(StandardField.TITLE, "systematic revie"); -// SearchQuery searchQuery = new SearchQuery("title=\"systematic review\"", EnumSet.noneOf(SearchFlags.class)); -// assertFalse(searchQuery.isMatch(entry)); -// } -// -// @Test -// public void grammarSearchFullEntry() { -// BibEntry entry = new BibEntry(); -// entry.setField(StandardField.TITLE, "systematic review"); -// SearchQuery searchQuery = new SearchQuery("title=\"systematic review\"", EnumSet.noneOf(SearchFlags.class)); -// assertTrue(searchQuery.isMatch(entry)); -// } -// -// @Test -// public void searchingForOpenBraketInBooktitle() { -// BibEntry e = new BibEntry(StandardEntryType.InProceedings); -// e.setField(StandardField.BOOKTITLE, "Super Conference (SC)"); -// -// SearchQuery searchQuery = new SearchQuery("booktitle=\"(\"", EnumSet.noneOf(SearchFlags.class)); -// assertTrue(searchQuery.isMatch(e)); -// } -// -// @Test -// public void searchMatchesSingleKeywordNotPart() { -// BibEntry e = new BibEntry(StandardEntryType.InProceedings); -// e.setField(StandardField.KEYWORDS, "banana, pineapple, orange"); -// -// SearchQuery searchQuery = new SearchQuery("anykeyword==apple", EnumSet.noneOf(SearchFlags.class)); -// assertFalse(searchQuery.isMatch(e)); -// } -// -// @Test -// public void searchMatchesSingleKeyword() { -// BibEntry e = new BibEntry(StandardEntryType.InProceedings); -// e.setField(StandardField.KEYWORDS, "banana, pineapple, orange"); -// -// SearchQuery searchQuery = new SearchQuery("anykeyword==pineapple", EnumSet.noneOf(SearchFlags.class)); -// assertTrue(searchQuery.isMatch(e)); -// } -// -// @Test -// public void searchAllFields() { -// BibEntry e = new BibEntry(StandardEntryType.InProceedings); -// e.setField(StandardField.TITLE, "Fruity features"); -// e.setField(StandardField.KEYWORDS, "banana, pineapple, orange"); -// -// SearchQuery searchQuery = new SearchQuery("anyfield==\"fruity features\"", EnumSet.noneOf(SearchFlags.class)); -// assertTrue(searchQuery.isMatch(e)); -// } -// -// @Test -// public void searchAllFieldsNotForSpecificField() { -// BibEntry e = new BibEntry(StandardEntryType.InProceedings); -// e.setField(StandardField.TITLE, "Fruity features"); -// e.setField(StandardField.KEYWORDS, "banana, pineapple, orange"); -// -// SearchQuery searchQuery = new SearchQuery("anyfield=fruit and keywords!=banana", EnumSet.noneOf(SearchFlags.class)); -// assertFalse(searchQuery.isMatch(e)); -// } -// -// @Test -// public void searchAllFieldsAndSpecificField() { -// BibEntry e = new BibEntry(StandardEntryType.InProceedings); -// e.setField(StandardField.TITLE, "Fruity features"); -// e.setField(StandardField.KEYWORDS, "banana, pineapple, orange"); -// -// SearchQuery searchQuery = new SearchQuery("anyfield=fruit and keywords=apple", EnumSet.noneOf(SearchFlags.class)); -// assertTrue(searchQuery.isMatch(e)); -// } -// -// @Test -// public void isMatch() { -// BibEntry entry = new BibEntry(); -// entry.setType(StandardEntryType.Article); -// entry.setField(StandardField.AUTHOR, "asdf"); -// -// assertFalse(new SearchQuery("BiblatexEntryType", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)).isMatch(entry)); -// assertTrue(new SearchQuery("asdf", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)).isMatch(entry)); -// assertTrue(new SearchQuery("author=asdf", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)).isMatch(entry)); -// } -// -// @Test -// public void isValidQueryNotAsRegEx() { -// assertTrue(new SearchQuery("asdf", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE)).isValid()); -// } -// -// @Test -// public void isValidQueryContainsBracketNotAsRegEx() { -// assertTrue(new SearchQuery("asdf[", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE)).isValid()); -// } -// -// @Test -// public void isNotValidQueryContainsBracketNotAsRegEx() { -// assertTrue(new SearchQuery("asdf[", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)).isValid()); -// } -// -// @Test -// public void isValidQueryAsRegEx() { -// assertTrue(new SearchQuery("asdf", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)).isValid()); -// } -// -// @Test -// public void isValidQueryWithNumbersAsRegEx() { -// assertTrue(new SearchQuery("123", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)).isValid()); -// } -// -// @Test -// public void isValidQueryContainsBracketAsRegEx() { -// assertTrue(new SearchQuery("asdf[", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)).isValid()); -// } -// -// @Test -// public void isValidQueryWithEqualSignAsRegEx() { -// assertTrue(new SearchQuery("author=asdf", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)).isValid()); -// } -// -// @Test -// public void isValidQueryWithNumbersAndEqualSignAsRegEx() { -// assertTrue(new SearchQuery("author=123", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)).isValid()); -// } -// -// @Test -// public void isValidQueryWithEqualSignNotAsRegEx() { -// assertTrue(new SearchQuery("author=asdf", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE)).isValid()); -// } -// -// @Test -// public void isValidQueryWithNumbersAndEqualSignNotAsRegEx() { -// assertTrue(new SearchQuery("author=123", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE)).isValid()); -// } -// -// @Test -// public void isMatchedForNormalAndFieldBasedSearchMixed() { -// BibEntry entry = new BibEntry(); -// entry.setType(StandardEntryType.Article); -// entry.setField(StandardField.AUTHOR, "asdf"); -// entry.setField(StandardField.ABSTRACT, "text"); -// -// assertTrue(new SearchQuery("text AND author=asdf", EnumSet.of(SearchRules.SearchFlags.CASE_SENSITIVE, SearchRules.SearchFlags.REGULAR_EXPRESSION)).isMatch(entry)); -// } -// -// @Test -// public void simpleTerm() { -// String query = "progress"; -// -// SearchQuery result = new SearchQuery(query, EnumSet.noneOf(SearchFlags.class)); -// assertFalse(result.isGrammarBasedSearch()); -// } -// -// @Test -// public void getPattern() { -// String query = "progress"; -// SearchQuery result = new SearchQuery(query, EnumSet.noneOf(SearchFlags.class)); -// Pattern pattern = Pattern.compile("(\\Qprogress\\E)"); -// // We can't directly compare the pattern objects -// assertEquals(Optional.of(pattern.toString()), result.getPatternForWords().map(Pattern::toString)); -// } -// -// @Test -// public void getRegexpPattern() { -// String queryText = "[a-c]\\d* \\d*"; -// SearchQuery regexQuery = new SearchQuery(queryText, EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)); -// Pattern pattern = Pattern.compile("([a-c]\\d* \\d*)"); -// assertEquals(Optional.of(pattern.toString()), regexQuery.getPatternForWords().map(Pattern::toString)); -// } -// -// @Test -// public void getRegexpJavascriptPattern() { -// String queryText = "[a-c]\\d* \\d*"; -// SearchQuery regexQuery = new SearchQuery(queryText, EnumSet.of(SearchRules.SearchFlags.REGULAR_EXPRESSION)); -// Pattern pattern = Pattern.compile("([a-c]\\d* \\d*)"); -// assertEquals(Optional.of(pattern.toString()), regexQuery.getJavaScriptPatternForWords().map(Pattern::toString)); -// } -// -// @Test -// public void escapingInPattern() { -// // first word contain all java special regex characters -// String queryText = "<([{\\\\^-=$!|]})?*+.> word1 word2."; -// SearchQuery textQueryWithSpecialChars = new SearchQuery(queryText, EnumSet.noneOf(SearchFlags.class)); -// String pattern = "(\\Q<([{\\^-=$!|]})?*+.>\\E)|(\\Qword1\\E)|(\\Qword2.\\E)"; -// assertEquals(Optional.of(pattern), textQueryWithSpecialChars.getPatternForWords().map(Pattern::toString)); -// } -// -// @Test -// public void escapingInJavascriptPattern() { -// // first word contain all javascript special regex characters that should be escaped individually in text based search -// String queryText = "([{\\\\^$|]})?*+./ word1 word2."; -// SearchQuery textQueryWithSpecialChars = new SearchQuery(queryText, EnumSet.noneOf(SearchFlags.class)); -// String pattern = "(\\(\\[\\{\\\\\\^\\$\\|\\]\\}\\)\\?\\*\\+\\.\\/)|(word1)|(word2\\.)"; -// assertEquals(Optional.of(pattern), textQueryWithSpecialChars.getJavaScriptPatternForWords().map(Pattern::toString)); -// } -} From 5d0118d951318195f09d86fc7ee8d75291a27b61 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Thu, 29 Aug 2024 01:10:07 +0200 Subject: [PATCH 194/256] Discard changes to src/test/resources/tinylog-test.properties --- src/test/resources/tinylog-test.properties | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/resources/tinylog-test.properties b/src/test/resources/tinylog-test.properties index ed6c4288b2a..640a69a1005 100644 --- a/src/test/resources/tinylog-test.properties +++ b/src/test/resources/tinylog-test.properties @@ -4,4 +4,3 @@ writer = console #level@org.jabref.model.entry.BibEntry = debug #level@org.jabref.logic.importer.fetcher.ResearchGate = trace #level@org.jabref.logic.importer.fetcher.DoiFetcher = trace -level@org.jabref.model.search.LatexToUnicodeFoldingFilter= debug\ From 78264d72868c5b12afbec7c2412badb05bf1ebcc Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Thu, 29 Aug 2024 01:10:48 +0200 Subject: [PATCH 195/256] Remove completely disabled code --- .../search/retrieval/LuceneSearcherTest.java | 115 ------------------ 1 file changed, 115 deletions(-) delete mode 100644 src/test/java/org/jabref/logic/search/retrieval/LuceneSearcherTest.java diff --git a/src/test/java/org/jabref/logic/search/retrieval/LuceneSearcherTest.java b/src/test/java/org/jabref/logic/search/retrieval/LuceneSearcherTest.java deleted file mode 100644 index 02ab3868336..00000000000 --- a/src/test/java/org/jabref/logic/search/retrieval/LuceneSearcherTest.java +++ /dev/null @@ -1,115 +0,0 @@ -package org.jabref.logic.search.retrieval; - -public class LuceneSearcherTest { - -// private LuceneSearcher searcher; -// private PreferencesService preferencesService; -// private BibDatabase bibDatabase; -// private BibDatabaseContext bibDatabaseContext; -// -// @BeforeEach -// public void setUp(@TempDir Path indexDir) throws IOException { -// preferencesService = mock(PreferencesService.class); -// when(preferencesService.getFilePreferences()).thenReturn(mock(FilePreferences.class)); -// -// bibDatabase = new BibDatabase(); -// bibDatabaseContext = mock(BibDatabaseContext.class); -// when(bibDatabaseContext.getFileDirectories(Mockito.any())).thenReturn(Collections.singletonList(Path.of("src/test/resources/pdfs"))); -// when(bibDatabaseContext.getFulltextIndexPath()).thenReturn(indexDir); -// when(bibDatabaseContext.getDatabase()).thenReturn(bibDatabase); -// when(bibDatabaseContext.getEntries()).thenReturn(bibDatabase.getEntries()); -// } -// -// private void initIndexer() throws IOException { -// LuceneIndexer indexer = LuceneIndexer.of(bibDatabaseContext, preferencesService); -// searcher = LuceneSearcher.of(bibDatabaseContext); -// -// for (BibEntry bibEntry : bibDatabaseContext.getEntries()) { -// indexer.addBibFieldsToIndex(bibEntry); -// indexer.addLinkedFilesToIndex(bibEntry); -// } -// } -// -// @Test -// public void searchForTest() throws IOException { -// insertPdfsForSearch(); -// initIndexer(); -// -// HashMap searchResults = searcher.search(new SearchQuery("", EnumSet.noneOf(SearchFlags.class))); -// int hits = searchResults.keySet().stream().mapToInt(key -> searchResults.get(key).numSearchResults()).sum(); -// assertEquals(8, hits); -// } -// -// @Test -// public void searchForUniversity() throws IOException { -// insertPdfsForSearch(); -// initIndexer(); -// -// HashMap searchResults = searcher.search(new SearchQuery("University", EnumSet.noneOf(SearchFlags.class))); -// int hits = searchResults.keySet().stream().mapToInt(key -> searchResults.get(key).numSearchResults()).sum(); -// assertEquals(1, hits); -// } -// -// @Test -// public void searchForStopWord() throws IOException { -// insertPdfsForSearch(); -// initIndexer(); -// -// HashMap searchResults = searcher.search(new SearchQuery("and", EnumSet.noneOf(SearchFlags.class))); -// int hits = searchResults.keySet().stream().mapToInt(key -> searchResults.get(key).numSearchResults()).sum(); -// assertEquals(0, hits); -// } -// -// @Test -// public void searchForSecond() throws IOException { -// insertPdfsForSearch(); -// initIndexer(); -// -// HashMap searchResults = searcher.search(new SearchQuery("second", EnumSet.noneOf(SearchFlags.class))); -// int hits = searchResults.keySet().stream().mapToInt(key -> searchResults.get(key).numSearchResults()).sum(); -// assertEquals(4, hits); -// } -// -// @Test -// public void searchForAnnotation() throws IOException { -// insertPdfsForSearch(); -// initIndexer(); -// -// HashMap searchResults = searcher.search(new SearchQuery("annotation", EnumSet.noneOf(SearchFlags.class))); -// int hits = searchResults.keySet().stream().mapToInt(key -> searchResults.get(key).numSearchResults()).sum(); -// assertEquals(2, hits); -// } -// -// @Test -// public void searchForEmptyString() throws IOException { -// insertPdfsForSearch(); -// initIndexer(); -// -// HashMap searchResults = searcher.search(new SearchQuery("", EnumSet.noneOf(SearchFlags.class))); -// int hits = searchResults.keySet().stream().mapToInt(key -> searchResults.get(key).numSearchResults()).sum(); -// assertEquals(0, hits); -// } -// -// @Test -// public void searchWithNullString() { -// assertThrows(NullPointerException.class, () -> searcher.search(null)); -// } -// -// private void insertPdfsForSearch() { -// when(preferencesService.getFilePreferences().shouldFulltextIndexLinkedFiles()).thenReturn(true); -// -// BibEntry examplePdf = new BibEntry(StandardEntryType.Article); -// examplePdf.setFiles(Collections.singletonList(new LinkedFile("Example Entry", "example.pdf", StandardFileType.PDF.getName()))); -// bibDatabase.insertEntry(examplePdf); -// -// BibEntry metaDataEntry = new BibEntry(StandardEntryType.Article); -// metaDataEntry.setFiles(Collections.singletonList(new LinkedFile("Metadata Entry", "metaData.pdf", StandardFileType.PDF.getName()))); -// metaDataEntry.setCitationKey("MetaData2017"); -// bibDatabase.insertEntry(metaDataEntry); -// -// BibEntry exampleThesis = new BibEntry(StandardEntryType.PhdThesis); -// exampleThesis.setFiles(Collections.singletonList(new LinkedFile("Example Thesis", "thesis-example.pdf", StandardFileType.PDF.getName()))); -// exampleThesis.setCitationKey("ExampleThesis"); -// bibDatabase.insertEntry(exampleThesis); -// } -} From 3dfd01e5dc01935eada891b36c41e9f46622308d Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Thu, 29 Aug 2024 02:18:00 +0300 Subject: [PATCH 196/256] Rename "all" to "any" --- .../jabref/migrations/SearchToLuceneVisitor.java | 2 +- .../model/search/SearchFieldConstants.java | 2 +- .../logic/search/LuceneQueryParserTest.java | 12 ++++++------ .../migrations/SearchToLuceneMigrationTest.java | 16 ++++++++-------- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/jabref/migrations/SearchToLuceneVisitor.java b/src/main/java/org/jabref/migrations/SearchToLuceneVisitor.java index 6e3c75bfcf6..aabdf570185 100644 --- a/src/main/java/org/jabref/migrations/SearchToLuceneVisitor.java +++ b/src/main/java/org/jabref/migrations/SearchToLuceneVisitor.java @@ -51,7 +51,7 @@ public QueryNode visitStart(SearchParser.StartContext ctx) { if (result instanceof AndQueryNode andQueryNode) { if (andQueryNode.getChildren().stream().allMatch(child -> child instanceof ModifierQueryNode modifierQueryNode && modifierQueryNode.getModifier() == ModifierQueryNode.Modifier.MOD_NOT)) { List children = andQueryNode.getChildren().stream() - // prepend "all:* AND" to each child + // prepend "any:* AND" to each child .map(child -> new AndQueryNode(List.of(new FieldQueryNode(SearchFieldConstants.DEFAULT_FIELD.toString(), "*", 0, 0), child))) .map(child -> (QueryNode) child) .toList(); diff --git a/src/main/java/org/jabref/model/search/SearchFieldConstants.java b/src/main/java/org/jabref/model/search/SearchFieldConstants.java index cb9f3b21d7f..f28adbf704c 100644 --- a/src/main/java/org/jabref/model/search/SearchFieldConstants.java +++ b/src/main/java/org/jabref/model/search/SearchFieldConstants.java @@ -8,7 +8,7 @@ public enum SearchFieldConstants { VERSION("99"), - DEFAULT_FIELD("all"), + DEFAULT_FIELD("any"), ENTRY_ID("id"), ENTRY_TYPE("entrytype"), PATH("path"), diff --git a/src/test/java/org/jabref/logic/search/LuceneQueryParserTest.java b/src/test/java/org/jabref/logic/search/LuceneQueryParserTest.java index 0ce6a3bda13..8d495609da4 100644 --- a/src/test/java/org/jabref/logic/search/LuceneQueryParserTest.java +++ b/src/test/java/org/jabref/logic/search/LuceneQueryParserTest.java @@ -17,14 +17,14 @@ public class LuceneQueryParserTest { public static Stream searchQuires() { return Stream.of( // unicode - Arguments.of("all:preissinger", "preißinger"), - Arguments.of("all:jesus", "jesús"), - Arguments.of("all:breitenbucher", "breitenbücher"), + Arguments.of("any:preissinger", "preißinger"), + Arguments.of("any:jesus", "jesús"), + Arguments.of("any:breitenbucher", "breitenbücher"), // latex - Arguments.of("all:preissinger", "\"prei{\\\\ss}inger\""), - Arguments.of("all:jesus", "\"jes{\\\\'{u}}s\""), - Arguments.of("all:breitenbucher", "\"breitenb{\\\\\\\"{u}}cher\""), + Arguments.of("any:preissinger", "\"prei{\\\\ss}inger\""), + Arguments.of("any:jesus", "\"jes{\\\\'{u}}s\""), + Arguments.of("any:breitenbucher", "\"breitenb{\\\\\\\"{u}}cher\""), Arguments.of("groups:/exclude", "groups:\\/exclude") ); diff --git a/src/test/java/org/jabref/migrations/SearchToLuceneMigrationTest.java b/src/test/java/org/jabref/migrations/SearchToLuceneMigrationTest.java index 8fea29c57c9..9147c8c655b 100644 --- a/src/test/java/org/jabref/migrations/SearchToLuceneMigrationTest.java +++ b/src/test/java/org/jabref/migrations/SearchToLuceneMigrationTest.java @@ -12,8 +12,8 @@ class SearchToLuceneMigrationTest { public static Stream transformationNormal() { return Stream.of( - // If "all:" should not be added, see e46e0a23d7b9526bd449cbecfb189ae6dbc40a28 for a fix - Arguments.of("all:chocolate", "chocolate"), + // If "any:" should not be added, see e46e0a23d7b9526bd449cbecfb189ae6dbc40a28 for a fix + Arguments.of("any:chocolate", "chocolate"), Arguments.of("title:chocolate", "title=chocolate"), Arguments.of("title:chocolate OR author:smith", "title = chocolate or author = smith"), @@ -22,11 +22,11 @@ public static Stream transformationNormal() { Arguments.of("title:chocolate AND author:smith", "title contains \"chocolate\" AND author matches \"smith\""), Arguments.of("( title:chocolate ) OR ( author:smith )", "(title == chocolate) or (author == smith)"), Arguments.of("( title:chocolate OR author:smith ) AND ( year:2024 )", "(title contains chocolate or author matches smith) AND (year = 2024)"), - Arguments.of("all:video AND year:1932", "video and year == 1932"), + Arguments.of("any:video AND year:1932", "video and year == 1932"), Arguments.of("title:neighbou?r", "title =neighbou?r"), Arguments.of("abstract:model\\{1,2\\}ing", "abstract = model{1,2}ing"), - Arguments.of("all:* AND -title:chocolate", "title != chocolate"), - Arguments.of("all:* AND -title:chocolate", "not title contains chocolate"), + Arguments.of("any:* AND -title:chocolate", "title != chocolate"), + Arguments.of("any:* AND -title:chocolate", "not title contains chocolate"), // https://github.com/JabRef/jabref/issues/11654#issuecomment-2313178736 Arguments.of("( groups:\\:paywall AND -file:/.+/ ) AND -groups:\\/exclude", "groups=:paywall and file!=\"\" and groups!=/exclude"), @@ -40,7 +40,7 @@ public static Stream transformationNormal() { Arguments.of("title:image\\ processing AND author:smith", "title = \"image processing\" AND author= smith"), // We renamed fields - Arguments.of("all:somecontent", "anyfield = somecontent"), + Arguments.of("any:somecontent", "anyfield = somecontent"), Arguments.of("keywords:somekeyword", "anykeyword = somekeyword"), Arguments.of("citationkey:somebibtexkey", "key = somebibtexkey") ); @@ -55,8 +55,8 @@ void transformationNormal(String expected, String query) { public static Stream transformationRegularExpression() { return Stream.of( - Arguments.of("all:* AND -groups:/.+/", "groups != .+"), - Arguments.of("( all:* AND -groups:/.+/ ) AND ( all:* AND -readstatus:/.+/ )", "groups != .+ and readstatus != .+"), + Arguments.of("any:* AND -groups:/.+/", "groups != .+"), + Arguments.of("( any:* AND -groups:/.+/ ) AND ( any:* AND -readstatus:/.+/ )", "groups != .+ and readstatus != .+"), Arguments.of("author:/(John|Doe).+(John|Doe)/", "author = \"(John|Doe).+(John|Doe)\"") ); } From e2c430b7751eea6c6bf84e0394f02530962c039e Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Thu, 29 Aug 2024 02:24:36 +0300 Subject: [PATCH 197/256] Catch thrown exception Invalid regex queries throws an exception --- src/main/java/org/jabref/model/search/SearchQuery.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/org/jabref/model/search/SearchQuery.java b/src/main/java/org/jabref/model/search/SearchQuery.java index 04e1846d36e..f8c2078bd73 100644 --- a/src/main/java/org/jabref/model/search/SearchQuery.java +++ b/src/main/java/org/jabref/model/search/SearchQuery.java @@ -13,7 +13,6 @@ import java.util.stream.Stream; import org.apache.lucene.queryparser.classic.MultiFieldQueryParser; -import org.apache.lucene.queryparser.classic.ParseException; import org.apache.lucene.search.Query; import org.apache.lucene.search.highlight.QueryTermExtractor; import org.apache.lucene.search.highlight.WeightedTerm; @@ -85,7 +84,7 @@ public SearchQuery(String query, EnumSet searchFlags) { try { parsedQuery = queryParser.parse(query); parseError = null; - } catch (ParseException e) { + } catch (Exception e) { parsedQuery = null; parseError = e.getMessage(); } From 05b6270967de11eef19a62ab2456099402565cc4 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Thu, 29 Aug 2024 02:31:14 +0300 Subject: [PATCH 198/256] Remove groups field from the default field https://github.com/JabRef/jabref/issues/7996 --- .../org/jabref/logic/search/indexing/BibFieldsIndexer.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java b/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java index 5277fba3f19..a47a02e2d23 100644 --- a/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java @@ -10,6 +10,7 @@ import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.Field; +import org.jabref.model.entry.field.StandardField; import org.jabref.model.search.LuceneIndexer; import org.jabref.model.search.SearchFieldConstants; @@ -83,6 +84,10 @@ private void addToIndex(BibEntry bibEntry) { StringBuilder allFields = new StringBuilder(bibEntry.getType().getName()); for (Map.Entry mapEntry : bibEntry.getFieldMap().entrySet()) { document.add(new TextField(mapEntry.getKey().getName(), mapEntry.getValue(), storeDisabled)); + if (mapEntry.getKey().equals(StandardField.GROUPS)) { + // Do not add groups to the allFields field: https://github.com/JabRef/jabref/issues/7996 + continue; + } allFields.append('\n').append(mapEntry.getValue()); } document.add(new TextField(SearchFieldConstants.DEFAULT_FIELD.toString(), allFields.toString(), storeDisabled)); From cde42f0a0c05d70224be7b5251df5f2a353ed5f4 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Thu, 29 Aug 2024 07:56:25 +0300 Subject: [PATCH 199/256] Remove SearchGroupsListener --- .../jabref/gui/groups/GroupNodeViewModel.java | 5 +---- .../jabref/gui/maintable/MainTableDataModel.java | 16 ---------------- .../groups/event/SearchGroupUpdatedEvent.java | 6 ------ 3 files changed, 1 insertion(+), 26 deletions(-) delete mode 100644 src/main/java/org/jabref/model/groups/event/SearchGroupUpdatedEvent.java diff --git a/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java b/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java index 7458aa0b455..b5fc1fc11df 100644 --- a/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java +++ b/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java @@ -45,7 +45,6 @@ import org.jabref.model.groups.RegexKeywordGroup; import org.jabref.model.groups.SearchGroup; import org.jabref.model.groups.TexGroup; -import org.jabref.model.groups.event.SearchGroupUpdatedEvent; import org.jabref.model.search.event.IndexAddedOrUpdatedEvent; import org.jabref.model.search.event.IndexClosedEvent; import org.jabref.model.search.event.IndexRemovedEvent; @@ -538,7 +537,6 @@ public void listen(IndexStartedEvent event) { stateManager.getLuceneManager(databaseContext).ifPresent(searchGroup::setLuceneManager); searchGroup.updateMatches(); refreshGroup(); - databaseContext.getDatabase().postEvent(new SearchGroupUpdatedEvent(groupNode)); databaseContext.getMetaData().groupsBinding().invalidate(); } } @@ -557,8 +555,7 @@ public void listen(IndexAddedOrUpdatedEvent event) { matchedEntries.remove(System.identityHashCode(entry)); } } - databaseContext.getMetaData().groupsBinding().invalidate(); - }).onFinished(() -> databaseContext.getDatabase().postEvent(new SearchGroupUpdatedEvent(groupNode))).executeWith(taskExecutor); + }).executeWith(taskExecutor); } } diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index 371caed6736..30d0cf288bc 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -26,7 +26,6 @@ import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.groups.GroupTreeNode; -import org.jabref.model.groups.event.SearchGroupUpdatedEvent; import org.jabref.model.search.SearchFieldConstants; import org.jabref.model.search.SearchQuery; import org.jabref.model.search.SearchResults; @@ -60,9 +59,7 @@ public class MainTableDataModel { private final Subscription selectedGroupsSubscription; private final Subscription groupViewModeSubscription; private final LuceneIndexListener indexUpdatedListener; - private final SearchGroupsListener searchGroupsListener; private final OptionalObjectProperty searchQueryProperty; - private final ListProperty selectedGroupsProperty; private Optional groupsMatcher; public MainTableDataModel(BibDatabaseContext context, @@ -81,13 +78,10 @@ public MainTableDataModel(BibDatabaseContext context, this.luceneManager = luceneManager; this.bibDatabaseContext = context; this.groupsMatcher = createGroupMatcher(selectedGroupsProperty.get(), groupsPreferences); - this.selectedGroupsProperty = selectedGroupsProperty; this.searchQueryProperty = searchQueryProperty; this.indexUpdatedListener = new LuceneIndexListener(); - this.searchGroupsListener = new SearchGroupsListener(); this.bibDatabaseContext.getDatabase().registerListener(indexUpdatedListener); - this.bibDatabaseContext.getDatabase().registerListener(searchGroupsListener); resetFieldFormatter(); allEntries = BindingsHelper.forUI(context.getDatabase().getEntries()); @@ -191,7 +185,6 @@ public void unbind() { groupViewModeSubscription.unsubscribe(); bibDatabaseContext.getDatabase().unregisterListener(indexUpdatedListener); - bibDatabaseContext.getDatabase().unregisterListener(searchGroupsListener); } public SortedList getEntriesFilteredAndSorted() { @@ -254,13 +247,4 @@ public void listen(IndexStartedEvent indexStartedEvent) { updateSearchMatches(searchQueryProperty.get()); } } - - class SearchGroupsListener { - @Subscribe - public void listen(SearchGroupUpdatedEvent event) { - if (selectedGroupsProperty.get().contains(event.group())) { - updateGroupMatches(selectedGroupsProperty.get()); - } - } - } } diff --git a/src/main/java/org/jabref/model/groups/event/SearchGroupUpdatedEvent.java b/src/main/java/org/jabref/model/groups/event/SearchGroupUpdatedEvent.java deleted file mode 100644 index 56df3d4c609..00000000000 --- a/src/main/java/org/jabref/model/groups/event/SearchGroupUpdatedEvent.java +++ /dev/null @@ -1,6 +0,0 @@ -package org.jabref.model.groups.event; - -import org.jabref.model.groups.GroupTreeNode; - -public record SearchGroupUpdatedEvent(GroupTreeNode group) { -} From 5cc131205f6a5a37c5837e7985e5a12fd20b324f Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Thu, 29 Aug 2024 08:01:24 +0300 Subject: [PATCH 200/256] Update Benchmarks.java --- src/jmh/java/org/jabref/benchmarks/Benchmarks.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/jmh/java/org/jabref/benchmarks/Benchmarks.java b/src/jmh/java/org/jabref/benchmarks/Benchmarks.java index e1b5bbb061b..d2e79b24c63 100644 --- a/src/jmh/java/org/jabref/benchmarks/Benchmarks.java +++ b/src/jmh/java/org/jabref/benchmarks/Benchmarks.java @@ -101,14 +101,14 @@ public String write() throws Exception { @Benchmark public List search() { - // FixMe: Create Benchmark for LuceneSearch - return null; + // TODO: Create Benchmark for LuceneSearch + return List.of(); } @Benchmark public List index() { - // FixMe: Create Benchmark for LuceneIndexer - return null; + // TODO: Create Benchmark for LuceneIndexer + return List.of(); } @Benchmark From 58184525427032042800b95cc29f6e135d7e2b5e Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Thu, 29 Aug 2024 08:06:47 +0300 Subject: [PATCH 201/256] Update module-info.java --- src/main/java/module-info.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/module-info.java b/src/main/java/module-info.java index ff8a8244dc8..ed8cb6f1f31 100644 --- a/src/main/java/module-info.java +++ b/src/main/java/module-info.java @@ -153,15 +153,15 @@ requires langchain4j.open.ai; // endregion - // region: fulltext search + // region: Lucene /** * In case the version is updated, please also adapt {@link org.jabref.model.search.SearchFieldConstants#VERSION} to the newly used version. */ uses org.apache.lucene.codecs.lucene99.Lucene99Codec; + requires org.apache.lucene.analysis.common; requires org.apache.lucene.core; - requires org.apache.lucene.queryparser; requires org.apache.lucene.highlighter; - requires org.apache.lucene.analysis.common; + requires org.apache.lucene.queryparser; // endregion requires net.harawata.appdirs; From ff56ea9179c398a8456be801d7faa1cb07b9e11f Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Thu, 29 Aug 2024 10:10:03 +0300 Subject: [PATCH 202/256] Fixes from code review on LibraryTab --- src/main/java/org/jabref/gui/LibraryTab.java | 72 +++++++------------ .../gui/exporter/SaveDatabaseAction.java | 2 +- .../gui/maintable/MainTableDataModel.java | 8 ++- .../model/database/BibDatabaseContext.java | 2 +- 4 files changed, 34 insertions(+), 50 deletions(-) diff --git a/src/main/java/org/jabref/gui/LibraryTab.java b/src/main/java/org/jabref/gui/LibraryTab.java index 2545add20d4..567ca8495a7 100644 --- a/src/main/java/org/jabref/gui/LibraryTab.java +++ b/src/main/java/org/jabref/gui/LibraryTab.java @@ -127,7 +127,6 @@ private enum PanelMode { MAIN_TABLE, MAIN_TABLE_AND_ENTRY_EDITOR } private BibDatabaseContext bibDatabaseContext; private MainTableDataModel tableModel; - private CitationStyleCache citationStyleCache; private FileAnnotationCache annotationCache; private EntryEditor entryEditor; private MainTable mainTable; @@ -165,6 +164,12 @@ private enum PanelMode { MAIN_TABLE, MAIN_TABLE_AND_ENTRY_EDITOR } private final DirectoryMonitorManager directoryMonitorManager; private LuceneManager luceneManager; + /** + * @param isDummyContext Indicates whether the database context is a dummy. A dummy context is used to display a progress indicator while parsing the database. + * If the context is a dummy, the Lucene index should not be created, as both the dummy context and the actual context share the same index path {@link BibDatabaseContext#getFulltextIndexPath()}. + * If the index is created for the dummy context, the actual context will not be able to open the index until it is closed by the dummy context. + * Closing the index takes time and will slow down opening the library. + */ private LibraryTab(BibDatabaseContext bibDatabaseContext, LibraryTabContainer tabContainer, DialogService dialogService, @@ -190,17 +195,32 @@ private LibraryTab(BibDatabaseContext bibDatabaseContext, this.taskExecutor = taskExecutor; this.directoryMonitorManager = new DirectoryMonitorManager(Injector.instantiateModelOrService(DirectoryMonitor.class)); - bibDatabaseContext.getDatabase().registerListener(this); - bibDatabaseContext.getMetaData().registerListener(this); + initializeComponentsAndListeners(isDummyContext); + + // set LibraryTab ID for drag'n'drop + // ID content doesn't matter, we only need different tabs to have different ID + this.setId(Long.valueOf(new Random().nextLong()).toString()); + setOnCloseRequest(this::onCloseRequest); + setOnClosed(this::onClosed); + } + + private void initializeComponentsAndListeners(boolean isDummyContext) { if (!isDummyContext) { - setLuceneManager(); + createLuceneManager(); + } + + if (tableModel != null) { + tableModel.unbind(); } + bibDatabaseContext.getDatabase().registerListener(this); + bibDatabaseContext.getMetaData().registerListener(this); + this.selectedGroupsProperty = new SimpleListProperty<>(stateManager.getSelectedGroups(bibDatabaseContext)); this.tableModel = new MainTableDataModel(getBibDatabaseContext(), preferencesService, taskExecutor, stateManager, getLuceneManager(), selectedGroupsProperty(), searchQueryProperty(), resultSizeProperty()); - citationStyleCache = new CitationStyleCache(bibDatabaseContext); + new CitationStyleCache(bibDatabaseContext); annotationCache = new FileAnnotationCache(bibDatabaseContext, preferencesService.getFilePreferences()); setupMainPanel(); @@ -218,18 +238,11 @@ private LibraryTab(BibDatabaseContext bibDatabaseContext, this.entryEditor = createEntryEditor(); - // set LibraryTab ID for drag'n'drop - // ID content doesn't matter, we only need different tabs to have different ID - this.setId(Long.valueOf(new Random().nextLong()).toString()); - Platform.runLater(() -> { EasyBind.subscribe(changedProperty, this::updateTabTitle); stateManager.getOpenDatabases().addListener((ListChangeListener) c -> updateTabTitle(changedProperty.getValue())); }); - - setOnCloseRequest(this::onCloseRequest); - setOnClosed(this::onClosed); } private EntryEditor createEntryEditor() { @@ -291,7 +304,7 @@ private void onDatabaseLoadingSucceed(ParserResult result) { dataLoadingTask = null; } - public void setLuceneManager() { + public void createLuceneManager() { luceneManager = new LuceneManager(bibDatabaseContext, taskExecutor, preferencesService.getFilePreferences()); stateManager.setLuceneManager(bibDatabaseContext, luceneManager); luceneManager.updateOnStart(); @@ -334,38 +347,7 @@ private void setDatabaseContext(BibDatabaseContext bibDatabaseContext) { stateManager.getOpenDatabases().add(bibDatabaseContext); - bibDatabaseContext.getDatabase().registerListener(this); - bibDatabaseContext.getMetaData().registerListener(this); - - setLuceneManager(); - this.tableModel.unbind(); - this.selectedGroupsProperty = new SimpleListProperty<>(stateManager.getSelectedGroups(bibDatabaseContext)); - this.tableModel = new MainTableDataModel(getBibDatabaseContext(), preferencesService, taskExecutor, stateManager, getLuceneManager(), selectedGroupsProperty(), searchQueryProperty(), resultSizeProperty()); - - citationStyleCache = new CitationStyleCache(bibDatabaseContext); - annotationCache = new FileAnnotationCache(bibDatabaseContext, preferencesService.getFilePreferences()); - - setupMainPanel(); - setupAutoCompletion(); - - this.getDatabase().registerListener(new IndexUpdateListener()); - this.getDatabase().registerListener(new EntriesRemovedListener()); - - // ensure that at each addition of a new entry, the entry is added to the groups interface - this.bibDatabaseContext.getDatabase().registerListener(new GroupTreeListener()); - // ensure that all entry changes mark the panel as changed - this.bibDatabaseContext.getDatabase().registerListener(this); - - this.getDatabase().registerListener(new UpdateTimestampListener(preferencesService)); - - this.entryEditor = createEntryEditor(); - - Platform.runLater(() -> { - EasyBind.subscribe(changedProperty, this::updateTabTitle); - stateManager.getOpenDatabases().addListener((ListChangeListener) c -> - updateTabTitle(changedProperty.getValue())); - }); - + initializeComponentsAndListeners(false); installAutosaveManagerAndBackupManager(); } diff --git a/src/main/java/org/jabref/gui/exporter/SaveDatabaseAction.java b/src/main/java/org/jabref/gui/exporter/SaveDatabaseAction.java index 098fca04a72..68833517b57 100644 --- a/src/main/java/org/jabref/gui/exporter/SaveDatabaseAction.java +++ b/src/main/java/org/jabref/gui/exporter/SaveDatabaseAction.java @@ -163,7 +163,7 @@ boolean saveAs(Path file, SaveDatabaseMode mode) { // Reset (here: uninstall and install again) AutosaveManager, BackupManager and LuceneManager for the new file name libraryTab.resetChangeMonitor(); libraryTab.installAutosaveManagerAndBackupManager(); - libraryTab.setLuceneManager(); + libraryTab.createLuceneManager(); preferences.getGuiPreferences().getFileHistory().newFile(file); } diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index 30d0cf288bc..bf89c819b38 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -39,6 +39,7 @@ import com.google.common.eventbus.Subscribe; import com.tobiasdiez.easybind.EasyBind; import com.tobiasdiez.easybind.Subscription; +import org.jspecify.annotations.Nullable; public class MainTableDataModel { @@ -52,7 +53,6 @@ public class MainTableDataModel { private final NameDisplayPreferences nameDisplayPreferences; private final BibDatabaseContext bibDatabaseContext; private final StateManager stateManager; - private final LuceneManager luceneManager; private final TaskExecutor taskExecutor; private final Subscription searchQuerySubscription; private final Subscription searchDisplayModeSubscription; @@ -60,13 +60,15 @@ public class MainTableDataModel { private final Subscription groupViewModeSubscription; private final LuceneIndexListener indexUpdatedListener; private final OptionalObjectProperty searchQueryProperty; + @Nullable private final LuceneManager luceneManager; + private Optional groupsMatcher; public MainTableDataModel(BibDatabaseContext context, PreferencesService preferencesService, TaskExecutor taskExecutor, StateManager stateManager, - LuceneManager luceneManager, + @Nullable LuceneManager luceneManager, ListProperty selectedGroupsProperty, OptionalObjectProperty searchQueryProperty, IntegerProperty resultSizeProperty) { @@ -77,9 +79,9 @@ public MainTableDataModel(BibDatabaseContext context, this.stateManager = stateManager; this.luceneManager = luceneManager; this.bibDatabaseContext = context; - this.groupsMatcher = createGroupMatcher(selectedGroupsProperty.get(), groupsPreferences); this.searchQueryProperty = searchQueryProperty; this.indexUpdatedListener = new LuceneIndexListener(); + this.groupsMatcher = createGroupMatcher(selectedGroupsProperty.get(), groupsPreferences); this.bibDatabaseContext.getDatabase().registerListener(indexUpdatedListener); resetFieldFormatter(); diff --git a/src/main/java/org/jabref/model/database/BibDatabaseContext.java b/src/main/java/org/jabref/model/database/BibDatabaseContext.java index b412533a11b..7917163e662 100644 --- a/src/main/java/org/jabref/model/database/BibDatabaseContext.java +++ b/src/main/java/org/jabref/model/database/BibDatabaseContext.java @@ -253,7 +253,7 @@ public List getEntries() { } /** - * @return The path to store the lucene index files and embeddings. One directory for each library. + * @return The path to store the lucene index files. One directory for each library. */ @NonNull public Path getFulltextIndexPath() { From 231062501c987c79cbedca672895eeffcdbecfd7 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Thu, 29 Aug 2024 10:34:24 +0300 Subject: [PATCH 203/256] Remove regex button from search bar --- .../jabref/gui/search/GlobalSearchBar.java | 40 ++----------------- .../org/jabref/model/search/SearchQuery.java | 4 -- .../jabref/preferences/JabRefPreferences.java | 4 -- .../jabref/preferences/SearchPreferences.java | 10 +---- src/main/resources/l10n/JabRef_en.properties | 1 - 5 files changed, 5 insertions(+), 54 deletions(-) diff --git a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java index f0c7720e5cc..e54bd28b22a 100644 --- a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java +++ b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java @@ -16,7 +16,6 @@ import javafx.beans.property.BooleanProperty; import javafx.beans.property.SimpleBooleanProperty; import javafx.collections.ListChangeListener; -import javafx.collections.SetChangeListener; import javafx.css.PseudoClass; import javafx.event.Event; import javafx.geometry.Insets; @@ -82,7 +81,6 @@ public class GlobalSearchBar extends HBox { private static final PseudoClass ILLEGAL_SEARCH = PseudoClass.getPseudoClass("illegal-search"); private final CustomTextField searchField; - private final ToggleButton regularExpressionButton; private final ToggleButton fulltextButton; private final Button openGlobalSearchButton; private final ToggleButton keepSearchString; @@ -96,7 +94,6 @@ public class GlobalSearchBar extends HBox { private final DialogService dialogService; private final BooleanProperty globalSearchActive = new SimpleBooleanProperty(false); private final BooleanProperty illegalSearch = new SimpleBooleanProperty(false); - private final BooleanProperty invalidRegex = new SimpleBooleanProperty(false); private GlobalSearchResultDialog globalSearchResultDialog; private final SearchType searchType; @@ -125,14 +122,11 @@ public GlobalSearchBar(LibraryTabContainer tabContainer, currentResults.setPrefWidth(150); currentResults.textProperty().bind(EasyBind.combine( - stateManager.activeSearchQuery(searchType), stateManager.searchResultSize(searchType), illegalSearch, invalidRegex, - (searchQuery, matched, illegal, invalid) -> { + stateManager.activeSearchQuery(searchType), stateManager.searchResultSize(searchType), illegalSearch, + (searchQuery, matched, illegal) -> { if (illegal) { searchField.pseudoClassStateChanged(ILLEGAL_SEARCH, true); return Localization.lang("Search failed: illegal search expression"); - } else if (invalid) { - searchField.pseudoClassStateChanged(ILLEGAL_SEARCH, true); - return Localization.lang("Invalid regular expression."); } else if (searchQuery.isEmpty()) { searchField.pseudoClassStateChanged(ILLEGAL_SEARCH, false); return ""; @@ -169,7 +163,6 @@ public GlobalSearchBar(LibraryTabContainer tabContainer, searchField.getContextMenu().getItems().add(SearchFieldRightClickMenu.createSearchFromHistorySubMenu(stateManager, searchField)); }); - regularExpressionButton = IconTheme.JabRefIcons.REG_EX.asToggleButton(); fulltextButton = IconTheme.JabRefIcons.FULLTEXT.asToggleButton(); openGlobalSearchButton = IconTheme.JabRefIcons.OPEN_GLOBAL_SEARCH.asButton(); keepSearchString = IconTheme.JabRefIcons.KEEP_SEARCH_STRING.asToggleButton(); @@ -178,14 +171,11 @@ public GlobalSearchBar(LibraryTabContainer tabContainer, initSearchModifierButtons(); BooleanBinding focusedOrActive = searchField.focusedProperty() - .or(regularExpressionButton.focusedProperty()) .or(fulltextButton.focusedProperty()) .or(keepSearchString.focusedProperty()) .or(filterModeButton.focusedProperty()) .or(searchField.textProperty().isNotEmpty()); - regularExpressionButton.visibleProperty().unbind(); - regularExpressionButton.visibleProperty().bind(focusedOrActive); fulltextButton.visibleProperty().unbind(); fulltextButton.visibleProperty().bind(focusedOrActive); keepSearchString.visibleProperty().unbind(); @@ -195,9 +185,9 @@ public GlobalSearchBar(LibraryTabContainer tabContainer, StackPane modifierButtons; if (searchType == SearchType.NORMAL_SEARCH) { - modifierButtons = new StackPane(new HBox(regularExpressionButton, fulltextButton, keepSearchString, filterModeButton)); + modifierButtons = new StackPane(new HBox(fulltextButton, keepSearchString, filterModeButton)); } else { - modifierButtons = new StackPane(new HBox(regularExpressionButton, fulltextButton)); + modifierButtons = new StackPane(new HBox(fulltextButton)); } modifierButtons.setAlignment(Pos.CENTER); searchField.setRight(new HBox(searchField.getRight(), modifierButtons)); @@ -239,14 +229,6 @@ public GlobalSearchBar(LibraryTabContainer tabContainer, } private void initSearchModifierButtons() { - regularExpressionButton.setSelected(searchPreferences.isRegularExpression()); - regularExpressionButton.setTooltip(new Tooltip(Localization.lang("regular expression"))); - initSearchModifierButton(regularExpressionButton); - regularExpressionButton.selectedProperty().addListener((obs, oldVal, newVal) -> { - searchPreferences.setSearchFlag(SearchFlags.REGULAR_EXPRESSION, newVal); - updateSearchQuery(); - }); - fulltextButton.setSelected(searchPreferences.isFulltext()); fulltextButton.setTooltip(new Tooltip(Localization.lang("Fulltext search"))); initSearchModifierButton(fulltextButton); @@ -269,11 +251,6 @@ private void initSearchModifierButtons() { openGlobalSearchButton.setTooltip(new Tooltip(Localization.lang("Search across libraries in a new window"))); initSearchModifierButton(openGlobalSearchButton); openGlobalSearchButton.setOnAction(evt -> openGlobalSearchDialog()); - - searchPreferences.getObservableSearchFlags().addListener((SetChangeListener.Change change) -> { - regularExpressionButton.setSelected(searchPreferences.isRegularExpression()); - fulltextButton.setSelected(searchPreferences.isFulltext()); - }); } public void openGlobalSearchDialog() { @@ -322,18 +299,9 @@ public void updateSearchQuery() { setSearchFieldHintTooltip(); stateManager.activeSearchQuery(searchType).set(Optional.empty()); illegalSearch.set(false); - invalidRegex.set(false); return; } - // Invalid regular expression - if (regularExpressionButton.isSelected() && !validRegex()) { - invalidRegex.setValue(true); - return; - } else { - invalidRegex.setValue(false); - } - SearchQuery searchQuery = new SearchQuery(this.searchField.getText(), searchPreferences.getSearchFlags()); if (!searchQuery.isValid()) { illegalSearch.set(true); diff --git a/src/main/java/org/jabref/model/search/SearchQuery.java b/src/main/java/org/jabref/model/search/SearchQuery.java index f8c2078bd73..7f566d786e2 100644 --- a/src/main/java/org/jabref/model/search/SearchQuery.java +++ b/src/main/java/org/jabref/model/search/SearchQuery.java @@ -74,10 +74,6 @@ public SearchQuery(String query, EnumSet searchFlags) { String[] fieldsToSearchArray = new String[boosts.size()]; boosts.keySet().toArray(fieldsToSearchArray); - if (searchFlags.contains(SearchFlags.REGULAR_EXPRESSION)) { - query = '/' + query + '/'; - } - MultiFieldQueryParser queryParser = new MultiFieldQueryParser(fieldsToSearchArray, SearchFieldConstants.LATEX_AWARE_ANALYZER, boosts); queryParser.setAllowLeadingWildcard(true); diff --git a/src/main/java/org/jabref/preferences/JabRefPreferences.java b/src/main/java/org/jabref/preferences/JabRefPreferences.java index 60fd2275b5a..7a370e2dd49 100644 --- a/src/main/java/org/jabref/preferences/JabRefPreferences.java +++ b/src/main/java/org/jabref/preferences/JabRefPreferences.java @@ -292,7 +292,6 @@ public class JabRefPreferences implements PreferencesService, AiApiKeyProvider { public static final String MAIN_FILE_DIRECTORY = "fileDirectory"; public static final String SEARCH_DISPLAY_MODE = "searchDisplayMode"; - public static final String SEARCH_REG_EXP = "regExpSearch"; public static final String SEARCH_FULLTEXT = "fulltextSearch"; public static final String SEARCH_KEEP_SEARCH_STRING = "keepSearchString"; public static final String SEARCH_KEEP_GLOBAL_WINDOW_ON_TOP = "keepOnTop"; @@ -573,7 +572,6 @@ private JabRefPreferences() { Localization.setLanguage(getLanguage()); defaults.put(SEARCH_DISPLAY_MODE, Boolean.TRUE); - defaults.put(SEARCH_REG_EXP, Boolean.FALSE); defaults.put(SEARCH_FULLTEXT, Boolean.FALSE); defaults.put(SEARCH_KEEP_SEARCH_STRING, Boolean.FALSE); defaults.put(SEARCH_KEEP_GLOBAL_WINDOW_ON_TOP, Boolean.TRUE); @@ -2880,7 +2878,6 @@ public SearchPreferences getSearchPreferences() { searchPreferences = new SearchPreferences( getBoolean(SEARCH_DISPLAY_MODE) ? SearchDisplayMode.FILTER : SearchDisplayMode.FLOAT, - getBoolean(SEARCH_REG_EXP), getBoolean(SEARCH_FULLTEXT), getBoolean(SEARCH_KEEP_SEARCH_STRING), getBoolean(SEARCH_KEEP_GLOBAL_WINDOW_ON_TOP), @@ -2889,7 +2886,6 @@ public SearchPreferences getSearchPreferences() { getDouble(SEARCH_WINDOW_DIVIDER_POS)); searchPreferences.getObservableSearchFlags().addListener((SetChangeListener) c -> { - putBoolean(SEARCH_REG_EXP, searchPreferences.getObservableSearchFlags().contains(SearchFlags.REGULAR_EXPRESSION)); putBoolean(SEARCH_FULLTEXT, searchPreferences.getObservableSearchFlags().contains(SearchFlags.FULLTEXT)); }); EasyBind.listen(searchPreferences.searchDisplayModeProperty(), (obs, oldValue, newValue) -> putBoolean(SEARCH_DISPLAY_MODE, newValue == SearchDisplayMode.FILTER)); diff --git a/src/main/java/org/jabref/preferences/SearchPreferences.java b/src/main/java/org/jabref/preferences/SearchPreferences.java index 7bbfef3d626..94b69292225 100644 --- a/src/main/java/org/jabref/preferences/SearchPreferences.java +++ b/src/main/java/org/jabref/preferences/SearchPreferences.java @@ -26,12 +26,8 @@ public class SearchPreferences { private final BooleanProperty keepSearchSting; private final ObjectProperty searchDisplayMode; - public SearchPreferences(SearchDisplayMode searchDisplayMode, boolean isRegularExpression, boolean isFulltext, boolean keepSearchString, boolean keepWindowOnTop, double searchWindowHeight, double searchWindowWidth, double searchWindowDividerPosition) { + public SearchPreferences(SearchDisplayMode searchDisplayMode, boolean isFulltext, boolean keepSearchString, boolean keepWindowOnTop, double searchWindowHeight, double searchWindowWidth, double searchWindowDividerPosition) { this(searchDisplayMode, EnumSet.noneOf(SearchFlags.class), keepSearchString, keepWindowOnTop, searchWindowHeight, searchWindowWidth, searchWindowDividerPosition); - - if (isRegularExpression) { - searchFlags.add(SearchFlags.REGULAR_EXPRESSION); - } if (isFulltext) { searchFlags.add(SearchFlags.FULLTEXT); } @@ -69,10 +65,6 @@ public void setSearchFlag(SearchFlags flag, boolean value) { } } - public boolean isRegularExpression() { - return searchFlags.contains(SearchFlags.REGULAR_EXPRESSION); - } - public boolean isFulltext() { return searchFlags.contains(SearchFlags.FULLTEXT); } diff --git a/src/main/resources/l10n/JabRef_en.properties b/src/main/resources/l10n/JabRef_en.properties index 8338ee0ecb3..9d572ce8a6e 100644 --- a/src/main/resources/l10n/JabRef_en.properties +++ b/src/main/resources/l10n/JabRef_en.properties @@ -840,7 +840,6 @@ Search\ for\ unlinked\ local\ files=Search for unlinked local files Search\ full\ text\ documents\ online=Search full text documents online Hint\:\n\nTo\ search\ all\ fields\ for\ Smith,\ enter\:\nsmith\n\nTo\ search\ the\ field\ author\ for\ Smith\ and\ the\ field\ title\ for\ electrical,\ enter\:\nauthor\:Smith\ AND\ title\:electrical=Hint:\n\nTo search all fields for Smith, enter:\nsmith\n\nTo search the field author for Smith and the field title for electrical, enter:\nauthor:Smith AND title:electrical Search\ term\ is\ empty.=Search term is empty. -Invalid\ regular\ expression.=Invalid regular expression. Searching\ for\ a\ keyword=Searching for a keyword Open\ global\ search\ window=Open global search window From f369b09c48eaae48f0177bbbed2a31a240a0ca51 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Thu, 29 Aug 2024 10:42:28 +0300 Subject: [PATCH 204/256] Use BibEntry.getId instead of System.identityHashCode --- ...s.md => 0038-use-entryId-for-bibentries.md} | 8 ++++---- .../gui/duplicationFinder/DuplicateSearch.java | 6 +++--- .../jabref/gui/groups/GroupNodeViewModel.java | 18 +++++++++--------- .../org/jabref/model/groups/SearchGroup.java | 10 +++++----- 4 files changed, 21 insertions(+), 21 deletions(-) rename docs/decisions/{0038-use-identityhashcode-for-bibentries.md => 0038-use-entryId-for-bibentries.md} (60%) diff --git a/docs/decisions/0038-use-identityhashcode-for-bibentries.md b/docs/decisions/0038-use-entryId-for-bibentries.md similarity index 60% rename from docs/decisions/0038-use-identityhashcode-for-bibentries.md rename to docs/decisions/0038-use-entryId-for-bibentries.md index d5c9fb25062..f709ccf9d1d 100644 --- a/docs/decisions/0038-use-identityhashcode-for-bibentries.md +++ b/docs/decisions/0038-use-entryId-for-bibentries.md @@ -1,11 +1,11 @@ --- -title: Use System.identityHashCode for BibEntry at indexing +title: Use BibEntry.getId for BibEntry at indexing nav_order: 38 parent: Decision Records --- -# Use `System.identityHashCode` for BibEntries at Indexing +# Use `BibEntry.getId` for BibEntries at Indexing ## Context and Problem Statement @@ -22,9 +22,9 @@ This, however, is not useful in the UI, where equal entries are not the same ent ## Considered Options -* Use `System.identityHashCode` for indexing `BibEntry` +* Use `BibEntry.getId` for indexing `BibEntry` * Rewrite `BibEntry` logic ## Decision Outcome -Chosen option: "Use `System.identityHashCode` for indexing `BibEntry`", because is the "natural" thing to ensure distinction between two instances of a `BibEntry` object - regardless of equality. +Chosen option: "Use `BibEntry.getId` for indexing `BibEntry`", because is the "natural" thing to ensure distinction between two instances of a `BibEntry` object - regardless of equality. diff --git a/src/main/java/org/jabref/gui/duplicationFinder/DuplicateSearch.java b/src/main/java/org/jabref/gui/duplicationFinder/DuplicateSearch.java index de084e65aff..96aa9ec959b 100644 --- a/src/main/java/org/jabref/gui/duplicationFinder/DuplicateSearch.java +++ b/src/main/java/org/jabref/gui/duplicationFinder/DuplicateSearch.java @@ -219,7 +219,7 @@ private void handleDuplicates(DuplicateSearchResult result) { */ static class DuplicateSearchResult { - private final Map toRemove = new HashMap<>(); + private final Map toRemove = new HashMap<>(); private final List toAdd = new ArrayList<>(); private int duplicates = 0; @@ -234,7 +234,7 @@ public synchronized List getToAdd() { public synchronized void remove(BibEntry entry) { // ADR-0038 - toRemove.put(System.identityHashCode(entry), entry); + toRemove.put(entry.getId(), entry); duplicates++; } @@ -252,7 +252,7 @@ public synchronized void replace(BibEntry entry, BibEntry replacement) { public synchronized boolean isToRemove(BibEntry entry) { // ADR-0038 - return toRemove.containsKey(System.identityHashCode(entry)); + return toRemove.containsKey(entry.getId()); } public synchronized int getDuplicateCount() { diff --git a/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java b/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java index b5fc1fc11df..37fdca01663 100644 --- a/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java +++ b/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java @@ -64,7 +64,7 @@ public class GroupNodeViewModel { private final BibDatabaseContext databaseContext; private final StateManager stateManager; private final GroupTreeNode groupNode; - private final ObservableMap matchedEntries = FXCollections.observableHashMap(); + private final ObservableMap matchedEntries = FXCollections.observableHashMap(); private final SimpleBooleanProperty hasChildren; private final SimpleBooleanProperty expandedProperty = new SimpleBooleanProperty(); private final BooleanBinding anySelectedEntriesMatched; @@ -258,21 +258,21 @@ private void onDatabaseChanged(ListChangeListener.Change cha for (BibEntry changedEntry : change.getList().subList(change.getFrom(), change.getTo())) { if (groupNode.matches(changedEntry)) { // ADR-0038 - matchedEntries.put(System.identityHashCode(changedEntry), changedEntry); + matchedEntries.put(changedEntry.getId(), changedEntry); } else { // ADR-0038 - matchedEntries.remove(System.identityHashCode(changedEntry)); + matchedEntries.remove(changedEntry.getId()); } } } else { for (BibEntry removedEntry : change.getRemoved()) { // ADR-0038 - matchedEntries.remove(System.identityHashCode(removedEntry)); + matchedEntries.remove(removedEntry.getId()); } for (BibEntry addedEntry : change.getAddedSubList()) { if (groupNode.matches(addedEntry)) { // ADR-0038 - matchedEntries.put(System.identityHashCode(addedEntry), addedEntry); + matchedEntries.put(addedEntry.getId(), addedEntry); } } } @@ -300,7 +300,7 @@ private void updateMatchedEntries() { .onSuccess(entries -> { matchedEntries.clear(); // ADR-0038 - entries.forEach(entry -> matchedEntries.put(System.identityHashCode(entry), entry)); + entries.forEach(entry -> matchedEntries.put(entry.getId(), entry)); }) .executeWith(taskExecutor); } @@ -549,10 +549,10 @@ public void listen(IndexAddedOrUpdatedEvent event) { searchGroup.updateMatches(entry); if (groupNode.matches(entry)) { // ADR-0038 - matchedEntries.put(System.identityHashCode(entry), entry); + matchedEntries.put(entry.getId(), entry); } else { // ADR-0038 - matchedEntries.remove(System.identityHashCode(entry)); + matchedEntries.remove(entry.getId()); } } }).executeWith(taskExecutor); @@ -566,7 +566,7 @@ public void listen(IndexRemovedEvent event) { for (BibEntry entry : event.entries()) { searchGroup.updateMatches(entry); // ADR-0038 - matchedEntries.remove(System.identityHashCode(entry)); + matchedEntries.remove(entry.getId()); } }).executeWith(taskExecutor); } diff --git a/src/main/java/org/jabref/model/groups/SearchGroup.java b/src/main/java/org/jabref/model/groups/SearchGroup.java index eaf0e9412de..ba865148d65 100644 --- a/src/main/java/org/jabref/model/groups/SearchGroup.java +++ b/src/main/java/org/jabref/model/groups/SearchGroup.java @@ -28,7 +28,7 @@ public class SearchGroup extends AbstractGroup { public static final Version VERSION_6_0_ALPHA = Version.parse("6.0-alpha"); private static final Logger LOGGER = LoggerFactory.getLogger(SearchGroup.class); - private final ObservableMap matchedEntries = FXCollections.observableHashMap(); + private final ObservableMap matchedEntries = FXCollections.observableHashMap(); private SearchQuery query; private LuceneManager luceneManager; @@ -75,7 +75,7 @@ public void updateMatches() { matchedEntries.clear(); // TODO: Search should be done in a background thread // ADR-0038 - luceneManager.search(query).getMatchedEntries().forEach(entry -> matchedEntries.put(System.identityHashCode(entry), entry)); + luceneManager.search(query).getMatchedEntries().forEach(entry -> matchedEntries.put(entry.getId(), entry)); } public void updateMatches(BibEntry entry) { @@ -84,10 +84,10 @@ public void updateMatches(BibEntry entry) { } if (luceneManager.isMatched(entry, query)) { // ADR-0038 - matchedEntries.put(System.identityHashCode(entry), entry); + matchedEntries.put(entry.getId(), entry); } else { // ADR-0038 - matchedEntries.remove(System.identityHashCode(entry)); + matchedEntries.remove(entry.getId()); } } @@ -108,7 +108,7 @@ public boolean equals(Object o) { @Override public boolean contains(BibEntry entry) { // ADR-0038 - return matchedEntries.containsKey(System.identityHashCode(entry)); + return matchedEntries.containsKey(entry.getId()); } @Override From 27ed10599d826af18995c0de61ff480c1ff90274 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Thu, 29 Aug 2024 11:01:29 +0300 Subject: [PATCH 205/256] Add BibEntry index map --- .../gui/maintable/MainTableDataModel.java | 49 ++++++++++++++----- 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index bf89c819b38..76aa011776c 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -1,6 +1,8 @@ package org.jabref.gui.maintable; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import javafx.beans.binding.Bindings; @@ -8,6 +10,7 @@ import javafx.beans.property.ListProperty; import javafx.beans.property.ObjectProperty; import javafx.beans.property.SimpleObjectProperty; +import javafx.collections.ListChangeListener; import javafx.collections.ObservableList; import javafx.collections.transformation.FilteredList; import javafx.collections.transformation.SortedList; @@ -62,6 +65,9 @@ public class MainTableDataModel { private final OptionalObjectProperty searchQueryProperty; @Nullable private final LuceneManager luceneManager; + // see ADR-0038 + private final Map entryIndexMap = new HashMap<>(); + private Optional groupsMatcher; public MainTableDataModel(BibDatabaseContext context, @@ -98,6 +104,35 @@ public MainTableDataModel(BibDatabaseContext context, resultSizeProperty.bind(Bindings.size(entriesFiltered.filtered(entry -> entry.matchCategory().isEqualTo(MatchCategory.MATCHING_SEARCH_AND_GROUPS).get()))); // We need to wrap the list since otherwise sorting in the table does not work entriesFilteredAndSorted = new SortedList<>(entriesFiltered); + + initializeEntryIndexMap(); + } + + private void initializeEntryIndexMap() { + for (int i = 0; i < allEntries.size(); i++) { + entryIndexMap.put(allEntries.get(i).getId(), i); + } + + allEntries.addListener((ListChangeListener.Change c) -> { + while (c.next()) { + if (c.wasPermutated()) { + for (int i = c.getFrom(); i < c.getTo(); ++i) { + entryIndexMap.put(allEntries.get(i).getId(), i); + } + } else { + if (c.wasRemoved()) { + for (BibEntry removedEntry : c.getRemoved()) { + entryIndexMap.remove(removedEntry.getId()); + } + } + if (c.wasAdded()) { + for (int i = c.getFrom(); i < c.getTo(); ++i) { + entryIndexMap.put(allEntries.get(i).getId(), i); + } + } + } + } + }); } private void updateSearchMatches(Optional query) { @@ -202,18 +237,8 @@ class LuceneIndexListener { public void listen(IndexAddedOrUpdatedEvent indexAddedOrUpdatedEvent) { indexAddedOrUpdatedEvent.entries().forEach(entry -> { BackgroundTask.wrap(() -> { - // Find the index of the entry in the list. - // The indexOf() method is not used because it relies on the equals(), - // which can return the wrong index if some entries are equal but different instances. - // For example, two different instances of an empty entry would be treated as equal by .equals(). - // See also ADR-0038 - int index = -1; - for (int i = 0; i < allEntries.size(); i++) { - if (allEntries.get(i) == entry) { - index = i; - break; - } - } + // See ADR-0038 + int index = entryIndexMap.getOrDefault(entry.getId(), -1); if (index >= 0) { BibEntryTableViewModel viewModel = entriesViewModel.get(index); boolean isFloatingMode = searchPreferences.getSearchDisplayMode() == SearchDisplayMode.FLOAT; From a1461e59c34e4c9664ec6c6644e2ec7d5a00bc7e Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Thu, 29 Aug 2024 11:14:18 +0200 Subject: [PATCH 206/256] Readd option --- docs/decisions/0038-use-entryId-for-bibentries.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/decisions/0038-use-entryId-for-bibentries.md b/docs/decisions/0038-use-entryId-for-bibentries.md index f709ccf9d1d..94504a6b5a7 100644 --- a/docs/decisions/0038-use-entryId-for-bibentries.md +++ b/docs/decisions/0038-use-entryId-for-bibentries.md @@ -23,6 +23,7 @@ This, however, is not useful in the UI, where equal entries are not the same ent ## Considered Options * Use `BibEntry.getId` for indexing `BibEntry` +* Use `System.identityHashCode` for indexing `BibEntry` * Rewrite `BibEntry` logic ## Decision Outcome From 53e274d4eb904f9baddb3dbfe27790d5e70d0af3 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Thu, 29 Aug 2024 11:41:07 +0200 Subject: [PATCH 207/256] Add `@ADR` annotation --- build.gradle | 3 +++ external-libraries.md | 7 +++++++ src/main/java/module-info.java | 1 + .../java/org/jabref/gui/maintable/MainTableDataModel.java | 4 +++- 4 files changed, 14 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 6252c37b26e..df50881bedd 100644 --- a/build.gradle +++ b/build.gradle @@ -344,6 +344,9 @@ dependencies { implementation 'commons-io:commons-io:2.16.1' + // Even if "compileOnly" is used, IntelliJ always adds to module-info.java. To avoid issues during committing, we use "implementation" instead of "compileOnly" + implementation 'io.github.adr:e-adr:2.0.0-SNAPSHOT' + testImplementation 'io.github.classgraph:classgraph:4.8.175' testImplementation 'org.junit.jupiter:junit-jupiter:5.11.0' testImplementation 'org.junit.platform:junit-platform-launcher:1.10.3' diff --git a/external-libraries.md b/external-libraries.md index 683e1071e48..73bbbdbc2c7 100644 --- a/external-libraries.md +++ b/external-libraries.md @@ -342,6 +342,13 @@ URL: https://github.com/tdebatty/java-string-similarity License: MIT ``` +```yaml +Id:io.github.adr:e-adr +Project:EmbeddedArchitecturalDecisionRecords +URL:https://github.com/adr/e-adr/ +License:EPL-2.0 +``` + ```yaml Id: io.github.java-diff-utils:java-diff-utils Project: java-diff-utils diff --git a/src/main/java/module-info.java b/src/main/java/module-info.java index ed8cb6f1f31..040717bf907 100644 --- a/src/main/java/module-info.java +++ b/src/main/java/module-info.java @@ -177,6 +177,7 @@ // region: other libraries (alphabetically) requires cuid; requires dd.plist; + requires io.github.adr; // required by okhttp and some AI library requires kotlin.stdlib; requires mslinks; diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index 76aa011776c..c92bb71c57d 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -42,6 +42,7 @@ import com.google.common.eventbus.Subscribe; import com.tobiasdiez.easybind.EasyBind; import com.tobiasdiez.easybind.Subscription; +import io.github.adr.linked.ADR; import org.jspecify.annotations.Nullable; public class MainTableDataModel { @@ -65,7 +66,8 @@ public class MainTableDataModel { private final OptionalObjectProperty searchQueryProperty; @Nullable private final LuceneManager luceneManager; - // see ADR-0038 + // + @ADR(38) private final Map entryIndexMap = new HashMap<>(); private Optional groupsMatcher; From d4754caa31a22e86c70046167f08362168596390 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Thu, 29 Aug 2024 11:41:50 +0200 Subject: [PATCH 208/256] Add some comment --- src/main/java/org/jabref/gui/maintable/MainTableDataModel.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index c92bb71c57d..3a4e132450c 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -66,6 +66,8 @@ public class MainTableDataModel { private final OptionalObjectProperty searchQueryProperty; @Nullable private final LuceneManager luceneManager; + // This stores the index of each entry in the allEntries list. + // Requried for a faster lookup for search results // @ADR(38) private final Map entryIndexMap = new HashMap<>(); From 71732a6cfa941cc910b5766cb822db3bf5265213 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Thu, 29 Aug 2024 11:43:10 +0200 Subject: [PATCH 209/256] One more annotation --- src/main/java/org/jabref/gui/maintable/MainTableDataModel.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index 3a4e132450c..888d3b589a9 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -241,7 +241,7 @@ class LuceneIndexListener { public void listen(IndexAddedOrUpdatedEvent indexAddedOrUpdatedEvent) { indexAddedOrUpdatedEvent.entries().forEach(entry -> { BackgroundTask.wrap(() -> { - // See ADR-0038 + @ADR(38) int index = entryIndexMap.getOrDefault(entry.getId(), -1); if (index >= 0) { BibEntryTableViewModel viewModel = entriesViewModel.get(index); From f4a5567275b703fd30ce44818bb224f6a6708bfd Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Thu, 29 Aug 2024 11:46:12 +0200 Subject: [PATCH 210/256] Add CHANGELOG.md entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e6353ba68c..c7c50b1b819 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv ### Changed - The search syntax is changed to [Apache Lucene syntax](https://docs.jabref.org/collect/import-using-online-bibliographic-database#search-syntax) (also to be similar to the online search syntax). [#11542](https://github.com/JabRef/jabref/pull/11542/) +- A search in "all" fields ignores the [groups](https://docs.jabref.org/finding-sorting-and-cleaning-entries/groups). [#7996](https://github.com/JabRef/jabref/issues/7996) - When a communication error with an [online service](https://docs.jabref.org/collect/import-using-online-bibliographic-database) occurs, JabRef displays the HTTP error. [#11223](https://github.com/JabRef/jabref/issues/11223) - The Pubmed/Medline Plain importer now imports the PMID field as well [#11488](https://github.com/JabRef/jabref/issues/11488) - The 'Check for updates' menu bar button is now always enabled. [#11485](https://github.com/JabRef/jabref/pull/11485) From f3f454c27aaf535459f55633841a48105fb8cab2 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Thu, 29 Aug 2024 11:47:05 +0200 Subject: [PATCH 211/256] One more annotation --- src/main/java/org/jabref/model/groups/SearchGroup.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/org/jabref/model/groups/SearchGroup.java b/src/main/java/org/jabref/model/groups/SearchGroup.java index ba865148d65..ecc5d4407d8 100644 --- a/src/main/java/org/jabref/model/groups/SearchGroup.java +++ b/src/main/java/org/jabref/model/groups/SearchGroup.java @@ -13,6 +13,7 @@ import org.jabref.model.search.SearchFlags; import org.jabref.model.search.SearchQuery; +import io.github.adr.linked.ADR; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,10 +27,14 @@ public class SearchGroup extends AbstractGroup { // We cannot have this constant in Version java because of recursion errors // Thus, we keep it here, because it is (currently) used only in the context of groups. public static final Version VERSION_6_0_ALPHA = Version.parse("6.0-alpha"); + private static final Logger LOGGER = LoggerFactory.getLogger(SearchGroup.class); + @ADR(31) private final ObservableMap matchedEntries = FXCollections.observableHashMap(); + private SearchQuery query; + private LuceneManager luceneManager; public SearchGroup(String name, GroupHierarchyType context, String searchExpression, EnumSet searchFlags, LuceneManager luceneManager) { From a23e047b52363add4b54e57a379478c48916da7a Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Thu, 29 Aug 2024 11:48:28 +0200 Subject: [PATCH 212/256] Add CHANGELOG.md entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c7c50b1b819..94cf654e444 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv ### Changed - The search syntax is changed to [Apache Lucene syntax](https://docs.jabref.org/collect/import-using-online-bibliographic-database#search-syntax) (also to be similar to the online search syntax). [#11542](https://github.com/JabRef/jabref/pull/11542/) +- When searching using a regular expession, one needs to enclose the search string in `/`. [#11542](https://github.com/JabRef/jabref/pull/11542/) - A search in "all" fields ignores the [groups](https://docs.jabref.org/finding-sorting-and-cleaning-entries/groups). [#7996](https://github.com/JabRef/jabref/issues/7996) - When a communication error with an [online service](https://docs.jabref.org/collect/import-using-online-bibliographic-database) occurs, JabRef displays the HTTP error. [#11223](https://github.com/JabRef/jabref/issues/11223) - The Pubmed/Medline Plain importer now imports the PMID field as well [#11488](https://github.com/JabRef/jabref/issues/11488) From edfb3ccf091e45361c892e7a78a029db2d0cfa1d Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Thu, 29 Aug 2024 14:07:58 +0300 Subject: [PATCH 213/256] Revert "Add BibEntry index map" This reverts commit 27ed10599d826af18995c0de61ff480c1ff90274. --- .../gui/maintable/MainTableDataModel.java | 50 ++++--------------- 1 file changed, 11 insertions(+), 39 deletions(-) diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index 888d3b589a9..88a7412a0f7 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -1,8 +1,6 @@ package org.jabref.gui.maintable; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Optional; import javafx.beans.binding.Bindings; @@ -10,7 +8,6 @@ import javafx.beans.property.ListProperty; import javafx.beans.property.ObjectProperty; import javafx.beans.property.SimpleObjectProperty; -import javafx.collections.ListChangeListener; import javafx.collections.ObservableList; import javafx.collections.transformation.FilteredList; import javafx.collections.transformation.SortedList; @@ -66,12 +63,6 @@ public class MainTableDataModel { private final OptionalObjectProperty searchQueryProperty; @Nullable private final LuceneManager luceneManager; - // This stores the index of each entry in the allEntries list. - // Requried for a faster lookup for search results - // - @ADR(38) - private final Map entryIndexMap = new HashMap<>(); - private Optional groupsMatcher; public MainTableDataModel(BibDatabaseContext context, @@ -108,35 +99,6 @@ public MainTableDataModel(BibDatabaseContext context, resultSizeProperty.bind(Bindings.size(entriesFiltered.filtered(entry -> entry.matchCategory().isEqualTo(MatchCategory.MATCHING_SEARCH_AND_GROUPS).get()))); // We need to wrap the list since otherwise sorting in the table does not work entriesFilteredAndSorted = new SortedList<>(entriesFiltered); - - initializeEntryIndexMap(); - } - - private void initializeEntryIndexMap() { - for (int i = 0; i < allEntries.size(); i++) { - entryIndexMap.put(allEntries.get(i).getId(), i); - } - - allEntries.addListener((ListChangeListener.Change c) -> { - while (c.next()) { - if (c.wasPermutated()) { - for (int i = c.getFrom(); i < c.getTo(); ++i) { - entryIndexMap.put(allEntries.get(i).getId(), i); - } - } else { - if (c.wasRemoved()) { - for (BibEntry removedEntry : c.getRemoved()) { - entryIndexMap.remove(removedEntry.getId()); - } - } - if (c.wasAdded()) { - for (int i = c.getFrom(); i < c.getTo(); ++i) { - entryIndexMap.put(allEntries.get(i).getId(), i); - } - } - } - } - }); } private void updateSearchMatches(Optional query) { @@ -241,8 +203,18 @@ class LuceneIndexListener { public void listen(IndexAddedOrUpdatedEvent indexAddedOrUpdatedEvent) { indexAddedOrUpdatedEvent.entries().forEach(entry -> { BackgroundTask.wrap(() -> { + // Find the index of the entry in the list. + // The indexOf() method is not used because it relies on the equals(), + // which can return the wrong index if some entries are equal but different instances. + // For example, two different instances of an empty entry would be treated as equal by .equals(). @ADR(38) - int index = entryIndexMap.getOrDefault(entry.getId(), -1); + int index = -1; + for (int i = 0; i < allEntries.size(); i++) { + if (allEntries.get(i) == entry) { + index = i; + break; + } + } if (index >= 0) { BibEntryTableViewModel viewModel = entriesViewModel.get(index); boolean isFloatingMode = searchPreferences.getSearchDisplayMode() == SearchDisplayMode.FLOAT; From 94b27eb9350d821b57022e3348834b3aed11e9b2 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Thu, 29 Aug 2024 14:59:37 +0300 Subject: [PATCH 214/256] Use binary search to find the index of the entry --- .../gui/maintable/MainTableDataModel.java | 12 +----- .../jabref/model/database/BibDatabase.java | 12 ++++++ .../model/database/BibDatabaseTest.java | 37 +++++++++++++++++-- 3 files changed, 46 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index 88a7412a0f7..668dbe5afd4 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -203,18 +203,8 @@ class LuceneIndexListener { public void listen(IndexAddedOrUpdatedEvent indexAddedOrUpdatedEvent) { indexAddedOrUpdatedEvent.entries().forEach(entry -> { BackgroundTask.wrap(() -> { - // Find the index of the entry in the list. - // The indexOf() method is not used because it relies on the equals(), - // which can return the wrong index if some entries are equal but different instances. - // For example, two different instances of an empty entry would be treated as equal by .equals(). @ADR(38) - int index = -1; - for (int i = 0; i < allEntries.size(); i++) { - if (allEntries.get(i) == entry) { - index = i; - break; - } - } + int index = bibDatabaseContext.getDatabase().indexOf(entry); if (index >= 0) { BibEntryTableViewModel viewModel = entriesViewModel.get(index); boolean isFloatingMode = searchPreferences.getSearchDisplayMode() == SearchDisplayMode.FLOAT; diff --git a/src/main/java/org/jabref/model/database/BibDatabase.java b/src/main/java/org/jabref/model/database/BibDatabase.java index 3ec1adb20d3..623230e874b 100644 --- a/src/main/java/org/jabref/model/database/BibDatabase.java +++ b/src/main/java/org/jabref/model/database/BibDatabase.java @@ -634,6 +634,18 @@ public String getNewLineSeparator() { return newLineSeparator; } + /** + * Returns the index of the given entry in the list of entries. + * + * @implNote New entries are always added to the end of the list and always get a higher ID. + * See {@link org.jabref.model.entry.BibEntry#BibEntry(org.jabref.model.entry.types.EntryType)}. + * Therefore, using binary search to find the index. + */ + public int indexOf(BibEntry bibEntry) { + int index = Collections.binarySearch(entries, bibEntry, Comparator.comparingInt(entry -> Integer.parseInt(entry.getId()))); + return (index >= 0) ? index : -1; + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/src/test/java/org/jabref/model/database/BibDatabaseTest.java b/src/test/java/org/jabref/model/database/BibDatabaseTest.java index 44b2ca8518d..b3ce97fb440 100644 --- a/src/test/java/org/jabref/model/database/BibDatabaseTest.java +++ b/src/test/java/org/jabref/model/database/BibDatabaseTest.java @@ -228,7 +228,7 @@ void hasStringLabelFindsString() { @Test void setSingleStringAsCollection() { - List strings = Arrays.asList(bibtexString); + List strings = List.of(bibtexString); database.setStrings(strings); assertEquals(Optional.of(bibtexString), database.getStringByName("DSP")); } @@ -406,7 +406,7 @@ void getUsedStrings() { database.addString(tripleC); database.insertEntry(entry); - Set usedStrings = new HashSet<>(database.getUsedStrings(Arrays.asList(entry))); + Set usedStrings = new HashSet<>(database.getUsedStrings(Collections.singletonList(entry))); assertEquals(stringSet, usedStrings); } @@ -423,7 +423,7 @@ void getUsedStringsSingleString() { database.addString(tripleB); database.insertEntry(entry); - List usedStrings = (List) database.getUsedStrings(Arrays.asList(entry)); + List usedStrings = (List) database.getUsedStrings(Collections.singletonList(entry)); assertEquals(strings, usedStrings); } @@ -434,7 +434,7 @@ void getUsedStringsNoString() { BibtexString string = new BibtexString("AAA", "Some other text"); database.addString(string); database.insertEntry(entry); - Collection usedStrings = database.getUsedStrings(Arrays.asList(entry)); + Collection usedStrings = database.getUsedStrings(Collections.singletonList(entry)); assertEquals(Collections.emptyList(), usedStrings); } @@ -458,4 +458,33 @@ void setPreambleWorks() { database.setPreamble("Oh yeah!"); assertEquals(Optional.of("Oh yeah!"), database.getPreamble()); } + + @Test + void getIndex() { + BibEntry entryA = new BibEntry(StandardEntryType.Article); + BibEntry entryB = new BibEntry(StandardEntryType.Article); + BibEntry entryC = new BibEntry(StandardEntryType.Article); + + database.insertEntries(entryA, entryB, entryC); + assertEquals(0, database.indexOf(entryA)); + assertEquals(1, database.indexOf(entryB)); + assertEquals(2, database.indexOf(entryC)); + assertEquals(-1, database.indexOf(new BibEntry())); + + database.removeEntry(entryB); + assertEquals(-1, database.indexOf(entryB)); + assertEquals(0, database.indexOf(entryA)); + assertEquals(1, database.indexOf(entryC)); + + database.removeEntry(entryA); + assertEquals(-1, database.indexOf(entryA)); + assertEquals(0, database.indexOf(entryC)); + + BibEntry entryD = new BibEntry(StandardEntryType.Article); + database.insertEntry(entryD); + + assertEquals(0, database.indexOf(entryC)); + assertEquals(1, database.indexOf(entryD)); + assertEquals(-1, database.indexOf(entryA)); + } } From edb2ab689356761963db40a7f91a7a45c1a6963d Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Thu, 29 Aug 2024 15:19:13 +0300 Subject: [PATCH 215/256] openrewrite --- src/main/java/org/jabref/gui/maintable/MainTableDataModel.java | 2 -- src/main/java/org/jabref/model/database/BibDatabase.java | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index 668dbe5afd4..35c647bcf00 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -39,7 +39,6 @@ import com.google.common.eventbus.Subscribe; import com.tobiasdiez.easybind.EasyBind; import com.tobiasdiez.easybind.Subscription; -import io.github.adr.linked.ADR; import org.jspecify.annotations.Nullable; public class MainTableDataModel { @@ -203,7 +202,6 @@ class LuceneIndexListener { public void listen(IndexAddedOrUpdatedEvent indexAddedOrUpdatedEvent) { indexAddedOrUpdatedEvent.entries().forEach(entry -> { BackgroundTask.wrap(() -> { - @ADR(38) int index = bibDatabaseContext.getDatabase().indexOf(entry); if (index >= 0) { BibEntryTableViewModel viewModel = entriesViewModel.get(index); diff --git a/src/main/java/org/jabref/model/database/BibDatabase.java b/src/main/java/org/jabref/model/database/BibDatabase.java index 623230e874b..46948455047 100644 --- a/src/main/java/org/jabref/model/database/BibDatabase.java +++ b/src/main/java/org/jabref/model/database/BibDatabase.java @@ -643,7 +643,7 @@ public String getNewLineSeparator() { */ public int indexOf(BibEntry bibEntry) { int index = Collections.binarySearch(entries, bibEntry, Comparator.comparingInt(entry -> Integer.parseInt(entry.getId()))); - return (index >= 0) ? index : -1; + return index >= 0 ? index : -1; } @Override From 24cf9f7eab940ec6087c3fec5f3b1574c8c1bd43 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Thu, 29 Aug 2024 16:50:43 +0300 Subject: [PATCH 216/256] Tests for LinkedFilesIndexer --- .../indexing/LinkedFilesIndexerTest.java | 180 +++++++++++++++++ .../search/indexing/LuceneIndexerTest.java | 187 ------------------ 2 files changed, 180 insertions(+), 187 deletions(-) create mode 100644 src/test/java/org/jabref/logic/search/indexing/LinkedFilesIndexerTest.java delete mode 100644 src/test/java/org/jabref/logic/search/indexing/LuceneIndexerTest.java diff --git a/src/test/java/org/jabref/logic/search/indexing/LinkedFilesIndexerTest.java b/src/test/java/org/jabref/logic/search/indexing/LinkedFilesIndexerTest.java new file mode 100644 index 00000000000..00438627649 --- /dev/null +++ b/src/test/java/org/jabref/logic/search/indexing/LinkedFilesIndexerTest.java @@ -0,0 +1,180 @@ +package org.jabref.logic.search.indexing; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import org.jabref.gui.util.BackgroundTask; +import org.jabref.logic.util.StandardFileType; +import org.jabref.model.database.BibDatabaseContext; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.LinkedFile; +import org.jabref.model.entry.types.StandardEntryType; +import org.jabref.model.search.LuceneIndexer; +import org.jabref.preferences.FilePreferences; +import org.jabref.preferences.PreferencesService; + +import org.apache.lucene.index.IndexReader; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mockito; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class LinkedFilesIndexerTest { + private static final PreferencesService PREFERENCES_SERVICE = mock(PreferencesService.class); + private static final FilePreferences FILE_PREFERENCES = mock(FilePreferences.class); + + private LuceneIndexer indexer; + + @BeforeEach + void setUp(@TempDir Path indexDir) throws IOException { + when(FILE_PREFERENCES.shouldFulltextIndexLinkedFiles()).thenReturn(true); + when(PREFERENCES_SERVICE.getFilePreferences()).thenReturn(FILE_PREFERENCES); + + BibDatabaseContext context = mock(BibDatabaseContext.class); + when(context.getDatabasePath()).thenReturn(Optional.of(Path.of("src/test/resources/pdfs/"))); + when(context.getFileDirectories(Mockito.any())).thenReturn(Collections.singletonList(Path.of("src/test/resources/pdfs"))); + when(context.getFulltextIndexPath()).thenReturn(indexDir); + + this.indexer = new DefaultLinkedFilesIndexer(context, FILE_PREFERENCES); + } + + @Test + void exampleThesisIndex() throws IOException { + // given + BibEntry entry = new BibEntry(StandardEntryType.PhdThesis); + entry.setFiles(Collections.singletonList(new LinkedFile("Example Thesis", "thesis-example.pdf", StandardFileType.PDF.getName()))); + + // when + indexer.addToIndex(List.of(entry), mock(BackgroundTask.class)); + + // then + indexer.getSearcherManager().maybeRefreshBlocking(); + try (IndexReader reader = indexer.getSearcherManager().acquire().getIndexReader()) { + assertEquals(33, reader.numDocs()); + } + } + + @Test + void dontIndexNonPdf() throws IOException { + // given + BibEntry entry = new BibEntry(StandardEntryType.PhdThesis); + entry.setFiles(Collections.singletonList(new LinkedFile("Example Thesis", "thesis-example.pdf", StandardFileType.AUX.getName()))); + + // when + indexer.addToIndex(List.of(entry), mock(BackgroundTask.class)); + + // then + indexer.getSearcherManager().maybeRefreshBlocking(); + try (IndexReader reader = indexer.getSearcherManager().acquire().getIndexReader()) { + assertEquals(0, reader.numDocs()); + } + } + + @Test + void dontIndexOnlineLinks() throws IOException { + // given + BibEntry entry = new BibEntry(StandardEntryType.PhdThesis); + entry.setFiles(Collections.singletonList(new LinkedFile("Example Thesis", "https://raw.githubusercontent.com/JabRef/jabref/main/src/test/resources/pdfs/thesis-example.pdf", StandardFileType.PDF.getName()))); + + // when + indexer.addToIndex(List.of(entry), mock(BackgroundTask.class)); + + // then + indexer.getSearcherManager().maybeRefreshBlocking(); + try (IndexReader reader = indexer.getSearcherManager().acquire().getIndexReader()) { + assertEquals(0, reader.numDocs()); + } + } + + @Test + void exampleThesisIndexWithKey() throws IOException { + // given + BibEntry entry = new BibEntry(StandardEntryType.PhdThesis); + entry.setCitationKey("Example2017"); + entry.setFiles(Collections.singletonList(new LinkedFile("Example Thesis", "thesis-example.pdf", StandardFileType.PDF.getName()))); + + // when + indexer.addToIndex(List.of(entry), mock(BackgroundTask.class)); + + // then + indexer.getSearcherManager().maybeRefreshBlocking(); + try (IndexReader reader = indexer.getSearcherManager().acquire().getIndexReader()) { + assertEquals(33, reader.numDocs()); + } + } + + @Test + void metaDataIndex() throws IOException { + // given + BibEntry entry = new BibEntry(StandardEntryType.Article); + entry.setFiles(Collections.singletonList(new LinkedFile("Example Thesis", "metaData.pdf", StandardFileType.PDF.getName()))); + + // when + indexer.addToIndex(List.of(entry), mock(BackgroundTask.class)); + + // then + indexer.getSearcherManager().maybeRefreshBlocking(); + try (IndexReader reader = indexer.getSearcherManager().acquire().getIndexReader()) { + assertEquals(1, reader.numDocs()); + } + } + + @Test + void exampleThesisIndexAppendMetaData() throws IOException { + // given + BibEntry exampleThesis = new BibEntry(StandardEntryType.PhdThesis); + exampleThesis.setCitationKey("ExampleThesis2017"); + exampleThesis.setFiles(Collections.singletonList(new LinkedFile("Example Thesis", "thesis-example.pdf", StandardFileType.PDF.getName()))); + + // when + indexer.addToIndex(List.of(exampleThesis), mock(BackgroundTask.class)); + + // index with first entry + indexer.getSearcherManager().maybeRefreshBlocking(); + try (IndexReader reader = indexer.getSearcherManager().acquire().getIndexReader()) { + assertEquals(33, reader.numDocs()); + } + + BibEntry metadata = new BibEntry(StandardEntryType.Article); + metadata.setCitationKey("MetaData2017"); + metadata.setFiles(Collections.singletonList(new LinkedFile("Metadata file", "metaData.pdf", StandardFileType.PDF.getName()))); + + // when + indexer.addToIndex(List.of(metadata), mock(BackgroundTask.class)); + + // then + indexer.getSearcherManager().maybeRefreshBlocking(); + try (IndexReader reader = indexer.getSearcherManager().acquire().getIndexReader()) { + assertEquals(34, reader.numDocs()); + } + } + + @Test + public void flushIndex() throws IOException { + // given + BibEntry entry = new BibEntry(StandardEntryType.PhdThesis); + entry.setCitationKey("Example2017"); + entry.setFiles(Collections.singletonList(new LinkedFile("Example Thesis", "thesis-example.pdf", StandardFileType.PDF.getName()))); + + indexer.addToIndex(List.of(entry), mock(BackgroundTask.class)); + + indexer.getSearcherManager().maybeRefreshBlocking(); + try (IndexReader reader = indexer.getSearcherManager().acquire().getIndexReader()) { + assertEquals(33, reader.numDocs()); + } + + indexer.removeAllFromIndex(); + + indexer.getSearcherManager().maybeRefreshBlocking(); + try (IndexReader reader = indexer.getSearcherManager().acquire().getIndexReader()) { + assertEquals(0, reader.numDocs()); + } + } +} diff --git a/src/test/java/org/jabref/logic/search/indexing/LuceneIndexerTest.java b/src/test/java/org/jabref/logic/search/indexing/LuceneIndexerTest.java deleted file mode 100644 index d651d736b73..00000000000 --- a/src/test/java/org/jabref/logic/search/indexing/LuceneIndexerTest.java +++ /dev/null @@ -1,187 +0,0 @@ -package org.jabref.logic.search.indexing; - -public class LuceneIndexerTest { - /* - private LuceneIndexer indexer; - private BibDatabase database; - private BibDatabaseContext context = mock(BibDatabaseContext.class); - - @BeforeEach - void setUp(@TempDir Path indexDir) throws IOException { - FilePreferences filePreferences = mock(FilePreferences.class); - when(filePreferences.shouldFulltextIndexLinkedFiles()).thenReturn(true); - PreferencesService preferencesService = mock(PreferencesService.class); - when(preferencesService.getFilePreferences()).thenReturn(filePreferences); - this.database = new BibDatabase(); - - this.context = mock(BibDatabaseContext.class); - when(context.getDatabasePath()).thenReturn(Optional.of(Path.of("src/test/resources/pdfs/"))); - when(context.getFileDirectories(Mockito.any())).thenReturn(Collections.singletonList(Path.of("src/test/resources/pdfs"))); - when(context.getFulltextIndexPath()).thenReturn(indexDir); - when(context.getDatabase()).thenReturn(database); - when(context.getEntries()).thenReturn(database.getEntries()); - this.indexer = LuceneIndexer.of(context, preferencesService); - } - - @Test - void exampleThesisIndex() throws IOException { - // given - BibEntry entry = new BibEntry(StandardEntryType.PhdThesis); - entry.setFiles(Collections.singletonList(new LinkedFile("Example Thesis", "thesis-example.pdf", StandardFileType.PDF.getName()))); - database.insertEntry(entry); - - // when - indexer.createIndex(); - for (BibEntry bibEntry : context.getEntries()) { - indexer.addBibFieldsToIndex(bibEntry); - indexer.addLinkedFilesToIndex(bibEntry); - } - - // then - try (IndexReader reader = DirectoryReader.open(new NIOFSDirectory(context.getFulltextIndexPath()))) { - assertEquals(34, reader.numDocs()); - } - } - - @Test - void dontIndexNonPdf() throws IOException { - // given - BibEntry entry = new BibEntry(StandardEntryType.PhdThesis); - entry.setFiles(Collections.singletonList(new LinkedFile("Example Thesis", "thesis-example.pdf", StandardFileType.AUX.getName()))); - database.insertEntry(entry); - - // when - indexer.createIndex(); - for (BibEntry bibEntry : context.getEntries()) { - indexer.addBibFieldsToIndex(bibEntry); - indexer.addLinkedFilesToIndex(bibEntry); - } - - // then - try (IndexReader reader = DirectoryReader.open(new NIOFSDirectory(context.getFulltextIndexPath()))) { - assertEquals(1, reader.numDocs()); - } - } - - @Test - void dontIndexOnlineLinks() throws IOException { - // given - BibEntry entry = new BibEntry(StandardEntryType.PhdThesis); - entry.setFiles(Collections.singletonList(new LinkedFile("Example Thesis", "https://raw.githubusercontent.com/JabRef/jabref/main/src/test/resources/pdfs/thesis-example.pdf", StandardFileType.PDF.getName()))); - database.insertEntry(entry); - - // when - indexer.createIndex(); - for (BibEntry bibEntry : context.getEntries()) { - indexer.addBibFieldsToIndex(bibEntry); - indexer.addLinkedFilesToIndex(bibEntry); - } - - // then - try (IndexReader reader = DirectoryReader.open(new NIOFSDirectory(context.getFulltextIndexPath()))) { - assertEquals(1, reader.numDocs()); - } - } - - @Test - void exampleThesisIndexWithKey() throws IOException { - // given - BibEntry entry = new BibEntry(StandardEntryType.PhdThesis); - entry.setCitationKey("Example2017"); - entry.setFiles(Collections.singletonList(new LinkedFile("Example Thesis", "thesis-example.pdf", StandardFileType.PDF.getName()))); - database.insertEntry(entry); - - // when - indexer.createIndex(); - for (BibEntry bibEntry : context.getEntries()) { - indexer.addBibFieldsToIndex(bibEntry); - indexer.addLinkedFilesToIndex(bibEntry); - } - - // then - try (IndexReader reader = DirectoryReader.open(new NIOFSDirectory(context.getFulltextIndexPath()))) { - assertEquals(34, reader.numDocs()); - } - } - - @Test - void metaDataIndex() throws IOException { - // given - BibEntry entry = new BibEntry(StandardEntryType.Article); - entry.setFiles(Collections.singletonList(new LinkedFile("Example Thesis", "metaData.pdf", StandardFileType.PDF.getName()))); - - database.insertEntry(entry); - - // when - indexer.createIndex(); - for (BibEntry bibEntry : context.getEntries()) { - indexer.addBibFieldsToIndex(bibEntry); - indexer.addLinkedFilesToIndex(bibEntry); - } - - // then - try (IndexReader reader = DirectoryReader.open(new NIOFSDirectory(context.getFulltextIndexPath()))) { - assertEquals(2, reader.numDocs()); - } - } - - @Test - public void flushIndex() throws IOException { - // given - BibEntry entry = new BibEntry(StandardEntryType.PhdThesis); - entry.setCitationKey("Example2017"); - entry.setFiles(Collections.singletonList(new LinkedFile("Example Thesis", "thesis-example.pdf", StandardFileType.PDF.getName()))); - database.insertEntry(entry); - - indexer.createIndex(); - for (BibEntry bibEntry : context.getEntries()) { - indexer.addBibFieldsToIndex(bibEntry); - indexer.addLinkedFilesToIndex(bibEntry); - } - // index actually exists - try (IndexReader reader = DirectoryReader.open(new NIOFSDirectory(context.getFulltextIndexPath()))) { - assertEquals(34, reader.numDocs()); - } - - // when - indexer.flushIndex(); - - // then - try (IndexReader reader = DirectoryReader.open(new NIOFSDirectory(context.getFulltextIndexPath()))) { - assertEquals(0, reader.numDocs()); - } - } - - @Test - void exampleThesisIndexAppendMetaData() throws IOException { - // given - BibEntry exampleThesis = new BibEntry(StandardEntryType.PhdThesis); - exampleThesis.setCitationKey("ExampleThesis2017"); - exampleThesis.setFiles(Collections.singletonList(new LinkedFile("Example Thesis", "thesis-example.pdf", StandardFileType.PDF.getName()))); - database.insertEntry(exampleThesis); - indexer.createIndex(); - for (BibEntry bibEntry : context.getEntries()) { - indexer.addBibFieldsToIndex(bibEntry); - indexer.addLinkedFilesToIndex(bibEntry); - } - - // index with first entry - try (IndexReader reader = DirectoryReader.open(new NIOFSDirectory(context.getFulltextIndexPath()))) { - assertEquals(34, reader.numDocs()); - } - - BibEntry metadata = new BibEntry(StandardEntryType.Article); - metadata.setCitationKey("MetaData2017"); - metadata.setFiles(Collections.singletonList(new LinkedFile("Metadata file", "metaData.pdf", StandardFileType.PDF.getName()))); - - // when - indexer.addBibFieldsToIndex(metadata); - indexer.addLinkedFilesToIndex(metadata); - - // then - try (IndexReader reader = DirectoryReader.open(new NIOFSDirectory(context.getFulltextIndexPath()))) { - assertEquals(36, reader.numDocs()); - } - } - */ -} From c3c9d1271a43b54c643e12cb105950be2dae2f1b Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Thu, 29 Aug 2024 17:48:53 +0300 Subject: [PATCH 217/256] Fix DatabaseSearcher --- .../org/jabref/cli/ArgumentProcessor.java | 10 ++- .../jabref/logic/search/DatabaseSearcher.java | 32 ++++++--- .../search/retrieval/LuceneSearcher.java | 3 +- src/main/resources/l10n/JabRef_en.properties | 1 + .../logic/search/DatabaseSearcherTest.java | 71 ++++++++++--------- 5 files changed, 71 insertions(+), 46 deletions(-) diff --git a/src/main/java/org/jabref/cli/ArgumentProcessor.java b/src/main/java/org/jabref/cli/ArgumentProcessor.java index 80fb9ad8d1e..0443befbf39 100644 --- a/src/main/java/org/jabref/cli/ArgumentProcessor.java +++ b/src/main/java/org/jabref/cli/ArgumentProcessor.java @@ -454,11 +454,17 @@ private boolean exportMatches(List loaded) { // $ stands for a blank ParserResult pr = loaded.getLast(); BibDatabaseContext databaseContext = pr.getDatabaseContext(); - BibDatabase dataBase = pr.getDatabase(); SearchPreferences searchPreferences = preferencesService.getSearchPreferences(); SearchQuery query = new SearchQuery(searchTerm, searchPreferences.getSearchFlags()); - List matches = new DatabaseSearcher(query, dataBase).getMatches(); + + List matches; + try { + matches = new DatabaseSearcher(query, databaseContext, preferencesService.getFilePreferences()).getMatches(); + } catch (IOException e) { + System.err.println(Localization.lang("Error occurred when searching") + ": " + e.getLocalizedMessage()); + return false; + } // export matches if (!matches.isEmpty()) { diff --git a/src/main/java/org/jabref/logic/search/DatabaseSearcher.java b/src/main/java/org/jabref/logic/search/DatabaseSearcher.java index 72770c1b00e..cbdc16b8b7c 100644 --- a/src/main/java/org/jabref/logic/search/DatabaseSearcher.java +++ b/src/main/java/org/jabref/logic/search/DatabaseSearcher.java @@ -1,27 +1,40 @@ package org.jabref.logic.search; +import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.Objects; -import org.jabref.model.database.BibDatabase; +import org.jabref.gui.util.BackgroundTask; +import org.jabref.logic.search.indexing.BibFieldsIndexer; +import org.jabref.logic.search.indexing.DefaultLinkedFilesIndexer; +import org.jabref.logic.search.retrieval.LuceneSearcher; +import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.database.BibDatabases; import org.jabref.model.entry.BibEntry; +import org.jabref.model.search.LuceneIndexer; import org.jabref.model.search.SearchQuery; +import org.jabref.preferences.FilePreferences; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class DatabaseSearcher { - private static final Logger LOGGER = LoggerFactory.getLogger(DatabaseSearcher.class); - private final SearchQuery query; + private static final BackgroundTask DUMMY_TASK = BackgroundTask.wrap(() -> null); - private final BibDatabase database; + private final SearchQuery query; + private final LuceneIndexer bibFieldsIndexer; + private final LuceneIndexer linkedFilesIndexer; + private final LuceneSearcher luceneSearcher; - public DatabaseSearcher(SearchQuery query, BibDatabase database) { + public DatabaseSearcher(SearchQuery query, BibDatabaseContext databaseContext, FilePreferences filePreferences) throws IOException { this.query = Objects.requireNonNull(query); - this.database = Objects.requireNonNull(database); + bibFieldsIndexer = new BibFieldsIndexer(databaseContext); + bibFieldsIndexer.updateOnStart(DUMMY_TASK); + linkedFilesIndexer = new DefaultLinkedFilesIndexer(databaseContext, filePreferences); + linkedFilesIndexer.updateOnStart(DUMMY_TASK); + this.luceneSearcher = new LuceneSearcher(databaseContext, bibFieldsIndexer, linkedFilesIndexer); } /** @@ -34,10 +47,9 @@ public List getMatches() { LOGGER.warn("Search failed: invalid search expression"); return Collections.emptyList(); } - - List matchEntries = List.of(); - // List matchEntries = database.getEntries().stream().filter(query::isMatch).toList(); - // TODO btut: is this for CLI? We need the databasecontext to access the index + List matchEntries = luceneSearcher.search(query.getParsedQuery(), query.getSearchFlags()).getMatchedEntries().stream().toList(); + bibFieldsIndexer.close(); + linkedFilesIndexer.close(); return BibDatabases.purgeEmptyEntries(matchEntries); } } diff --git a/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java b/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java index 84e74b97b2d..093db0e696b 100644 --- a/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java +++ b/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java @@ -134,7 +134,8 @@ private SearchResults getSearchResults(TopDocs topDocs, StoredFields storedField return searchResults; } - private static String getFieldContents(Document document, SearchFieldConstants field) { + private static String getFieldContents(Document document, SearchFieldConstants + field) { return Optional.ofNullable(document.get(field.toString())).orElse(""); } diff --git a/src/main/resources/l10n/JabRef_en.properties b/src/main/resources/l10n/JabRef_en.properties index 9d572ce8a6e..0b8bf867757 100644 --- a/src/main/resources/l10n/JabRef_en.properties +++ b/src/main/resources/l10n/JabRef_en.properties @@ -1075,6 +1075,7 @@ Use\ custom\ file\ browser=Use custom file browser Use\ custom\ terminal\ emulator=Use custom terminal emulator exportFormat=exportFormat Output\ file\ missing=Output file missing +Error\ occurred\ when\ searching=Error occurred when searching The\ output\ option\ depends\ on\ a\ valid\ input\ option.=The output option depends on a valid input option. Linked\ file\ name\ conventions=Linked file name conventions Filename\ format\ pattern=Filename format pattern diff --git a/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java b/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java index 3b38aed3f84..142d440079e 100644 --- a/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java +++ b/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java @@ -1,122 +1,127 @@ package org.jabref.logic.search; +import java.io.IOException; import java.util.Collections; import java.util.EnumSet; import java.util.List; -import org.jabref.model.database.BibDatabase; +import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.types.StandardEntryType; import org.jabref.model.search.SearchFlags; import org.jabref.model.search.SearchQuery; +import org.jabref.preferences.FilePreferences; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class DatabaseSearcherTest { - public static final SearchQuery INVALID_SEARCH_QUERY = new SearchQuery("\\asd123{}asdf", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)); - - private BibDatabase database; + private static final SearchQuery INVALID_SEARCH_QUERY = new SearchQuery("\\asd123{}asdf", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)); + private static final FilePreferences FILE_PREFERENCES = mock(FilePreferences.class); + private BibDatabaseContext databaseContext; @BeforeEach void setUp() { - database = new BibDatabase(); + when(FILE_PREFERENCES.shouldFulltextIndexLinkedFiles()).thenReturn(false); + databaseContext = new BibDatabaseContext(); } @Test - void noMatchesFromEmptyDatabase() { - List matches = new DatabaseSearcher(new SearchQuery("whatever", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)), database).getMatches(); + void noMatchesFromEmptyDatabase() throws IOException { + List matches = new DatabaseSearcher(new SearchQuery("whatever", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)), databaseContext, FILE_PREFERENCES).getMatches(); assertEquals(Collections.emptyList(), matches); } @Test - void noMatchesFromEmptyDatabaseWithInvalidSearchExpression() { - List matches = new DatabaseSearcher(INVALID_SEARCH_QUERY, database).getMatches(); + void noMatchesFromEmptyDatabaseWithInvalidSearchExpression() throws IOException { + List matches = new DatabaseSearcher(INVALID_SEARCH_QUERY, databaseContext, FILE_PREFERENCES).getMatches(); assertEquals(Collections.emptyList(), matches); } @Test - void getDatabaseFromMatchesDatabaseWithEmptyEntries() { - database.insertEntry(new BibEntry()); - List matches = new DatabaseSearcher(new SearchQuery("whatever", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)), database).getMatches(); + void getDatabaseFromMatchesDatabaseWithEmptyEntries() throws IOException { + databaseContext.getDatabase().insertEntry(new BibEntry()); + List matches = new DatabaseSearcher(new SearchQuery("whatever", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)), databaseContext, FILE_PREFERENCES).getMatches(); assertEquals(Collections.emptyList(), matches); } @Test - void noMatchesFromDatabaseWithArticleTypeEntry() { + void noMatchesFromDatabaseWithArticleTypeEntry() throws IOException { BibEntry entry = new BibEntry(StandardEntryType.Article); entry.setField(StandardField.AUTHOR, "harrer"); - database.insertEntry(entry); - List matches = new DatabaseSearcher(new SearchQuery("whatever", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)), database).getMatches(); + databaseContext.getDatabase().insertEntry(entry); + List matches = new DatabaseSearcher(new SearchQuery("whatever", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)), databaseContext, FILE_PREFERENCES).getMatches(); assertEquals(Collections.emptyList(), matches); } @Test - void correctMatchFromDatabaseWithArticleTypeEntry() { + void correctMatchFromDatabaseWithArticleTypeEntry() throws IOException { BibEntry entry = new BibEntry(StandardEntryType.Article); entry.setField(StandardField.AUTHOR, "harrer"); - database.insertEntry(entry); - List matches = new DatabaseSearcher(new SearchQuery("harrer", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)), database).getMatches(); + databaseContext.getDatabase().insertEntry(entry); + List matches = new DatabaseSearcher(new SearchQuery("harrer", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)), databaseContext, FILE_PREFERENCES).getMatches(); assertEquals(Collections.singletonList(entry), matches); } @Test - void noMatchesFromEmptyDatabaseWithInvalidQuery() { + void noMatchesFromEmptyDatabaseWithInvalidQuery() throws IOException { SearchQuery query = new SearchQuery("asdf[", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)); - DatabaseSearcher databaseSearcher = new DatabaseSearcher(query, database); + DatabaseSearcher databaseSearcher = new DatabaseSearcher(query, databaseContext, FILE_PREFERENCES); assertEquals(Collections.emptyList(), databaseSearcher.getMatches()); } @Test - void correctMatchFromDatabaseWithIncollectionTypeEntry() { + void correctMatchFromDatabaseWithIncollectionTypeEntry() throws IOException { BibEntry entry = new BibEntry(StandardEntryType.InCollection); entry.setField(StandardField.AUTHOR, "tonho"); - database.insertEntry(entry); + databaseContext.getDatabase().insertEntry(entry); SearchQuery query = new SearchQuery("tonho", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)); - List matches = new DatabaseSearcher(query, database).getMatches(); + List matches = new DatabaseSearcher(query, databaseContext, FILE_PREFERENCES).getMatches(); assertEquals(Collections.singletonList(entry), matches); } @Test - void noMatchesFromDatabaseWithTwoEntries() { + void noMatchesFromDatabaseWithTwoEntries() throws IOException { BibEntry entry = new BibEntry(); - database.insertEntry(entry); + databaseContext.getDatabase().insertEntry(entry); entry = new BibEntry(StandardEntryType.InCollection); entry.setField(StandardField.AUTHOR, "tonho"); - database.insertEntry(entry); + databaseContext.getDatabase().insertEntry(entry); SearchQuery query = new SearchQuery("tonho", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)); - DatabaseSearcher databaseSearcher = new DatabaseSearcher(query, database); + DatabaseSearcher databaseSearcher = new DatabaseSearcher(query, databaseContext, FILE_PREFERENCES); assertEquals(Collections.singletonList(entry), databaseSearcher.getMatches()); } @Test - void noMatchesFromDabaseWithIncollectionTypeEntry() { + void noMatchesFromDabaseWithIncollectionTypeEntry() throws IOException { BibEntry entry = new BibEntry(StandardEntryType.InCollection); entry.setField(StandardField.AUTHOR, "tonho"); - database.insertEntry(entry); + databaseContext.getDatabase().insertEntry(entry); SearchQuery query = new SearchQuery("asdf", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)); - DatabaseSearcher databaseSearcher = new DatabaseSearcher(query, database); + DatabaseSearcher databaseSearcher = new DatabaseSearcher(query, databaseContext, FILE_PREFERENCES); assertEquals(Collections.emptyList(), databaseSearcher.getMatches()); } @Test - void noMatchFromDatabaseWithEmptyEntry() { + void noMatchFromDatabaseWithEmptyEntry() throws IOException { BibEntry entry = new BibEntry(); - database.insertEntry(entry); + databaseContext.getDatabase().insertEntry(entry); SearchQuery query = new SearchQuery("tonho", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)); - DatabaseSearcher databaseSearcher = new DatabaseSearcher(query, database); + DatabaseSearcher databaseSearcher = new DatabaseSearcher(query, databaseContext, FILE_PREFERENCES); assertEquals(Collections.emptyList(), databaseSearcher.getMatches()); } From e3c3256a0cf0a6a1f38d7dff47264ddbe51410e9 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Thu, 29 Aug 2024 18:27:38 +0300 Subject: [PATCH 218/256] LocalizationConsistencyTest --- src/main/resources/l10n/JabRef_en.properties | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/resources/l10n/JabRef_en.properties b/src/main/resources/l10n/JabRef_en.properties index 0b8bf867757..0103de6b4c2 100644 --- a/src/main/resources/l10n/JabRef_en.properties +++ b/src/main/resources/l10n/JabRef_en.properties @@ -2355,6 +2355,7 @@ Presets=Presets Generate\ groups\ from\ keywords\ in\ the\ following\ field=Generate groups from keywords in the following field Generate\ groups\ for\ author\ last\ names=Generate groups for author last names Regular\ expression=Regular expression +Invalid\ regular\ expression.=Invalid regular expression. Error\ importing.\ See\ the\ error\ log\ for\ details.=Error importing. See the error log for details. From 12ff246dc47d07ac5ffca167726f18e3e6b5a9f9 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Thu, 29 Aug 2024 18:28:50 +0300 Subject: [PATCH 219/256] DatabaseSearcherWithBibFilesTest --- .../DatabaseSearcherWithBibFilesTest.java | 65 +++++++------------ 1 file changed, 24 insertions(+), 41 deletions(-) diff --git a/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java b/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java index da3674f96ca..3d84c79aa2c 100644 --- a/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java +++ b/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java @@ -7,12 +7,10 @@ import java.util.Objects; import java.util.stream.Stream; -import org.jabref.gui.StateManager; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.ParserResult; import org.jabref.logic.importer.fileformat.BibtexImporter; import org.jabref.logic.util.StandardFileType; -import org.jabref.model.database.BibDatabase; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.LinkedFile; @@ -22,9 +20,7 @@ import org.jabref.model.search.SearchQuery; import org.jabref.model.util.DummyFileUpdateMonitor; import org.jabref.preferences.FilePreferences; -import org.jabref.preferences.PreferencesService; -import com.airhacks.afterburner.injection.Injector; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -38,66 +34,53 @@ class DatabaseSearcherWithBibFilesTest { - private static BibEntry titleSentenceCased = new BibEntry(StandardEntryType.Misc) + private static final BibEntry TITLE_SENTENCE_CASED = new BibEntry(StandardEntryType.Misc) .withCitationKey("title-sentence-cased") .withField(StandardField.TITLE, "Title Sentence Cased"); - private static BibEntry titleMixedCased = new BibEntry(StandardEntryType.Misc) + private static final BibEntry TITLE_MIXED_CASED = new BibEntry(StandardEntryType.Misc) .withCitationKey("title-mixed-cased") .withField(StandardField.TITLE, "TiTle MiXed CaSed"); - private static BibEntry titleUpperCased = new BibEntry(StandardEntryType.Misc) + private static final BibEntry TITLE_UPPER_CASED = new BibEntry(StandardEntryType.Misc) .withCitationKey("title-upper-cased") .withField(StandardField.TITLE, "TITLE UPPER CASED"); - private static BibEntry mininimalSentenceCase = new BibEntry(StandardEntryType.Misc) + private static final BibEntry MINIMAL_SENTENCE_CASE = new BibEntry(StandardEntryType.Misc) .withCitationKey("minimal-sentence-case") .withFiles(Collections.singletonList(new LinkedFile("", "minimal-sentence-case.pdf", StandardFileType.PDF.getName()))); - private static BibEntry minimalAllUpperCase = new BibEntry(StandardEntryType.Misc) + private static final BibEntry MINIMAL_ALL_UPPER_CASE = new BibEntry(StandardEntryType.Misc) .withCitationKey("minimal-all-upper-case") .withFiles(Collections.singletonList(new LinkedFile("", "minimal-all-upper-case.pdf", StandardFileType.PDF.getName()))); - private static BibEntry minimalMixedCase = new BibEntry(StandardEntryType.Misc) + private static final BibEntry MINIMAL_MIXED_CASE = new BibEntry(StandardEntryType.Misc) .withCitationKey("minimal-mixed-case") .withFiles(Collections.singletonList(new LinkedFile("", "minimal-mixed-case.pdf", StandardFileType.PDF.getName()))); - private static BibEntry minimalNoteSentenceCase = new BibEntry(StandardEntryType.Misc) + private static final BibEntry MINIMAL_NOTE_SENTENCE_CASE = new BibEntry(StandardEntryType.Misc) .withCitationKey("minimal-note-sentence-case") .withFiles(Collections.singletonList(new LinkedFile("", "minimal-note-sentence-case.pdf", StandardFileType.PDF.getName()))); - private static BibEntry minimalNoteAllUpperCase = new BibEntry(StandardEntryType.Misc) + private static final BibEntry MINIMAL_NOTE_ALL_UPPER_CASE = new BibEntry(StandardEntryType.Misc) .withCitationKey("minimal-note-all-upper-case") .withFiles(Collections.singletonList(new LinkedFile("", "minimal-note-all-upper-case.pdf", StandardFileType.PDF.getName()))); - private static BibEntry minimalNoteMixedCase = new BibEntry(StandardEntryType.Misc) + private static final BibEntry MINIMAL_NOTE_MIXED_CASE = new BibEntry(StandardEntryType.Misc) .withCitationKey("minimal-note-mixed-case") .withFiles(Collections.singletonList(new LinkedFile("", "minimal-note-mixed-case.pdf", StandardFileType.PDF.getName()))); - + private static final FilePreferences FILE_PREFERENCES = mock(FilePreferences.class); @TempDir private Path indexDir; - private StateManager stateManager; - private PreferencesService preferencesService; - private BibDatabase initializeDatabaseFromPath(String testFile) throws Exception { + private BibDatabaseContext initializeDatabaseFromPath(String testFile) throws Exception { return initializeDatabaseFromPath(Path.of(Objects.requireNonNull(DatabaseSearcherWithBibFilesTest.class.getResource(testFile)).toURI())); } - private BibDatabase initializeDatabaseFromPath(Path testFile) throws Exception { + private BibDatabaseContext initializeDatabaseFromPath(Path testFile) throws Exception { ParserResult result = new BibtexImporter(mock(ImportFormatPreferences.class, Answers.RETURNS_DEEP_STUBS), new DummyFileUpdateMonitor()).importDatabase(testFile); - BibDatabase database = result.getDatabase(); + BibDatabaseContext databaseContext = result.getDatabaseContext(); BibDatabaseContext context = mock(BibDatabaseContext.class); when(context.getFileDirectories(Mockito.any())).thenReturn(List.of(testFile.getParent())); when(context.getFulltextIndexPath()).thenReturn(indexDir); - when(context.getDatabase()).thenReturn(database); - when(context.getEntries()).thenReturn(database.getEntries()); - - // Required because of {@Link org.jabref.model.search.rules.FullTextSearchRule.FullTextSearchRule} - stateManager = new StateManager(); - stateManager.setActiveDatabase(context); - Injector.setModelOrService(StateManager.class, stateManager); - - preferencesService = mock(PreferencesService.class); - FilePreferences filePreferences = mock(FilePreferences.class); - when(preferencesService.getFilePreferences()).thenReturn(filePreferences); - Injector.setModelOrService(PreferencesService.class, preferencesService); + when(FILE_PREFERENCES.shouldFulltextIndexLinkedFiles()).thenReturn(true); - return database; + return databaseContext; } private static Stream searchLibrary() { @@ -108,10 +91,10 @@ private static Stream searchLibrary() { // test-library-title-casing Arguments.of(List.of(), "test-library-title-casing.bib", "NotExisting", EnumSet.noneOf(SearchFlags.class)), - Arguments.of(List.of(titleSentenceCased, titleMixedCased, titleUpperCased), "test-library-title-casing.bib", "Title", EnumSet.noneOf(SearchFlags.class)), + Arguments.of(List.of(TITLE_SENTENCE_CASED, TITLE_MIXED_CASED, TITLE_UPPER_CASED), "test-library-title-casing.bib", "Title", EnumSet.noneOf(SearchFlags.class)), Arguments.of(List.of(), "test-library-title-casing.bib", "title=NotExisting", EnumSet.noneOf(SearchFlags.class)), - Arguments.of(List.of(titleSentenceCased, titleMixedCased, titleUpperCased), "test-library-title-casing.bib", "title=Title", EnumSet.noneOf(SearchFlags.class)), + Arguments.of(List.of(TITLE_SENTENCE_CASED, TITLE_MIXED_CASED, TITLE_UPPER_CASED), "test-library-title-casing.bib", "title=Title", EnumSet.noneOf(SearchFlags.class)), // Arguments.of(List.of(), "test-library-title-casing.bib", "title=TiTLE", EnumSet.of(SearchFlags.CASE_SENSITIVE)), // Arguments.of(List.of(titleSentenceCased), "test-library-title-casing.bib", "title=Title", EnumSet.of(SearchFlags.CASE_SENSITIVE)), @@ -123,7 +106,7 @@ private static Stream searchLibrary() { // Arguments.of(List.of(titleMixedCased), "test-library-title-casing.bib", "title=TiTle", EnumSet.of(SearchFlags.CASE_SENSITIVE)), Arguments.of(List.of(), "test-library-title-casing.bib", "[Y]", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)), - Arguments.of(List.of(titleUpperCased), "test-library-title-casing.bib", "[U]", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)), + Arguments.of(List.of(TITLE_UPPER_CASED), "test-library-title-casing.bib", "[U]", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)), // Word boundaries // Arguments.of(List.of(), "test-library-title-casing.bib", "\\bTit\\b", EnumSet.of(SearchFlags.REGULAR_EXPRESSION, SearchFlags.CASE_SENSITIVE)), @@ -133,8 +116,8 @@ private static Stream searchLibrary() { // Arguments.of(List.of(), "test-library-with-attached-files.bib", "This is a test.", EnumSet.of(SearchFlags.FULLTEXT, SearchFlags.CASE_SENSITIVE)), - Arguments.of(List.of(mininimalSentenceCase, minimalAllUpperCase, minimalMixedCase), "test-library-with-attached-files.bib", "This is a short sentence, comma included.", EnumSet.of(SearchFlags.FULLTEXT)), - Arguments.of(List.of(mininimalSentenceCase, minimalAllUpperCase, minimalMixedCase), "test-library-with-attached-files.bib", "comma", EnumSet.of(SearchFlags.FULLTEXT)), + Arguments.of(List.of(MINIMAL_SENTENCE_CASE, MINIMAL_ALL_UPPER_CASE, MINIMAL_MIXED_CASE), "test-library-with-attached-files.bib", "This is a short sentence, comma included.", EnumSet.of(SearchFlags.FULLTEXT)), + Arguments.of(List.of(MINIMAL_SENTENCE_CASE, MINIMAL_ALL_UPPER_CASE, MINIMAL_MIXED_CASE), "test-library-with-attached-files.bib", "comma", EnumSet.of(SearchFlags.FULLTEXT)), // TODO: PDF search does not support case sensitive search (yet) // Arguments.of(List.of(minimalAllUpperCase, minimalMixedCase), "test-library-with-attached-files.bib", "THIS", EnumSet.of(SearchFlags.FULLTEXT, SearchFlags.CASE_SENSITIVE)), // Arguments.of(List.of(minimalAllUpperCase), "test-library-with-attached-files.bib", "THIS is a short sentence, comma included.", EnumSet.of(SearchFlags.FULLTEXT, SearchFlags.CASE_SENSITIVE)), @@ -143,8 +126,8 @@ private static Stream searchLibrary() { Arguments.of(List.of(), "test-library-with-attached-files.bib", "NotExisting", EnumSet.of(SearchFlags.FULLTEXT)), - Arguments.of(List.of(minimalNoteSentenceCase, minimalNoteAllUpperCase, minimalNoteMixedCase), "test-library-with-attached-files.bib", "world", EnumSet.of(SearchFlags.FULLTEXT)), - Arguments.of(List.of(minimalNoteSentenceCase, minimalNoteAllUpperCase, minimalNoteMixedCase), "test-library-with-attached-files.bib", "Hello World", EnumSet.of(SearchFlags.FULLTEXT)) + Arguments.of(List.of(MINIMAL_NOTE_SENTENCE_CASE, MINIMAL_NOTE_ALL_UPPER_CASE, MINIMAL_NOTE_MIXED_CASE), "test-library-with-attached-files.bib", "world", EnumSet.of(SearchFlags.FULLTEXT)), + Arguments.of(List.of(MINIMAL_NOTE_SENTENCE_CASE, MINIMAL_NOTE_ALL_UPPER_CASE, MINIMAL_NOTE_MIXED_CASE), "test-library-with-attached-files.bib", "Hello World", EnumSet.of(SearchFlags.FULLTEXT)) // TODO: PDF search does not support case sensitive search (yet) // Arguments.of(List.of(minimalNoteAllUpperCase), "test-library-with-attached-files.bib", "HELLO WORLD", EnumSet.of(SearchFlags.FULLTEXT, SearchFlags.CASE_SENSITIVE)), // Arguments.of(List.of(), "test-library-with-attached-files.bib", "NotExisting", EnumSet.of(SearchFlags.FULLTEXT, SearchFlags.CASE_SENSITIVE)) @@ -154,8 +137,8 @@ private static Stream searchLibrary() { @ParameterizedTest(name = "{index} => query={2}, searchFlags={3}, testFile={1}, expected={0}") @MethodSource void searchLibrary(List expected, String testFile, String query, EnumSet searchFlags) throws Exception { - BibDatabase database = initializeDatabaseFromPath(testFile); - List matches = new DatabaseSearcher(new SearchQuery(query, searchFlags), database).getMatches(); + BibDatabaseContext databaseContext = initializeDatabaseFromPath(testFile); + List matches = new DatabaseSearcher(new SearchQuery(query, searchFlags), databaseContext, FILE_PREFERENCES).getMatches(); assertEquals(expected, matches); } } From 36ee11276e181b5ad15a4004bbaa3da694652286 Mon Sep 17 00:00:00 2001 From: Oliver Kopp Date: Thu, 29 Aug 2024 20:53:43 +0200 Subject: [PATCH 220/256] Fix typo in CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94cf654e444..7a8e89ffe47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,7 +63,7 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv - We removed support for case-insentive and exact search. [#11542](https://github.com/JabRef/jabref/pull/11542) - We removed support for importing using the SilverPlatterImporter (`Record INSPEC`). [#11576](https://github.com/JabRef/jabref/pull/11576) -- We removed the descroption of search strings. [#11565](https://github.com/JabRef/jabref/pull/11565) +- We removed the description of search strings. [#11565](https://github.com/JabRef/jabref/pull/11565) From 15ed52b7d741974eb4f85089abb66c6b9e439341 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Fri, 30 Aug 2024 06:06:53 +0300 Subject: [PATCH 221/256] Fix typo --- src/main/java/org/jabref/logic/search/DatabaseSearcher.java | 2 ++ .../java/org/jabref/logic/search/DatabaseSearcherTest.java | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/DatabaseSearcher.java b/src/main/java/org/jabref/logic/search/DatabaseSearcher.java index cbdc16b8b7c..840f2f9d47a 100644 --- a/src/main/java/org/jabref/logic/search/DatabaseSearcher.java +++ b/src/main/java/org/jabref/logic/search/DatabaseSearcher.java @@ -32,8 +32,10 @@ public DatabaseSearcher(SearchQuery query, BibDatabaseContext databaseContext, F this.query = Objects.requireNonNull(query); bibFieldsIndexer = new BibFieldsIndexer(databaseContext); bibFieldsIndexer.updateOnStart(DUMMY_TASK); + linkedFilesIndexer = new DefaultLinkedFilesIndexer(databaseContext, filePreferences); linkedFilesIndexer.updateOnStart(DUMMY_TASK); + this.luceneSearcher = new LuceneSearcher(databaseContext, bibFieldsIndexer, linkedFilesIndexer); } diff --git a/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java b/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java index 142d440079e..00a5f1f90ac 100644 --- a/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java +++ b/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java @@ -77,7 +77,7 @@ void noMatchesFromEmptyDatabaseWithInvalidQuery() throws IOException { } @Test - void correctMatchFromDatabaseWithIncollectionTypeEntry() throws IOException { + void correctMatchFromDatabaseWithInCollectionTypeEntry() throws IOException { BibEntry entry = new BibEntry(StandardEntryType.InCollection); entry.setField(StandardField.AUTHOR, "tonho"); databaseContext.getDatabase().insertEntry(entry); @@ -104,7 +104,7 @@ void noMatchesFromDatabaseWithTwoEntries() throws IOException { } @Test - void noMatchesFromDabaseWithIncollectionTypeEntry() throws IOException { + void noMatchesFromDatabaseWithInCollectionTypeEntry() throws IOException { BibEntry entry = new BibEntry(StandardEntryType.InCollection); entry.setField(StandardField.AUTHOR, "tonho"); databaseContext.getDatabase().insertEntry(entry); From cd6e55fa2504b4e2d667ccc70e1a25c262d0f7e9 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Fri, 30 Aug 2024 06:26:40 +0300 Subject: [PATCH 222/256] Use parameterized test for DatabaseSearcherTest --- .../logic/search/DatabaseSearcherTest.java | 114 +++++------------- 1 file changed, 28 insertions(+), 86 deletions(-) diff --git a/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java b/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java index 00a5f1f90ac..c7accc41761 100644 --- a/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java +++ b/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java @@ -1,9 +1,9 @@ package org.jabref.logic.search; import java.io.IOException; -import java.util.Collections; import java.util.EnumSet; import java.util.List; +import java.util.stream.Stream; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; @@ -14,7 +14,9 @@ import org.jabref.preferences.FilePreferences; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.mock; @@ -22,7 +24,6 @@ public class DatabaseSearcherTest { - private static final SearchQuery INVALID_SEARCH_QUERY = new SearchQuery("\\asd123{}asdf", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)); private static final FilePreferences FILE_PREFERENCES = mock(FilePreferences.class); private BibDatabaseContext databaseContext; @@ -32,97 +33,38 @@ void setUp() { databaseContext = new BibDatabaseContext(); } - @Test - void noMatchesFromEmptyDatabase() throws IOException { - List matches = new DatabaseSearcher(new SearchQuery("whatever", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)), databaseContext, FILE_PREFERENCES).getMatches(); - assertEquals(Collections.emptyList(), matches); - } - - @Test - void noMatchesFromEmptyDatabaseWithInvalidSearchExpression() throws IOException { - List matches = new DatabaseSearcher(INVALID_SEARCH_QUERY, databaseContext, FILE_PREFERENCES).getMatches(); - assertEquals(Collections.emptyList(), matches); - } - - @Test - void getDatabaseFromMatchesDatabaseWithEmptyEntries() throws IOException { - databaseContext.getDatabase().insertEntry(new BibEntry()); - List matches = new DatabaseSearcher(new SearchQuery("whatever", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)), databaseContext, FILE_PREFERENCES).getMatches(); - assertEquals(Collections.emptyList(), matches); - } - - @Test - void noMatchesFromDatabaseWithArticleTypeEntry() throws IOException { - BibEntry entry = new BibEntry(StandardEntryType.Article); - entry.setField(StandardField.AUTHOR, "harrer"); - databaseContext.getDatabase().insertEntry(entry); - List matches = new DatabaseSearcher(new SearchQuery("whatever", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)), databaseContext, FILE_PREFERENCES).getMatches(); - assertEquals(Collections.emptyList(), matches); - } - - @Test - void correctMatchFromDatabaseWithArticleTypeEntry() throws IOException { - BibEntry entry = new BibEntry(StandardEntryType.Article); - entry.setField(StandardField.AUTHOR, "harrer"); - databaseContext.getDatabase().insertEntry(entry); - List matches = new DatabaseSearcher(new SearchQuery("harrer", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)), databaseContext, FILE_PREFERENCES).getMatches(); - assertEquals(Collections.singletonList(entry), matches); - } - - @Test - void noMatchesFromEmptyDatabaseWithInvalidQuery() throws IOException { - SearchQuery query = new SearchQuery("asdf[", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)); - DatabaseSearcher databaseSearcher = new DatabaseSearcher(query, databaseContext, FILE_PREFERENCES); - assertEquals(Collections.emptyList(), databaseSearcher.getMatches()); - } - - @Test - void correctMatchFromDatabaseWithInCollectionTypeEntry() throws IOException { - BibEntry entry = new BibEntry(StandardEntryType.InCollection); - entry.setField(StandardField.AUTHOR, "tonho"); - databaseContext.getDatabase().insertEntry(entry); - - SearchQuery query = new SearchQuery("tonho", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)); + @ParameterizedTest + @MethodSource + void testDatabaseSearcher(List expectedMatches, SearchQuery query, List entries) throws IOException { + for (BibEntry entry : entries) { + databaseContext.getDatabase().insertEntry(entry); + } List matches = new DatabaseSearcher(query, databaseContext, FILE_PREFERENCES).getMatches(); - - assertEquals(Collections.singletonList(entry), matches); + assertEquals(expectedMatches, matches); } - @Test - void noMatchesFromDatabaseWithTwoEntries() throws IOException { - BibEntry entry = new BibEntry(); - databaseContext.getDatabase().insertEntry(entry); + private static Stream testDatabaseSearcher() { + BibEntry emptyEntry = new BibEntry(); - entry = new BibEntry(StandardEntryType.InCollection); - entry.setField(StandardField.AUTHOR, "tonho"); - databaseContext.getDatabase().insertEntry(entry); + BibEntry articleEntry = new BibEntry(StandardEntryType.Article); + articleEntry.setField(StandardField.AUTHOR, "harrer"); - SearchQuery query = new SearchQuery("tonho", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)); - DatabaseSearcher databaseSearcher = new DatabaseSearcher(query, databaseContext, FILE_PREFERENCES); + BibEntry inCollectionEntry = new BibEntry(StandardEntryType.InCollection); + inCollectionEntry.setField(StandardField.AUTHOR, "tonho"); - assertEquals(Collections.singletonList(entry), databaseSearcher.getMatches()); - } - - @Test - void noMatchesFromDatabaseWithInCollectionTypeEntry() throws IOException { - BibEntry entry = new BibEntry(StandardEntryType.InCollection); - entry.setField(StandardField.AUTHOR, "tonho"); - databaseContext.getDatabase().insertEntry(entry); - - SearchQuery query = new SearchQuery("asdf", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)); - DatabaseSearcher databaseSearcher = new DatabaseSearcher(query, databaseContext, FILE_PREFERENCES); - - assertEquals(Collections.emptyList(), databaseSearcher.getMatches()); - } + return Stream.of( + Arguments.of(List.of(), new SearchQuery("whatever", EnumSet.noneOf(SearchFlags.class)), List.of()), + Arguments.of(List.of(), new SearchQuery("whatever", EnumSet.noneOf(SearchFlags.class)), List.of(emptyEntry)), + Arguments.of(List.of(), new SearchQuery("whatever", EnumSet.noneOf(SearchFlags.class)), List.of(emptyEntry, articleEntry, inCollectionEntry)), - @Test - void noMatchFromDatabaseWithEmptyEntry() throws IOException { - BibEntry entry = new BibEntry(); - databaseContext.getDatabase().insertEntry(entry); + // invalid search syntax + Arguments.of(List.of(), new SearchQuery("author:", EnumSet.noneOf(SearchFlags.class)), List.of(articleEntry)), - SearchQuery query = new SearchQuery("tonho", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)); - DatabaseSearcher databaseSearcher = new DatabaseSearcher(query, databaseContext, FILE_PREFERENCES); + Arguments.of(List.of(articleEntry), new SearchQuery("harrer", EnumSet.noneOf(SearchFlags.class)), List.of(articleEntry)), + Arguments.of(List.of(), new SearchQuery("title: harrer", EnumSet.noneOf(SearchFlags.class)), List.of(articleEntry)), - assertEquals(Collections.emptyList(), databaseSearcher.getMatches()); + Arguments.of(List.of(inCollectionEntry), new SearchQuery("tonho", EnumSet.noneOf(SearchFlags.class)), List.of(inCollectionEntry)), + Arguments.of(List.of(inCollectionEntry), new SearchQuery("tonho", EnumSet.noneOf(SearchFlags.class)), List.of(articleEntry, inCollectionEntry)) + ); } } From df5d7c1bd6596574e615f38d799fdf291550178e Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Fri, 30 Aug 2024 07:24:42 +0300 Subject: [PATCH 223/256] Fix DatabaseSearcherWithBibFiles tests --- .../jabref/logic/search/DatabaseSearcher.java | 4 +- .../search/indexing/BibFieldsIndexer.java | 29 +++++++---- .../indexing/DefaultLinkedFilesIndexer.java | 37 +++++++------ .../indexing/ReadOnlyLinkedFilesIndexer.java | 23 +++++--- .../jabref/model/search/LuceneIndexer.java | 2 + .../DatabaseSearcherWithBibFilesTest.java | 52 +++++++------------ 6 files changed, 79 insertions(+), 68 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/DatabaseSearcher.java b/src/main/java/org/jabref/logic/search/DatabaseSearcher.java index 840f2f9d47a..1f151416d48 100644 --- a/src/main/java/org/jabref/logic/search/DatabaseSearcher.java +++ b/src/main/java/org/jabref/logic/search/DatabaseSearcher.java @@ -50,8 +50,8 @@ public List getMatches() { return Collections.emptyList(); } List matchEntries = luceneSearcher.search(query.getParsedQuery(), query.getSearchFlags()).getMatchedEntries().stream().toList(); - bibFieldsIndexer.close(); - linkedFilesIndexer.close(); + bibFieldsIndexer.closeAndWait(); + linkedFilesIndexer.closeAndWait(); return BibDatabases.purgeEmptyEntries(matchEntries); } } diff --git a/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java b/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java index a47a02e2d23..7811859be3f 100644 --- a/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/BibFieldsIndexer.java @@ -153,16 +153,23 @@ public SearcherManager getSearcherManager() { @Override public void close() { - LOGGER.debug("Closing bib fields index"); - HeadlessExecutorService.INSTANCE.execute(() -> { - try { - searcherManager.close(); - indexWriter.close(); - indexDirectory.close(); - LOGGER.debug("Bib fields index closed"); - } catch (IOException e) { - LOGGER.error("Error while closing bib fields index", e); - } - }); + HeadlessExecutorService.INSTANCE.execute(this::closeIndex); + } + + @Override + public void closeAndWait() { + HeadlessExecutorService.INSTANCE.executeAndWait(this::closeIndex); + } + + private void closeIndex() { + try { + LOGGER.debug("Closing bib fields index"); + searcherManager.close(); + indexWriter.close(); + indexDirectory.close(); + LOGGER.debug("Bib fields index closed"); + } catch (IOException e) { + LOGGER.error("Error while closing bib fields index", e); + } } } diff --git a/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java b/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java index b53f602628f..ded53d2381f 100644 --- a/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java @@ -313,21 +313,28 @@ public SearcherManager getSearcherManager() { @Override public void close() { - HeadlessExecutorService.INSTANCE.execute(() -> { - try { - LOGGER.debug("Closing linked files index"); - searcherManager.close(); - optimizeIndex(); - indexWriter.close(); - indexDirectory.close(); - LOGGER.debug("Linked files index closed"); - if ("unsaved".equals(databaseContext.getFulltextIndexPath().getFileName().toString())) { - LOGGER.debug("Deleting unsaved index directory"); - FileUtils.deleteDirectory(indexDirectoryPath.toFile()); - } - } catch (IOException e) { - LOGGER.error("Error while closing linked files index", e); + HeadlessExecutorService.INSTANCE.execute(this::closeIndex); + } + + @Override + public void closeAndWait() { + HeadlessExecutorService.INSTANCE.executeAndWait(this::closeIndex); + } + + private void closeIndex() { + try { + LOGGER.debug("Closing linked files index"); + searcherManager.close(); + optimizeIndex(); + indexWriter.close(); + indexDirectory.close(); + LOGGER.debug("Linked files index closed"); + if ("unsaved".equals(databaseContext.getFulltextIndexPath().getFileName().toString())) { + LOGGER.debug("Deleting unsaved index directory"); + FileUtils.deleteDirectory(indexDirectoryPath.toFile()); } - }); + } catch (IOException e) { + LOGGER.error("Error while closing linked files index", e); + } } } diff --git a/src/main/java/org/jabref/logic/search/indexing/ReadOnlyLinkedFilesIndexer.java b/src/main/java/org/jabref/logic/search/indexing/ReadOnlyLinkedFilesIndexer.java index 8c4e9200330..aa94960d52a 100644 --- a/src/main/java/org/jabref/logic/search/indexing/ReadOnlyLinkedFilesIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/ReadOnlyLinkedFilesIndexer.java @@ -60,13 +60,20 @@ public SearcherManager getSearcherManager() { @Override public void close() { - HeadlessExecutorService.INSTANCE.execute(() -> { - try { - searcherManager.close(); - indexDirectory.close(); - } catch (IOException e) { - LOGGER.error("Error closing index", e); - } - }); + HeadlessExecutorService.INSTANCE.execute(this::closeIndex); + } + + @Override + public void closeAndWait() { + HeadlessExecutorService.INSTANCE.executeAndWait(this::closeIndex); + } + + private void closeIndex() { + try { + searcherManager.close(); + indexDirectory.close(); + } catch (IOException e) { + LOGGER.error("Error closing index", e); + } } } diff --git a/src/main/java/org/jabref/model/search/LuceneIndexer.java b/src/main/java/org/jabref/model/search/LuceneIndexer.java index 1f0dfbd6edb..2b71016e935 100644 --- a/src/main/java/org/jabref/model/search/LuceneIndexer.java +++ b/src/main/java/org/jabref/model/search/LuceneIndexer.java @@ -23,4 +23,6 @@ public interface LuceneIndexer { SearcherManager getSearcherManager(); void close(); + + void closeAndWait(); } diff --git a/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java b/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java index 3d84c79aa2c..f23092b2da2 100644 --- a/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java +++ b/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java @@ -1,12 +1,12 @@ package org.jabref.logic.search; import java.nio.file.Path; -import java.util.Collections; import java.util.EnumSet; import java.util.List; import java.util.Objects; import java.util.stream.Stream; +import org.jabref.logic.bibtex.comparator.IdComparator; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.ParserResult; import org.jabref.logic.importer.fileformat.BibtexImporter; @@ -26,14 +26,12 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.mockito.Answers; -import org.mockito.Mockito; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; class DatabaseSearcherWithBibFilesTest { - private static final BibEntry TITLE_SENTENCE_CASED = new BibEntry(StandardEntryType.Misc) .withCitationKey("title-sentence-cased") .withField(StandardField.TITLE, "Title Sentence Cased"); @@ -46,22 +44,22 @@ class DatabaseSearcherWithBibFilesTest { private static final BibEntry MINIMAL_SENTENCE_CASE = new BibEntry(StandardEntryType.Misc) .withCitationKey("minimal-sentence-case") - .withFiles(Collections.singletonList(new LinkedFile("", "minimal-sentence-case.pdf", StandardFileType.PDF.getName()))); + .withFiles(List.of(new LinkedFile("", "minimal-sentence-case.pdf", StandardFileType.PDF.getName()))); private static final BibEntry MINIMAL_ALL_UPPER_CASE = new BibEntry(StandardEntryType.Misc) .withCitationKey("minimal-all-upper-case") - .withFiles(Collections.singletonList(new LinkedFile("", "minimal-all-upper-case.pdf", StandardFileType.PDF.getName()))); + .withFiles(List.of(new LinkedFile("", "minimal-all-upper-case.pdf", StandardFileType.PDF.getName()))); private static final BibEntry MINIMAL_MIXED_CASE = new BibEntry(StandardEntryType.Misc) .withCitationKey("minimal-mixed-case") - .withFiles(Collections.singletonList(new LinkedFile("", "minimal-mixed-case.pdf", StandardFileType.PDF.getName()))); + .withFiles(List.of(new LinkedFile("", "minimal-mixed-case.pdf", StandardFileType.PDF.getName()))); private static final BibEntry MINIMAL_NOTE_SENTENCE_CASE = new BibEntry(StandardEntryType.Misc) .withCitationKey("minimal-note-sentence-case") - .withFiles(Collections.singletonList(new LinkedFile("", "minimal-note-sentence-case.pdf", StandardFileType.PDF.getName()))); + .withFiles(List.of(new LinkedFile("", "minimal-note-sentence-case.pdf", StandardFileType.PDF.getName()))); private static final BibEntry MINIMAL_NOTE_ALL_UPPER_CASE = new BibEntry(StandardEntryType.Misc) .withCitationKey("minimal-note-all-upper-case") - .withFiles(Collections.singletonList(new LinkedFile("", "minimal-note-all-upper-case.pdf", StandardFileType.PDF.getName()))); + .withFiles(List.of(new LinkedFile("", "minimal-note-all-upper-case.pdf", StandardFileType.PDF.getName()))); private static final BibEntry MINIMAL_NOTE_MIXED_CASE = new BibEntry(StandardEntryType.Misc) .withCitationKey("minimal-note-mixed-case") - .withFiles(Collections.singletonList(new LinkedFile("", "minimal-note-mixed-case.pdf", StandardFileType.PDF.getName()))); + .withFiles(List.of(new LinkedFile("", "minimal-note-mixed-case.pdf", StandardFileType.PDF.getName()))); private static final FilePreferences FILE_PREFERENCES = mock(FilePreferences.class); @TempDir @@ -75,11 +73,7 @@ private BibDatabaseContext initializeDatabaseFromPath(Path testFile) throws Exce ParserResult result = new BibtexImporter(mock(ImportFormatPreferences.class, Answers.RETURNS_DEEP_STUBS), new DummyFileUpdateMonitor()).importDatabase(testFile); BibDatabaseContext databaseContext = result.getDatabaseContext(); - BibDatabaseContext context = mock(BibDatabaseContext.class); - when(context.getFileDirectories(Mockito.any())).thenReturn(List.of(testFile.getParent())); - when(context.getFulltextIndexPath()).thenReturn(indexDir); when(FILE_PREFERENCES.shouldFulltextIndexLinkedFiles()).thenReturn(true); - return databaseContext; } @@ -89,35 +83,28 @@ private static Stream searchLibrary() { Arguments.of(List.of(), "empty.bib", "Test", EnumSet.noneOf(SearchFlags.class)), // test-library-title-casing - Arguments.of(List.of(), "test-library-title-casing.bib", "NotExisting", EnumSet.noneOf(SearchFlags.class)), Arguments.of(List.of(TITLE_SENTENCE_CASED, TITLE_MIXED_CASED, TITLE_UPPER_CASED), "test-library-title-casing.bib", "Title", EnumSet.noneOf(SearchFlags.class)), - Arguments.of(List.of(), "test-library-title-casing.bib", "title=NotExisting", EnumSet.noneOf(SearchFlags.class)), - Arguments.of(List.of(TITLE_SENTENCE_CASED, TITLE_MIXED_CASED, TITLE_UPPER_CASED), "test-library-title-casing.bib", "title=Title", EnumSet.noneOf(SearchFlags.class)), + Arguments.of(List.of(), "test-library-title-casing.bib", "title:NotExisting", EnumSet.noneOf(SearchFlags.class)), + Arguments.of(List.of(TITLE_SENTENCE_CASED, TITLE_MIXED_CASED, TITLE_UPPER_CASED), "test-library-title-casing.bib", "title:Title", EnumSet.noneOf(SearchFlags.class)), - // Arguments.of(List.of(), "test-library-title-casing.bib", "title=TiTLE", EnumSet.of(SearchFlags.CASE_SENSITIVE)), - // Arguments.of(List.of(titleSentenceCased), "test-library-title-casing.bib", "title=Title", EnumSet.of(SearchFlags.CASE_SENSITIVE)), + // Arguments.of(List.of(), "test-library-title-casing.bib", "title:TiTLE", EnumSet.of(SearchFlags.CASE_SENSITIVE)), + // Arguments.of(List.of(titleSentenceCased), "test-library-title-casing.bib", "title:Title", EnumSet.of(SearchFlags.CASE_SENSITIVE)), // Arguments.of(List.of(), "test-library-title-casing.bib", "TiTLE", EnumSet.of(SearchFlags.CASE_SENSITIVE)), // Arguments.of(List.of(titleMixedCased), "test-library-title-casing.bib", "TiTle", EnumSet.of(SearchFlags.CASE_SENSITIVE)), - // Arguments.of(List.of(), "test-library-title-casing.bib", "title=NotExisting", EnumSet.of(SearchFlags.CASE_SENSITIVE)), - // Arguments.of(List.of(titleMixedCased), "test-library-title-casing.bib", "title=TiTle", EnumSet.of(SearchFlags.CASE_SENSITIVE)), + // Arguments.of(List.of(), "test-library-title-casing.bib", "title:NotExisting", EnumSet.of(SearchFlags.CASE_SENSITIVE)), + // Arguments.of(List.of(titleMixedCased), "test-library-title-casing.bib", "title:TiTle", EnumSet.of(SearchFlags.CASE_SENSITIVE)), - Arguments.of(List.of(), "test-library-title-casing.bib", "[Y]", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)), - Arguments.of(List.of(TITLE_UPPER_CASED), "test-library-title-casing.bib", "[U]", EnumSet.of(SearchFlags.REGULAR_EXPRESSION)), - - // Word boundaries - // Arguments.of(List.of(), "test-library-title-casing.bib", "\\bTit\\b", EnumSet.of(SearchFlags.REGULAR_EXPRESSION, SearchFlags.CASE_SENSITIVE)), - // Arguments.of(List.of(titleSentenceCased), "test-library-title-casing.bib", "\\bTitle\\b", EnumSet.of(SearchFlags.REGULAR_EXPRESSION, SearchFlags.CASE_SENSITIVE)), + Arguments.of(List.of(), "test-library-title-casing.bib", "/[Y]/", EnumSet.noneOf(SearchFlags.class)), // test-library-with-attached-files - - // Arguments.of(List.of(), "test-library-with-attached-files.bib", "This is a test.", EnumSet.of(SearchFlags.FULLTEXT, SearchFlags.CASE_SENSITIVE)), - + Arguments.of(List.of(), "test-library-with-attached-files.bib", "NotExisting.", EnumSet.of(SearchFlags.FULLTEXT)), Arguments.of(List.of(MINIMAL_SENTENCE_CASE, MINIMAL_ALL_UPPER_CASE, MINIMAL_MIXED_CASE), "test-library-with-attached-files.bib", "This is a short sentence, comma included.", EnumSet.of(SearchFlags.FULLTEXT)), Arguments.of(List.of(MINIMAL_SENTENCE_CASE, MINIMAL_ALL_UPPER_CASE, MINIMAL_MIXED_CASE), "test-library-with-attached-files.bib", "comma", EnumSet.of(SearchFlags.FULLTEXT)), + // TODO: PDF search does not support case sensitive search (yet) // Arguments.of(List.of(minimalAllUpperCase, minimalMixedCase), "test-library-with-attached-files.bib", "THIS", EnumSet.of(SearchFlags.FULLTEXT, SearchFlags.CASE_SENSITIVE)), // Arguments.of(List.of(minimalAllUpperCase), "test-library-with-attached-files.bib", "THIS is a short sentence, comma included.", EnumSet.of(SearchFlags.FULLTEXT, SearchFlags.CASE_SENSITIVE)), @@ -125,21 +112,22 @@ private static Stream searchLibrary() { // Arguments.of(List.of(minimalNoteAllUpperCase), "test-library-with-attached-files.bib", "THIS IS A SHORT SENTENCE, COMMA INCLUDED.", EnumSet.of(SearchFlags.FULLTEXT, SearchFlags.CASE_SENSITIVE)), Arguments.of(List.of(), "test-library-with-attached-files.bib", "NotExisting", EnumSet.of(SearchFlags.FULLTEXT)), - Arguments.of(List.of(MINIMAL_NOTE_SENTENCE_CASE, MINIMAL_NOTE_ALL_UPPER_CASE, MINIMAL_NOTE_MIXED_CASE), "test-library-with-attached-files.bib", "world", EnumSet.of(SearchFlags.FULLTEXT)), Arguments.of(List.of(MINIMAL_NOTE_SENTENCE_CASE, MINIMAL_NOTE_ALL_UPPER_CASE, MINIMAL_NOTE_MIXED_CASE), "test-library-with-attached-files.bib", "Hello World", EnumSet.of(SearchFlags.FULLTEXT)) + // TODO: PDF search does not support case sensitive search (yet) // Arguments.of(List.of(minimalNoteAllUpperCase), "test-library-with-attached-files.bib", "HELLO WORLD", EnumSet.of(SearchFlags.FULLTEXT, SearchFlags.CASE_SENSITIVE)), // Arguments.of(List.of(), "test-library-with-attached-files.bib", "NotExisting", EnumSet.of(SearchFlags.FULLTEXT, SearchFlags.CASE_SENSITIVE)) ); } - @ParameterizedTest(name = "{index} => query={2}, searchFlags={3}, testFile={1}, expected={0}") + @ParameterizedTest @MethodSource void searchLibrary(List expected, String testFile, String query, EnumSet searchFlags) throws Exception { BibDatabaseContext databaseContext = initializeDatabaseFromPath(testFile); List matches = new DatabaseSearcher(new SearchQuery(query, searchFlags), databaseContext, FILE_PREFERENCES).getMatches(); - assertEquals(expected, matches); + // assert that both lists has the same items, ignoring the order + assertEquals(expected.stream().sorted(new IdComparator()).toList(), matches.stream().sorted(new IdComparator()).toList()); } } From 4872dc517c435f2feb5997f6f4f6b61306ca577a Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Fri, 30 Aug 2024 07:38:57 +0300 Subject: [PATCH 224/256] Fix exportMatches test --- src/test/java/org/jabref/cli/ArgumentProcessorTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/jabref/cli/ArgumentProcessorTest.java b/src/test/java/org/jabref/cli/ArgumentProcessorTest.java index 70423c9a748..8b18bb9826d 100644 --- a/src/test/java/org/jabref/cli/ArgumentProcessorTest.java +++ b/src/test/java/org/jabref/cli/ArgumentProcessorTest.java @@ -96,7 +96,7 @@ void exportMatches(@TempDir Path tempDir) throws Exception { Path outputBib = tempDir.resolve("output.bib").toAbsolutePath(); String outputBibFile = outputBib.toAbsolutePath().toString(); - List args = List.of("-n", "--debug", "--exportMatches", "Author=Einstein," + outputBibFile, originBibFile); + List args = List.of("-n", "--debug", "--exportMatches", "author:Einstein," + outputBibFile, originBibFile); ArgumentProcessor processor = new ArgumentProcessor( args.toArray(String[]::new), From b12fa5697deb0fbfda6614cd1e1e80d6de1b815e Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Fri, 30 Aug 2024 09:33:30 +0300 Subject: [PATCH 225/256] Remove regex check box from search groups dialog --- .../org/jabref/gui/groups/GroupDialog.fxml | 3 -- .../jabref/gui/groups/GroupDialogView.java | 13 -------- .../gui/groups/GroupDialogViewModel.java | 30 ++----------------- 3 files changed, 3 insertions(+), 43 deletions(-) diff --git a/src/main/java/org/jabref/gui/groups/GroupDialog.fxml b/src/main/java/org/jabref/gui/groups/GroupDialog.fxml index c4504b939f6..9ec2a307ad1 100644 --- a/src/main/java/org/jabref/gui/groups/GroupDialog.fxml +++ b/src/main/java/org/jabref/gui/groups/GroupDialog.fxml @@ -121,9 +121,6 @@ ", " "))); - content.getChildren().addAll(new Text(System.lineSeparator()), lineSeparator(0.8), createPageLink(linkedFile, searchResult.getPageNumber())); - } - if (!searchResult.getAnnotationsResultStringsHtml().isEmpty()) { - Text annotationsText = new Text(System.lineSeparator() + Localization.lang("Found matches in annotations:") + System.lineSeparator() + System.lineSeparator()); - annotationsText.setStyle("-fx-font-style: italic;"); - content.getChildren().add(annotationsText); - } - for (String resultTextHtml : searchResult.getAnnotationsResultStringsHtml()) { - content.getChildren().addAll(TooltipTextUtil.createTextsFromHtml(resultTextHtml.replace(" ", " "))); - content.getChildren().addAll(new Text(System.lineSeparator()), lineSeparator(0.8), createPageLink(linkedFile, searchResult.getPageNumber())); + stateManager.activeSearchQuery(SearchType.NORMAL_SEARCH).get().ifPresent(searchQuery -> { + SearchResults searchResults = searchQuery.getSearchResults(); + if (searchResults != null) { + Map> searchResultsForEntry = searchResults.getFileSearchResultsForEntry(entry); + if (searchResultsForEntry.isEmpty()) { + content.getChildren().add(new Text(Localization.lang("No search matches."))); + } else { + // Iterate through files with search hits + for (Map.Entry> iterator : searchResultsForEntry.entrySet()) { + entry.getFiles().stream().filter(file -> file.getLink().equals(iterator.getKey())).findFirst().ifPresent(linkedFile -> { + content.getChildren().addAll(createFileLink(linkedFile), lineSeparator()); + // Iterate through pages (within file) with search hits + for (SearchResult searchResult : iterator.getValue()) { + for (String resultTextHtml : searchResult.getContentResultStringsHtml()) { + content.getChildren().addAll(TooltipTextUtil.createTextsFromHtml(resultTextHtml.replace(" ", " "))); + content.getChildren().addAll(new Text(System.lineSeparator()), lineSeparator(0.8), createPageLink(linkedFile, searchResult.getPageNumber())); + } + if (!searchResult.getAnnotationsResultStringsHtml().isEmpty()) { + Text annotationsText = new Text(System.lineSeparator() + Localization.lang("Found matches in annotations:") + System.lineSeparator() + System.lineSeparator()); + annotationsText.setStyle("-fx-font-style: italic;"); + content.getChildren().add(annotationsText); + } + for (String resultTextHtml : searchResult.getAnnotationsResultStringsHtml()) { + content.getChildren().addAll(TooltipTextUtil.createTextsFromHtml(resultTextHtml.replace(" ", " "))); + content.getChildren().addAll(new Text(System.lineSeparator()), lineSeparator(0.8), createPageLink(linkedFile, searchResult.getPageNumber())); + } } - } - }); + }); + } } } }); diff --git a/src/main/java/org/jabref/gui/groups/GroupDialogViewModel.java b/src/main/java/org/jabref/gui/groups/GroupDialogViewModel.java index 4a2942e3129..e21ea47332a 100644 --- a/src/main/java/org/jabref/gui/groups/GroupDialogViewModel.java +++ b/src/main/java/org/jabref/gui/groups/GroupDialogViewModel.java @@ -30,6 +30,7 @@ import org.jabref.logic.auxparser.DefaultAuxParser; import org.jabref.logic.groups.DefaultGroupsFactory; import org.jabref.logic.l10n.Localization; +import org.jabref.logic.search.LuceneManager; import org.jabref.logic.util.StandardFileType; import org.jabref.logic.util.io.FileUtil; import org.jabref.model.database.BibDatabase; @@ -316,8 +317,13 @@ public AbstractGroup resultConverter(ButtonType button) { groupName, groupHierarchySelectedProperty.getValue(), searchGroupSearchTermProperty.getValue().trim(), - searchFlagsProperty.getValue(), - stateManager.getLuceneManager(currentDatabase).orElse(null)); + searchFlagsProperty.getValue()); + + Optional luceneManager = stateManager.getLuceneManager(currentDatabase); + if (luceneManager.isPresent()) { + SearchGroup searchGroup = (SearchGroup) resultingGroup; + searchGroup.setMatchedEntries(luceneManager.get().search(searchGroup.getQuery()).getMatchedEntries()); + } } else if (typeAutoProperty.getValue()) { if (autoGroupKeywordsOptionProperty.getValue()) { // Set default value for delimiters: ',' for base and '>' for hierarchical diff --git a/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java b/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java index 37fdca01663..75a8ed5417b 100644 --- a/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java +++ b/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java @@ -14,7 +14,7 @@ import javafx.collections.FXCollections; import javafx.collections.ListChangeListener; import javafx.collections.ObservableList; -import javafx.collections.ObservableMap; +import javafx.collections.ObservableSet; import javafx.scene.input.Dragboard; import javafx.scene.paint.Color; @@ -55,6 +55,7 @@ import com.google.common.eventbus.Subscribe; import com.tobiasdiez.easybind.EasyBind; import com.tobiasdiez.easybind.EasyObservableList; +import io.github.adr.linked.ADR; public class GroupNodeViewModel { @@ -64,7 +65,8 @@ public class GroupNodeViewModel { private final BibDatabaseContext databaseContext; private final StateManager stateManager; private final GroupTreeNode groupNode; - private final ObservableMap matchedEntries = FXCollections.observableHashMap(); + @ADR(38) + private final ObservableSet matchedEntries = FXCollections.observableSet(); private final SimpleBooleanProperty hasChildren; private final SimpleBooleanProperty expandedProperty = new SimpleBooleanProperty(); private final BooleanBinding anySelectedEntriesMatched; @@ -99,10 +101,14 @@ public GroupNodeViewModel(BibDatabaseContext databaseContext, StateManager state if (groupNode.getGroup() instanceof TexGroup) { databaseContext.getMetaData().groupsBinding().addListener(new WeakInvalidationListener(onInvalidatedGroup)); } else if (groupNode.getGroup() instanceof SearchGroup searchGroup) { - stateManager.getLuceneManager(databaseContext).ifPresent(searchGroup::setLuceneManager); - searchGroup.updateMatches(); - refreshGroup(); - databaseContext.getMetaData().groupsBinding().invalidate(); + stateManager.getLuceneManager(databaseContext).ifPresent(luceneManager -> { + BackgroundTask.wrap(() -> { + searchGroup.setMatchedEntries(luceneManager.search(searchGroup.getQuery()).getMatchedEntries()); + }).onSuccess(success -> { + refreshGroup(); + databaseContext.getMetaData().groupsBinding().invalidate(); + }).executeWith(taskExecutor); + }); } hasChildren = new SimpleBooleanProperty(); @@ -258,7 +264,7 @@ private void onDatabaseChanged(ListChangeListener.Change cha for (BibEntry changedEntry : change.getList().subList(change.getFrom(), change.getTo())) { if (groupNode.matches(changedEntry)) { // ADR-0038 - matchedEntries.put(changedEntry.getId(), changedEntry); + matchedEntries.add(changedEntry.getId()); } else { // ADR-0038 matchedEntries.remove(changedEntry.getId()); @@ -272,7 +278,7 @@ private void onDatabaseChanged(ListChangeListener.Change cha for (BibEntry addedEntry : change.getAddedSubList()) { if (groupNode.matches(addedEntry)) { // ADR-0038 - matchedEntries.put(addedEntry.getId(), addedEntry); + matchedEntries.add(addedEntry.getId()); } } } @@ -300,7 +306,7 @@ private void updateMatchedEntries() { .onSuccess(entries -> { matchedEntries.clear(); // ADR-0038 - entries.forEach(entry -> matchedEntries.put(entry.getId(), entry)); + entries.forEach(entry -> matchedEntries.add(entry.getId())); }) .executeWith(taskExecutor); } @@ -534,48 +540,49 @@ class LuceneIndexListener { @Subscribe public void listen(IndexStartedEvent event) { if (groupNode.getGroup() instanceof SearchGroup searchGroup) { - stateManager.getLuceneManager(databaseContext).ifPresent(searchGroup::setLuceneManager); - searchGroup.updateMatches(); - refreshGroup(); - databaseContext.getMetaData().groupsBinding().invalidate(); + stateManager.getLuceneManager(databaseContext).ifPresent(luceneManager -> { + BackgroundTask.wrap(() -> { + searchGroup.setMatchedEntries(luceneManager.search(searchGroup.getQuery()).getMatchedEntries()); + }).onSuccess(success -> { + refreshGroup(); + databaseContext.getMetaData().groupsBinding().invalidate(); + }).executeWith(taskExecutor); + }); } } @Subscribe public void listen(IndexAddedOrUpdatedEvent event) { if (groupNode.getGroup() instanceof SearchGroup searchGroup) { - BackgroundTask.wrap(() -> { - for (BibEntry entry : event.entries()) { - searchGroup.updateMatches(entry); - if (groupNode.matches(entry)) { - // ADR-0038 - matchedEntries.put(entry.getId(), entry); - } else { - // ADR-0038 - matchedEntries.remove(entry.getId()); + stateManager.getLuceneManager(databaseContext).ifPresent(luceneManager -> { + BackgroundTask.wrap(() -> { + for (BibEntry entry : event.entries()) { + boolean matched = luceneManager.isMatched(entry, searchGroup.getQuery()); + searchGroup.updateMatches(entry, matched); + if (matched) { + matchedEntries.add(entry.getId()); + } else { + matchedEntries.remove(entry.getId()); + } } - } - }).executeWith(taskExecutor); + }).executeWith(taskExecutor); + }); } } @Subscribe public void listen(IndexRemovedEvent event) { if (groupNode.getGroup() instanceof SearchGroup searchGroup) { - BackgroundTask.wrap(() -> { - for (BibEntry entry : event.entries()) { - searchGroup.updateMatches(entry); - // ADR-0038 - matchedEntries.remove(entry.getId()); - } - }).executeWith(taskExecutor); + for (BibEntry entry : event.entries()) { + searchGroup.updateMatches(entry, false); + matchedEntries.remove(entry.getId()); + } } } @Subscribe public void listen(IndexClosedEvent event) { if (groupNode.getGroup() instanceof SearchGroup group) { - group.setLuceneManager(null); databaseContext.getDatabase().unregisterListener(this); } } diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index 9bba80eecd6..af46c4f6d48 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -104,7 +104,6 @@ private void updateSearchMatches(Optional query) { boolean isFloatingMode = searchPreferences.getSearchDisplayMode() == SearchDisplayMode.FLOAT; if (query.isPresent()) { SearchResults results = luceneManager.search(query.get()); - stateManager.getSearchResults().put(bibDatabaseContext.getUid(), results); entriesViewModel.forEach(entry -> { entry.searchScoreProperty().set(results.getSearchScoreForEntry(entry.getEntry())); entry.hasFullTextResultsProperty().set(results.hasFulltextResults(entry.getEntry())); diff --git a/src/main/java/org/jabref/logic/search/DatabaseSearcher.java b/src/main/java/org/jabref/logic/search/DatabaseSearcher.java index 598eb73c3ee..bf5e74e070b 100644 --- a/src/main/java/org/jabref/logic/search/DatabaseSearcher.java +++ b/src/main/java/org/jabref/logic/search/DatabaseSearcher.java @@ -18,10 +18,12 @@ public class DatabaseSearcher { private static final Logger LOGGER = LoggerFactory.getLogger(DatabaseSearcher.class); + private final BibDatabaseContext databaseContext; private final SearchQuery query; private final LuceneManager luceneManager; public DatabaseSearcher(SearchQuery query, BibDatabaseContext databaseContext, TaskExecutor taskExecutor, FilePreferences filePreferences) throws IOException { + this.databaseContext = databaseContext; this.query = Objects.requireNonNull(query); this.luceneManager = new LuceneManager(databaseContext, taskExecutor, filePreferences); } @@ -36,7 +38,11 @@ public List getMatches() { LOGGER.warn("Search failed: invalid search expression"); return Collections.emptyList(); } - List matchEntries = luceneManager.search(query).getMatchedEntries().stream().toList(); + List matchEntries = luceneManager.search(query) + .getMatchedEntries() + .stream() + .map(entryId -> databaseContext.getDatabase().getEntryById(entryId)) + .toList(); luceneManager.close(); return BibDatabases.purgeEmptyEntries(matchEntries); } diff --git a/src/main/java/org/jabref/logic/search/LuceneManager.java b/src/main/java/org/jabref/logic/search/LuceneManager.java index 8ea520603e3..086130ee7e0 100644 --- a/src/main/java/org/jabref/logic/search/LuceneManager.java +++ b/src/main/java/org/jabref/logic/search/LuceneManager.java @@ -217,9 +217,11 @@ public AutoCloseable blockLinkedFileIndexer() { public SearchResults search(SearchQuery query) { if (query.isValid()) { - return luceneSearcher.search(query.getParsedQuery(), query.getSearchFlags()); + query.setSearchResults(luceneSearcher.search(query.getParsedQuery(), query.getSearchFlags())); + } else { + query.setSearchResults(new SearchResults()); } - return new SearchResults(); + return query.getSearchResults(); } public boolean isMatched(BibEntry entry, SearchQuery query) { diff --git a/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java b/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java index a11ceea1f12..c37b878d001 100644 --- a/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java @@ -65,7 +65,7 @@ public DefaultLinkedFilesIndexer(BibDatabaseContext databaseContext, FilePrefere this.indexedFiles = new ConcurrentHashMap<>(); indexDirectoryPath = databaseContext.getFulltextIndexPath(); - IndexWriterConfig config = new IndexWriterConfig(SearchFieldConstants.LATEX_AWARE_ANALYZER); + IndexWriterConfig config = new IndexWriterConfig(SearchFieldConstants.LINKED_FILES_ANALYZER); if ("unsaved".equals(indexDirectoryPath.getFileName().toString())) { config.setOpenMode(IndexWriterConfig.OpenMode.CREATE); indexDirectoryPath = indexDirectoryPath.resolveSibling("unsaved" + NUMBER_OF_UNSAVED_LIBRARIES++); diff --git a/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java b/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java index 84e74b97b2d..a293a7f5cea 100644 --- a/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java +++ b/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java @@ -98,13 +98,12 @@ private SearchResults search(IndexSearcher indexSearcher, Query searchQuery) thr private SearchResults getSearchResults(TopDocs topDocs, StoredFields storedFields, Query searchQuery) throws IOException { SearchResults searchResults = new SearchResults(); - Map entriesMap = new HashMap<>(); - Map> linkedFilesMap = new HashMap<>(); + // fileLink to List of entry IDs + Map> linkedFilesMap = new HashMap<>(); for (BibEntry bibEntry : databaseContext.getEntries()) { - entriesMap.put(bibEntry.getId(), bibEntry); for (LinkedFile linkedFile : bibEntry.getFiles()) { - linkedFilesMap.computeIfAbsent(linkedFile.getLink(), k -> new ArrayList<>()).add(bibEntry); + linkedFilesMap.computeIfAbsent(linkedFile.getLink(), k -> new ArrayList<>()).add(bibEntry.getId()); } } @@ -116,7 +115,7 @@ private SearchResults getSearchResults(TopDocs topDocs, StoredFields storedField String fileLink = getFieldContents(document, SearchFieldConstants.PATH); if (!fileLink.isEmpty()) { - List entriesWithFile = linkedFilesMap.get(fileLink); + List entriesWithFile = linkedFilesMap.get(fileLink); if (!entriesWithFile.isEmpty()) { SearchResult searchResult = new SearchResult(score, fileLink, getFieldContents(document, SearchFieldConstants.CONTENT), @@ -127,7 +126,7 @@ private SearchResults getSearchResults(TopDocs topDocs, StoredFields storedField } } else { String entryId = getFieldContents(document, SearchFieldConstants.ENTRY_ID); - searchResults.addSearchResult(entriesMap.get(entryId), new SearchResult(score)); + searchResults.addSearchResult(entryId, new SearchResult(score)); } } LOGGER.debug("Mapping search results took {} ms", System.currentTimeMillis() - startTime); diff --git a/src/main/java/org/jabref/model/database/BibDatabase.java b/src/main/java/org/jabref/model/database/BibDatabase.java index a7fc8a57599..a5d1d461851 100644 --- a/src/main/java/org/jabref/model/database/BibDatabase.java +++ b/src/main/java/org/jabref/model/database/BibDatabase.java @@ -7,6 +7,7 @@ import java.util.Collection; import java.util.Collections; import java.util.Comparator; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -52,6 +53,9 @@ public class BibDatabase { * State attributes */ private final ObservableList entries = FXCollections.synchronizedObservableList(FXCollections.observableArrayList(BibEntry::getObservables)); + + // BibEntryId to BibEntry + private final Map entriesId = new HashMap<>(); private Map bibtexStrings = new ConcurrentHashMap<>(); // Not included in equals, because it is not relevant for the content of the database @@ -204,6 +208,7 @@ public synchronized void insertEntries(List newEntries, EntriesEventSo eventBus.post(new EntriesAddedEvent(newEntries, newEntries.getFirst(), eventSource)); } entries.addAll(newEntries); + newEntries.forEach(entry -> entriesId.put(entry.getId(), entry)); } public synchronized void removeEntry(BibEntry bibEntry) { @@ -240,6 +245,7 @@ public synchronized void removeEntries(List toBeDeleted, EntriesEventS } boolean anyRemoved = entries.removeIf(entry -> ids.contains(entry.getId())); if (anyRemoved) { + toBeDeleted.forEach(entry -> entriesId.remove(entry.getId())); eventBus.post(new EntriesRemovedEvent(toBeDeleted, eventSource)); } } @@ -650,6 +656,10 @@ public int indexOf(BibEntry bibEntry) { return index >= 0 ? index : -1; } + public BibEntry getEntryById(String id) { + return entriesId.get(id); + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/src/main/java/org/jabref/model/groups/SearchGroup.java b/src/main/java/org/jabref/model/groups/SearchGroup.java index ecc5d4407d8..fe7e8132357 100644 --- a/src/main/java/org/jabref/model/groups/SearchGroup.java +++ b/src/main/java/org/jabref/model/groups/SearchGroup.java @@ -1,13 +1,11 @@ package org.jabref.model.groups; +import java.util.Collection; import java.util.EnumSet; +import java.util.HashSet; import java.util.Objects; +import java.util.Set; -import javafx.collections.FXCollections; -import javafx.collections.ObservableMap; - -import org.jabref.architecture.AllowedToUseLogic; -import org.jabref.logic.search.LuceneManager; import org.jabref.logic.util.Version; import org.jabref.model.entry.BibEntry; import org.jabref.model.search.SearchFlags; @@ -21,31 +19,21 @@ * This group matches entries by a complex search pattern, which might include conditions about the values of * multiple fields. */ -@AllowedToUseLogic("because it needs access to lucene manager") public class SearchGroup extends AbstractGroup { // We cannot have this constant in Version java because of recursion errors // Thus, we keep it here, because it is (currently) used only in the context of groups. public static final Version VERSION_6_0_ALPHA = Version.parse("6.0-alpha"); - private static final Logger LOGGER = LoggerFactory.getLogger(SearchGroup.class); - @ADR(31) - private final ObservableMap matchedEntries = FXCollections.observableHashMap(); + @ADR(38) + private final Set matchedEntries = new HashSet<>(); private SearchQuery query; - private LuceneManager luceneManager; - - public SearchGroup(String name, GroupHierarchyType context, String searchExpression, EnumSet searchFlags, LuceneManager luceneManager) { + public SearchGroup(String name, GroupHierarchyType context, String searchExpression, EnumSet searchFlags) { super(name, context); this.query = new SearchQuery(searchExpression, searchFlags); - this.luceneManager = luceneManager; - updateMatches(); - } - - public SearchGroup(String name, GroupHierarchyType context, String searchExpression, EnumSet searchFlags) { - this(name, context, searchExpression, searchFlags, null); } public String getSearchExpression() { @@ -69,29 +57,15 @@ public EnumSet getSearchFlags() { return query.getSearchFlags(); } - public void setLuceneManager(LuceneManager luceneManager) { - this.luceneManager = luceneManager; - } - - public void updateMatches() { - if (luceneManager == null) { - return; - } + public void setMatchedEntries(Collection entriesId) { matchedEntries.clear(); - // TODO: Search should be done in a background thread - // ADR-0038 - luceneManager.search(query).getMatchedEntries().forEach(entry -> matchedEntries.put(entry.getId(), entry)); + matchedEntries.addAll(entriesId); } - public void updateMatches(BibEntry entry) { - if (luceneManager == null) { - return; - } - if (luceneManager.isMatched(entry, query)) { - // ADR-0038 - matchedEntries.put(entry.getId(), entry); + public void updateMatches(BibEntry entry, boolean matched) { + if (matched) { + matchedEntries.add(entry.getId()); } else { - // ADR-0038 matchedEntries.remove(entry.getId()); } } @@ -112,14 +86,13 @@ public boolean equals(Object o) { @Override public boolean contains(BibEntry entry) { - // ADR-0038 - return matchedEntries.containsKey(entry.getId()); + return matchedEntries.contains(entry.getId()); } @Override public AbstractGroup deepCopy() { try { - return new SearchGroup(getName(), getHierarchicalContext(), getSearchExpression(), getSearchFlags(), luceneManager); + return new SearchGroup(getName(), getHierarchicalContext(), getSearchExpression(), getSearchFlags()); } catch (Throwable t) { // this should never happen, because the constructor obviously // succeeded in creating _this_ instance! diff --git a/src/main/java/org/jabref/model/search/LatexAwareAnalyzer.java b/src/main/java/org/jabref/model/search/Analyzer/LatexAwareAnalyzer.java similarity index 76% rename from src/main/java/org/jabref/model/search/LatexAwareAnalyzer.java rename to src/main/java/org/jabref/model/search/Analyzer/LatexAwareAnalyzer.java index 2d306b89036..a67f1883b80 100644 --- a/src/main/java/org/jabref/model/search/LatexAwareAnalyzer.java +++ b/src/main/java/org/jabref/model/search/Analyzer/LatexAwareAnalyzer.java @@ -1,12 +1,12 @@ -package org.jabref.model.search; +package org.jabref.model.search.Analyzer; import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.CharArraySet; import org.apache.lucene.analysis.LowerCaseFilter; import org.apache.lucene.analysis.StopFilter; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.core.WhitespaceTokenizer; +import org.apache.lucene.analysis.en.EnglishAnalyzer; import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter; /** @@ -14,16 +14,11 @@ * Especially, LaTeX-encoded text. */ public class LatexAwareAnalyzer extends Analyzer { - private final CharArraySet stopWords; - public LatexAwareAnalyzer(CharArraySet stopWords) { - this.stopWords = stopWords; - } - @Override protected TokenStreamComponents createComponents(String fieldName) { Tokenizer source = new WhitespaceTokenizer(); TokenStream result = new LatexToUnicodeFoldingFilter(source); - result = new StopFilter(result, stopWords); + result = new StopFilter(result, EnglishAnalyzer.ENGLISH_STOP_WORDS_SET); result = new ASCIIFoldingFilter(result); result = new LowerCaseFilter(result); return new TokenStreamComponents(source, result); diff --git a/src/main/java/org/jabref/model/search/LatexAwareNGramAnalyzer.java b/src/main/java/org/jabref/model/search/Analyzer/LatexAwareNGramAnalyzer.java similarity index 77% rename from src/main/java/org/jabref/model/search/LatexAwareNGramAnalyzer.java rename to src/main/java/org/jabref/model/search/Analyzer/LatexAwareNGramAnalyzer.java index c4db64b6aca..961b9ce301b 100644 --- a/src/main/java/org/jabref/model/search/LatexAwareNGramAnalyzer.java +++ b/src/main/java/org/jabref/model/search/Analyzer/LatexAwareNGramAnalyzer.java @@ -1,31 +1,29 @@ -package org.jabref.model.search; +package org.jabref.model.search.Analyzer; import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.CharArraySet; import org.apache.lucene.analysis.LowerCaseFilter; import org.apache.lucene.analysis.StopFilter; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.core.WhitespaceTokenizer; +import org.apache.lucene.analysis.en.EnglishAnalyzer; import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter; import org.apache.lucene.analysis.ngram.NGramTokenFilter; public class LatexAwareNGramAnalyzer extends Analyzer { private final int minGram; private final int maxGram; - private final CharArraySet stopWords; - public LatexAwareNGramAnalyzer(int minGram, int maxGram, CharArraySet stopWords) { + public LatexAwareNGramAnalyzer(int minGram, int maxGram) { this.minGram = minGram; this.maxGram = maxGram; - this.stopWords = stopWords; } @Override protected TokenStreamComponents createComponents(String fieldName) { Tokenizer source = new WhitespaceTokenizer(); TokenStream result = new LatexToUnicodeFoldingFilter(source); - result = new StopFilter(result, stopWords); + result = new StopFilter(result, EnglishAnalyzer.ENGLISH_STOP_WORDS_SET); result = new ASCIIFoldingFilter(result); result = new LowerCaseFilter(result); result = new NGramTokenFilter(result, minGram, maxGram, true); diff --git a/src/main/java/org/jabref/model/search/LatexToUnicodeFoldingFilter.java b/src/main/java/org/jabref/model/search/Analyzer/LatexToUnicodeFoldingFilter.java similarity index 98% rename from src/main/java/org/jabref/model/search/LatexToUnicodeFoldingFilter.java rename to src/main/java/org/jabref/model/search/Analyzer/LatexToUnicodeFoldingFilter.java index dcb4de0d3fe..786378ed344 100644 --- a/src/main/java/org/jabref/model/search/LatexToUnicodeFoldingFilter.java +++ b/src/main/java/org/jabref/model/search/Analyzer/LatexToUnicodeFoldingFilter.java @@ -1,4 +1,4 @@ -package org.jabref.model.search; +package org.jabref.model.search.Analyzer; import java.io.IOException; import java.util.Arrays; diff --git a/src/main/java/org/jabref/model/search/Analyzer/LinkedFilesAnalyzer.java b/src/main/java/org/jabref/model/search/Analyzer/LinkedFilesAnalyzer.java new file mode 100644 index 00000000000..d7c930adcd0 --- /dev/null +++ b/src/main/java/org/jabref/model/search/Analyzer/LinkedFilesAnalyzer.java @@ -0,0 +1,19 @@ +package org.jabref.model.search.Analyzer; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.LowerCaseFilter; +import org.apache.lucene.analysis.StopFilter; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.analysis.en.EnglishAnalyzer; +import org.apache.lucene.analysis.standard.StandardTokenizer; + +public class LinkedFilesAnalyzer extends Analyzer { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer source = new StandardTokenizer(); + TokenStream result = new StopFilter(source, EnglishAnalyzer.ENGLISH_STOP_WORDS_SET); + result = new LowerCaseFilter(result); + return new TokenStreamComponents(source, result); + } +} diff --git a/src/main/java/org/jabref/model/search/SearchFieldConstants.java b/src/main/java/org/jabref/model/search/SearchFieldConstants.java index f28adbf704c..c40935987ed 100644 --- a/src/main/java/org/jabref/model/search/SearchFieldConstants.java +++ b/src/main/java/org/jabref/model/search/SearchFieldConstants.java @@ -2,8 +2,11 @@ import java.util.List; +import org.jabref.model.search.Analyzer.LatexAwareAnalyzer; +import org.jabref.model.search.Analyzer.LatexAwareNGramAnalyzer; +import org.jabref.model.search.Analyzer.LinkedFilesAnalyzer; + import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.en.EnglishAnalyzer; public enum SearchFieldConstants { @@ -17,9 +20,10 @@ public enum SearchFieldConstants { PAGE_NUMBER("pageNumber"), MODIFIED("modified"); - public static final Analyzer LATEX_AWARE_ANALYZER = new LatexAwareAnalyzer(EnglishAnalyzer.ENGLISH_STOP_WORDS_SET); - public static final Analyzer LATEX_AWARE_NGRAM_ANALYZER = new LatexAwareNGramAnalyzer(1, Integer.MAX_VALUE, EnglishAnalyzer.ENGLISH_STOP_WORDS_SET); - public static final List PDF_FIELDS = List.of(PATH.toString(), CONTENT.toString(), ANNOTATIONS.toString()); + public static final Analyzer LINKED_FILES_ANALYZER = new LinkedFilesAnalyzer(); + public static final Analyzer LATEX_AWARE_ANALYZER = new LatexAwareAnalyzer(); + public static final Analyzer LATEX_AWARE_NGRAM_ANALYZER = new LatexAwareNGramAnalyzer(1, Integer.MAX_VALUE); + public static final List PDF_FIELDS = List.of(CONTENT.toString(), ANNOTATIONS.toString()); private final String field; SearchFieldConstants(String field) { diff --git a/src/main/java/org/jabref/model/search/SearchQuery.java b/src/main/java/org/jabref/model/search/SearchQuery.java index 7f566d786e2..6fd46706626 100644 --- a/src/main/java/org/jabref/model/search/SearchQuery.java +++ b/src/main/java/org/jabref/model/search/SearchQuery.java @@ -55,10 +55,12 @@ String format(String regex) { abstract String format(String regex); } - protected final String query; - protected Query parsedQuery; - protected String parseError; - protected EnumSet searchFlags; + private final String query; + private final EnumSet searchFlags; + + private Query parsedQuery; + private String parseError; + private SearchResults searchResults; public SearchQuery(String query, EnumSet searchFlags) { this.query = Objects.requireNonNull(query); @@ -90,6 +92,14 @@ public String getSearchExpression() { return query; } + public SearchResults getSearchResults() { + return searchResults; + } + + public void setSearchResults(SearchResults searchResults) { + this.searchResults = searchResults; + } + @Override public String toString() { return query; diff --git a/src/main/java/org/jabref/model/search/SearchResult.java b/src/main/java/org/jabref/model/search/SearchResult.java index 5789b48aead..8e5f91114d0 100644 --- a/src/main/java/org/jabref/model/search/SearchResult.java +++ b/src/main/java/org/jabref/model/search/SearchResult.java @@ -11,7 +11,7 @@ public final class SearchResult { - private final float luceneScore; + private final float searchScore; private final boolean hasFulltextResults; private final String path; private final String pageContent; @@ -21,14 +21,14 @@ public final class SearchResult { private List contentResultStringsHtml; private List annotationsResultStringsHtml; - private SearchResult(float luceneScore, + private SearchResult(float searchScore, boolean hasFulltextResults, String path, String pageContent, String annotation, int pageNumber, Highlighter highlighter) { - this.luceneScore = luceneScore; + this.searchScore = searchScore; this.hasFulltextResults = hasFulltextResults; this.path = path; this.pageContent = pageContent; @@ -37,12 +37,12 @@ private SearchResult(float luceneScore, this.highlighter = highlighter; } - public SearchResult(float luceneScore) { - this(luceneScore, false, "", "", "", -1, null); + public SearchResult(float searchScore) { + this(searchScore, false, "", "", "", -1, null); } - public SearchResult(float luceneScore, String path, String pageContent, String annotation, int pageNumber, Highlighter highlighter) { - this(luceneScore, true, path, pageContent, annotation, pageNumber, highlighter); + public SearchResult(float searchScore, String path, String pageContent, String annotation, int pageNumber, Highlighter highlighter) { + this(searchScore, true, path, pageContent, annotation, pageNumber, highlighter); } public List getContentResultStringsHtml() { @@ -59,8 +59,8 @@ public List getAnnotationsResultStringsHtml() { return annotationsResultStringsHtml; } - public float getLuceneScore() { - return luceneScore; + public float getSearchScore() { + return searchScore; } public boolean hasFulltextResults() { diff --git a/src/main/java/org/jabref/model/search/SearchResults.java b/src/main/java/org/jabref/model/search/SearchResults.java index cb0962c54b6..3c0acb85ed7 100644 --- a/src/main/java/org/jabref/model/search/SearchResults.java +++ b/src/main/java/org/jabref/model/search/SearchResults.java @@ -11,38 +11,43 @@ public class SearchResults { - private final Map> searchResults = new HashMap<>(); + private final Map> searchResults = new HashMap<>(); public void mergeSearchResults(SearchResults additionalResults) { this.searchResults.putAll(additionalResults.searchResults); } - public void addSearchResult(BibEntry entry, SearchResult result) { - searchResults.computeIfAbsent(entry, k -> new ArrayList<>()).add(result); + public void addSearchResult(String entryId, SearchResult result) { + searchResults.computeIfAbsent(entryId, k -> new ArrayList<>()).add(result); } - public void addSearchResult(Collection entries, SearchResult result) { + public void addSearchResult(Collection entries, SearchResult result) { entries.forEach(entry -> addSearchResult(entry, result)); } public float getSearchScoreForEntry(BibEntry entry) { - return searchResults.containsKey(entry) ? - searchResults.get(entry).stream() - .map(SearchResult::getLuceneScore) - .reduce(0f, Float::sum) : 0f; + if (searchResults.containsKey(entry.getId())) { + return searchResults.get(entry.getId()) + .stream() + .map(SearchResult::getSearchScore) + .reduce(0f, Float::max); + } + return 0f; } public boolean hasFulltextResults(BibEntry entry) { - if (searchResults.containsKey(entry)) { - return searchResults.get(entry).stream().anyMatch(SearchResult::hasFulltextResults); + if (searchResults.containsKey(entry.getId())) { + return searchResults.get(entry.getId()) + .stream() + .anyMatch(SearchResult::hasFulltextResults); } return false; } public Map> getFileSearchResultsForEntry(BibEntry entry) { Map> results = new HashMap<>(); - if (searchResults.containsKey(entry)) { - for (SearchResult result : searchResults.get(entry)) { + if (searchResults.containsKey(entry.getId())) { + for (SearchResult result : searchResults.get(entry.getId())) { if (result.hasFulltextResults()) { results.computeIfAbsent(result.getPath(), k -> new ArrayList<>()).add(result); } @@ -51,11 +56,7 @@ public Map> getFileSearchResultsForEntry(BibEntry ent return results; } - public Set getMatchedEntries() { + public Set getMatchedEntries() { return searchResults.keySet(); } - - public int getNumberOfResults() { - return searchResults.size(); - } } diff --git a/src/test/java/org/jabref/model/groups/SearchGroupTest.java b/src/test/java/org/jabref/model/groups/SearchGroupTest.java index f79e3ffaab3..a23fb440d35 100644 --- a/src/test/java/org/jabref/model/groups/SearchGroupTest.java +++ b/src/test/java/org/jabref/model/groups/SearchGroupTest.java @@ -6,10 +6,8 @@ import javafx.beans.property.BooleanProperty; -import org.jabref.architecture.AllowedToUseLogic; import org.jabref.gui.util.CurrentThreadTaskExecutor; import org.jabref.gui.util.TaskExecutor; -import org.jabref.logic.search.LuceneManager; import org.jabref.model.database.BibDatabaseContext; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; @@ -27,7 +25,6 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -@AllowedToUseLogic("because it needs access to lucene manager") class SearchGroupTest { private static final TaskExecutor TASK_EXECUTOR = new CurrentThreadTaskExecutor(); private static final FilePreferences FILE_PREFERENCES = mock(FilePreferences.class); @@ -73,9 +70,7 @@ void testSearchGroup(String searchTerm, List entries, boolean expected BibDatabaseContext databaseContext = new BibDatabaseContext(); databaseContext.getDatabase().insertEntries(entries); - LuceneManager luceneManager = new LuceneManager(databaseContext, TASK_EXECUTOR, FILE_PREFERENCES); - - SearchGroup group = new SearchGroup("TestGroup", GroupHierarchyType.INDEPENDENT, searchTerm, EnumSet.noneOf(SearchFlags.class), luceneManager); + SearchGroup group = new SearchGroup("TestGroup", GroupHierarchyType.INDEPENDENT, searchTerm, EnumSet.noneOf(SearchFlags.class)); assertEquals(expectedResult, group.containsAll(entries)); } } From 84a275492d31a008980b3f95bd617e3d3b0c6803 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Mon, 2 Sep 2024 13:50:07 +0300 Subject: [PATCH 238/256] Use non-static preferences variables --- .../org/jabref/logic/search/DatabaseSearcherTest.java | 8 ++++---- .../search/DatabaseSearcherWithBibFilesTest.java | 7 ++++--- .../logic/search/indexing/LinkedFilesIndexerTest.java | 10 +++++----- .../java/org/jabref/model/groups/SearchGroupTest.java | 11 ++++------- 4 files changed, 17 insertions(+), 19 deletions(-) diff --git a/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java b/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java index 8a81846dec1..919b002c54a 100644 --- a/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java +++ b/src/test/java/org/jabref/logic/search/DatabaseSearcherTest.java @@ -28,13 +28,13 @@ public class DatabaseSearcherTest { private static final TaskExecutor TASK_EXECUTOR = new CurrentThreadTaskExecutor(); - private static final FilePreferences FILE_PREFERENCES = mock(FilePreferences.class); private BibDatabaseContext databaseContext; + private final FilePreferences filePreferences = mock(FilePreferences.class); @BeforeEach void setUp() { - when(FILE_PREFERENCES.shouldFulltextIndexLinkedFiles()).thenReturn(false); - when(FILE_PREFERENCES.fulltextIndexLinkedFilesProperty()).thenReturn(mock(BooleanProperty.class)); + when(filePreferences.shouldFulltextIndexLinkedFiles()).thenReturn(false); + when(filePreferences.fulltextIndexLinkedFilesProperty()).thenReturn(mock(BooleanProperty.class)); databaseContext = new BibDatabaseContext(); } @@ -44,7 +44,7 @@ void testDatabaseSearcher(List expectedMatches, SearchQuery query, Lis for (BibEntry entry : entries) { databaseContext.getDatabase().insertEntry(entry); } - List matches = new DatabaseSearcher(query, databaseContext, TASK_EXECUTOR, FILE_PREFERENCES).getMatches(); + List matches = new DatabaseSearcher(query, databaseContext, TASK_EXECUTOR, filePreferences).getMatches(); assertEquals(expectedMatches, matches); } diff --git a/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java b/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java index b81374d3720..d928d60cb94 100644 --- a/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java +++ b/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java @@ -66,7 +66,8 @@ class DatabaseSearcherWithBibFilesTest { .withCitationKey("minimal-note-mixed-case") .withFiles(List.of(new LinkedFile("", "minimal-note-mixed-case.pdf", StandardFileType.PDF.getName()))); - private static final FilePreferences FILE_PREFERENCES = mock(FilePreferences.class); + private final FilePreferences filePreferences = mock(FilePreferences.class); + @TempDir private Path indexDir; @@ -78,7 +79,7 @@ private BibDatabaseContext initializeDatabaseFromPath(Path testFile) throws Exce ParserResult result = new BibtexImporter(mock(ImportFormatPreferences.class, Answers.RETURNS_DEEP_STUBS), new DummyFileUpdateMonitor()).importDatabase(testFile); BibDatabaseContext databaseContext = result.getDatabaseContext(); - when(FILE_PREFERENCES.fulltextIndexLinkedFilesProperty()).thenReturn(new SimpleBooleanProperty(true)); + when(filePreferences.fulltextIndexLinkedFilesProperty()).thenReturn(new SimpleBooleanProperty(true)); return databaseContext; } @@ -130,7 +131,7 @@ private static Stream searchLibrary() { @MethodSource void searchLibrary(List expected, String testFile, String query, EnumSet searchFlags) throws Exception { BibDatabaseContext databaseContext = initializeDatabaseFromPath(testFile); - List matches = new DatabaseSearcher(new SearchQuery(query, searchFlags), databaseContext, TASK_EXECUTOR, FILE_PREFERENCES).getMatches(); + List matches = new DatabaseSearcher(new SearchQuery(query, searchFlags), databaseContext, TASK_EXECUTOR, filePreferences).getMatches(); assertThat(expected, Matchers.containsInAnyOrder(matches.toArray())); } } diff --git a/src/test/java/org/jabref/logic/search/indexing/LinkedFilesIndexerTest.java b/src/test/java/org/jabref/logic/search/indexing/LinkedFilesIndexerTest.java index 00438627649..b9dbac5548b 100644 --- a/src/test/java/org/jabref/logic/search/indexing/LinkedFilesIndexerTest.java +++ b/src/test/java/org/jabref/logic/search/indexing/LinkedFilesIndexerTest.java @@ -27,22 +27,22 @@ import static org.mockito.Mockito.when; public class LinkedFilesIndexerTest { - private static final PreferencesService PREFERENCES_SERVICE = mock(PreferencesService.class); - private static final FilePreferences FILE_PREFERENCES = mock(FilePreferences.class); + private final PreferencesService preferencesService = mock(PreferencesService.class); + private final FilePreferences filePreferences = mock(FilePreferences.class); private LuceneIndexer indexer; @BeforeEach void setUp(@TempDir Path indexDir) throws IOException { - when(FILE_PREFERENCES.shouldFulltextIndexLinkedFiles()).thenReturn(true); - when(PREFERENCES_SERVICE.getFilePreferences()).thenReturn(FILE_PREFERENCES); + when(filePreferences.shouldFulltextIndexLinkedFiles()).thenReturn(true); + when(preferencesService.getFilePreferences()).thenReturn(filePreferences); BibDatabaseContext context = mock(BibDatabaseContext.class); when(context.getDatabasePath()).thenReturn(Optional.of(Path.of("src/test/resources/pdfs/"))); when(context.getFileDirectories(Mockito.any())).thenReturn(Collections.singletonList(Path.of("src/test/resources/pdfs"))); when(context.getFulltextIndexPath()).thenReturn(indexDir); - this.indexer = new DefaultLinkedFilesIndexer(context, FILE_PREFERENCES); + this.indexer = new DefaultLinkedFilesIndexer(context, filePreferences); } @Test diff --git a/src/test/java/org/jabref/model/groups/SearchGroupTest.java b/src/test/java/org/jabref/model/groups/SearchGroupTest.java index a23fb440d35..a2264921cd9 100644 --- a/src/test/java/org/jabref/model/groups/SearchGroupTest.java +++ b/src/test/java/org/jabref/model/groups/SearchGroupTest.java @@ -18,8 +18,6 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.mock; @@ -27,8 +25,6 @@ class SearchGroupTest { private static final TaskExecutor TASK_EXECUTOR = new CurrentThreadTaskExecutor(); - private static final FilePreferences FILE_PREFERENCES = mock(FilePreferences.class); - private static final BibEntry ENTRY_1 = new BibEntry(StandardEntryType.Misc) .withCitationKey("entry1") .withField(StandardField.AUTHOR, "Test") @@ -40,7 +36,8 @@ class SearchGroupTest { .withField(StandardField.AUTHOR, "TEST") .withField(StandardField.TITLE, "CASE") .withField(StandardField.GROUPS, "A"); - private static final Logger log = LoggerFactory.getLogger(SearchGroupTest.class); + + private final FilePreferences filePreferences = mock(FilePreferences.class); private static Stream testSearchGroup() { return Stream.of( @@ -64,8 +61,8 @@ private static Stream testSearchGroup() { @ParameterizedTest @MethodSource void testSearchGroup(String searchTerm, List entries, boolean expectedResult) { - when(FILE_PREFERENCES.fulltextIndexLinkedFilesProperty()).thenReturn(mock(BooleanProperty.class)); - when(FILE_PREFERENCES.shouldFulltextIndexLinkedFiles()).thenReturn(false); + when(filePreferences.fulltextIndexLinkedFilesProperty()).thenReturn(mock(BooleanProperty.class)); + when(filePreferences.shouldFulltextIndexLinkedFiles()).thenReturn(false); BibDatabaseContext databaseContext = new BibDatabaseContext(); databaseContext.getDatabase().insertEntries(entries); From a8a845f7dff53b2f0954f2fbd5145209646a1300 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Mon, 2 Sep 2024 14:35:00 +0300 Subject: [PATCH 239/256] Update CHANGELOG.md --- CHANGELOG.md | 7 ++++--- .../java/org/jabref/gui/groups/GroupNodeViewModel.java | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b80005b94d..830dac0cb3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv ### Added -- We added probable search hits instead of exact matches. Sorting by hit score can be done by the new score table column. [#11326](https://github.com/JabRef/jabref/pull/11326) +- We added probable search hits instead of exact matches. Sorting by hit score can be done by the new score table column. [#11542](https://github.com/JabRef/jabref/pull/11542) - We added support finding LaTeX-encoded special characters based on plain Unicode and vice versa. [#11542](https://github.com/JabRef/jabref/pull/11542) - When a search hits a file, the file icon of that entry is changed accordingly. [#11542](https://github.com/JabRef/jabref/pull/11542) - We added an AI-based chat for entries with linked PDF files. [#11430](https://github.com/JabRef/jabref/pull/11430) @@ -62,9 +62,10 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv ### Removed -- We removed support for case-insentive and exact search. [#11542](https://github.com/JabRef/jabref/pull/11542) +- We removed support for case-sensitive and exact search. [#11542](https://github.com/JabRef/jabref/pull/11542) +- We removed the description of search strings. [#11542](https://github.com/JabRef/jabref/pull/11542) - We removed support for importing using the SilverPlatterImporter (`Record INSPEC`). [#11576](https://github.com/JabRef/jabref/pull/11576) -- We removed the description of search strings. [#11565](https://github.com/JabRef/jabref/pull/11565) + diff --git a/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java b/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java index 75a8ed5417b..2d71b54d257 100644 --- a/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java +++ b/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java @@ -560,9 +560,9 @@ public void listen(IndexAddedOrUpdatedEvent event) { boolean matched = luceneManager.isMatched(entry, searchGroup.getQuery()); searchGroup.updateMatches(entry, matched); if (matched) { - matchedEntries.add(entry.getId()); + matchedEntries.add(entry.getId()); } else { - matchedEntries.remove(entry.getId()); + matchedEntries.remove(entry.getId()); } } }).executeWith(taskExecutor); From d6fbf5505a522a64946e2992679560dc9dff1130 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Mon, 2 Sep 2024 15:28:15 +0300 Subject: [PATCH 240/256] Delete SearchGroupTest.java --- .../jabref/model/groups/SearchGroupTest.java | 73 ------------------- 1 file changed, 73 deletions(-) delete mode 100644 src/test/java/org/jabref/model/groups/SearchGroupTest.java diff --git a/src/test/java/org/jabref/model/groups/SearchGroupTest.java b/src/test/java/org/jabref/model/groups/SearchGroupTest.java deleted file mode 100644 index a2264921cd9..00000000000 --- a/src/test/java/org/jabref/model/groups/SearchGroupTest.java +++ /dev/null @@ -1,73 +0,0 @@ -package org.jabref.model.groups; - -import java.util.EnumSet; -import java.util.List; -import java.util.stream.Stream; - -import javafx.beans.property.BooleanProperty; - -import org.jabref.gui.util.CurrentThreadTaskExecutor; -import org.jabref.gui.util.TaskExecutor; -import org.jabref.model.database.BibDatabaseContext; -import org.jabref.model.entry.BibEntry; -import org.jabref.model.entry.field.StandardField; -import org.jabref.model.entry.types.StandardEntryType; -import org.jabref.model.search.SearchFlags; -import org.jabref.preferences.FilePreferences; - -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -class SearchGroupTest { - private static final TaskExecutor TASK_EXECUTOR = new CurrentThreadTaskExecutor(); - private static final BibEntry ENTRY_1 = new BibEntry(StandardEntryType.Misc) - .withCitationKey("entry1") - .withField(StandardField.AUTHOR, "Test") - .withField(StandardField.TITLE, "Case") - .withField(StandardField.GROUPS, "A"); - - private static final BibEntry ENTRY_2 = new BibEntry(StandardEntryType.Misc) - .withCitationKey("entry2") - .withField(StandardField.AUTHOR, "TEST") - .withField(StandardField.TITLE, "CASE") - .withField(StandardField.GROUPS, "A"); - - private final FilePreferences filePreferences = mock(FilePreferences.class); - - private static Stream testSearchGroup() { - return Stream.of( - // containsFindsWords - Arguments.of("Test", List.of(ENTRY_1, ENTRY_2), true), - // containsDoesNotFindWords - Arguments.of("Unknown", List.of(ENTRY_1, ENTRY_2), false), - // containsFindsWordWithRegularExpression - Arguments.of("any:/rev*/", List.of(new BibEntry().withField(StandardField.KEYWORDS, "review")), true), - // containsDoesNotFindsWordWithInvalidRegularExpression - Arguments.of("any:/[rev/", List.of(new BibEntry().withField(StandardField.KEYWORDS, "review")), false), - // notQueryWorksWithLeftPartOfQuery - Arguments.of("(any:* AND -groups:alpha) AND (any:* AND -groups:beta)", List.of(new BibEntry().withField(StandardField.GROUPS, "alpha")), false), - // notQueryWorksWithLRightPartOfQuery - Arguments.of("(any:* AND -groups:alpha) AND (any:* AND -groups:beta)", List.of(new BibEntry().withField(StandardField.GROUPS, "beta")), false), - // notQueryWorksWithBothPartsOfQuery - Arguments.of("(any:* AND -groups:alpha) AND (any:* AND -groups:beta)", List.of(new BibEntry().withField(StandardField.GROUPS, "gamma")), true) - ); - } - - @ParameterizedTest - @MethodSource - void testSearchGroup(String searchTerm, List entries, boolean expectedResult) { - when(filePreferences.fulltextIndexLinkedFilesProperty()).thenReturn(mock(BooleanProperty.class)); - when(filePreferences.shouldFulltextIndexLinkedFiles()).thenReturn(false); - - BibDatabaseContext databaseContext = new BibDatabaseContext(); - databaseContext.getDatabase().insertEntries(entries); - - SearchGroup group = new SearchGroup("TestGroup", GroupHierarchyType.INDEPENDENT, searchTerm, EnumSet.noneOf(SearchFlags.class)); - assertEquals(expectedResult, group.containsAll(entries)); - } -} From eeadaee893045166f05e68afdd3be906cea14bf7 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Mon, 2 Sep 2024 15:32:40 +0300 Subject: [PATCH 241/256] fix typo --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 830dac0cb3d..d67759c5ff3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,8 +32,8 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv ### Changed - The search syntax is changed to [Apache Lucene syntax](https://docs.jabref.org/collect/import-using-online-bibliographic-database#search-syntax) (also to be similar to the online search syntax). [#11542](https://github.com/JabRef/jabref/pull/11542/) -- When searching using a regular expession, one needs to enclose the search string in `/`. [#11542](https://github.com/JabRef/jabref/pull/11542/) -- A search in "all" fields ignores the [groups](https://docs.jabref.org/finding-sorting-and-cleaning-entries/groups). [#7996](https://github.com/JabRef/jabref/issues/7996) +- When searching using a regular expression, one needs to enclose the search string in `/`. [#11542](https://github.com/JabRef/jabref/pull/11542/) +- A search in "any" fields ignores the [groups](https://docs.jabref.org/finding-sorting-and-cleaning-entries/groups). [#7996](https://github.com/JabRef/jabref/issues/7996) - When a communication error with an [online service](https://docs.jabref.org/collect/import-using-online-bibliographic-database) occurs, JabRef displays the HTTP error. [#11223](https://github.com/JabRef/jabref/issues/11223) - The Pubmed/Medline Plain importer now imports the PMID field as well [#11488](https://github.com/JabRef/jabref/issues/11488) - The 'Check for updates' menu bar button is now always enabled. [#11485](https://github.com/JabRef/jabref/pull/11485) From 0959592b86472ead70d95f4bdecf28d4e40c4f8b Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Mon, 2 Sep 2024 15:41:54 +0300 Subject: [PATCH 242/256] fix indentation --- .../java/org/jabref/gui/entryeditor/UserDefinedFieldsTab.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/jabref/gui/entryeditor/UserDefinedFieldsTab.java b/src/main/java/org/jabref/gui/entryeditor/UserDefinedFieldsTab.java index 0ce88f13e9f..26ae2b54ba5 100644 --- a/src/main/java/org/jabref/gui/entryeditor/UserDefinedFieldsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/UserDefinedFieldsTab.java @@ -38,7 +38,7 @@ public UserDefinedFieldsTab(String name, TaskExecutor taskExecutor, JournalAbbreviationRepository journalAbbreviationRepository, LuceneManager luceneManager, - OptionalObjectProperty searchQueryProperty) { + OptionalObjectProperty searchQueryProperty) { super(false, databaseContext, suggestionProviders, undoManager, undoAction, redoAction, dialogService, preferences, themeManager, taskExecutor, journalAbbreviationRepository, luceneManager, searchQueryProperty); this.fields = new LinkedHashSet<>(fields); From 29d2ad33016a3bc9935311907fbcfe9a7b750bfa Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Mon, 2 Sep 2024 16:42:52 +0300 Subject: [PATCH 243/256] Update matchedEntries on the UI thread matchedEntries should be updated on the UI thread because the size binding of matchedEntries will be reflected in the UI. --- .../java/org/jabref/gui/groups/GroupNodeViewModel.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java b/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java index 2d71b54d257..a9847cd7318 100644 --- a/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java +++ b/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java @@ -557,9 +557,11 @@ public void listen(IndexAddedOrUpdatedEvent event) { stateManager.getLuceneManager(databaseContext).ifPresent(luceneManager -> { BackgroundTask.wrap(() -> { for (BibEntry entry : event.entries()) { - boolean matched = luceneManager.isMatched(entry, searchGroup.getQuery()); - searchGroup.updateMatches(entry, matched); - if (matched) { + searchGroup.updateMatches(entry, luceneManager.isMatched(entry, searchGroup.getQuery())); + } + }).onFinished(() -> { + for (BibEntry entry : event.entries()) { + if (groupNode.matches(entry)) { matchedEntries.add(entry.getId()); } else { matchedEntries.remove(entry.getId()); From 0a7b92fb4ea8b19f4ddd449df58a21dad23f7753 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb <52158423+LoayGhreeb@users.noreply.github.com> Date: Mon, 2 Sep 2024 18:03:52 +0300 Subject: [PATCH 244/256] Discard changes to src/main/java/org/jabref/gui/importer/actions/GUIPostOpenAction.java --- .../java/org/jabref/gui/importer/actions/GUIPostOpenAction.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/jabref/gui/importer/actions/GUIPostOpenAction.java b/src/main/java/org/jabref/gui/importer/actions/GUIPostOpenAction.java index 0e8fa96fd3b..3b392a1bc5a 100644 --- a/src/main/java/org/jabref/gui/importer/actions/GUIPostOpenAction.java +++ b/src/main/java/org/jabref/gui/importer/actions/GUIPostOpenAction.java @@ -31,6 +31,8 @@ public interface GUIPostOpenAction { * Note: if several such methods need to be called sequentially, it is * important that all implementations of this method do not return * until the operation is finished. + * + * @param pr The result of the BIB parse operation. */ void performAction(ParserResult pr, DialogService dialogService, PreferencesService preferencesService); } From 5bcb82d5f0fbff403bf253fad0527b508bd743f5 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Mon, 2 Sep 2024 18:28:15 +0300 Subject: [PATCH 245/256] Fix LoayGhreeb#12 --- .../gui/maintable/MainTableDataModel.java | 31 ++++++++++++------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java index af46c4f6d48..48da98309e4 100644 --- a/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java +++ b/src/main/java/org/jabref/gui/maintable/MainTableDataModel.java @@ -101,24 +101,33 @@ public MainTableDataModel(BibDatabaseContext context, private void updateSearchMatches(Optional query) { BackgroundTask.wrap(() -> { - boolean isFloatingMode = searchPreferences.getSearchDisplayMode() == SearchDisplayMode.FLOAT; if (query.isPresent()) { SearchResults results = luceneManager.search(query.get()); - entriesViewModel.forEach(entry -> { - entry.searchScoreProperty().set(results.getSearchScoreForEntry(entry.getEntry())); - entry.hasFullTextResultsProperty().set(results.hasFulltextResults(entry.getEntry())); - updateEntrySearchMatch(entry, entry.searchScoreProperty().get() > 0, isFloatingMode); - }); + setSearchMatches(results); } else { - entriesViewModel.forEach(entry -> { - entry.searchScoreProperty().set(0); - entry.hasFullTextResultsProperty().set(false); - updateEntrySearchMatch(entry, true, isFloatingMode); - }); + clearSearchMatches(); } }).onSuccess(result -> FilteredListProxy.refilterListReflection(entriesFiltered)).executeWith(taskExecutor); } + private void setSearchMatches(SearchResults results) { + boolean isFloatingMode = searchPreferences.getSearchDisplayMode() == SearchDisplayMode.FLOAT; + entriesViewModel.forEach(entry -> { + entry.searchScoreProperty().set(results.getSearchScoreForEntry(entry.getEntry())); + entry.hasFullTextResultsProperty().set(results.hasFulltextResults(entry.getEntry())); + updateEntrySearchMatch(entry, entry.searchScoreProperty().get() > 0, isFloatingMode); + }); + } + + private void clearSearchMatches() { + boolean isFloatingMode = searchPreferences.getSearchDisplayMode() == SearchDisplayMode.FLOAT; + entriesViewModel.forEach(entry -> { + entry.searchScoreProperty().set(0); + entry.hasFullTextResultsProperty().set(false); + updateEntrySearchMatch(entry, true, isFloatingMode); + }); + } + private static void updateEntrySearchMatch(BibEntryTableViewModel entry, boolean isMatched, boolean isFloatingMode) { entry.isMatchedBySearch().set(isMatched); entry.updateMatchCategory(); From 09d51482bd3db874a6f9b49dfddf7a42ac84f235 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Mon, 2 Sep 2024 19:27:22 +0300 Subject: [PATCH 246/256] Sync search flags between search bar and global search bar --- .../java/org/jabref/gui/search/GlobalSearchBar.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java index e54bd28b22a..4a021b75dc6 100644 --- a/src/main/java/org/jabref/gui/search/GlobalSearchBar.java +++ b/src/main/java/org/jabref/gui/search/GlobalSearchBar.java @@ -16,6 +16,7 @@ import javafx.beans.property.BooleanProperty; import javafx.beans.property.SimpleBooleanProperty; import javafx.collections.ListChangeListener; +import javafx.collections.SetChangeListener; import javafx.css.PseudoClass; import javafx.event.Event; import javafx.geometry.Insets; @@ -251,6 +252,10 @@ private void initSearchModifierButtons() { openGlobalSearchButton.setTooltip(new Tooltip(Localization.lang("Search across libraries in a new window"))); initSearchModifierButton(openGlobalSearchButton); openGlobalSearchButton.setOnAction(evt -> openGlobalSearchDialog()); + + searchPreferences.getObservableSearchFlags().addListener((SetChangeListener.Change change) -> { + fulltextButton.setSelected(searchPreferences.isFulltext()); + }); } public void openGlobalSearchDialog() { @@ -296,7 +301,6 @@ public void updateSearchQuery() { // An empty search field should cause the search to be cleared. if (searchField.getText().isEmpty()) { - setSearchFieldHintTooltip(); stateManager.activeSearchQuery(searchType).set(Optional.empty()); illegalSearch.set(false); return; @@ -349,7 +353,7 @@ private AutoCompletePopup getPopup(AutoCompletionBinding autoCompletio } } - private void setSearchFieldHintTooltip() { + public void updateHintVisibility() { if (preferencesService.getWorkspacePreferences().shouldShowAdvancedHints()) { String genericDescription = Localization.lang("Hint:\n\nTo search all fields for Smith, enter:\nsmith\n\nTo search the field author for Smith and the field title for electrical, enter:\nauthor:Smith AND title:electrical"); List genericDescriptionTexts = TooltipTextUtil.createTextsFromHtml(genericDescription); @@ -360,10 +364,6 @@ private void setSearchFieldHintTooltip() { } } - public void updateHintVisibility() { - setSearchFieldHintTooltip(); - } - public void setSearchTerm(SearchQuery searchQuery) { if (searchQuery.toString().equals(searchField.getText())) { return; From a9a8a7c8fa13aaee0a027f3808fc9338398224d3 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Mon, 2 Sep 2024 20:36:46 +0300 Subject: [PATCH 247/256] Move VERSION_6_0_ALPHA const to SearchGroupsMigrationAction --- .../gui/importer/actions/SearchGroupsMigrationAction.java | 4 +++- src/main/java/org/jabref/logic/util/Version.java | 2 +- src/main/java/org/jabref/model/groups/SearchGroup.java | 5 ----- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java b/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java index ed720a8f41b..b8d32f1529a 100644 --- a/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java +++ b/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java @@ -6,6 +6,7 @@ import org.jabref.gui.DialogService; import org.jabref.logic.importer.ParserResult; import org.jabref.logic.l10n.Localization; +import org.jabref.logic.util.Version; import org.jabref.migrations.SearchToLuceneMigration; import org.jabref.model.groups.GroupTreeNode; import org.jabref.model.groups.SearchGroup; @@ -17,6 +18,7 @@ * If not we ask the user whether to migrate. */ public class SearchGroupsMigrationAction implements GUIPostOpenAction { + public static final Version VERSION_6_0_ALPHA = Version.parse("6.0-alpha"); @Override public boolean isActionNecessary(ParserResult parserResult, PreferencesService preferencesService) { @@ -50,7 +52,7 @@ public void performAction(ParserResult parserResult, DialogService dialogService } parserResult.getMetaData().getGroups().ifPresent(this::migrateGroups); - parserResult.getMetaData().setGroupSearchSyntaxVersion(SearchGroup.VERSION_6_0_ALPHA); + parserResult.getMetaData().setGroupSearchSyntaxVersion(VERSION_6_0_ALPHA); parserResult.setChangedOnMigration(true); } diff --git a/src/main/java/org/jabref/logic/util/Version.java b/src/main/java/org/jabref/logic/util/Version.java index 15f360a1ac5..ef0e73953cd 100644 --- a/src/main/java/org/jabref/logic/util/Version.java +++ b/src/main/java/org/jabref/logic/util/Version.java @@ -58,7 +58,7 @@ private static Logger getLogger() { * @return the parsed version or {@link Version#UNKNOWN_VERSION} if an error occurred */ public static Version parse(String version) { - if ((version == null) || "".equals(version) || version.equals(BuildInfo.UNKNOWN_VERSION) + if ((version == null) || version.isEmpty() || version.equals(BuildInfo.UNKNOWN_VERSION) || "${version}".equals(version)) { return UNKNOWN_VERSION; } diff --git a/src/main/java/org/jabref/model/groups/SearchGroup.java b/src/main/java/org/jabref/model/groups/SearchGroup.java index fe7e8132357..9823ae5a7e8 100644 --- a/src/main/java/org/jabref/model/groups/SearchGroup.java +++ b/src/main/java/org/jabref/model/groups/SearchGroup.java @@ -6,7 +6,6 @@ import java.util.Objects; import java.util.Set; -import org.jabref.logic.util.Version; import org.jabref.model.entry.BibEntry; import org.jabref.model.search.SearchFlags; import org.jabref.model.search.SearchQuery; @@ -20,10 +19,6 @@ * multiple fields. */ public class SearchGroup extends AbstractGroup { - - // We cannot have this constant in Version java because of recursion errors - // Thus, we keep it here, because it is (currently) used only in the context of groups. - public static final Version VERSION_6_0_ALPHA = Version.parse("6.0-alpha"); private static final Logger LOGGER = LoggerFactory.getLogger(SearchGroup.class); @ADR(38) From 146e3f707df045e4dd24c2db4db4ca789dad8381 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Tue, 3 Sep 2024 01:02:03 +0300 Subject: [PATCH 248/256] Refactor LuceneSearcher --- .../jabref/gui/groups/GroupNodeViewModel.java | 2 +- .../jabref/logic/search/LuceneManager.java | 6 +- .../search/retrieval/LuceneSearcher.java | 155 ++++++++++++------ 3 files changed, 105 insertions(+), 58 deletions(-) diff --git a/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java b/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java index a9847cd7318..50d7cc83e0b 100644 --- a/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java +++ b/src/main/java/org/jabref/gui/groups/GroupNodeViewModel.java @@ -557,7 +557,7 @@ public void listen(IndexAddedOrUpdatedEvent event) { stateManager.getLuceneManager(databaseContext).ifPresent(luceneManager -> { BackgroundTask.wrap(() -> { for (BibEntry entry : event.entries()) { - searchGroup.updateMatches(entry, luceneManager.isMatched(entry, searchGroup.getQuery())); + searchGroup.updateMatches(entry, luceneManager.isEntryMatched(entry, searchGroup.getQuery())); } }).onFinished(() -> { for (BibEntry entry : event.entries()) { diff --git a/src/main/java/org/jabref/logic/search/LuceneManager.java b/src/main/java/org/jabref/logic/search/LuceneManager.java index 086130ee7e0..eb0b3ca4dc3 100644 --- a/src/main/java/org/jabref/logic/search/LuceneManager.java +++ b/src/main/java/org/jabref/logic/search/LuceneManager.java @@ -57,7 +57,7 @@ public LuceneManager(BibDatabaseContext databaseContext, TaskExecutor executor, } linkedFilesIndexer = indexer; - this.luceneSearcher = new LuceneSearcher(databaseContext, bibFieldsIndexer, linkedFilesIndexer); + this.luceneSearcher = new LuceneSearcher(databaseContext, bibFieldsIndexer, linkedFilesIndexer, preferences); updateOnStart(); } @@ -224,7 +224,7 @@ public SearchResults search(SearchQuery query) { return query.getSearchResults(); } - public boolean isMatched(BibEntry entry, SearchQuery query) { - return luceneSearcher.isMatched(entry, query); + public boolean isEntryMatched(BibEntry entry, SearchQuery query) { + return luceneSearcher.isEntryMatched(entry, query); } } diff --git a/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java b/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java index a293a7f5cea..c0f91e7852b 100644 --- a/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java +++ b/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java @@ -17,6 +17,7 @@ import org.jabref.model.search.SearchQuery; import org.jabref.model.search.SearchResult; import org.jabref.model.search.SearchResults; +import org.jabref.preferences.FilePreferences; import org.apache.lucene.document.Document; import org.apache.lucene.index.MultiReader; @@ -39,105 +40,151 @@ public final class LuceneSearcher { private static final Logger LOGGER = LoggerFactory.getLogger(LuceneSearcher.class); + private final FilePreferences filePreferences; private final BibDatabaseContext databaseContext; private final SearcherManager bibFieldsSearcherManager; private final SearcherManager linkedFilesSearcherManager; - public LuceneSearcher(BibDatabaseContext databaseContext, LuceneIndexer bibFieldsIndexer, LuceneIndexer linkedFilesIndexer) { + public LuceneSearcher(BibDatabaseContext databaseContext, LuceneIndexer bibFieldsIndexer, LuceneIndexer linkedFilesIndexer, FilePreferences filePreferences) { this.bibFieldsSearcherManager = bibFieldsIndexer.getSearcherManager(); this.linkedFilesSearcherManager = linkedFilesIndexer.getSearcherManager(); this.databaseContext = databaseContext; + this.filePreferences = filePreferences; } - public boolean isMatched(BibEntry entry, SearchQuery searchQuery) { - BooleanQuery booleanQuery = createBooleanQuery(entry, searchQuery); + public boolean isEntryMatched(BibEntry entry, SearchQuery searchQuery) { + BooleanQuery booleanQuery = buildBooleanQueryForEntry(entry, searchQuery); SearchResults results = search(booleanQuery, searchQuery.getSearchFlags()); return results.getSearchScoreForEntry(entry) > 0; } - private BooleanQuery createBooleanQuery(BibEntry entry, SearchQuery searchQuery) { - Query query = searchQuery.getParsedQuery(); - TermQuery termQuery = new TermQuery(new Term(SearchFieldConstants.ENTRY_ID.toString(), entry.getId())); + private BooleanQuery buildBooleanQueryForEntry(BibEntry entry, SearchQuery searchQuery) { + Query parsedQuery = searchQuery.getParsedQuery(); + TermQuery entryIdQuery = new TermQuery(new Term(SearchFieldConstants.ENTRY_ID.toString(), entry.getId())); return new BooleanQuery.Builder() - .add(query, BooleanClause.Occur.MUST) - .add(termQuery, BooleanClause.Occur.MUST) + .add(parsedQuery, BooleanClause.Occur.MUST) + .add(entryIdQuery, BooleanClause.Occur.MUST) .build(); } public SearchResults search(Query searchQuery, EnumSet searchFlags) { - LOGGER.debug("Searching for entries matching query: {}", searchQuery); + LOGGER.debug("Executing search with query: {}", searchQuery); try { - if (searchFlags.contains(SearchFlags.FULLTEXT)) { - IndexSearcher bibFieldsIndexSearcher = getIndexedSearcher(bibFieldsSearcherManager); - IndexSearcher linkedFilesIndexSearcher = getIndexedSearcher(linkedFilesSearcherManager); - MultiReader multiReader = new MultiReader(bibFieldsIndexSearcher.getIndexReader(), linkedFilesIndexSearcher.getIndexReader()); - IndexSearcher indexSearcher = new IndexSearcher(multiReader); - - SearchResults searchResults = search(indexSearcher, searchQuery); - releaseIndexSearcher(bibFieldsSearcherManager, bibFieldsIndexSearcher); - releaseIndexSearcher(linkedFilesSearcherManager, linkedFilesIndexSearcher); - return searchResults; - } else { - IndexSearcher indexSearcher = getIndexedSearcher(bibFieldsSearcherManager); - SearchResults searchResults = search(indexSearcher, searchQuery); - releaseIndexSearcher(bibFieldsSearcherManager, indexSearcher); - return searchResults; - } + boolean shouldSearchInLinkedFiles = searchFlags.contains(SearchFlags.FULLTEXT) && filePreferences.shouldFulltextIndexLinkedFiles(); + return performSearch(searchQuery, shouldSearchInLinkedFiles); } catch (IOException | IndexSearcher.TooManyClauses e) { - LOGGER.error("Error while searching", e); + LOGGER.error("Error during search execution", e); } return new SearchResults(); } - private SearchResults search(IndexSearcher indexSearcher, Query searchQuery) throws IOException { + private SearchResults performSearch(Query searchQuery, boolean shouldSearchInLinkedFiles) throws IOException { + if (shouldSearchInLinkedFiles) { + return searchInBibFieldsAndLinkedFiles(searchQuery); + } else { + return searchInBibFields(searchQuery); + } + } + + private SearchResults searchInBibFieldsAndLinkedFiles(Query searchQuery) throws IOException { + IndexSearcher bibFieldsIndexSearcher = acquireIndexedSearcher(bibFieldsSearcherManager); + IndexSearcher linkedFilesIndexSearcher = acquireIndexedSearcher(linkedFilesSearcherManager); + try { + MultiReader multiReader = new MultiReader(bibFieldsIndexSearcher.getIndexReader(), linkedFilesIndexSearcher.getIndexReader()); + IndexSearcher indexSearcher = new IndexSearcher(multiReader); + return search(indexSearcher, searchQuery, true); + } finally { + releaseIndexSearcher(bibFieldsSearcherManager, bibFieldsIndexSearcher); + releaseIndexSearcher(linkedFilesSearcherManager, linkedFilesIndexSearcher); + } + } + + private SearchResults searchInBibFields(Query searchQuery) throws IOException { + IndexSearcher indexSearcher = acquireIndexedSearcher(bibFieldsSearcherManager); + try { + return search(indexSearcher, searchQuery, false); + } finally { + releaseIndexSearcher(bibFieldsSearcherManager, indexSearcher); + } + } + + private SearchResults search(IndexSearcher indexSearcher, Query searchQuery, boolean shouldSearchInLinkedFiles) throws IOException { TopDocs topDocs = indexSearcher.search(searchQuery, Integer.MAX_VALUE); StoredFields storedFields = indexSearcher.storedFields(); - LOGGER.debug("Found {} matches", topDocs.totalHits.value); - return getSearchResults(topDocs, storedFields, searchQuery); + LOGGER.debug("Found {} matching documents", topDocs.totalHits.value); + return getSearchResults(topDocs, storedFields, searchQuery, shouldSearchInLinkedFiles); } - private SearchResults getSearchResults(TopDocs topDocs, StoredFields storedFields, Query searchQuery) throws IOException { + private SearchResults getSearchResults(TopDocs topDocs, StoredFields storedFields, Query searchQuery, boolean shouldSearchInLinkedFiles) throws IOException { SearchResults searchResults = new SearchResults(); - // fileLink to List of entry IDs - Map> linkedFilesMap = new HashMap<>(); + long startTime = System.currentTimeMillis(); - for (BibEntry bibEntry : databaseContext.getEntries()) { - for (LinkedFile linkedFile : bibEntry.getFiles()) { - linkedFilesMap.computeIfAbsent(linkedFile.getLink(), k -> new ArrayList<>()).add(bibEntry.getId()); - } + if (shouldSearchInLinkedFiles) { + getBibFieldsAndLinkedFilesResults(topDocs, storedFields, searchQuery, searchResults); + } else { + getBibFieldsResults(topDocs, storedFields, searchResults); } - long startTime = System.currentTimeMillis(); + LOGGER.debug("Getting search results took {} ms", System.currentTimeMillis() - startTime); + return searchResults; + } + + private void getBibFieldsAndLinkedFilesResults(TopDocs topDocs, StoredFields storedFields, Query searchQuery, SearchResults searchResults) throws IOException { + Map> linkedFilesMap = getLinkedFilesMap(); Highlighter highlighter = new Highlighter(new SimpleHTMLFormatter("", ""), new QueryScorer(searchQuery)); + for (ScoreDoc scoreDoc : topDocs.scoreDocs) { - float score = scoreDoc.score; Document document = storedFields.document(scoreDoc.doc); - String fileLink = getFieldContents(document, SearchFieldConstants.PATH); + if (!fileLink.isEmpty()) { - List entriesWithFile = linkedFilesMap.get(fileLink); - if (!entriesWithFile.isEmpty()) { - SearchResult searchResult = new SearchResult(score, fileLink, - getFieldContents(document, SearchFieldConstants.CONTENT), - getFieldContents(document, SearchFieldConstants.ANNOTATIONS), - Integer.parseInt(getFieldContents(document, SearchFieldConstants.PAGE_NUMBER)), - highlighter); - searchResults.addSearchResult(entriesWithFile, searchResult); - } + addLinkedFileToResults(document, fileLink, linkedFilesMap, highlighter, searchResults, scoreDoc.score); } else { - String entryId = getFieldContents(document, SearchFieldConstants.ENTRY_ID); - searchResults.addSearchResult(entryId, new SearchResult(score)); + addBibEntryToResults(document, searchResults, scoreDoc.score); } } - LOGGER.debug("Mapping search results took {} ms", System.currentTimeMillis() - startTime); - return searchResults; } - private static String getFieldContents(Document document, SearchFieldConstants field) { + private void getBibFieldsResults(TopDocs topDocs, StoredFields storedFields, SearchResults searchResults) throws IOException { + for (ScoreDoc scoreDoc : topDocs.scoreDocs) { + Document document = storedFields.document(scoreDoc.doc); + addBibEntryToResults(document, searchResults, scoreDoc.score); + } + } + + private void addLinkedFileToResults(Document document, String fileLink, Map> linkedFilesMap, Highlighter highlighter, SearchResults searchResults, float score) { + List entriesWithFile = linkedFilesMap.get(fileLink); + if (entriesWithFile != null && !entriesWithFile.isEmpty()) { + SearchResult searchResult = new SearchResult(score, fileLink, + getFieldContents(document, SearchFieldConstants.CONTENT), + getFieldContents(document, SearchFieldConstants.ANNOTATIONS), + Integer.parseInt(getFieldContents(document, SearchFieldConstants.PAGE_NUMBER)), + highlighter); + searchResults.addSearchResult(entriesWithFile, searchResult); + } + } + + private void addBibEntryToResults(Document document, SearchResults searchResults, float score) { + String entryId = getFieldContents(document, SearchFieldConstants.ENTRY_ID); + searchResults.addSearchResult(entryId, new SearchResult(score)); + } + + private Map> getLinkedFilesMap() { + Map> linkedFilesMap = new HashMap<>(); + for (BibEntry bibEntry : databaseContext.getEntries()) { + for (LinkedFile linkedFile : bibEntry.getFiles()) { + linkedFilesMap.computeIfAbsent(linkedFile.getLink(), k -> new ArrayList<>()).add(bibEntry.getId()); + } + } + return linkedFilesMap; + } + + private static String getFieldContents(Document document, SearchFieldConstants + field) { return Optional.ofNullable(document.get(field.toString())).orElse(""); } - private static IndexSearcher getIndexedSearcher(SearcherManager searcherManager) throws IOException { + private static IndexSearcher acquireIndexedSearcher(SearcherManager searcherManager) throws IOException { searcherManager.maybeRefreshBlocking(); return searcherManager.acquire(); } From 558b1b3ed4a9530ab072aafcfe1807f28063965c Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Tue, 3 Sep 2024 01:32:29 +0300 Subject: [PATCH 249/256] Use linked files analyzer for highlighting full-text results --- .../fileannotationtab/FulltextSearchResultsTab.java | 9 +++++---- src/main/java/org/jabref/model/search/SearchResult.java | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java b/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java index 215dbfc3a47..c6ef8dbadc9 100644 --- a/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java +++ b/src/main/java/org/jabref/gui/entryeditor/fileannotationtab/FulltextSearchResultsTab.java @@ -117,10 +117,11 @@ protected void bindToEntry(BibEntry entry) { Text annotationsText = new Text(System.lineSeparator() + Localization.lang("Found matches in annotations:") + System.lineSeparator() + System.lineSeparator()); annotationsText.setStyle("-fx-font-style: italic;"); content.getChildren().add(annotationsText); - } - for (String resultTextHtml : searchResult.getAnnotationsResultStringsHtml()) { - content.getChildren().addAll(TooltipTextUtil.createTextsFromHtml(resultTextHtml.replace(" ", " "))); - content.getChildren().addAll(new Text(System.lineSeparator()), lineSeparator(0.8), createPageLink(linkedFile, searchResult.getPageNumber())); + + for (String resultTextHtml : searchResult.getAnnotationsResultStringsHtml()) { + content.getChildren().addAll(TooltipTextUtil.createTextsFromHtml(resultTextHtml.replace(" ", " "))); + content.getChildren().addAll(new Text(System.lineSeparator()), lineSeparator(0.8), createPageLink(linkedFile, searchResult.getPageNumber())); + } } } }); diff --git a/src/main/java/org/jabref/model/search/SearchResult.java b/src/main/java/org/jabref/model/search/SearchResult.java index 8e5f91114d0..a26a443d21a 100644 --- a/src/main/java/org/jabref/model/search/SearchResult.java +++ b/src/main/java/org/jabref/model/search/SearchResult.java @@ -76,7 +76,7 @@ public int getPageNumber() { } private static List getHighlighterFragments(Highlighter highlighter, SearchFieldConstants field, String content) { - try (TokenStream contentStream = SearchFieldConstants.LATEX_AWARE_ANALYZER.tokenStream(field.toString(), content)) { + try (TokenStream contentStream = SearchFieldConstants.LINKED_FILES_ANALYZER.tokenStream(field.toString(), content)) { TextFragment[] frags = highlighter.getBestTextFragments(contentStream, content, true, 10); return Arrays.stream(frags).map(TextFragment::toString).toList(); } catch (IOException | InvalidTokenOffsetsException e) { From e6f3b5d6f1a806fd906c3a6be34c09532da4baa9 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Tue, 3 Sep 2024 01:37:13 +0300 Subject: [PATCH 250/256] Fix line break --- .../logic/search/retrieval/LuceneSearcher.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java b/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java index c0f91e7852b..25264cc0770 100644 --- a/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java +++ b/src/main/java/org/jabref/logic/search/retrieval/LuceneSearcher.java @@ -87,8 +87,8 @@ private SearchResults performSearch(Query searchQuery, boolean shouldSearchInLin } private SearchResults searchInBibFieldsAndLinkedFiles(Query searchQuery) throws IOException { - IndexSearcher bibFieldsIndexSearcher = acquireIndexedSearcher(bibFieldsSearcherManager); - IndexSearcher linkedFilesIndexSearcher = acquireIndexedSearcher(linkedFilesSearcherManager); + IndexSearcher bibFieldsIndexSearcher = acquireIndexSearcher(bibFieldsSearcherManager); + IndexSearcher linkedFilesIndexSearcher = acquireIndexSearcher(linkedFilesSearcherManager); try { MultiReader multiReader = new MultiReader(bibFieldsIndexSearcher.getIndexReader(), linkedFilesIndexSearcher.getIndexReader()); IndexSearcher indexSearcher = new IndexSearcher(multiReader); @@ -100,7 +100,7 @@ private SearchResults searchInBibFieldsAndLinkedFiles(Query searchQuery) throws } private SearchResults searchInBibFields(Query searchQuery) throws IOException { - IndexSearcher indexSearcher = acquireIndexedSearcher(bibFieldsSearcherManager); + IndexSearcher indexSearcher = acquireIndexSearcher(bibFieldsSearcherManager); try { return search(indexSearcher, searchQuery, false); } finally { @@ -170,6 +170,7 @@ private void addBibEntryToResults(Document document, SearchResults searchResults } private Map> getLinkedFilesMap() { + // fileLink to List of entry IDs Map> linkedFilesMap = new HashMap<>(); for (BibEntry bibEntry : databaseContext.getEntries()) { for (LinkedFile linkedFile : bibEntry.getFiles()) { @@ -179,12 +180,11 @@ private Map> getLinkedFilesMap() { return linkedFilesMap; } - private static String getFieldContents(Document document, SearchFieldConstants - field) { + private static String getFieldContents(Document document, SearchFieldConstants field) { return Optional.ofNullable(document.get(field.toString())).orElse(""); } - private static IndexSearcher acquireIndexedSearcher(SearcherManager searcherManager) throws IOException { + private static IndexSearcher acquireIndexSearcher(SearcherManager searcherManager) throws IOException { searcherManager.maybeRefreshBlocking(); return searcherManager.acquire(); } From 95458d3141960d97432420265527adff39089551 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Tue, 3 Sep 2024 01:54:30 +0300 Subject: [PATCH 251/256] Fix tests --- .../jabref/logic/search/DatabaseSearcherWithBibFilesTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java b/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java index d928d60cb94..fe4d3203f0f 100644 --- a/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java +++ b/src/test/java/org/jabref/logic/search/DatabaseSearcherWithBibFilesTest.java @@ -79,6 +79,7 @@ private BibDatabaseContext initializeDatabaseFromPath(Path testFile) throws Exce ParserResult result = new BibtexImporter(mock(ImportFormatPreferences.class, Answers.RETURNS_DEEP_STUBS), new DummyFileUpdateMonitor()).importDatabase(testFile); BibDatabaseContext databaseContext = result.getDatabaseContext(); + when(filePreferences.shouldFulltextIndexLinkedFiles()).thenReturn(true); when(filePreferences.fulltextIndexLinkedFilesProperty()).thenReturn(new SimpleBooleanProperty(true)); return databaseContext; } From e5665ce5092efe7d984019c6f3b325dec15360be Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Tue, 3 Sep 2024 02:33:19 +0300 Subject: [PATCH 252/256] Use EnglishAnalyzer for indexing/searching linked files https://github.com/apache/lucene/blob/68cc8734ca28a9db800e4192a636d3b490cfd41a/lucene/analysis/common/src/java/org/apache/lucene/analysis/en/EnglishAnalyzer.java#L101-L110 --- .../search/Analyzer/LinkedFilesAnalyzer.java | 19 ------------------- .../model/search/SearchFieldConstants.java | 4 ++-- .../org/jabref/model/search/SearchQuery.java | 13 ++++++++++--- 3 files changed, 12 insertions(+), 24 deletions(-) delete mode 100644 src/main/java/org/jabref/model/search/Analyzer/LinkedFilesAnalyzer.java diff --git a/src/main/java/org/jabref/model/search/Analyzer/LinkedFilesAnalyzer.java b/src/main/java/org/jabref/model/search/Analyzer/LinkedFilesAnalyzer.java deleted file mode 100644 index d7c930adcd0..00000000000 --- a/src/main/java/org/jabref/model/search/Analyzer/LinkedFilesAnalyzer.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.jabref.model.search.Analyzer; - -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.LowerCaseFilter; -import org.apache.lucene.analysis.StopFilter; -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.Tokenizer; -import org.apache.lucene.analysis.en.EnglishAnalyzer; -import org.apache.lucene.analysis.standard.StandardTokenizer; - -public class LinkedFilesAnalyzer extends Analyzer { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer source = new StandardTokenizer(); - TokenStream result = new StopFilter(source, EnglishAnalyzer.ENGLISH_STOP_WORDS_SET); - result = new LowerCaseFilter(result); - return new TokenStreamComponents(source, result); - } -} diff --git a/src/main/java/org/jabref/model/search/SearchFieldConstants.java b/src/main/java/org/jabref/model/search/SearchFieldConstants.java index c40935987ed..8becbeb1851 100644 --- a/src/main/java/org/jabref/model/search/SearchFieldConstants.java +++ b/src/main/java/org/jabref/model/search/SearchFieldConstants.java @@ -4,9 +4,9 @@ import org.jabref.model.search.Analyzer.LatexAwareAnalyzer; import org.jabref.model.search.Analyzer.LatexAwareNGramAnalyzer; -import org.jabref.model.search.Analyzer.LinkedFilesAnalyzer; import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.en.EnglishAnalyzer; public enum SearchFieldConstants { @@ -20,7 +20,7 @@ public enum SearchFieldConstants { PAGE_NUMBER("pageNumber"), MODIFIED("modified"); - public static final Analyzer LINKED_FILES_ANALYZER = new LinkedFilesAnalyzer(); + public static final Analyzer LINKED_FILES_ANALYZER = new EnglishAnalyzer(); public static final Analyzer LATEX_AWARE_ANALYZER = new LatexAwareAnalyzer(); public static final Analyzer LATEX_AWARE_NGRAM_ANALYZER = new LatexAwareNGramAnalyzer(1, Integer.MAX_VALUE); public static final List PDF_FIELDS = List.of(CONTENT.toString(), ANNOTATIONS.toString()); diff --git a/src/main/java/org/jabref/model/search/SearchQuery.java b/src/main/java/org/jabref/model/search/SearchQuery.java index 6fd46706626..0428631564f 100644 --- a/src/main/java/org/jabref/model/search/SearchQuery.java +++ b/src/main/java/org/jabref/model/search/SearchQuery.java @@ -12,6 +12,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper; import org.apache.lucene.queryparser.classic.MultiFieldQueryParser; import org.apache.lucene.search.Query; import org.apache.lucene.search.highlight.QueryTermExtractor; @@ -67,16 +69,21 @@ public SearchQuery(String query, EnumSet searchFlags) { this.searchFlags = searchFlags; Map boosts = new HashMap<>(); - boosts.put(SearchFieldConstants.DEFAULT_FIELD.toString(), 4F); + Map fieldAnalyzers = new HashMap<>(); + boosts.put(SearchFieldConstants.DEFAULT_FIELD.toString(), 4F); if (searchFlags.contains(SearchFlags.FULLTEXT)) { - SearchFieldConstants.PDF_FIELDS.forEach(field -> boosts.put(field, 1F)); + SearchFieldConstants.PDF_FIELDS.forEach(field -> { + boosts.put(field, 1F); + fieldAnalyzers.put(field, SearchFieldConstants.LINKED_FILES_ANALYZER); + }); } String[] fieldsToSearchArray = new String[boosts.size()]; boosts.keySet().toArray(fieldsToSearchArray); - MultiFieldQueryParser queryParser = new MultiFieldQueryParser(fieldsToSearchArray, SearchFieldConstants.LATEX_AWARE_ANALYZER, boosts); + PerFieldAnalyzerWrapper analyzerWrapper = new PerFieldAnalyzerWrapper(SearchFieldConstants.LATEX_AWARE_ANALYZER, fieldAnalyzers); + MultiFieldQueryParser queryParser = new MultiFieldQueryParser(fieldsToSearchArray, analyzerWrapper, boosts); queryParser.setAllowLeadingWildcard(true); try { From ce21f4a26a563ce514f7e12f3fbdab32c688a180 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Wed, 4 Sep 2024 09:24:13 +0300 Subject: [PATCH 253/256] Ask to wait for linked files indexing on shutdown When closing JabRef, only ask users to wait for the linked files indexer to finish. The bib fields indexer is recalculated on startup, so it doesn't need to be completed before shutdown. --- src/main/java/org/jabref/logic/search/LuceneManager.java | 1 + .../jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/jabref/logic/search/LuceneManager.java b/src/main/java/org/jabref/logic/search/LuceneManager.java index eb0b3ca4dc3..9801a97667d 100644 --- a/src/main/java/org/jabref/logic/search/LuceneManager.java +++ b/src/main/java/org/jabref/logic/search/LuceneManager.java @@ -83,6 +83,7 @@ protected Object call() { return null; } }.showToUser(true) + .willBeRecoveredAutomatically(true) .onFinished(() -> this.databaseContext.getDatabase().postEvent(new IndexStartedEvent())) .executeWith(taskExecutor); diff --git a/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java b/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java index c37b878d001..49cd13fa5fb 100644 --- a/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java +++ b/src/main/java/org/jabref/logic/search/indexing/DefaultLinkedFilesIndexer.java @@ -133,7 +133,6 @@ private void addToIndex(Map> linkedFiles, BackgroundTas return; } - task.willBeRecoveredAutomatically(true); task.setTitle(Localization.lang("Indexing PDF files for %0", libraryName)); LOGGER.debug("Adding {} files to index", linkedFiles.size()); int i = 1; From 6dbc187621338d46e03cf81855a63802c8270e48 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Wed, 4 Sep 2024 14:05:10 +0300 Subject: [PATCH 254/256] Use EdgeNGram instead of NGram --- .../jabref/model/search/Analyzer/LatexAwareNGramAnalyzer.java | 4 ++-- src/main/java/org/jabref/model/search/SearchQuery.java | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/jabref/model/search/Analyzer/LatexAwareNGramAnalyzer.java b/src/main/java/org/jabref/model/search/Analyzer/LatexAwareNGramAnalyzer.java index 961b9ce301b..f1f9ddabc3f 100644 --- a/src/main/java/org/jabref/model/search/Analyzer/LatexAwareNGramAnalyzer.java +++ b/src/main/java/org/jabref/model/search/Analyzer/LatexAwareNGramAnalyzer.java @@ -8,7 +8,7 @@ import org.apache.lucene.analysis.core.WhitespaceTokenizer; import org.apache.lucene.analysis.en.EnglishAnalyzer; import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter; -import org.apache.lucene.analysis.ngram.NGramTokenFilter; +import org.apache.lucene.analysis.ngram.EdgeNGramTokenFilter; public class LatexAwareNGramAnalyzer extends Analyzer { private final int minGram; @@ -26,7 +26,7 @@ protected TokenStreamComponents createComponents(String fieldName) { result = new StopFilter(result, EnglishAnalyzer.ENGLISH_STOP_WORDS_SET); result = new ASCIIFoldingFilter(result); result = new LowerCaseFilter(result); - result = new NGramTokenFilter(result, minGram, maxGram, true); + result = new EdgeNGramTokenFilter(result, minGram, maxGram, true); return new TokenStreamComponents(source, result); } } diff --git a/src/main/java/org/jabref/model/search/SearchQuery.java b/src/main/java/org/jabref/model/search/SearchQuery.java index 0428631564f..b76fc036fce 100644 --- a/src/main/java/org/jabref/model/search/SearchQuery.java +++ b/src/main/java/org/jabref/model/search/SearchQuery.java @@ -71,12 +71,14 @@ public SearchQuery(String query, EnumSet searchFlags) { Map boosts = new HashMap<>(); Map fieldAnalyzers = new HashMap<>(); - boosts.put(SearchFieldConstants.DEFAULT_FIELD.toString(), 4F); if (searchFlags.contains(SearchFlags.FULLTEXT)) { + boosts.put(SearchFieldConstants.DEFAULT_FIELD.toString(), 4F); SearchFieldConstants.PDF_FIELDS.forEach(field -> { boosts.put(field, 1F); fieldAnalyzers.put(field, SearchFieldConstants.LINKED_FILES_ANALYZER); }); + } else { + boosts.put(SearchFieldConstants.DEFAULT_FIELD.toString(), 1F); } String[] fieldsToSearchArray = new String[boosts.size()]; From c9d82300c21cfd08d0fd0b94fee9706ec666a50c Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Wed, 4 Sep 2024 14:12:53 +0300 Subject: [PATCH 255/256] Return comment --- .../gui/importer/actions/SearchGroupsMigrationAction.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java b/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java index b8d32f1529a..63b157c66c1 100644 --- a/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java +++ b/src/main/java/org/jabref/gui/importer/actions/SearchGroupsMigrationAction.java @@ -18,6 +18,9 @@ * If not we ask the user whether to migrate. */ public class SearchGroupsMigrationAction implements GUIPostOpenAction { + + // We cannot have this constant in `Version.java` because of recursion errors + // Thus, we keep it here, because it is (currently) used only in the context of groups migration. public static final Version VERSION_6_0_ALPHA = Version.parse("6.0-alpha"); @Override From fa8344b0cd35ddfc591b5bad9226b78a873de7f7 Mon Sep 17 00:00:00 2001 From: Loay Ghreeb Date: Wed, 4 Sep 2024 16:19:18 +0300 Subject: [PATCH 256/256] Update CHANGELOG.md --- CHANGELOG.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d67759c5ff3..3ff60969f28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,7 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv ### Changed -- The search syntax is changed to [Apache Lucene syntax](https://docs.jabref.org/collect/import-using-online-bibliographic-database#search-syntax) (also to be similar to the online search syntax). [#11542](https://github.com/JabRef/jabref/pull/11542/) +- The search syntax is changed to [Apache Lucene syntax](https://lucene.apache.org/core/9_11_1/queryparser/org/apache/lucene/queryparser/classic/package-summary.html#Overview) (also to be similar to the [online search syntax](https://docs.jabref.org/collect/import-using-online-bibliographic-database#search-syntax)). [#11542](https://github.com/JabRef/jabref/pull/11542/) - When searching using a regular expression, one needs to enclose the search string in `/`. [#11542](https://github.com/JabRef/jabref/pull/11542/) - A search in "any" fields ignores the [groups](https://docs.jabref.org/finding-sorting-and-cleaning-entries/groups). [#7996](https://github.com/JabRef/jabref/issues/7996) - When a communication error with an [online service](https://docs.jabref.org/collect/import-using-online-bibliographic-database) occurs, JabRef displays the HTTP error. [#11223](https://github.com/JabRef/jabref/issues/11223) @@ -54,11 +54,16 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv - We fixed an issue with query transformers (JStor and others). [#11643](https://github.com/JabRef/jabref/pull/11643) - We fixed an issue where a new unsaved library was not marked with an asterisk. [#11519](https://github.com/JabRef/jabref/pull/11519) - We fixed an issue where JabRef starts without window decorations. [#11440](https://github.com/JabRef/jabref/pull/11440) -- We fixed an issue when the library was not marked changed after a migration. [#11542](https://github.com/JabRef/jabref/pull/11542) - We fixed an issue where the entry preview highlight was not working when searching before opening the entry editor. [#11659](https://github.com/JabRef/jabref/pull/11659) - We fixed an issue where text in Dark mode inside "Citation information" was not readable. [#11512](https://github.com/JabRef/jabref/issues/11512) - We fixed an issue where the selection of an entry in the table lost after searching for a group. [#3176](https://github.com/JabRef/jabref/issues/3176) - We fixed the non-functionality of the option "Automatically sync bibliography when inserting citations" in the OpenOffice panel, when enabled in case of JStyles. [#11684](https://github.com/JabRef/jabref/issues/11684) +- We fixed an issue where the library was not marked changed after a migration. [#11542](https://github.com/JabRef/jabref/pull/11542) +- We fixed an issue where rebuilding the full-text search index was not working. [#11374](https://github.com/JabRef/jabref/issues/11374) +- We fixed an issue where the progress of indexing linked files showed an incorrect number of files. [#11378](https://github.com/JabRef/jabref/issues/11378) +- We fixed an issue where the full-text search results were incomplete. [#8626](https://github.com/JabRef/jabref/issues/8626) +- We fixed an issue where search result highlighting was incorrectly highlighting the boolean operators. [#11595](https://github.com/JabRef/jabref/issues/11595) +- We fixed an issue where search result highlighting was broken at complex searches. [#8067](https://github.com/JabRef/jabref/issues/8067) ### Removed