diff --git a/src/main/java/org/apache/maven/plugin/compiler/AbstractCompilerMojo.java b/src/main/java/org/apache/maven/plugin/compiler/AbstractCompilerMojo.java
index b12f6a192..e9290bcf9 100644
--- a/src/main/java/org/apache/maven/plugin/compiler/AbstractCompilerMojo.java
+++ b/src/main/java/org/apache/maven/plugin/compiler/AbstractCompilerMojo.java
@@ -1840,10 +1840,10 @@ private boolean hasInputFileTreeChanged(IncrementalBuildHelper ibh, Set<File> in
         }
         Path mojoConfigFile = mojoConfigBase.resolve(INPUT_FILES_LST_FILENAME);
 
-        List<String> oldInputFiles = Collections.emptyList();
+        Set<String> oldInputFiles = Collections.emptySet();
         if (Files.isRegularFile(mojoConfigFile)) {
             try {
-                oldInputFiles = Files.readAllLines(mojoConfigFile);
+                oldInputFiles = new HashSet<>(Files.readAllLines(mojoConfigFile));
             } catch (IOException e) {
                 // we cannot read the mojo config file, so don't do anything beside logging
                 getLog().warn("Error while reading old mojo status: " + mojoConfigFile);
@@ -1851,8 +1851,10 @@ private boolean hasInputFileTreeChanged(IncrementalBuildHelper ibh, Set<File> in
             }
         }
 
-        List<String> newInputFiles =
-                inputFiles.stream().sorted().map(File::getAbsolutePath).collect(Collectors.toList());
+        Set<String> newInputFiles = inputFiles.stream()
+                .sorted()
+                .map(File::getAbsolutePath)
+                .collect(Collectors.toCollection(LinkedHashSet::new));
 
         try {
             Files.write(mojoConfigFile, newInputFiles);
diff --git a/src/main/java/org/apache/maven/plugin/compiler/DeltaList.java b/src/main/java/org/apache/maven/plugin/compiler/DeltaList.java
index 18943fab9..a4e7b3daa 100644
--- a/src/main/java/org/apache/maven/plugin/compiler/DeltaList.java
+++ b/src/main/java/org/apache/maven/plugin/compiler/DeltaList.java
@@ -18,25 +18,31 @@
  */
 package org.apache.maven.plugin.compiler;
 
-import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
 
 /**
  * Show the modifications between two lists.
  */
-final class DeltaList<E> {
+final class DeltaList<E extends Comparable<E>> {
 
-    private final List<E> added;
-    private final List<E> removed;
+    private final Set<E> added = new TreeSet<>();
+    private final Set<E> removed = new TreeSet<>();
     private final boolean hasChanged;
 
     DeltaList(Collection<E> oldList, Collection<E> newList) {
-        this.added = new ArrayList<>(newList);
-        this.removed = new ArrayList<>(oldList);
-        added.removeAll(oldList);
-        removed.removeAll(newList);
+        for (E newListItem : newList) {
+            if (!oldList.contains(newListItem)) {
+                added.add(newListItem);
+            }
+        }
+        for (E oldListItem : oldList) {
+            if (!newList.contains(oldListItem)) {
+                removed.add(oldListItem);
+            }
+        }
         this.hasChanged = !added.isEmpty() || !removed.isEmpty();
     }