Add soft-deletes upgrade tests (#36286)

This change adds a rolling-upgrade and full-cluster-restart test with
soft-deletes enabled.

Author: dnhatn

qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java

@@ -953,6 +953,56 @@ public void testHistoryUUIDIsAdded() throws Exception {
         }
     }
 
+    public void testSoftDeletes() throws Exception {
+        if (isRunningAgainstOldCluster()) {
+            XContentBuilder mappingsAndSettings = jsonBuilder();
+            mappingsAndSettings.startObject();
+            {
+                mappingsAndSettings.startObject("settings");
+                mappingsAndSettings.field("number_of_shards", 1);
+                mappingsAndSettings.field("number_of_replicas", 1);
+                if (getOldClusterVersion().onOrAfter(Version.V_6_5_0)) {
+                    mappingsAndSettings.field("soft_deletes.enabled", true);
+                }
+                mappingsAndSettings.endObject();
+            }
+            mappingsAndSettings.endObject();
+            Request createIndex = new Request("PUT", "/" + index);
+            createIndex.setJsonEntity(Strings.toString(mappingsAndSettings));
+            client().performRequest(createIndex);
+            int numDocs = between(10, 100);
+            for (int i = 0; i < numDocs; i++) {
+                String doc = Strings.toString(JsonXContent.contentBuilder().startObject().field("field", "v1").endObject());
+                Request request = new Request("POST", "/" + index + "/doc/" + i);
+                request.setJsonEntity(doc);
+                client().performRequest(request);
+                if (rarely()) {
+                    refresh();
+                }
+            }
+            client().performRequest(new Request("POST", "/" + index + "/_flush"));
+            int liveDocs = numDocs;
+            assertTotalHits(liveDocs, entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search"))));
+            for (int i = 0; i < numDocs; i++) {
+                if (randomBoolean()) {
+                    String doc = Strings.toString(JsonXContent.contentBuilder().startObject().field("field", "v2").endObject());
+                    Request request = new Request("POST", "/" + index + "/doc/" + i);
+                    request.setJsonEntity(doc);
+                    client().performRequest(request);
+                } else if (randomBoolean()) {
+                    client().performRequest(new Request("DELETE", "/" + index + "/doc/" + i));
+                    liveDocs--;
+                }
+            }
+            refresh();
+            assertTotalHits(liveDocs, entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search"))));
+            saveInfoDocument("doc_count", Integer.toString(liveDocs));
+        } else {
+            int liveDocs = Integer.parseInt(loadInfoDocument("doc_count"));
+            assertTotalHits(liveDocs, entityAsMap(client().performRequest(new Request("GET", "/" + index + "/_search"))));
+        }
+    }
+
     private void checkSnapshot(String snapshotName, int count, Version tookOnVersion) throws IOException {
         // Check the snapshot metadata, especially the version
         Request listSnapshotRequest = new Request("GET", "/_snapshot/repo/" + snapshotName);

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java

@@ -26,6 +26,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.test.rest.yaml.ObjectPath;
 
 import java.io.IOException;
@@ -272,4 +273,35 @@ public void testRecoverSyncedFlushIndex() throws Exception {
         ensureGreen(index);
     }
 
+    public void testRecoveryWithSoftDeletes() throws Exception {
+        final String index = "recover_with_soft_deletes";
+        if (CLUSTER_TYPE == ClusterType.OLD) {
+            Settings.Builder settings = Settings.builder()
+                .put(IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
+                .put(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 1)
+                // if the node with the replica is the first to be restarted, while a replica is still recovering
+                // then delayed allocation will kick in. When the node comes back, the master will search for a copy
+                // but the recovering copy will be seen as invalid and the cluster health won't return to GREEN
+                // before timing out
+                .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms")
+                .put(SETTING_ALLOCATION_MAX_RETRY.getKey(), "0"); // fail faster
+            if (getNodeId(v -> v.onOrAfter(Version.V_6_5_0)) != null) {
+                settings.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true);
+            }
+            createIndex(index, settings.build());
+            int numDocs = randomInt(10);
+            indexDocs(index, 0, numDocs);
+            if (randomBoolean()) {
+                client().performRequest(new Request("POST", "/" + index + "/_flush"));
+            }
+            for (int i = 0; i < numDocs; i++) {
+                if (randomBoolean()) {
+                    indexDocs(index, i, 1); // update
+                } else if (randomBoolean()) {
+                    client().performRequest(new Request("DELETE", index + "/test/" + i));
+                }
+            }
+        }
+        ensureGreen(index);
+    }
 }