[Bug Fix] fix skip_unavailable setting changing to default during node drop

CHANGELOG.md

@@ -22,6 +22,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix flaky tests in CloseIndexIT by addressing cluster state synchronization issues ([#18878](https://github.com/opensearch-project/OpenSearch/issues/18878))
 - [Tiered Caching] Handle  query execution exception ([#19000](https://github.com/opensearch-project/OpenSearch/issues/19000))
 - Grant access to testclusters dir for tests ([#19085](https://github.com/opensearch-project/OpenSearch/issues/19085))
+- Fix skip_unavailable setting changing to default during node drop issue ([#18766](https://github.com/opensearch-project/OpenSearch/pull/18766))
 
 ### Dependencies
 - Bump `com.netflix.nebula.ospackage-base` from 12.0.0 to 12.1.0 ([#19019](https://github.com/opensearch-project/OpenSearch/pull/19019))

qa/ccs-unavailable-clusters/src/test/java/org/opensearch/search/CrossClusterSearchUnavailableClusterIT.java

@@ -294,6 +294,63 @@ public void testSkipUnavailableDependsOnSeeds() throws IOException {
         }
     }
 
+    public void testCrossClusterConnectionWithSkipUnavailable() throws IOException {
+        try (MockTransportService remoteTransport1 = startTransport("node1", new CopyOnWriteArrayList<>(), Version.CURRENT, threadPool);
+             MockTransportService remoteTransport2 = startTransport("node2", new CopyOnWriteArrayList<>(), Version.CURRENT, threadPool)) {
+
+            DiscoveryNode remoteNode1 = remoteTransport1.getLocalDiscoNode();
+            DiscoveryNode remoteNode2 = remoteTransport2.getLocalDiscoNode();
+
+            Map<String, Object> initialSettings = new HashMap<>();
+            initialSettings.put("seeds", remoteNode1.getAddress().toString());
+            initialSettings.put("skip_unavailable", true);
+            updateRemoteClusterSettings(initialSettings);
+
+            for (int i = 0; i < 5; i++) {
+                restHighLevelClient.index(
+                    new IndexRequest("index").id(String.valueOf(i)).source("field", "value"),
+                    RequestOptions.DEFAULT
+                );
+            }
+
+            Response refreshResponse = client().performRequest(new Request("POST", "/index/_refresh"));
+            assertEquals(200, refreshResponse.getStatusLine().getStatusCode());
+
+            SearchResponse response1 = restHighLevelClient.search(
+                new SearchRequest("index", "remote1:index"),
+                RequestOptions.DEFAULT
+            );
+
+            assertEquals(2, response1.getClusters().getTotal());
+            assertEquals(2, response1.getClusters().getSuccessful());
+            assertEquals(0, response1.getClusters().getSkipped());
+            assertEquals(5, response1.getHits().getTotalHits().value());
+
+            Map<String, Object> clusterSettings = new HashMap<>();
+            clusterSettings.put("seeds", remoteNode2.getAddress().toString());
+            clusterSettings.put("skip_unavailable", true);
+            updateRemoteClusterSettings(clusterSettings);
+
+            remoteTransport1.close();
+            remoteTransport2.close();
+
+            SearchResponse response2 = restHighLevelClient.search(
+                new SearchRequest("index", "remote1:index"),
+                RequestOptions.DEFAULT
+            );
+
+            assertEquals(2, response2.getClusters().getTotal());
+            assertEquals(1, response2.getClusters().getSuccessful());
+            assertEquals(1, response2.getClusters().getSkipped());
+            assertEquals(5, response2.getHits().getTotalHits().value());
+
+            Map<String, Object> cleanupSettings = new HashMap<>();
+            cleanupSettings.put("seeds", null);
+            cleanupSettings.put("skip_unavailable", null);
+            updateRemoteClusterSettings(cleanupSettings);
+        }
+    }
+
     private static void assertSearchConnectFailure() {
         {
             OpenSearchException exception = expectThrows(OpenSearchException.class,

server/src/main/java/org/opensearch/transport/RemoteClusterAware.java

@@ -146,6 +146,7 @@ public void listenForUpdates(ClusterSettings clusterSettings) {
             RemoteClusterService.REMOTE_CLUSTER_COMPRESS,
             RemoteClusterService.REMOTE_CLUSTER_PING_SCHEDULE,
             RemoteConnectionStrategy.REMOTE_CONNECTION_MODE,
+            RemoteClusterService.REMOTE_CLUSTER_SKIP_UNAVAILABLE,
             SniffConnectionStrategy.REMOTE_CLUSTERS_PROXY,
             SniffConnectionStrategy.REMOTE_CLUSTER_SEEDS,
             SniffConnectionStrategy.REMOTE_NODE_CONNECTIONS,