add proper reason when removing nodes

Author: ywelsch

server/src/main/java/org/elasticsearch/cluster/coordination/FollowersChecker.java

@@ -240,7 +240,7 @@ private void handleDisconnectedNode(DiscoveryNode discoveryNode) {
         synchronized (mutex) {
             FollowerChecker followerChecker = followerCheckers.get(discoveryNode);
             if (followerChecker != null && followerChecker.running()) {
-                followerChecker.failNode();
+                followerChecker.failNode("disconnected");
             }
         }
     }
@@ -320,18 +320,21 @@ public void handleException(TransportException exp) {
 
                         failureCountSinceLastSuccess++;
 
+                        final String reason;
                         if (failureCountSinceLastSuccess >= followerCheckRetryCount) {
                             logger.debug(() -> new ParameterizedMessage("{} failed too many times", FollowerChecker.this), exp);
+                            reason = "followers check retry count exceeded";
                         } else if (exp instanceof ConnectTransportException
                             || exp.getCause() instanceof ConnectTransportException) {
                             logger.debug(() -> new ParameterizedMessage("{} disconnected", FollowerChecker.this), exp);
+                            reason = "disconnected";
                         } else {
                             logger.debug(() -> new ParameterizedMessage("{} failed, retrying", FollowerChecker.this), exp);
                             scheduleNextWakeUp();
                             return;
                         }
 
-                        failNode();
+                        failNode(reason);
                     }
 
 
@@ -342,7 +345,7 @@ public String executor() {
                 });
         }
 
-        void failNode() {
+        void failNode(String reason) {
             transportService.getThreadPool().generic().execute(new Runnable() {
                 @Override
                 public void run() {
@@ -354,7 +357,7 @@ public void run() {
                         faultyNodes.add(discoveryNode);
                         followerCheckers.remove(discoveryNode);
                     }
-                    onNodeFailure.accept(discoveryNode, "followers_checker");
+                    onNodeFailure.accept(discoveryNode, reason);
                 }
 
                 @Override

server/src/test/java/org/elasticsearch/cluster/coordination/FollowersCheckerTests.java

@@ -165,6 +165,7 @@ public void testFailsNodeThatDoesNotRespond() {
         final Settings settings = settingsBuilder.build();
 
         testBehaviourOfFailingNode(settings, () -> null,
+            "followers check retry count exceeded",
             (FOLLOWER_CHECK_RETRY_COUNT_SETTING.get(settings) - 1) * FOLLOWER_CHECK_INTERVAL_SETTING.get(settings).millis()
                 + FOLLOWER_CHECK_RETRY_COUNT_SETTING.get(settings) * FOLLOWER_CHECK_TIMEOUT_SETTING.get(settings).millis());
     }
@@ -182,6 +183,7 @@ public void testFailsNodeThatRejectsCheck() {
         testBehaviourOfFailingNode(settings, () -> {
                 throw new ElasticsearchException("simulated exception");
             },
+            "followers check retry count exceeded",
             (FOLLOWER_CHECK_RETRY_COUNT_SETTING.get(settings) - 1) * FOLLOWER_CHECK_INTERVAL_SETTING.get(settings).millis());
     }
 
@@ -213,14 +215,15 @@ public Empty get() {
                     throw new ElasticsearchException("simulated exception");
                 }
             },
+            "followers check retry count exceeded",
             (FOLLOWER_CHECK_RETRY_COUNT_SETTING.get(settings) * (maxRecoveries + 1) - 1)
                 * FOLLOWER_CHECK_INTERVAL_SETTING.get(settings).millis());
     }
 
     public void testFailsNodeThatIsDisconnected() {
         testBehaviourOfFailingNode(Settings.EMPTY, () -> {
             throw new ConnectTransportException(null, "simulated exception");
-        }, 0);
+        }, "disconnected", 0);
     }
 
     public void testFailsNodeThatDisconnects() {
@@ -262,6 +265,7 @@ public String toString() {
             assert false : fcr;
         }, (node, reason) -> {
             assertTrue(nodeFailed.compareAndSet(false, true));
+            assertThat(reason, equalTo("disconnected"));
         });
 
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder().add(localNode).add(otherNode).localNodeId(localNode.getId()).build();
@@ -274,7 +278,8 @@ public String toString() {
         assertThat(followersChecker.getFaultyNodes(), contains(otherNode));
     }
 
-    private void testBehaviourOfFailingNode(Settings testSettings, Supplier<TransportResponse.Empty> responder, long expectedFailureTime) {
+    private void testBehaviourOfFailingNode(Settings testSettings, Supplier<TransportResponse.Empty> responder, String failureReason,
+                                            long expectedFailureTime) {
         final DiscoveryNode localNode = new DiscoveryNode("local-node", buildNewFakeTransportAddress(), Version.CURRENT);
         final DiscoveryNode otherNode = new DiscoveryNode("other-node", buildNewFakeTransportAddress(), Version.CURRENT);
         final Settings settings = Settings.builder().put(NODE_NAME_SETTING.getKey(), localNode.getName()).put(testSettings).build();
@@ -321,6 +326,7 @@ public String toString() {
             assert false : fcr;
         }, (node, reason) -> {
             assertTrue(nodeFailed.compareAndSet(false, true));
+            assertThat(reason, equalTo(failureReason));
         });
 
         DiscoveryNodes discoveryNodes = DiscoveryNodes.builder().add(localNode).add(otherNode).localNodeId(localNode.getId()).build();