hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
@Test
  public void testScmDetectStaleAndDeadNode() throws IOException,
      InterruptedException, TimeoutException {
    final int interval = 100;
    final int nodeCount = 10;

    OzoneConfiguration conf = getConf();
    conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, interval,
        MILLISECONDS);
    conf.setTimeDuration(OZONE_SCM_HEARTBEAT_INTERVAL, 1, SECONDS);
    conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, SECONDS);
    conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, SECONDS);


    try (SCMNodeManager nodeManager = createNodeManager(conf)) {
      List<DatanodeDetails> nodeList = createNodeSet(nodeManager, nodeCount);


      DatanodeDetails staleNode = TestUtils.getDatanodeDetails(nodeManager);

      // Heartbeat once
      nodeManager.sendHeartbeat(staleNode.getProtoBufMessage(),
          null, reportState);

      // Heartbeat all other nodes.
      for (DatanodeDetails dn : nodeList) {
        nodeManager.sendHeartbeat(dn.getProtoBufMessage(), null, reportState);
      }

      // Wait for 2 seconds .. and heartbeat good nodes again.
      Thread.sleep(2 * 1000);

      for (DatanodeDetails dn : nodeList) {
        nodeManager.sendHeartbeat(dn.getProtoBufMessage(), null, reportState);
      }

      // Wait for 2 seconds, wait a total of 4 seconds to make sure that the
      // node moves into stale state.
      Thread.sleep(2 * 1000);
      List<DatanodeDetails> staleNodeList = nodeManager.getNodes(STALE);
      assertEquals("Expected to find 1 stale node",
          1, nodeManager.getNodeCount(STALE));
      assertEquals("Expected to find 1 stale node",
          1, staleNodeList.size());
      assertEquals("Stale node is not the expected ID", staleNode
          .getUuid(), staleNodeList.get(0).getUuid());
      Thread.sleep(1000);

      // heartbeat good nodes again.
      for (DatanodeDetails dn : nodeList) {
        nodeManager.sendHeartbeat(dn.getProtoBufMessage(), null, reportState);
      }

      //  6 seconds is the dead window for this test , so we wait a total of
      // 7 seconds to make sure that the node moves into dead state.
      Thread.sleep(2 * 1000);

      // the stale node has been removed
      staleNodeList = nodeManager.getNodes(STALE);
      assertEquals("Expected to find 1 stale node",
          0, nodeManager.getNodeCount(STALE));
      assertEquals("Expected to find 1 stale node",
          0, staleNodeList.size());

      // Check for the dead node now.
      List<DatanodeDetails> deadNodeList = nodeManager.getNodes(DEAD);
      assertEquals("Expected to find 1 dead node", 1,
          nodeManager.getNodeCount(DEAD));
      assertEquals("Expected to find 1 dead node",
          1, deadNodeList.size());
      assertEquals("Dead node is not the expected ID", staleNode
          .getUuid(), deadNodeList.get(0).getUuid());
    }
  }
/**
   * Asserts that a single node moves from Healthy to stale node, then from
   * stale node to dead node if it misses enough heartbeats.
   *
   * @throws IOException
   * @throws InterruptedException
   * @throws TimeoutException
   */
Asserts that a single node moves from Healthy to stale node, then from
stale node to dead node if it misses enough heartbeats.


Asserts that a single node moves from Healthy to stale node then from stale node to dead node if it misses enough heartbeats .
@Test
  public void testScmDetectStaleAndDeadNode() throws IOException,
      InterruptedException, TimeoutException {
    final int interval = 100;
    final int nodeCount = 10;
    OzoneConfiguration conf = getConf();
    conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, interval,
        MILLISECONDS);
    conf.setTimeDuration(OZONE_SCM_HEARTBEAT_INTERVAL, 1, SECONDS);
    conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, SECONDS);
    conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, SECONDS);
    try (SCMNodeManager nodeManager = createNodeManager(conf)) {
      List<DatanodeDetails> nodeList = createNodeSet(nodeManager, nodeCount);
      DatanodeDetails staleNode = TestUtils.getDatanodeDetails(nodeManager);
      nodeManager.sendHeartbeat(staleNode.getProtoBufMessage(),
          null, reportState);
      for (DatanodeDetails dn : nodeList) {
        nodeManager.sendHeartbeat(dn.getProtoBufMessage(), null, reportState);
      }
      Thread.sleep(2 * 1000);
      for (DatanodeDetails dn : nodeList) {
        nodeManager.sendHeartbeat(dn.getProtoBufMessage(), null, reportState);
      }
      Thread.sleep(2 * 1000);
      List<DatanodeDetails> staleNodeList = nodeManager.getNodes(STALE);
      assertEquals("Expected to find 1 stale node",
          1, nodeManager.getNodeCount(STALE));
      assertEquals("Expected to find 1 stale node",
          1, staleNodeList.size());
      assertEquals("Stale node is not the expected ID", staleNode
          .getUuid(), staleNodeList.get(0).getUuid());
      Thread.sleep(1000);
      for (DatanodeDetails dn : nodeList) {
        nodeManager.sendHeartbeat(dn.getProtoBufMessage(), null, reportState);
      }
      Thread.sleep(2 * 1000);
      staleNodeList = nodeManager.getNodes(STALE);
      assertEquals("Expected to find 1 stale node",
          0, nodeManager.getNodeCount(STALE));
      assertEquals("Expected to find 1 stale node",
          0, staleNodeList.size());
      List<DatanodeDetails> deadNodeList = nodeManager.getNodes(DEAD);
      assertEquals("Expected to find 1 dead node", 1,
          nodeManager.getNodeCount(DEAD));
      assertEquals("Expected to find 1 dead node",
          1, deadNodeList.size());
      assertEquals("Dead node is not the expected ID", staleNode
          .getUuid(), deadNodeList.get(0).getUuid());
    }
  }
@ Test public void testScmDetectStaleAndDeadNode ( ) throws IOException , InterruptedException , TimeoutException { final int interval = 100 ; final int nodeCount = 10 ; OzoneConfiguration conf = getConf ( ) ; conf . setTimeDuration ( OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL , interval , MILLISECONDS ) ; conf . setTimeDuration ( OZONE_SCM_HEARTBEAT_INTERVAL , 1 , SECONDS ) ; conf . setTimeDuration ( OZONE_SCM_STALENODE_INTERVAL , 3 , SECONDS ) ; conf . setTimeDuration ( OZONE_SCM_DEADNODE_INTERVAL , 6 , SECONDS ) ; try ( SCMNodeManager nodeManager = createNodeManager ( conf ) ) { List < DatanodeDetails > nodeList = createNodeSet ( nodeManager , nodeCount ) ; DatanodeDetails staleNode = TestUtils . getDatanodeDetails ( nodeManager ) ; nodeManager . sendHeartbeat ( staleNode . getProtoBufMessage ( ) , null , reportState ) ; for ( DatanodeDetails dn : nodeList ) { nodeManager . sendHeartbeat ( dn . getProtoBufMessage ( ) , null , reportState ) ; } Thread . sleep ( 2 * 1000 ) ; for ( DatanodeDetails dn : nodeList ) { nodeManager . sendHeartbeat ( dn . getProtoBufMessage ( ) , null , reportState ) ; } Thread . sleep ( 2 * 1000 ) ; List < DatanodeDetails > staleNodeList = nodeManager . getNodes ( STALE ) ; assertEquals ( "Expected to find 1 stale node" , 1 , nodeManager . getNodeCount ( STALE ) ) ; assertEquals ( "Expected to find 1 stale node" , 1 , staleNodeList . size ( ) ) ; assertEquals ( "Stale node is not the expected ID" , staleNode . getUuid ( ) , staleNodeList . get ( 0 ) . getUuid ( ) ) ; Thread . sleep ( 1000 ) ; for ( DatanodeDetails dn : nodeList ) { nodeManager . sendHeartbeat ( dn . getProtoBufMessage ( ) , null , reportState ) ; } Thread . sleep ( 2 * 1000 ) ; staleNodeList = nodeManager . getNodes ( STALE ) ; assertEquals ( "Expected to find 1 stale node" , 0 , nodeManager . getNodeCount ( STALE ) ) ; assertEquals ( "Expected to find 1 stale node" , 0 , staleNodeList . size ( ) ) ; List < DatanodeDetails > deadNodeList = nodeManager . getNodes ( DEAD ) ; assertEquals ( "Expected to find 1 dead node" , 1 , nodeManager . getNodeCount ( DEAD ) ) ; assertEquals ( "Expected to find 1 dead node" , 1 , deadNodeList . size ( ) ) ; assertEquals ( "Dead node is not the expected ID" , staleNode . getUuid ( ) , deadNodeList . get ( 0 ) . getUuid ( ) ) ; } }
Asserts that a single node moves from Healthy to stale node, then from
 stale node to dead node if it misses enough heartbeats.
Asserts that a single node moves from Healthy to stale node then from stale node to dead node if it misses enough heartbeats .
// Heartbeat once // Heartbeat all other nodes. // Wait for 2 seconds .. and heartbeat good nodes again. // Wait for 2 seconds, wait a total of 4 seconds to make sure that the // node moves into stale state. // heartbeat good nodes again. //  6 seconds is the dead window for this test , so we wait a total of // 7 seconds to make sure that the node moves into dead state. // the stale node has been removed // Check for the dead node now.