hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeManager.java
@Test
  public void testScmClusterIsInExpectedState1() throws IOException,
      InterruptedException, TimeoutException {
    /**
     * These values are very important. Here is what it means so you don't
     * have to look it up while reading this code.
     *
     *  OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL - This the frequency of the
     *  HB processing thread that is running in the SCM. This thread must run
     *  for the SCM  to process the Heartbeats.
     *
     *  OZONE_SCM_HEARTBEAT_INTERVAL - This is the frequency at which
     *  datanodes will send heartbeats to SCM. Please note: This is the only
     *  config value for node manager that is specified in seconds. We don't
     *  want SCM heartbeat resolution to be more than in seconds.
     *  In this test it is not used, but we are forced to set it because we
     *  have validation code that checks Stale Node interval and Dead Node
     *  interval is larger than the value of
     *  OZONE_SCM_HEARTBEAT_INTERVAL.
     *
     *  OZONE_SCM_STALENODE_INTERVAL - This is the time that must elapse
     *  from the last heartbeat for us to mark a node as stale. In this test
     *  we set that to 3. That is if a node has not heartbeat SCM for last 3
     *  seconds we will mark it as stale.
     *
     *  OZONE_SCM_DEADNODE_INTERVAL - This is the time that must elapse
     *  from the last heartbeat for a node to be marked dead. We have an
     *  additional constraint that this must be at least 2 times bigger than
     *  Stale node Interval.
     *
     *  With these we are trying to explore the state of this cluster with
     *  various timeouts. Each section is commented so that you can keep
     *  track of the state of the cluster nodes.
     *
     */

    OzoneConfiguration conf = getConf();
    conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 100,
        MILLISECONDS);
    conf.setTimeDuration(OZONE_SCM_HEARTBEAT_INTERVAL, 1, SECONDS);
    conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, SECONDS);
    conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, SECONDS);


    /**
     * Cluster state: Healthy: All nodes are heartbeat-ing like normal.
     */
    try (SCMNodeManager nodeManager = createNodeManager(conf)) {
      DatanodeDetails healthyNode =
          TestUtils.getDatanodeDetails(nodeManager);
      DatanodeDetails staleNode =
          TestUtils.getDatanodeDetails(nodeManager);
      DatanodeDetails deadNode =
          TestUtils.getDatanodeDetails(nodeManager);
      nodeManager.sendHeartbeat(
          healthyNode.getProtoBufMessage(), null, reportState);
      nodeManager.sendHeartbeat(
          staleNode.getProtoBufMessage(), null, reportState);
      nodeManager.sendHeartbeat(
          deadNode.getProtoBufMessage(), null, reportState);

      // Sleep so that heartbeat processing thread gets to run.
      Thread.sleep(500);

      //Assert all nodes are healthy.
      assertEquals(3, nodeManager.getAllNodes().size());
      assertEquals(3, nodeManager.getNodeCount(HEALTHY));

      /**
       * Cluster state: Quiesced: We are going to sleep for 3 seconds. Which
       * means that no node is heartbeating. All nodes should move to Stale.
       */
      Thread.sleep(3 * 1000);
      assertEquals(3, nodeManager.getAllNodes().size());
      assertEquals(3, nodeManager.getNodeCount(STALE));


      /**
       * Cluster State : Move healthy node back to healthy state, move other 2
       * nodes to Stale State.
       *
       * We heartbeat healthy node after 1 second and let other 2 nodes elapse
       * the 3 second windows.
       */

      nodeManager.sendHeartbeat(
          healthyNode.getProtoBufMessage(), null, reportState);
      nodeManager.sendHeartbeat(
          staleNode.getProtoBufMessage(), null, reportState);
      nodeManager.sendHeartbeat(
          deadNode.getProtoBufMessage(), null, reportState);

      Thread.sleep(1500);
      nodeManager.sendHeartbeat(
          healthyNode.getProtoBufMessage(), null, reportState);
      Thread.sleep(2 * 1000);
      assertEquals(1, nodeManager.getNodeCount(HEALTHY));


      // 3.5 seconds from last heartbeat for the stale and deadNode. So those
      //  2 nodes must move to Stale state and the healthy node must
      // remain in the healthy State.
      List<DatanodeDetails> healthyList = nodeManager.getNodes(HEALTHY);
      assertEquals("Expected one healthy node", 1, healthyList.size());
      assertEquals("Healthy node is not the expected ID", healthyNode
          .getUuid(), healthyList.get(0).getUuid());

      assertEquals(2, nodeManager.getNodeCount(STALE));

      /**
       * Cluster State: Allow healthyNode to remain in healthy state and
       * staleNode to move to stale state and deadNode to move to dead state.
       */

      nodeManager.sendHeartbeat(
          healthyNode.getProtoBufMessage(), null, reportState);
      nodeManager.sendHeartbeat(
          staleNode.getProtoBufMessage(), null, reportState);
      Thread.sleep(1500);
      nodeManager.sendHeartbeat(
          healthyNode.getProtoBufMessage(), null, reportState);
      Thread.sleep(2 * 1000);

      // 3.5 seconds have elapsed for stale node, so it moves into Stale.
      // 7 seconds have elapsed for dead node, so it moves into dead.
      // 2 Seconds have elapsed for healthy node, so it stays in healhty state.
      healthyList = nodeManager.getNodes(HEALTHY);
      List<DatanodeDetails> staleList = nodeManager.getNodes(STALE);
      List<DatanodeDetails> deadList = nodeManager.getNodes(DEAD);

      assertEquals(3, nodeManager.getAllNodes().size());
      assertEquals(1, nodeManager.getNodeCount(HEALTHY));
      assertEquals(1, nodeManager.getNodeCount(STALE));
      assertEquals(1, nodeManager.getNodeCount(DEAD));

      assertEquals("Expected one healthy node",
          1, healthyList.size());
      assertEquals("Healthy node is not the expected ID", healthyNode
          .getUuid(), healthyList.get(0).getUuid());

      assertEquals("Expected one stale node",
          1, staleList.size());
      assertEquals("Stale node is not the expected ID", staleNode
          .getUuid(), staleList.get(0).getUuid());

      assertEquals("Expected one dead node",
          1, deadList.size());
      assertEquals("Dead node is not the expected ID", deadNode
          .getUuid(), deadList.get(0).getUuid());
      /**
       * Cluster State : let us heartbeat all the nodes and verify that we get
       * back all the nodes in healthy state.
       */
      nodeManager.sendHeartbeat(
          healthyNode.getProtoBufMessage(), null, reportState);
      nodeManager.sendHeartbeat(
          staleNode.getProtoBufMessage(), null, reportState);
      nodeManager.sendHeartbeat(
          deadNode.getProtoBufMessage(), null, reportState);
      Thread.sleep(500);
      //Assert all nodes are healthy.
      assertEquals(3, nodeManager.getAllNodes().size());
      assertEquals(3, nodeManager.getNodeCount(HEALTHY));
    }
  }
/**
   * Asserts that a dead node, stale node and healthy nodes co-exist. The counts
   * , lists and node ID match the expected node state.
   * <p/>
   * This test is pretty complicated because it explores all states of Node
   * manager in a single test. Please read thru the comments to get an idea of
   * the current state of the node Manager.
   * <p/>
   * This test is written like a state machine to avoid threads and concurrency
   * issues. This test is replicated below with the use of threads. Avoiding
   * threads make it easy to debug the state machine.
   *
   * @throws IOException
   * @throws InterruptedException
   * @throws TimeoutException
   */
Asserts that a dead node, stale node and healthy nodes co-exist. The counts
, lists and node ID match the expected node state.

This test is pretty complicated because it explores all states of Node
manager in a single test. Please read thru the comments to get an idea of
the current state of the node Manager.

This test is written like a state machine to avoid threads and concurrency
issues. This test is replicated below with the use of threads. Avoiding
threads make it easy to debug the state machine.


Asserts that a dead node stale node and healthy nodes co - exist . The counts lists and node ID match the expected node state . This test is pretty complicated because it explores all states of Node manager in a single test . Please read thru the comments to get an idea of the current state of the node Manager . This test is written like a state machine to avoid threads and concurrency issues . This test is replicated below with the use of threads . Avoiding threads make it easy to debug the state machine .
@Test
  public void testScmClusterIsInExpectedState1() throws IOException,
      InterruptedException, TimeoutException {
    /**
     * These values are very important. Here is what it means so you don't
     * have to look it up while reading this code.
     *
     *  OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL - This the frequency of the
     *  HB processing thread that is running in the SCM. This thread must run
     *  for the SCM  to process the Heartbeats.
     *
     *  OZONE_SCM_HEARTBEAT_INTERVAL - This is the frequency at which
     *  datanodes will send heartbeats to SCM. Please note: This is the only
     *  config value for node manager that is specified in seconds. We don't
     *  want SCM heartbeat resolution to be more than in seconds.
     *  In this test it is not used, but we are forced to set it because we
     *  have validation code that checks Stale Node interval and Dead Node
     *  interval is larger than the value of
     *  OZONE_SCM_HEARTBEAT_INTERVAL.
     *
     *  OZONE_SCM_STALENODE_INTERVAL - This is the time that must elapse
     *  from the last heartbeat for us to mark a node as stale. In this test
     *  we set that to 3. That is if a node has not heartbeat SCM for last 3
     *  seconds we will mark it as stale.
     *
     *  OZONE_SCM_DEADNODE_INTERVAL - This is the time that must elapse
     *  from the last heartbeat for a node to be marked dead. We have an
     *  additional constraint that this must be at least 2 times bigger than
     *  Stale node Interval.
     *
     *  With these we are trying to explore the state of this cluster with
     *  various timeouts. Each section is commented so that you can keep
     *  track of the state of the cluster nodes.
     *
     */
    OzoneConfiguration conf = getConf();
    conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 100,
        MILLISECONDS);
    conf.setTimeDuration(OZONE_SCM_HEARTBEAT_INTERVAL, 1, SECONDS);
    conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, SECONDS);
    conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, SECONDS);
    /**
     * Cluster state: Healthy: All nodes are heartbeat-ing like normal.
     */
    try (SCMNodeManager nodeManager = createNodeManager(conf)) {
      DatanodeDetails healthyNode =
          TestUtils.getDatanodeDetails(nodeManager);
      DatanodeDetails staleNode =
          TestUtils.getDatanodeDetails(nodeManager);
      DatanodeDetails deadNode =
          TestUtils.getDatanodeDetails(nodeManager);
      nodeManager.sendHeartbeat(
          healthyNode.getProtoBufMessage(), null, reportState);
      nodeManager.sendHeartbeat(
          staleNode.getProtoBufMessage(), null, reportState);
      nodeManager.sendHeartbeat(
          deadNode.getProtoBufMessage(), null, reportState);
      Thread.sleep(500);
      assertEquals(3, nodeManager.getAllNodes().size());
      assertEquals(3, nodeManager.getNodeCount(HEALTHY));
      /**
       * Cluster state: Quiesced: We are going to sleep for 3 seconds. Which
       * means that no node is heartbeating. All nodes should move to Stale.
       */
      Thread.sleep(3 * 1000);
      assertEquals(3, nodeManager.getAllNodes().size());
      assertEquals(3, nodeManager.getNodeCount(STALE));
      /**
       * Cluster State : Move healthy node back to healthy state, move other 2
       * nodes to Stale State.
       *
       * We heartbeat healthy node after 1 second and let other 2 nodes elapse
       * the 3 second windows.
       */
      nodeManager.sendHeartbeat(
          healthyNode.getProtoBufMessage(), null, reportState);
      nodeManager.sendHeartbeat(
          staleNode.getProtoBufMessage(), null, reportState);
      nodeManager.sendHeartbeat(
          deadNode.getProtoBufMessage(), null, reportState);
      Thread.sleep(1500);
      nodeManager.sendHeartbeat(
          healthyNode.getProtoBufMessage(), null, reportState);
      Thread.sleep(2 * 1000);
      assertEquals(1, nodeManager.getNodeCount(HEALTHY));
      List<DatanodeDetails> healthyList = nodeManager.getNodes(HEALTHY);
      assertEquals("Expected one healthy node", 1, healthyList.size());
      assertEquals("Healthy node is not the expected ID", healthyNode
          .getUuid(), healthyList.get(0).getUuid());
      assertEquals(2, nodeManager.getNodeCount(STALE));
      /**
       * Cluster State: Allow healthyNode to remain in healthy state and
       * staleNode to move to stale state and deadNode to move to dead state.
       */
      nodeManager.sendHeartbeat(
          healthyNode.getProtoBufMessage(), null, reportState);
      nodeManager.sendHeartbeat(
          staleNode.getProtoBufMessage(), null, reportState);
      Thread.sleep(1500);
      nodeManager.sendHeartbeat(
          healthyNode.getProtoBufMessage(), null, reportState);
      Thread.sleep(2 * 1000);
      healthyList = nodeManager.getNodes(HEALTHY);
      List<DatanodeDetails> staleList = nodeManager.getNodes(STALE);
      List<DatanodeDetails> deadList = nodeManager.getNodes(DEAD);
      assertEquals(3, nodeManager.getAllNodes().size());
      assertEquals(1, nodeManager.getNodeCount(HEALTHY));
      assertEquals(1, nodeManager.getNodeCount(STALE));
      assertEquals(1, nodeManager.getNodeCount(DEAD));
      assertEquals("Expected one healthy node",
          1, healthyList.size());
      assertEquals("Healthy node is not the expected ID", healthyNode
          .getUuid(), healthyList.get(0).getUuid());
      assertEquals("Expected one stale node",
          1, staleList.size());
      assertEquals("Stale node is not the expected ID", staleNode
          .getUuid(), staleList.get(0).getUuid());
      assertEquals("Expected one dead node",
          1, deadList.size());
      assertEquals("Dead node is not the expected ID", deadNode
          .getUuid(), deadList.get(0).getUuid());
      /**
       * Cluster State : let us heartbeat all the nodes and verify that we get
       * back all the nodes in healthy state.
       */
      nodeManager.sendHeartbeat(
          healthyNode.getProtoBufMessage(), null, reportState);
      nodeManager.sendHeartbeat(
          staleNode.getProtoBufMessage(), null, reportState);
      nodeManager.sendHeartbeat(
          deadNode.getProtoBufMessage(), null, reportState);
      Thread.sleep(500);
      assertEquals(3, nodeManager.getAllNodes().size());
      assertEquals(3, nodeManager.getNodeCount(HEALTHY));
    }
  }
@ Test public void testScmClusterIsInExpectedState1 ( ) throws IOException , InterruptedException , TimeoutException { /**
     * These values are very important. Here is what it means so you don't
     * have to look it up while reading this code.
     *
     *  OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL - This the frequency of the
     *  HB processing thread that is running in the SCM. This thread must run
     *  for the SCM  to process the Heartbeats.
     *
     *  OZONE_SCM_HEARTBEAT_INTERVAL - This is the frequency at which
     *  datanodes will send heartbeats to SCM. Please note: This is the only
     *  config value for node manager that is specified in seconds. We don't
     *  want SCM heartbeat resolution to be more than in seconds.
     *  In this test it is not used, but we are forced to set it because we
     *  have validation code that checks Stale Node interval and Dead Node
     *  interval is larger than the value of
     *  OZONE_SCM_HEARTBEAT_INTERVAL.
     *
     *  OZONE_SCM_STALENODE_INTERVAL - This is the time that must elapse
     *  from the last heartbeat for us to mark a node as stale. In this test
     *  we set that to 3. That is if a node has not heartbeat SCM for last 3
     *  seconds we will mark it as stale.
     *
     *  OZONE_SCM_DEADNODE_INTERVAL - This is the time that must elapse
     *  from the last heartbeat for a node to be marked dead. We have an
     *  additional constraint that this must be at least 2 times bigger than
     *  Stale node Interval.
     *
     *  With these we are trying to explore the state of this cluster with
     *  various timeouts. Each section is commented so that you can keep
     *  track of the state of the cluster nodes.
     *
     */ OzoneConfiguration conf = getConf ( ) ; conf . setTimeDuration ( OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL , 100 , MILLISECONDS ) ; conf . setTimeDuration ( OZONE_SCM_HEARTBEAT_INTERVAL , 1 , SECONDS ) ; conf . setTimeDuration ( OZONE_SCM_STALENODE_INTERVAL , 3 , SECONDS ) ; conf . setTimeDuration ( OZONE_SCM_DEADNODE_INTERVAL , 6 , SECONDS ) ; /**
     * Cluster state: Healthy: All nodes are heartbeat-ing like normal.
     */ try ( SCMNodeManager nodeManager = createNodeManager ( conf ) ) { DatanodeDetails healthyNode = TestUtils . getDatanodeDetails ( nodeManager ) ; DatanodeDetails staleNode = TestUtils . getDatanodeDetails ( nodeManager ) ; DatanodeDetails deadNode = TestUtils . getDatanodeDetails ( nodeManager ) ; nodeManager . sendHeartbeat ( healthyNode . getProtoBufMessage ( ) , null , reportState ) ; nodeManager . sendHeartbeat ( staleNode . getProtoBufMessage ( ) , null , reportState ) ; nodeManager . sendHeartbeat ( deadNode . getProtoBufMessage ( ) , null , reportState ) ; Thread . sleep ( 500 ) ; assertEquals ( 3 , nodeManager . getAllNodes ( ) . size ( ) ) ; assertEquals ( 3 , nodeManager . getNodeCount ( HEALTHY ) ) ; /**
       * Cluster state: Quiesced: We are going to sleep for 3 seconds. Which
       * means that no node is heartbeating. All nodes should move to Stale.
       */ Thread . sleep ( 3 * 1000 ) ; assertEquals ( 3 , nodeManager . getAllNodes ( ) . size ( ) ) ; assertEquals ( 3 , nodeManager . getNodeCount ( STALE ) ) ; /**
       * Cluster State : Move healthy node back to healthy state, move other 2
       * nodes to Stale State.
       *
       * We heartbeat healthy node after 1 second and let other 2 nodes elapse
       * the 3 second windows.
       */ nodeManager . sendHeartbeat ( healthyNode . getProtoBufMessage ( ) , null , reportState ) ; nodeManager . sendHeartbeat ( staleNode . getProtoBufMessage ( ) , null , reportState ) ; nodeManager . sendHeartbeat ( deadNode . getProtoBufMessage ( ) , null , reportState ) ; Thread . sleep ( 1500 ) ; nodeManager . sendHeartbeat ( healthyNode . getProtoBufMessage ( ) , null , reportState ) ; Thread . sleep ( 2 * 1000 ) ; assertEquals ( 1 , nodeManager . getNodeCount ( HEALTHY ) ) ; List < DatanodeDetails > healthyList = nodeManager . getNodes ( HEALTHY ) ; assertEquals ( "Expected one healthy node" , 1 , healthyList . size ( ) ) ; assertEquals ( "Healthy node is not the expected ID" , healthyNode . getUuid ( ) , healthyList . get ( 0 ) . getUuid ( ) ) ; assertEquals ( 2 , nodeManager . getNodeCount ( STALE ) ) ; /**
       * Cluster State: Allow healthyNode to remain in healthy state and
       * staleNode to move to stale state and deadNode to move to dead state.
       */ nodeManager . sendHeartbeat ( healthyNode . getProtoBufMessage ( ) , null , reportState ) ; nodeManager . sendHeartbeat ( staleNode . getProtoBufMessage ( ) , null , reportState ) ; Thread . sleep ( 1500 ) ; nodeManager . sendHeartbeat ( healthyNode . getProtoBufMessage ( ) , null , reportState ) ; Thread . sleep ( 2 * 1000 ) ; healthyList = nodeManager . getNodes ( HEALTHY ) ; List < DatanodeDetails > staleList = nodeManager . getNodes ( STALE ) ; List < DatanodeDetails > deadList = nodeManager . getNodes ( DEAD ) ; assertEquals ( 3 , nodeManager . getAllNodes ( ) . size ( ) ) ; assertEquals ( 1 , nodeManager . getNodeCount ( HEALTHY ) ) ; assertEquals ( 1 , nodeManager . getNodeCount ( STALE ) ) ; assertEquals ( 1 , nodeManager . getNodeCount ( DEAD ) ) ; assertEquals ( "Expected one healthy node" , 1 , healthyList . size ( ) ) ; assertEquals ( "Healthy node is not the expected ID" , healthyNode . getUuid ( ) , healthyList . get ( 0 ) . getUuid ( ) ) ; assertEquals ( "Expected one stale node" , 1 , staleList . size ( ) ) ; assertEquals ( "Stale node is not the expected ID" , staleNode . getUuid ( ) , staleList . get ( 0 ) . getUuid ( ) ) ; assertEquals ( "Expected one dead node" , 1 , deadList . size ( ) ) ; assertEquals ( "Dead node is not the expected ID" , deadNode . getUuid ( ) , deadList . get ( 0 ) . getUuid ( ) ) ; /**
       * Cluster State : let us heartbeat all the nodes and verify that we get
       * back all the nodes in healthy state.
       */ nodeManager . sendHeartbeat ( healthyNode . getProtoBufMessage ( ) , null , reportState ) ; nodeManager . sendHeartbeat ( staleNode . getProtoBufMessage ( ) , null , reportState ) ; nodeManager . sendHeartbeat ( deadNode . getProtoBufMessage ( ) , null , reportState ) ; Thread . sleep ( 500 ) ; assertEquals ( 3 , nodeManager . getAllNodes ( ) . size ( ) ) ; assertEquals ( 3 , nodeManager . getNodeCount ( HEALTHY ) ) ; } }
Asserts that a dead node, stale node and healthy nodes co-exist.
Asserts that a dead node stale node and healthy nodes co - exist .
// Sleep so that heartbeat processing thread gets to run. //Assert all nodes are healthy. // 3.5 seconds from last heartbeat for the stale and deadNode. So those //  2 nodes must move to Stale state and the healthy node must // remain in the healthy State. // 3.5 seconds have elapsed for stale node, so it moves into Stale. // 7 seconds have elapsed for dead node, so it moves into dead. // 2 Seconds have elapsed for healthy node, so it stays in healhty state. //Assert all nodes are healthy.