public void testGenerateHostLimit() throws Exception {
  ArrayList<URLCrawlDatum> list=new ArrayList<URLCrawlDatum>();
  list.add(createURLCrawlDatum("http://www.example.com/index1.html",1,1));
  list.add(createURLCrawlDatum("http://www.example.com/index2.html",1,1));
  list.add(createURLCrawlDatum("http://www.example.com/index3.html",1,1));
  createCrawlDB(list);
  Configuration myConfiguration=new Configuration(conf);
  myConfiguration.setInt(Generator.GENERATOR_MAX_COUNT,2);
  Path generatedSegment=generateFetchlist(Integer.MAX_VALUE,myConfiguration,false);
  Path fetchlistPath=new Path(new Path(generatedSegment,CrawlDatum.GENERATE_DIR_NAME),"part-00000");
  ArrayList<URLCrawlDatum> fetchList=readContents(fetchlistPath);
  assertEquals(1,fetchList.size());
  myConfiguration=new Configuration(conf);
  myConfiguration.setInt(Generator.GENERATOR_MAX_COUNT,3);
  generatedSegment=generateFetchlist(Integer.MAX_VALUE,myConfiguration,false);
  fetchlistPath=new Path(new Path(generatedSegment,CrawlDatum.GENERATE_DIR_NAME),"part-00000");
  fetchList=readContents(fetchlistPath);
  assertEquals(2,fetchList.size());
  myConfiguration=new Configuration(conf);
  myConfiguration.setInt(Generator.GENERATOR_MAX_COUNT,4);
  generatedSegment=generateFetchlist(Integer.MAX_VALUE,myConfiguration,false);
  fetchlistPath=new Path(new Path(generatedSegment,CrawlDatum.GENERATE_DIR_NAME),"part-00000");
  fetchList=readContents(fetchlistPath);
  assertEquals(3,fetchList.size());
}

public void testGenerateHostLimit ( ) throws Exception { ArrayList < URLCrawlDatum > list = new ArrayList < URLCrawlDatum > ( ) ; list . add ( createURLCrawlDatum ( "http://www.example.com/index1.html" , 1 , 1 ) ) ; list . add ( createURLCrawlDatum ( "http://www.example.com/index2.html" , 1 , 1 ) ) ; list . add ( createURLCrawlDatum ( "http://www.example.com/index3.html" , 1 , 1 ) ) ; createCrawlDB ( list ) ; Configuration myConfiguration = new Configuration ( conf ) ; myConfiguration . setInt ( Generator . GENERATOR_MAX_COUNT , 2 ) ; Path generatedSegment = generateFetchlist ( Integer . MAX_VALUE , myConfiguration , false ) ; Path fetchlistPath = new Path ( new Path ( generatedSegment , CrawlDatum . GENERATE_DIR_NAME ) , "part-00000" ) ; ArrayList < URLCrawlDatum > fetchList = readContents ( fetchlistPath ) ; assertEquals ( 1 , fetchList . size ( ) ) ; myConfiguration = new Configuration ( conf ) ; myConfiguration . setInt ( Generator . GENERATOR_MAX_COUNT , 3 ) ; generatedSegment = generateFetchlist ( Integer . MAX_VALUE , myConfiguration , false ) ; fetchlistPath = new Path ( new Path ( generatedSegment , CrawlDatum . GENERATE_DIR_NAME ) , "part-00000" ) ; fetchList = readContents ( fetchlistPath ) ; assertEquals ( 2 , fetchList . size ( ) ) ; myConfiguration = new Configuration ( conf ) ; myConfiguration . setInt ( Generator . GENERATOR_MAX_COUNT , 4 ) ; generatedSegment = generateFetchlist ( Integer . MAX_VALUE , myConfiguration , false ) ; fetchlistPath = new Path ( new Path ( generatedSegment , CrawlDatum . GENERATE_DIR_NAME ) , "part-00000" ) ; fetchList = readContents ( fetchlistPath ) ; assertEquals ( 3 , fetchList . size ( ) ) ; }
test that generator obeys the property " generate . max . per . host " .
