title: Mapreduce in Hadoop gives GC overhead limit exceeded when using files over 200MB
body: <p>I am running a Mapreduce code on Hadoop Multi-Node Cluster (2.4.1). I get the error GC overhead limit exceeded when I try to run with 2 input files of sizes 200MB and 200MB. This runs perfectly and got correct output when i use very small files.
  <BR>My aim is to compare every flow record in 1st file to every flow-record in 2nd file and compute distance, then take 10 maximum values and output to reducer based on these 10 max. values.<BR><BR>
  Example flow record in both files - 194.144.0.27|192.168.1.5|0.0.0.0|0|0|2|104|1410985350|1410985350|51915|51413|6|6<BR></p>
  
  <p>Few Snapshots: <a href="http://goo.gl/5tUhJJ" rel="nofollow">http://goo.gl/5tUhJJ</a>  and <a href="http://goo.gl/lh1Qvm" rel="nofollow">http://goo.gl/lh1Qvm</a>
  <BR></p>
  
  <p>Here is Mapper Class:</p>
  
  <p><strong>Mapper Class:</strong><BR><BR></p>
  
  <pre><code>public class mapper extends Mapper&lt;LongWritable, Text, Text, IntWritable&gt; 
  {
  
   private final static IntWritable five = new IntWritable(5);
  
   private Text counter1;
  
   ArrayList&lt;String&gt; lines = new ArrayList&lt;String&gt;();
   String str;
   BufferedReader br,in;
   int ddos_line = 0; 
   int normal_line = 0,total_testing_records=4000;
   int K = 10;
  
    @Override
    protected void setup(Context context) throws IOException, InterruptedException 
    { 
       //BufferedReader in = new BufferedReader(new FileReader("normal"));
  
        Configuration conf = context.getConfiguration();          
        URI[] cachefiles = context.getCacheFiles();
  
        FileSystem fs = FileSystem.get(new Configuration());          
        FileStatus[] status = fs.listStatus(new Path(cachefiles[0].toString()));            
        BufferedReader in=new BufferedReader(new InputStreamReader(fs.open(status[0].getPath()))); 
  
  
        while((str = in.readLine()) != null)
        {
            lines.add(str);
        }
        in.close();
        //System.out.println("na netti");
    }
  
  @Override
  public void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException 
  {
  
      String line1 = value.toString();
      ddos_line++;
      normal_line = 0;
  
      double[] count = {-1, -1, -1, -1, -1, -1, -1, -1, -1, -1};
      int[] lineIndex = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
  
      String[] parts = line1.split("\\|");
      String[] linesArray = lines.toArray(new String[lines.size()]);  
  
      boolean bool = true;
      int t1=0;
      double sum=0;
      while (bool) 
      {
          for(int i=0; i&lt;K;i++)
          {
                  if(bool==false) break;
                  sum = 0;
                  String[] parts2 = linesArray[normal_line].split("\\|");
  
                  for(int x=0;x&lt;13;x++)
                      {
                              if(parts[x].equals(parts2[x]))
                              {
                                  t1 = 1;
                              }
                              else t1 = 0;
  
                              sum += t1;
                      }
  
                      sum = Math.sqrt(sum);
  
                      if(count[K-1] &lt;= sum)
                      {
                          count[K-1] = sum;
                          lineIndex[K-1]=normal_line;
                      } 
  
  
  
                      for(int k=0;k&lt;K;k++)
                      {
                          for(int j=0;j&lt;K-1;j++)
                          {   
                              if(count[j] &lt; count[j+1]) 
                              {
                                  double temp2 = count[j+1];
                                  count[j+1] = count[j];
                                  count[j] = temp2;
  
                                  int temp3 = lineIndex[j+1];
                                  lineIndex[j+1] = lineIndex[j];
                                  lineIndex[j] = temp3;
                              }
                           }
                       }
  
                  //System.out.println(ddos_line + "   " + normal_line);
                  if (normal_line + 1 &lt; linesArray.length)
                  {
                      normal_line++;
                      continue;
                  } 
  
                  else bool = false;
              }
  
  
      } // while end
  
      char[] t = {'d','d','d','d','d','d','d','d','d','d'};
      for(int i=0;i&lt;K;i++)
      {
          if(lineIndex[i] &lt;= total_testing_records/2 ) t[i] = 'n'; 
      }
  
      int counter_normal=0, counter_ddos=0;
      for(int i=0;i&lt;K;i++)
      {
          if(t[i]=='n')
              counter_normal++;
          else
              counter_ddos++;
          //System.out.println("t[i]: "+t[i]+", counter: "+counter_ddos);
  
      }
  
      if(counter_normal&lt;=K/2)
      {
          counter1 = new Text(ddos_line + " : d : "+ counter_ddos);
      }
      else
      {
          counter1 = new Text(ddos_line + " : n : "+ (counter_normal));
      }
  
  
  
      context.write(counter1, five);
  
      //System.out.println("mapper finished");    
  }
    public void run(Context context) throws IOException, InterruptedException 
    {
        setup(context);
        while (context.nextKeyValue()) {
              map(context.getCurrentKey(), context.getCurrentValue(), context);
        }
        cleanup(context);
    }
  }
  </code></pre>