title: Computing Jaccard Similarity in Java
body: <p>I have following the code that loop through array list(mainItems) and find the most similar two arrays and put them in sortedTransactions. It is working fine for small data (10000 transactions) but it is running forever for 88000 transactions. What can be done to make it work for big data.</p>
  
  <pre><code>import java.util.*;
  
      public class Sort {
  
      static private  List&lt;Transactions&gt; trans = ReadFile.transactions;
      static public   List&lt;int[]&gt; mainItems;
      static public   ArrayList&lt;int[]&gt; sortedTransactions = new ArrayList&lt;int[]&gt;();
  
      static {
          mainItems = new ArrayList&lt;int[]&gt;();
  
          for (Transactions t : trans) {
              mainItems.add(t.getItems());
          }
      }
  
      static private  double jaccardSimilarity(int[] a, int[] b) {
  
          Set&lt;Integer&gt; s1 = new LinkedHashSet&lt;Integer&gt;();
          for(int i =0; i&lt; a.length; i++){
              s1.add(a[i]);
          }
          Set&lt;Integer&gt; s2 = new LinkedHashSet&lt;Integer&gt;();
          for(int i =0; i&lt; b.length; i++){
              s2.add(b[i]);
          }
  
          Set&lt;Integer&gt; intersection = new LinkedHashSet&lt;&gt;(s1);
          intersection.retainAll(s2);
  
          Set&lt;Integer&gt; union = new LinkedHashSet&lt;Integer&gt;(s1); 
          union.addAll(s2); 
  
          double jaccardSimilarity = (double)intersection.size()/ (double)union.size();
          //System.out.println(intersection);
          return jaccardSimilarity;
      }
  
      static private boolean isAllEqual(List&lt;Double&gt; a){
  
          for(int i=1; i&lt;a.size(); i++){
              if(a.get(0) != a.get(i)){
                  return false;
              }
          }
  
          return true;
      }
  
  
      static public void generatePairs() {
  
          for (int i = 0; i &lt; mainItems.size() - 1; i++) {
  
              if (!sortedTransactions.contains(mainItems.get(i))) {
  
                  List&lt;Double&gt; myd = new ArrayList&lt;Double&gt;();
                  List&lt;int[]&gt; mys = new ArrayList&lt;int[]&gt;();
  
                  for (int j = i + 1; j &lt; mainItems.size(); j++) {
  
                      if (!sortedTransactions.contains(mainItems.get(j))) {
  
                          myd.add(jaccardSimilarity(mainItems.get(i),mainItems.get(j)));
                          mys.add(mainItems.get(j));
                      }
                  }
  
                  if (isAllEqual(myd) == false) {
  
                      sortedTransactions.add(mainItems.get(i));
                      sortedTransactions.add(mys.get(maxValue(myd)));
                  }
              }
          }
      }
  
      static private int maxValue(List&lt;Double&gt; d) {
  
          double max = d.get(0);
          int f = 0;
  
          for(int i =1; i&lt; d.size(); i++){
  
              if(d.get(i) &gt; max){
  
                  max= d.get(i); 
                  f= i; 
              }
          }
          return f;
      }
  }
  </code></pre>