title: Convert Lucene Index to Mahout vectors
body: <p>I have a Spring Web Application. It maps the model Education to a lucene index through Hibernate Search:</p>
  
  <pre><code>@Entity
  @Table(name="educations")
  @Indexed
  public class Education {
  
      @Id
      @GeneratedValue(strategy = GenerationType.AUTO)
      @Field(termVector = TermVector.WITH_POSITION_OFFSETS)
      private long id;
  
      @Column(name = "name")
      @Field(termVector = TermVector.WITH_POSITION_OFFSETS)
      @Boost(value = 1.5f)
      private String name;
  
      @Column(name = "local_name")
      private String localName;
  
      @Column(name = "description", columnDefinition="TEXT")
      @Field(termVector = TermVector.WITH_POSITION_OFFSETS)
      private String description;
  </code></pre>
  
  <p>This works great!</p>
  
  <p>Now im trying to cluster my Lucene index through Mahout 0.9. I got a basic K-means clustering going but I don't know how to convert my Lucene Index to Mahout vectors.</p>
  
  <p>Here is my basic K-Means clustering class that works with some test data points as shown below:</p>
  
  <pre><code>package com.courseportal.project.utils.lsh.util;
  
  import org.apache.hadoop.conf.Configuration;
  import org.apache.hadoop.fs.FileSystem;
  import org.apache.hadoop.fs.Path;
  import org.apache.hadoop.io.IntWritable;
  import org.apache.hadoop.io.LongWritable;
  import org.apache.hadoop.io.SequenceFile;
  import org.apache.hadoop.io.Text;
  import org.apache.mahout.clustering.Cluster;
  import org.apache.mahout.clustering.classify.WeightedPropertyVectorWritable;
  import org.apache.mahout.clustering.kmeans.KMeansDriver;
  import org.apache.mahout.clustering.kmeans.Kluster;
  import org.apache.mahout.common.distance.EuclideanDistanceMeasure;
  import org.apache.mahout.math.RandomAccessSparseVector;
  import org.apache.mahout.math.Vector;
  import org.apache.mahout.math.VectorWritable;
  import java.io.File;
  import java.io.IOException;
  import java.util.ArrayList;
  import java.util.List;
  
  public class SimpleKMeansClustering {
  
      public static final double[][] points = {
              {1, 1}, {2, 1}, {1, 2},
              {2, 2}, {3, 3}, {8, 8},
              {9, 8}, {8, 9}, {9, 9}};
  
      public static void writePointsToFile(List&lt;Vector&gt; points,
                                           String fileName,
                                           FileSystem fs,
                                           Configuration conf) throws IOException {
          Path path = new Path(fileName);
          SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf,
                  path, LongWritable.class, VectorWritable.class);
          long recNum = 0;
          VectorWritable vec = new VectorWritable();
          for (Vector point : points) {
              vec.set(point);
              writer.append(new LongWritable(recNum++), vec);
          }
          writer.close();
      }
  
      public static List&lt;Vector&gt; getPoints(double[][] raw) {
          List&lt;Vector&gt; points = new ArrayList&lt;Vector&gt;();
          for (int i = 0; i &lt; raw.length; i++) {
              double[] fr = raw[i];
              Vector vec = new RandomAccessSparseVector(fr.length);
              vec.assign(fr);
              points.add(vec);
  
          }
          return points;
      }
  
      public static void main(String args[]) throws Exception {
  
          int k = 2;
  
          List&lt;Vector&gt; vectors = getPoints(points);
  
          File testData = new File("clustering/testdata");
          if (!testData.exists()) {
              testData.mkdir();
          }
          testData = new File("clustering/testdata/points");
          if (!testData.exists()) {
              testData.mkdir();
          }
  
          Configuration conf = new Configuration();
          FileSystem fs = FileSystem.get(conf);
          writePointsToFile(vectors, "clustering/testdata/points/file1", fs, conf);
  
          Path path = new Path("clustering/testdata/clusters/part-00000");
          SequenceFile.Writer writer = new SequenceFile.Writer(fs, conf, path, Text.class, Kluster.class);
  
          for (int i = 0; i &lt; k; i++) {
              Vector vec = vectors.get(i);
              Kluster cluster = new Kluster(vec, i, new EuclideanDistanceMeasure());
              writer.append(new Text(cluster.getIdentifier()), cluster);
          }
          writer.close();
  
          KMeansDriver.run(conf,
                  new Path("clustering/testdata/points"),
                  new Path("clustering/testdata/clusters"),
                  new Path("clustering/output"),
                  0.001,
                  10,
                  true,
                  0,
                  true);
  
          SequenceFile.Reader reader = new SequenceFile.Reader(fs,
                  new Path("clustering/output/" + Cluster.CLUSTERED_POINTS_DIR + "/part-m-0"), conf);
  
          IntWritable key = new IntWritable();
          WeightedPropertyVectorWritable value = new WeightedPropertyVectorWritable();
          while (reader.next(key, value)) {
              System.out.println(value.toString() + " belongs to cluster " + key.toString());
          }
          reader.close();
      }
  }
  </code></pre>
  
  <p>I read (<a href="https://www.google.com.hk/url?sa=t&amp;rct=j&amp;q=&amp;esrc=s&amp;source=web&amp;cd=1&amp;ved=0CCsQFjAA&amp;url=https://blog.trifork.com/2012/03/05/using-your-lucene-index-as-input-to-your-mahout-job-part-i/&amp;ei=XwE8U94dpv3KA4mVgPAK&amp;usg=AFQjCNFnC_nZ3osQENDKLeuMsduOH1J4jQ" rel="nofollow">here</a>) that I should use LuceneIndexToSequenceFiles to do this but I can't find that class in Mahout 0.9. Is this something that I have to pull in manually?</p>
  
  <p>How can I convert my index to work with my K-means clustering class?</p>