public void sample(String input,String output,int numPosts) throws IOException {
  Random random=new Random();
  JsonUtil J=new JsonUtil();
  Map<String,List<String>> userposts=new Hashtable<String,List<String>>();
  RecordReader reader=new RecordReader(input);
  while (reader.next()) {
    Post p=J.fromJson(reader.value(),Post.class);
    String userid=p.getUserId();
    List<String> posts=userposts.get(userid);
    if (posts == null) {
      posts=new ArrayList<String>();
      userposts.put(userid,posts);
    }
    posts.add(reader.value());
    if (reader.numRead() % 1000 == 0) {
      LOG.info("read: " + reader.numRead());
    }
  }
  LOG.info("total " + reader.numRead() + " posts and "+ userposts.size()+ "users");
  reader.close();
  RecordWriter writer=new RecordWriter(output);
  int total=0;
  for (  Entry<String,List<String>> e : userposts.entrySet()) {
    List<String> posts=e.getValue();
    int n=0;
    while (n < numPosts && posts.size() > 0) {
      writer.add(posts.remove(random.nextInt(posts.size())));
      n++;
    }
    total+=n;
  }
  LOG.info("write " + total + " posts.");
  writer.close();
}

public void sample ( String input , String output , int numPosts ) throws IOException { Random random = new Random ( ) ; JsonUtil J = new JsonUtil ( ) ; Map < String , List < String > > userposts = new Hashtable < String , List < String > > ( ) ; RecordReader reader = new RecordReader ( input ) ; while ( reader . next ( ) ) { Post p = J . fromJson ( reader . value ( ) , Post . class ) ; String userid = p . getUserId ( ) ; List < String > posts = userposts . get ( userid ) ; if ( posts == null ) { posts = new ArrayList < String > ( ) ; userposts . put ( userid , posts ) ; } posts . add ( reader . value ( ) ) ; if ( reader . numRead ( ) % 1000 == 0 ) { LOG . info ( "read: " + reader . numRead ( ) ) ; } } LOG . info ( "total " + reader . numRead ( ) + " posts and " + userposts . size ( ) + "users" ) ; reader . close ( ) ; RecordWriter writer = new RecordWriter ( output ) ; int total = 0 ; for ( Entry < String , List < String > > e : userposts . entrySet ( ) ) { List < String > posts = e . getValue ( ) ; int n = 0 ; while ( n < numPosts && posts . size ( ) > 0 ) { writer . add ( posts . remove ( random . nextInt ( posts . size ( ) ) ) ) ; n ++ ; } total += n ; } LOG . info ( "write " + total + " posts." ) ; writer . close ( ) ; }
load all data into the memory , then sample numposts posts for each user .
