src/preprocess/TermExtractor.java
/*********************************************
	 * for a status input file format:
	 * %USER_ID%USERNAME%COUNT
	 * %USER_ID%STATUS_ID%RETWEET...
	 * 
	 * grabs all status from user and sends a request
	 * @param statusesFile
	 * @param termsOutputfile
	 * @param fromUserId 	to begin from user id
	 */


grabs all status from user and sends a request
@param statusesFile
@param termsOutputfile
@param fromUserId    to begin from user id
grabs all status from user and sends a request @param statusesFile @param termsOutputfile @param fromUserId to begin from user id
public void extractTermsInBatch(String statusesFile, String termsOutputfile, String fromUserId) throws Exception { 
		BufferedWriter out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(termsOutputfile),"UTF8")); 
		HashMap<Long, List<Status>> userStatuses = Input.readStatuses(statusesFile);
		int errorCount = 0;
		boolean fromHere =false;
		for(Long userId : userStatuses.keySet()) {
			if (fromUserId != null && userId == Long.parseLong(fromUserId)) fromHere = true;
			if (fromUserId != null && !fromHere)  
				continue;
			if (!userStatuses.containsKey(userId)) {
				System.out.println("no status detected for user: "+userId);
				continue;
			}
			String tweets = "";
			for (Status s : userStatuses.get(userId)){
				tweets += s.getText() + "\n";
			}
			try { 
			String extractedTerms = this.yahooTermExtractor(tweets).join(",");
			out.write(userId + Constants.NEW_IDENTIFIER + extractedTerms + System.getProperty("line.separator"));
			System.out.println("done for: "+userId);
			} catch (Exception e) {
				errorCount++;
				System.err.println("ERROR: "+e.getMessage());
				System.out.println("not possible for user "+userId + " skipping...");
				if (errorCount == 10) {
					out.close();
					System.exit(1);
				}
			}
		}
		out.close();
	}
public void extractTermsInBatch ( String statusesFile , String termsOutputfile , String fromUserId ) throws Exception { BufferedWriter out = new BufferedWriter ( new OutputStreamWriter ( new FileOutputStream ( termsOutputfile ) , "UTF8" ) ) ; HashMap < Long , List < Status > > userStatuses = Input . readStatuses ( statusesFile ) ; int errorCount = 0 ; boolean fromHere = false ; for ( Long userId : userStatuses . keySet ( ) ) { if ( fromUserId != null && userId == Long . parseLong ( fromUserId ) ) fromHere = true ; if ( fromUserId != null && ! fromHere ) continue ; if ( ! userStatuses . containsKey ( userId ) ) { System . out . println ( "no status detected for user: " + userId ) ; continue ; } String tweets = "" ; for ( Status s : userStatuses . get ( userId ) ) { tweets += s . getText ( ) + "\n" ; } try { String extractedTerms = this . yahooTermExtractor ( tweets ) . join ( "," ) ; out . write ( userId + Constants . NEW_IDENTIFIER + extractedTerms + System . getProperty ( "line.separator" ) ) ; System . out . println ( "done for: " + userId ) ; } catch ( Exception e ) { errorCount ++ ; System . err . println ( "ERROR: " + e . getMessage ( ) ) ; System . out . println ( "not possible for user " + userId + " skipping..." ) ; if ( errorCount == 10 ) { out . close ( ) ; System . exit ( 1 ) ; } } } out . close ( ) ; }
for a status input file format:
 %USER_ID%USERNAME%COUNT
 %USER_ID%STATUS_ID%RETWEET...
for a status input file format : %USER_ID%USERNAME%COUNT %USER_ID%STATUS_ID%RETWEET ...
// not the user id yet, go to next 
 //String extractedTerms = this.alchemyTermExtractor(tweets, CONCEPT_TAGGING, true).join(",");
