src/preprocess/TermExtractor.java
/*
	 * Extract terms using Yahoo term extractor service
	 */
Extract terms using Yahoo term extractor service
Extract terms using Yahoo term extractor service
private JSONArray yahooTermExtractor(String tweet) throws Exception {
		connection = (HttpURLConnection) url.openConnection();
		connection.setDoOutput(true);
		connection.setDoInput(true);
		connection.setInstanceFollowRedirects(false);
		connection.setAllowUserInteraction(true);
		 String data = URLEncoder.encode("appid", "UTF-8") + "=" + URLEncoder.encode(Constants.YAHOO_API_KEY, "UTF-8");
		    data += "&" + URLEncoder.encode("context", "UTF-8") + "=" + URLEncoder.encode(tweet, "UTF-8");
		    data += "&" + URLEncoder.encode("output", "UTF-8") + "=" + URLEncoder.encode("json", "UTF-8");
		    OutputStreamWriter wr = new OutputStreamWriter(connection.getOutputStream());
		    wr.write(data);
		    wr.flush();
		    int tryouts = 0;
		    BufferedReader rd = null;
		    while (rd == null) {
			    try {
			    	 rd = new BufferedReader(new InputStreamReader(connection.getInputStream()));
			    }catch (IOException e) {
			    	System.err.println(e.getMessage());
			    	if (tryouts > 5) {
			    		System.out.println("impossible to fetch data, skipping...");
			    		return new JSONArray();
			    	}
			    	System.out.println("waiting 60 secondes before next try...");
			    	Thread.sleep(60*1000);
			    	tryouts++;
			    }
		    }
		    String line;
		    String jsonReponse = "";
		    while ((line = rd.readLine()) != null) {
		    	jsonReponse += line;
		    }
		    JSONObject obj = new JSONObject(jsonReponse);
		    JSONArray terms = (JSONArray) ((JSONObject) obj.get("ResultSet")).get("Result");
		    wr.close();
			rd.close();
		    return terms;
	}
private JSONArray yahooTermExtractor ( String tweet ) throws Exception { connection = ( HttpURLConnection ) url . openConnection ( ) ; connection . setDoOutput ( true ) ; connection . setDoInput ( true ) ; connection . setInstanceFollowRedirects ( false ) ; connection . setAllowUserInteraction ( true ) ; String data = URLEncoder . encode ( "appid" , "UTF-8" ) + "=" + URLEncoder . encode ( Constants . YAHOO_API_KEY , "UTF-8" ) ; data += "&" + URLEncoder . encode ( "context" , "UTF-8" ) + "=" + URLEncoder . encode ( tweet , "UTF-8" ) ; data += "&" + URLEncoder . encode ( "output" , "UTF-8" ) + "=" + URLEncoder . encode ( "json" , "UTF-8" ) ; OutputStreamWriter wr = new OutputStreamWriter ( connection . getOutputStream ( ) ) ; wr . write ( data ) ; wr . flush ( ) ; int tryouts = 0 ; BufferedReader rd = null ; while ( rd == null ) { try { rd = new BufferedReader ( new InputStreamReader ( connection . getInputStream ( ) ) ) ; } catch ( IOException e ) { System . err . println ( e . getMessage ( ) ) ; if ( tryouts > 5 ) { System . out . println ( "impossible to fetch data, skipping..." ) ; return new JSONArray ( ) ; } System . out . println ( "waiting 60 secondes before next try..." ) ; Thread . sleep ( 60 * 1000 ) ; tryouts ++ ; } } String line ; String jsonReponse = "" ; while ( ( line = rd . readLine ( ) ) != null ) { jsonReponse += line ; } JSONObject obj = new JSONObject ( jsonReponse ) ; JSONArray terms = ( JSONArray ) ( ( JSONObject ) obj . get ( "ResultSet" ) ) . get ( "Result" ) ; wr . close ( ) ; rd . close ( ) ; return terms ; }
Extract terms using Yahoo term extractor service
Extract terms using Yahoo term extractor service
// Get the response
 // Process line...
 //ps.print(line);
 //terms.join(",");
