public static void main(String [] args){
        if (args.length != 2){
            displayHelp();
            return;
        }
        
        //Read the input data
        try{
            String option = args[0];
            if (option.equalsIgnoreCase("-inputfile")){
                BufferedReader in = new BufferedReader(
                        new InputStreamReader(new FileInputStream(args[1]), "UTF-8"));
                BufferedWriter out = new BufferedWriter(
                        new OutputStreamWriter(new FileOutputStream(args[1] + ".tkn") , "UTF-8"));

                String line = "";
                while ((line = in.readLine()) != null){                
                    out.write(PennTokenizer.tokenize(line));
                    out.write("\n");
                }
                
                in.close();
                out.close();
            }
            
            else if (option.equalsIgnoreCase("-inputdir")){
                System.out.println("Tokenize input");
                //segment only files ends with .sent
                File inputDir = new File(args[1]);
                File [] childrent = inputDir.listFiles(new FilenameFilter() {
                    public boolean accept(File dir, String name) {
                        return name.endsWith(".sent");
                    }
                });
                
                for (int i = 0; i < childrent.length; ++i){                    
                    BufferedReader in = new BufferedReader(
                            new InputStreamReader(new FileInputStream(childrent[i]), "UTF-8"));
                    BufferedWriter out = new BufferedWriter(
                            new OutputStreamWriter(new FileOutputStream(childrent[i] + ".tkn") , "UTF-8"));

                    String line = "";
                    while ((line = in.readLine()) != null){                
                        out.write(PennTokenizer.tokenize(line));
                        out.write("\n");
                    }

                    in.close();
                    out.close();    
                }
            }           
        } catch (Exception e){
            System.out.println("Error:" + e.getMessage());
        }
    }
public static void main ( String [ ] args ) { if ( args . length != 2 ) { displayHelp ( ) ; return ; } try { String option = args [ 0 ] ; if ( option . equalsIgnoreCase ( "-inputfile" ) ) { BufferedReader in = new BufferedReader ( new InputStreamReader ( new FileInputStream ( args [ 1 ] ) , "UTF-8" ) ) ; BufferedWriter out = new BufferedWriter ( new OutputStreamWriter ( new FileOutputStream ( args [ 1 ] + ".tkn" ) , "UTF-8" ) ) ; String line = "" ; while ( ( line = in . readLine ( ) ) != null ) { out . write ( PennTokenizer . tokenize ( line ) ) ; out . write ( "\n" ) ; } in . close ( ) ; out . close ( ) ; } else if ( option . equalsIgnoreCase ( "-inputdir" ) ) { System . out . println ( "Tokenize input" ) ; File inputDir = new File ( args [ 1 ] ) ; File [ ] childrent = inputDir . listFiles ( new FilenameFilter ( ) { public boolean accept ( File dir , String name ) { return name . endsWith ( ".sent" ) ; } } ) ; for ( int i = 0 ; i < childrent . length ; ++ i ) { BufferedReader in = new BufferedReader ( new InputStreamReader ( new FileInputStream ( childrent [ i ] ) , "UTF-8" ) ) ; BufferedWriter out = new BufferedWriter ( new OutputStreamWriter ( new FileOutputStream ( childrent [ i ] + ".tkn" ) , "UTF-8" ) ) ; String line = "" ; while ( ( line = in . readLine ( ) ) != null ) { out . write ( PennTokenizer . tokenize ( line ) ) ; out . write ( "\n" ) ; } in . close ( ) ; out . close ( ) ; } } } catch ( Exception e ) { System . out . println ( "Error:" + e . getMessage ( ) ) ; } }
The main method.

@param args the arguments