<s> import argparse , collections <EOL> import nltk . corpus <EOL> from nltk . tree import Tree <EOL> from nltk . corpus . util import LazyCorpusLoader <EOL> from nltk_trainer import load_corpus_reader , simplify_wsj_tag <EOL> from nltk_trainer . chunking . transforms import node_label <EOL> parser = argparse . ArgumentParser ( description = '<STR_LIT>' , <EOL> formatter_class = argparse . RawTextHelpFormatter ) <EOL> parser . add_argument ( '<STR_LIT>' , <EOL> help = '''<STR_LIT>''' ) <EOL> parser . add_argument ( '<STR_LIT>' , default = <NUM_LIT:1> , type = int , <EOL> help = '<STR_LIT>' ) <EOL> corpus_group = parser . add_argument_group ( '<STR_LIT>' ) <EOL> corpus_group . add_argument ( '<STR_LIT>' , default = None , <EOL> help = '''<STR_LIT>''' ) <EOL> corpus_group . add_argument ( '<STR_LIT>' , default = None , <EOL> help = '<STR_LIT>' ) <EOL> if simplify_wsj_tag : <EOL> corpus_group . add_argument ( '<STR_LIT>' , action = '<STR_LIT:store_true>' , default = False , <EOL> help = '<STR_LIT>' ) <EOL> sort_group = parser . add_argument_group ( '<STR_LIT>' ) <EOL> sort_group . add_argument ( '<STR_LIT>' , default = '<STR_LIT>' , choices = [ '<STR_LIT>' , '<STR_LIT:count>' ] , <EOL> help = '<STR_LIT>' ) <EOL> sort_group . add_argument ( '<STR_LIT>' , action = '<STR_LIT:store_true>' , default = False , <EOL> help = '<STR_LIT>' ) <EOL> args = parser . parse_args ( ) <EOL> chunked_corpus = load_corpus_reader ( args . corpus , reader = args . reader , fileids = args . fileids ) <EOL> if not chunked_corpus : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> if args . trace : <EOL> print ( '<STR_LIT>' % args . corpus ) <EOL> wc = <NUM_LIT:0> <EOL> tag_counts = collections . defaultdict ( int ) <EOL> iob_counts = collections . defaultdict ( int ) <EOL> tag_iob_counts = collections . defaultdict ( lambda : collections . defaultdict ( int ) ) <EOL> word_set = set ( ) <EOL> for obj in chunked_corpus . chunked_words ( ) : <EOL> if isinstance ( obj , Tree ) : <EOL> label = node_label ( obj ) <EOL> iob_counts [ label ] += <NUM_LIT:1> <EOL> for word , tag in obj . leaves ( ) : <EOL> wc += <NUM_LIT:1> <EOL> word_set . add ( word ) <EOL> tag_counts [ tag ] += <NUM_LIT:1> <EOL> tag_iob_counts [ tag ] [ label ] += <NUM_LIT:1> <EOL> else : <EOL> word , tag = obj <EOL> wc += <NUM_LIT:1> <EOL> word_set . add ( word ) <EOL> tag_counts [ tag ] += <NUM_LIT:1> <EOL> print ( '<STR_LIT>' % wc ) <EOL> print ( '<STR_LIT>' % len ( word_set ) ) <EOL> print ( '<STR_LIT>' % len ( tag_counts ) ) <EOL> print ( '<STR_LIT>' % len ( iob_counts ) ) <EOL> if args . sort == '<STR_LIT>' : <EOL> sort_key = lambda tc : tc [ <NUM_LIT:0> ] <EOL> elif args . sort == '<STR_LIT:count>' : <EOL> sort_key = lambda tc : tc [ <NUM_LIT:1> ] <EOL> else : <EOL> raise ValueError ( '<STR_LIT>' % args . sort ) <EOL> line1 = '<STR_LIT>' <EOL> line2 = '<STR_LIT>' <EOL> iobs = sorted ( iob_counts . keys ( ) ) <EOL> for iob in iobs : <EOL> line1 += '<STR_LIT>' % iob <EOL> line2 += '<STR_LIT>' % ( '<STR_LIT:=>' * len ( iob ) ) <EOL> print ( line1 ) <EOL> print ( line2 ) <EOL> for tag , count in sorted ( tag_counts . items ( ) , key = sort_key , reverse = args . reverse ) : <EOL> iob_counts = [ str ( tag_iob_counts [ tag ] [ iob ] ) . rjust ( <NUM_LIT:4> + len ( iob ) ) for iob in iobs ] <EOL> print ( '<STR_LIT:U+0020>' . join ( [ tag . ljust ( <NUM_LIT:7> ) , str ( count ) . rjust ( <NUM_LIT:9> ) ] + iob_counts ) ) <EOL> print ( line2 ) </s>
