<s> """<STR_LIT>""" <EOL> from collections import Counter , defaultdict <EOL> import hashlib <EOL> import random <EOL> import re <EOL> import math <EOL> import nltk <EOL> import numpy as np <EOL> import pandas as pd <EOL> from . . common import smart_open , DocIDError <EOL> from . . common_abc import SaveLoad <EOL> from . import nlp <EOL> from . import streaming_filters <EOL> class BaseTokenizer ( SaveLoad ) : <EOL> """<STR_LIT>""" <EOL> def text_to_counter ( self , text ) : <EOL> """<STR_LIT>""" <EOL> return Counter ( self . text_to_token_list ( text ) ) <EOL> class MakeTokenizer ( BaseTokenizer ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , tokenizer_func ) : <EOL> """<STR_LIT>""" <EOL> self . text_to_token_list = tokenizer_func <EOL> class TokenizerBasic ( BaseTokenizer ) : <EOL> """<STR_LIT>""" <EOL> def text_to_token_list ( self , text ) : <EOL> """<STR_LIT>""" <EOL> tokens = nlp . word_tokenize ( text , L = <NUM_LIT:2> , numeric = False ) <EOL> return [ word . lower ( ) for word in tokens if not nlp . is_stopword ( word ) ] <EOL> class TokenizerPOSFilter ( BaseTokenizer ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( <EOL> self , pos_types = [ ] , sent_tokenizer = nltk . sent_tokenize , <EOL> word_tokenizer = TokenizerBasic ( ) , word_tokenizer_func = None , <EOL> pos_tagger = nltk . pos_tag ) : <EOL> """<STR_LIT>""" <EOL> self . pos_types = set ( pos_types ) <EOL> self . sent_tokenizer = sent_tokenizer <EOL> self . pos_tagger = pos_tagger <EOL> if not word_tokenizer : <EOL> self . word_tokenizer = MakeTokenizer ( word_tokenizer_func ) <EOL> else : <EOL> self . word_tokenizer = word_tokenizer <EOL> def text_to_token_list ( self , text ) : <EOL> """<STR_LIT>""" <EOL> sentences = self . sent_tokenizer ( text ) <EOL> func = self . word_tokenizer . text_to_token_list <EOL> tokenized_sentences = [ func ( sent ) for sent in sentences ] <EOL> tagged_sentences = [ <EOL> self . pos_tagger ( sent ) for sent in tokenized_sentences ] <EOL> token_list = sum ( <EOL> [ self . _sent_filter ( sent ) for sent in tagged_sentences ] , [ ] ) <EOL> return token_list <EOL> def _sent_filter ( self , tokenized_sent ) : <EOL> return [ <EOL> word for ( word , pos ) in tokenized_sent if pos in self . pos_types ] <EOL> class SparseFormatter ( object ) : <EOL> """<STR_LIT>""" <EOL> def _parse_feature_str ( self , feature_str ) : <EOL> """<STR_LIT>""" <EOL> assert feature_str [ <NUM_LIT:0> ] == '<STR_LIT:U+0020>' <EOL> feature_str = feature_str [ <NUM_LIT:1> : ] <EOL> fv_list = re . findall ( r'<STR_LIT>' , feature_str ) <EOL> feature_values = { <EOL> f : self . _string_to_number ( v , empty_sub = <NUM_LIT:1> ) for ( f , v ) in fv_list } <EOL> return feature_values <EOL> def sstr_to_dict ( self , sstr ) : <EOL> """<STR_LIT>""" <EOL> sstr = sstr . rstrip ( '<STR_LIT:\n>' ) . rstrip ( '<STR_LIT:\r>' ) <EOL> idx = sstr . index ( self . preamble_char ) <EOL> preamble , feature_str = sstr [ : idx ] , sstr [ idx + <NUM_LIT:1> : ] <EOL> record_dict = self . _parse_preamble ( preamble ) <EOL> record_dict [ '<STR_LIT>' ] = self . _parse_feature_str ( feature_str ) <EOL> return record_dict <EOL> def sstr_to_info ( self , sstr ) : <EOL> """<STR_LIT>""" <EOL> info = self . sstr_to_dict ( sstr ) <EOL> info [ '<STR_LIT>' ] = self . _dict_to_tokens ( info ) <EOL> return info <EOL> def _dict_to_tokens ( self , record_dict ) : <EOL> token_list = [ ] <EOL> if '<STR_LIT>' in record_dict : <EOL> for feature , value in record_dict [ '<STR_LIT>' ] . iteritems ( ) : <EOL> int_value = int ( value ) <EOL> assert int_value == value <EOL> token_list += [ feature ] * int_value <EOL> return token_list <EOL> def sstr_to_token_list ( self , sstr ) : <EOL> """<STR_LIT>""" <EOL> record_dict = self . sstr_to_dict ( sstr ) <EOL> return self . _dict_to_tokens ( record_dict ) <EOL> def sfile_to_token_iter ( self , filepath_or_buffer , limit = None ) : <EOL> """<STR_LIT>""" <EOL> with smart_open ( filepath_or_buffer ) as open_file : <EOL> for index , line in enumerate ( open_file ) : <EOL> if index == limit : <EOL> raise StopIteration <EOL> yield self . sstr_to_token_list ( line ) <EOL> def _string_to_number ( self , string , empty_sub = None ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> return int ( string ) <EOL> except ValueError : <EOL> pass <EOL> try : <EOL> return float ( string ) <EOL> except ValueError : <EOL> if ( string == '<STR_LIT>' ) and ( empty_sub is not None ) : <EOL> return empty_sub <EOL> else : <EOL> raise <EOL> class VWFormatter ( SparseFormatter ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self ) : <EOL> self . format_name = '<STR_LIT>' <EOL> self . preamble_char = '<STR_LIT:|>' <EOL> def get_sstr ( <EOL> self , feature_values = None , target = None , importance = None , doc_id = None ) : <EOL> """<STR_LIT>""" <EOL> if doc_id : <EOL> if re . search ( r"<STR_LIT>" , doc_id ) : <EOL> msg = ( <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> raise DocIDError ( msg ) <EOL> assert importance is not None <EOL> formatted = "<STR_LIT>" % doc_id <EOL> else : <EOL> formatted = "<STR_LIT>" <EOL> if importance : <EOL> formatted = "<STR_LIT:U+0020>" + str ( importance ) + formatted <EOL> if target : <EOL> formatted = str ( target ) + formatted <EOL> formatted += '<STR_LIT:U+0020>' <EOL> for word , count in feature_values . iteritems ( ) : <EOL> formatted += "<STR_LIT>" % ( word , count ) <EOL> if len ( feature_values ) > <NUM_LIT:0> : <EOL> formatted = formatted . rstrip ( ) <EOL> return formatted <EOL> def _parse_preamble ( self , preamble ) : <EOL> """<STR_LIT>""" <EOL> if preamble [ - <NUM_LIT:1> ] != '<STR_LIT:U+0020>' : <EOL> doc_id_left = preamble . rfind ( '<STR_LIT:U+0020>' ) <EOL> doc_id = preamble [ doc_id_left + <NUM_LIT:1> : ] <EOL> preamble = preamble [ : doc_id_left ] <EOL> else : <EOL> doc_id = None <EOL> in_target = True <EOL> target = '<STR_LIT>' <EOL> importance = '<STR_LIT>' <EOL> for char in preamble : <EOL> if char == '<STR_LIT:U+0020>' : <EOL> in_target = False <EOL> elif in_target : <EOL> target += char <EOL> else : <EOL> importance += char <EOL> parsed = { } <EOL> items = ( <EOL> ( '<STR_LIT>' , doc_id ) , ( '<STR_LIT:target>' , target ) , ( '<STR_LIT>' , importance ) ) <EOL> for key , value in items : <EOL> if value : <EOL> if key in [ '<STR_LIT:target>' , '<STR_LIT>' ] : <EOL> parsed [ key ] = self . _string_to_number ( value ) <EOL> else : <EOL> parsed [ key ] = value <EOL> return parsed <EOL> class SVMLightFormatter ( SparseFormatter ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self ) : <EOL> """<STR_LIT:U+0020>""" <EOL> self . format_name = '<STR_LIT>' <EOL> self . preamble_char = '<STR_LIT:U+0020>' <EOL> def get_sstr ( <EOL> self , feature_values = None , target = <NUM_LIT:1> , importance = None , doc_id = None ) : <EOL> """<STR_LIT>""" <EOL> formatted = str ( target ) + '<STR_LIT:U+0020>' <EOL> for word , count in feature_values . iteritems ( ) : <EOL> formatted += "<STR_LIT>" % ( word , count ) <EOL> return formatted <EOL> def _parse_preamble ( self , preamble ) : <EOL> return { '<STR_LIT:target>' : float ( preamble ) } <EOL> class SFileFilter ( SaveLoad ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , formatter , bit_precision = <NUM_LIT> , sfile = None , verbose = True ) : <EOL> """<STR_LIT>""" <EOL> assert isinstance ( bit_precision , int ) <EOL> self . formatter = formatter <EOL> self . bit_precision = bit_precision <EOL> self . verbose = verbose <EOL> self . precision = <NUM_LIT:2> ** bit_precision <EOL> self . sfile_loaded = False <EOL> self . bit_precision_required = bit_precision <EOL> if sfile is not None : <EOL> self . load_sfile ( sfile ) <EOL> def _get_hash_fun ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . bit_precision <= <NUM_LIT:64> : <EOL> hash_fun = lambda w : hash ( w ) % self . precision <EOL> elif self . bit_precision <= <NUM_LIT> : <EOL> hash_fun = lambda w : ( <EOL> int ( hashlib . sha224 ( w ) . hexdigest ( ) , <NUM_LIT:16> ) % self . precision ) <EOL> else : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> return hash_fun <EOL> def load_sfile ( self , sfile ) : <EOL> """<STR_LIT>""" <EOL> assert not self . sfile_loaded <EOL> token2id , token_score , doc_freq , num_docs , idf = ( <EOL> self . _load_sfile_fwd ( sfile ) ) <EOL> self . token2id = token2id <EOL> self . token_score = token_score <EOL> self . doc_freq = doc_freq <EOL> self . num_docs = num_docs <EOL> self . idf = idf <EOL> self . sfile_loaded = True <EOL> self . collisions_resolved = False <EOL> def _load_sfile_fwd ( self , sfile ) : <EOL> """<STR_LIT>""" <EOL> token2id = { } <EOL> token_score = defaultdict ( float ) <EOL> doc_freq = defaultdict ( int ) <EOL> num_docs = <NUM_LIT:0> <EOL> idf = defaultdict ( float ) <EOL> hash_fun = self . _get_hash_fun ( ) <EOL> with smart_open ( sfile ) as open_file : <EOL> for line in open_file : <EOL> num_docs += <NUM_LIT:1> <EOL> record_dict = self . formatter . sstr_to_dict ( line ) <EOL> for token , value in record_dict [ '<STR_LIT>' ] . iteritems ( ) : <EOL> hash_value = hash_fun ( token ) <EOL> token2id [ token ] = hash_value <EOL> token_score [ token ] += value <EOL> doc_freq [ token ] += <NUM_LIT:1> <EOL> idf [ token ] += <NUM_LIT:1> <EOL> for token in idf . iterkeys ( ) : <EOL> idf [ token ] = math . log ( num_docs / idf [ token ] ) <EOL> return token2id , token_score , doc_freq , num_docs , idf <EOL> def set_id2token ( self , seed = None ) : <EOL> """<STR_LIT>""" <EOL> self . _resolve_collisions ( seed = seed ) <EOL> self . id2token = { v : k for k , v in self . token2id . iteritems ( ) } <EOL> def _resolve_collisions ( self , seed = None ) : <EOL> """<STR_LIT>""" <EOL> id_counts = Counter ( self . token2id . values ( ) ) <EOL> vocab_size = self . vocab_size <EOL> num_collisions = vocab_size - len ( id_counts ) <EOL> self . _print ( <EOL> "<STR_LIT>" % ( num_collisions , vocab_size ) ) <EOL> if num_collisions > vocab_size / <NUM_LIT> : <EOL> msg = ( <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> % ( num_collisions , vocab_size ) ) <EOL> raise CollisionError ( msg ) <EOL> random . seed ( seed ) <EOL> collisions = ( <EOL> tok for tok in self . token2id if id_counts [ self . token2id [ tok ] ] > <NUM_LIT:1> ) <EOL> for token in collisions : <EOL> old_id = self . token2id [ token ] <EOL> new_id = old_id <EOL> if id_counts [ old_id ] > <NUM_LIT:1> : <EOL> while new_id in id_counts : <EOL> new_id = random . randint ( <NUM_LIT:0> , self . precision - <NUM_LIT:1> ) <EOL> new_id = new_id % self . precision <EOL> id_counts [ old_id ] -= <NUM_LIT:1> <EOL> id_counts [ new_id ] = <NUM_LIT:1> <EOL> self . token2id [ token ] = new_id <EOL> self . _print ( "<STR_LIT>" ) <EOL> self . collisions_resolved = True <EOL> def compactify ( self ) : <EOL> """<STR_LIT>""" <EOL> min_precision = int ( np . ceil ( np . log2 ( self . vocab_size ) ) ) <EOL> if self . bit_precision < min_precision : <EOL> raise CollisionError ( <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" % min_precision ) <EOL> new_token2id = { } <EOL> for i , tok in enumerate ( self . token2id ) : <EOL> new_token2id [ tok ] = i <EOL> self . token2id = new_token2id <EOL> if hasattr ( self , '<STR_LIT>' ) : <EOL> self . set_id2token ( ) <EOL> self . set_bit_precision_required ( ) <EOL> self . _print ( <EOL> "<STR_LIT>" <EOL> % self . bit_precision_required ) <EOL> def set_bit_precision_required ( self ) : <EOL> """<STR_LIT>""" <EOL> max_id = np . max ( self . token2id . values ( ) ) <EOL> self . bit_precision_required = int ( np . ceil ( np . log2 ( max_id ) ) ) <EOL> def filter_sfile ( <EOL> self , infile , outfile , doc_id_list = None , enforce_all_doc_id = True , <EOL> min_tf_idf = <NUM_LIT:0> , filters = None ) : <EOL> """<STR_LIT>""" <EOL> assert self . sfile_loaded , "<STR_LIT>" <EOL> if not hasattr ( self , '<STR_LIT>' ) : <EOL> self . _print ( <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> if filters is None : <EOL> filters = [ ] <EOL> prefilters = [ ] <EOL> if doc_id_list is not None : <EOL> doc_id_set = set ( doc_id_list ) <EOL> prefilters . append ( streaming_filters . get_doc_id_filter ( doc_id_set ) ) <EOL> else : <EOL> doc_id_set = set ( ) <EOL> if min_tf_idf != <NUM_LIT:0> : <EOL> prefilters . append ( <EOL> streaming_filters . get_tf_idf_filter ( self , min_tf_idf ) ) <EOL> postfilters = [ streaming_filters . get_token_to_id_filter ( self ) ] <EOL> filters = prefilters + filters + postfilters <EOL> doc_id_seen = set ( ) <EOL> with smart_open ( infile ) as f , smart_open ( outfile , '<STR_LIT:w>' ) as g : <EOL> for line in f : <EOL> record_dict = self . formatter . sstr_to_dict ( line ) <EOL> doc_id = record_dict [ '<STR_LIT>' ] <EOL> doc_id_seen . add ( doc_id ) <EOL> if all ( func ( record_dict ) for func in filters ) : <EOL> new_sstr = self . formatter . get_sstr ( ** record_dict ) <EOL> g . write ( new_sstr + '<STR_LIT:\n>' ) <EOL> if enforce_all_doc_id : <EOL> assert doc_id_set . issubset ( doc_id_seen ) , ( <EOL> "<STR_LIT>" ) <EOL> def filter_extremes ( <EOL> self , doc_freq_min = <NUM_LIT:0> , doc_freq_max = np . inf , doc_fraction_min = <NUM_LIT:0> , <EOL> doc_fraction_max = <NUM_LIT:1> , token_score_min = <NUM_LIT:0> , token_score_max = np . inf , <EOL> token_score_quantile_min = <NUM_LIT:0> , token_score_quantile_max = <NUM_LIT:1> ) : <EOL> """<STR_LIT>""" <EOL> frame = self . to_frame ( ) <EOL> to_remove_mask = ( <EOL> ( frame . doc_freq < doc_freq_min ) <EOL> | ( frame . doc_freq > doc_freq_max ) <EOL> | ( frame . doc_freq < ( doc_fraction_min * self . num_docs ) ) <EOL> | ( frame . doc_freq > ( doc_fraction_max * self . num_docs ) ) <EOL> | ( frame . token_score < token_score_min ) <EOL> | ( frame . token_score > token_score_max ) <EOL> | ( frame . token_score <EOL> < frame . token_score . quantile ( token_score_quantile_min ) ) <EOL> | ( frame . token_score <EOL> > frame . token_score . quantile ( token_score_quantile_max ) ) <EOL> ) <EOL> self . _print ( <EOL> "<STR_LIT>" % ( to_remove_mask . sum ( ) , len ( frame ) ) ) <EOL> self . filter_tokens ( frame [ to_remove_mask ] . index ) <EOL> def filter_tokens ( self , tokens ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( tokens , str ) : <EOL> tokens = [ tokens ] <EOL> for tok in tokens : <EOL> id_value = self . token2id [ tok ] <EOL> self . token2id . pop ( tok ) <EOL> self . token_score . pop ( tok ) <EOL> self . doc_freq . pop ( tok ) <EOL> if hasattr ( self , '<STR_LIT>' ) : <EOL> self . id2token . pop ( id_value ) <EOL> def _print ( self , msg ) : <EOL> if self . verbose : <EOL> print ( msg ) <EOL> def to_frame ( self ) : <EOL> """<STR_LIT>""" <EOL> token2id = self . token2id <EOL> token_score = self . token_score <EOL> doc_freq = self . doc_freq <EOL> frame = pd . DataFrame ( <EOL> { '<STR_LIT>' : [ token_score [ tok ] for tok in token2id ] , <EOL> '<STR_LIT>' : [ doc_freq [ tok ] for tok in token2id ] } , <EOL> index = [ tok for tok in token2id ] ) <EOL> frame [ '<STR_LIT>' ] = frame . doc_freq / float ( self . num_docs ) <EOL> frame . index . name = '<STR_LIT>' <EOL> return frame <EOL> @ property <EOL> def vocab_size ( self ) : <EOL> return len ( self . token2id ) <EOL> def save ( self , savepath , protocol = - <NUM_LIT:1> , set_id2token = True ) : <EOL> """<STR_LIT>""" <EOL> if set_id2token : <EOL> self . set_id2token ( ) <EOL> SaveLoad . save ( self , savepath , protocol = protocol ) <EOL> def collision_probability ( vocab_size , bit_precision ) : <EOL> """<STR_LIT>""" <EOL> exponent = - vocab_size * ( vocab_size - <NUM_LIT:1> ) / <NUM_LIT> ** bit_precision <EOL> return <NUM_LIT:1> - np . exp ( exponent ) <EOL> class CollisionError ( Exception ) : <EOL> pass </s>
