<s> """<STR_LIT>""" <EOL> from __future__ import division , print_function , unicode_literals <EOL> import numpy as np <EOL> try : <EOL> import cPickle <EOL> except ImportError : <EOL> import pickle as cPickle <EOL> try : <EOL> from itertools import izip_longest <EOL> except ImportError : <EOL> from itertools import zip_longest as izip_longest <EOL> import functools <EOL> from collections import defaultdict <EOL> def lazyprop ( fn ) : <EOL> """<STR_LIT>""" <EOL> attr_name = '<STR_LIT>' + fn . __name__ <EOL> @ property <EOL> def _lazyprop ( self ) : <EOL> if not hasattr ( self , attr_name ) : <EOL> setattr ( self , attr_name , fn ( self ) ) <EOL> return getattr ( self , attr_name ) <EOL> return _lazyprop <EOL> class smart_open ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , filename , * args ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( filename , basestring ) : <EOL> self . fh = open ( filename , * args ) <EOL> self . closing = True <EOL> else : <EOL> self . fh = filename <EOL> self . closing = False <EOL> def __enter__ ( self ) : <EOL> return self . fh <EOL> def __exit__ ( self , exc_type , exc_val , exc_tb ) : <EOL> if self . closing : <EOL> self . fh . close ( ) <EOL> return False <EOL> def get_list_from_filerows ( infile ) : <EOL> """<STR_LIT>""" <EOL> with smart_open ( infile , '<STR_LIT:rb>' ) as f : <EOL> kpv_list = [ ] <EOL> for line in f : <EOL> line = line . strip ( ) <EOL> if len ( line ) > <NUM_LIT:0> : <EOL> if line [ <NUM_LIT:0> ] != '<STR_LIT:#>' : <EOL> kpv_list . append ( line . rstrip ( '<STR_LIT:\n>' ) ) <EOL> return kpv_list <EOL> def write_list_to_filerows ( outfile , mylist ) : <EOL> """<STR_LIT>""" <EOL> with smart_open ( outfile , '<STR_LIT:wb>' ) as f : <EOL> for item in mylist : <EOL> f . write ( str ( item ) + '<STR_LIT:\n>' ) <EOL> def pickleme ( obj , pkl_file , protocol = <NUM_LIT:2> ) : <EOL> """<STR_LIT>""" <EOL> with smart_open ( pkl_file , '<STR_LIT:w>' ) as f : <EOL> cPickle . dump ( obj , f , protocol = protocol ) <EOL> def unpickleme ( pkl_file ) : <EOL> """<STR_LIT>""" <EOL> with smart_open ( pkl_file , '<STR_LIT:r>' ) as f : <EOL> return cPickle . load ( f ) <EOL> def get_structured_array ( listoflists , schema , dropmissing = False ) : <EOL> """<STR_LIT>""" <EOL> if dropmissing : <EOL> tuple_list = [ tuple ( row ) for row in listoflists if '<STR_LIT>' not in row ] <EOL> else : <EOL> tuple_list = [ tuple ( row ) for row in listoflists ] <EOL> return np . array ( tuple_list , schema ) <EOL> class BadDataError ( Exception ) : <EOL> """<STR_LIT>""" <EOL> pass <EOL> class TokenError ( Exception ) : <EOL> """<STR_LIT>""" <EOL> pass <EOL> class ConfigurationSyntaxError ( Exception ) : <EOL> """<STR_LIT>""" <EOL> pass <EOL> class DocIDError ( Exception ) : <EOL> pass <EOL> def printdict ( d , max_print_len = None ) : <EOL> s = '<STR_LIT>' <EOL> for key , value in d . iteritems ( ) : <EOL> s += str ( key ) + '<STR_LIT>' + str ( value ) + '<STR_LIT:\n>' <EOL> if max_print_len : <EOL> print ( s [ : max_print_len ] ) <EOL> else : <EOL> print ( s ) <EOL> def nested_defaultdict ( default_factory , levels = <NUM_LIT:1> ) : <EOL> """<STR_LIT>""" <EOL> if not isinstance ( levels , int ) or ( levels < <NUM_LIT:1> ) : <EOL> raise ValueError ( "<STR_LIT>" % levels ) <EOL> def nestone ( ) : <EOL> """<STR_LIT>""" <EOL> return nested_defaultdict ( default_factory , levels - <NUM_LIT:1> ) <EOL> if levels == <NUM_LIT:1> : <EOL> return defaultdict ( default_factory ) <EOL> else : <EOL> return defaultdict ( nestone ) <EOL> def nested_keysearch ( ndict , key_list ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( key_list , basestring ) : <EOL> key_list = [ key_list ] <EOL> first_key = key_list [ <NUM_LIT:0> ] <EOL> if len ( key_list ) == <NUM_LIT:1> : <EOL> return first_key in ndict <EOL> else : <EOL> if first_key in ndict : <EOL> return nested_keysearch ( ndict [ first_key ] , key_list [ <NUM_LIT:1> : ] ) <EOL> def grouper ( iterable , chunksize , fillvalue = None ) : <EOL> """<STR_LIT>""" <EOL> args = [ iter ( iterable ) ] * chunksize <EOL> return izip_longest ( fillvalue = fillvalue , * args ) <EOL> def compose ( * functions ) : <EOL> def compose2 ( f , g ) : <EOL> def f_g ( x ) : <EOL> return f ( g ( x ) ) <EOL> return f_g <EOL> return functools . reduce ( compose2 , functions ) </s>
