<s> """<STR_LIT>""" <EOL> import random <EOL> import string <EOL> import time <EOL> from mapreduce import context <EOL> from mapreduce import errors <EOL> from mapreduce import operation <EOL> from mapreduce . api import map_job <EOL> COUNTER_IO_READ_BYTES = "<STR_LIT>" <EOL> COUNTER_IO_READ_MSEC = "<STR_LIT>" <EOL> class SampleInputReader ( map_job . InputReader ) : <EOL> """<STR_LIT>""" <EOL> COUNT = "<STR_LIT:count>" <EOL> STRING_LENGTH = "<STR_LIT>" <EOL> _DEFAULT_STRING_LENGTH = <NUM_LIT:10> <EOL> def __init__ ( self , count , string_length ) : <EOL> """<STR_LIT>""" <EOL> self . _count = count <EOL> self . _string_length = string_length <EOL> def __iter__ ( self ) : <EOL> ctx = context . get ( ) <EOL> while self . _count : <EOL> self . _count -= <NUM_LIT:1> <EOL> start_time = time . time ( ) <EOL> content = "<STR_LIT>" . join ( random . choice ( string . ascii_lowercase ) <EOL> for _ in range ( self . _string_length ) ) <EOL> if ctx : <EOL> operation . counters . Increment ( <EOL> COUNTER_IO_READ_MSEC , int ( ( time . time ( ) - start_time ) * <NUM_LIT:1000> ) ) ( ctx ) <EOL> operation . counters . Increment ( COUNTER_IO_READ_BYTES , len ( content ) ) ( ctx ) <EOL> yield content <EOL> @ classmethod <EOL> def from_json ( cls , state ) : <EOL> """<STR_LIT>""" <EOL> return cls ( state [ cls . COUNT ] , state [ cls . STRING_LENGTH ] ) <EOL> def to_json ( self ) : <EOL> """<STR_LIT>""" <EOL> return { self . COUNT : self . _count , self . STRING_LENGTH : self . _string_length } <EOL> @ classmethod <EOL> def split_input ( cls , job_config ) : <EOL> """<STR_LIT>""" <EOL> params = job_config . input_reader_params <EOL> count = params [ cls . COUNT ] <EOL> string_length = params . get ( cls . STRING_LENGTH , cls . _DEFAULT_STRING_LENGTH ) <EOL> shard_count = job_config . shard_count <EOL> count_per_shard = count // shard_count <EOL> mr_input_readers = [ <EOL> cls ( count_per_shard , string_length ) for _ in range ( shard_count ) ] <EOL> left = count - count_per_shard * shard_count <EOL> if left > <NUM_LIT:0> : <EOL> mr_input_readers . append ( cls ( left , string_length ) ) <EOL> return mr_input_readers <EOL> @ classmethod <EOL> def validate ( cls , job_config ) : <EOL> """<STR_LIT>""" <EOL> super ( SampleInputReader , cls ) . validate ( job_config ) <EOL> params = job_config . input_reader_params <EOL> if cls . COUNT not in params : <EOL> raise errors . BadReaderParamsError ( "<STR_LIT>" % cls . COUNT ) <EOL> if not isinstance ( params [ cls . COUNT ] , int ) : <EOL> raise errors . BadReaderParamsError ( "<STR_LIT>" % <EOL> ( cls . COUNT , type ( params [ cls . COUNT ] ) ) ) <EOL> if params [ cls . COUNT ] <= <NUM_LIT:0> : <EOL> raise errors . BadReaderParamsError ( "<STR_LIT>" ) <EOL> if cls . STRING_LENGTH in params and not ( <EOL> isinstance ( params [ cls . STRING_LENGTH ] , int ) and <EOL> params [ cls . STRING_LENGTH ] > <NUM_LIT:0> ) : <EOL> raise errors . BadReaderParamsError ( "<STR_LIT>" <EOL> "<STR_LIT>" % <EOL> ( cls . STRING_LENGTH , <EOL> params [ cls . STRING_LENGTH ] ) ) </s>
