<s> """<STR_LIT>""" <EOL> import contextlib <EOL> import glob <EOL> import itertools <EOL> import logging <EOL> import os <EOL> import pipes <EOL> import random <EOL> import shlex <EOL> import shutil <EOL> import sys <EOL> import tarfile <EOL> import zipfile <EOL> import zlib <EOL> from collections import defaultdict <EOL> from copy import deepcopy <EOL> from datetime import timedelta <EOL> from distutils . spawn import find_executable <EOL> from logging import getLogger <EOL> try : <EOL> import bz2 <EOL> bz2 <EOL> except ImportError : <EOL> bz2 = None <EOL> from mrjob . py2 import PY2 <EOL> log = getLogger ( __name__ ) <EOL> class NullHandler ( logging . Handler ) : <EOL> def emit ( self , record ) : <EOL> pass <EOL> def args_for_opt_dest_subset ( option_parser , args , dests = None ) : <EOL> """<STR_LIT>""" <EOL> for dest , value in _args_for_opt_dest_subset ( option_parser , args , dests ) : <EOL> yield value <EOL> def _process_short_opts ( option_parser , rargs , values , dests ) : <EOL> """<STR_LIT>""" <EOL> arg = rargs . pop ( <NUM_LIT:0> ) <EOL> stop = False <EOL> i = <NUM_LIT:1> <EOL> for ch in arg [ <NUM_LIT:1> : ] : <EOL> opt = "<STR_LIT:->" + ch <EOL> option = option_parser . _short_opt . get ( opt ) <EOL> i += <NUM_LIT:1> <EOL> rargs_before_processing = [ x for x in rargs ] <EOL> args_from_smashed_short_opt = [ ] <EOL> if option . takes_value ( ) : <EOL> if i < len ( arg ) : <EOL> rargs . insert ( <NUM_LIT:0> , arg [ i : ] ) <EOL> args_from_smashed_short_opt . append ( arg [ i : ] ) <EOL> stop = True <EOL> nargs = option . nargs <EOL> if nargs == <NUM_LIT:1> : <EOL> value = rargs . pop ( <NUM_LIT:0> ) <EOL> else : <EOL> value = tuple ( rargs [ <NUM_LIT:0> : nargs ] ) <EOL> del rargs [ <NUM_LIT:0> : nargs ] <EOL> else : <EOL> value = None <EOL> option . process ( opt , value , values , option_parser ) <EOL> if dests is None or option . dest in dests : <EOL> length_difference = len ( rargs_before_processing ) - len ( rargs ) <EOL> for item in ( [ opt ] + args_from_smashed_short_opt + <EOL> rargs_before_processing [ : length_difference ] ) : <EOL> yield option . dest , item <EOL> if stop : <EOL> break <EOL> def _args_for_opt_dest_subset ( option_parser , args , dests = None ) : <EOL> """<STR_LIT>""" <EOL> values = deepcopy ( option_parser . get_default_values ( ) ) <EOL> rargs = [ x for x in args ] <EOL> option_parser . rargs = rargs <EOL> while rargs : <EOL> arg = rargs [ <NUM_LIT:0> ] <EOL> if arg == '<STR_LIT>' : <EOL> del rargs [ <NUM_LIT:0> ] <EOL> return <EOL> elif arg [ <NUM_LIT:0> : <NUM_LIT:2> ] == '<STR_LIT>' : <EOL> for item in _process_long_opt ( option_parser , rargs , values , dests ) : <EOL> yield item <EOL> elif arg [ : <NUM_LIT:1> ] == '<STR_LIT:->' and len ( arg ) > <NUM_LIT:1> : <EOL> for item in _process_short_opts ( option_parser , rargs , values , <EOL> dests ) : <EOL> yield item <EOL> else : <EOL> del rargs [ <NUM_LIT:0> ] <EOL> def bash_wrap ( cmd_str ) : <EOL> """<STR_LIT>""" <EOL> return "<STR_LIT>" % cmd_str . replace ( "<STR_LIT:'>" , "<STR_LIT>" ) <EOL> def buffer_iterator_to_line_iterator ( chunks ) : <EOL> """<STR_LIT>""" <EOL> log . warning ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> return to_lines ( chunks ) <EOL> def bunzip2_stream ( fileobj , bufsize = <NUM_LIT> ) : <EOL> """<STR_LIT>""" <EOL> if bz2 is None : <EOL> raise Exception ( <EOL> '<STR_LIT>' ) <EOL> d = bz2 . BZ2Decompressor ( ) <EOL> while True : <EOL> chunk = fileobj . read ( bufsize ) <EOL> if not chunk : <EOL> return <EOL> part = d . decompress ( chunk ) <EOL> if part : <EOL> yield part <EOL> def cmd_line ( args ) : <EOL> """<STR_LIT>""" <EOL> args = [ str ( x ) for x in args ] <EOL> return '<STR_LIT:U+0020>' . join ( pipes . quote ( x ) for x in args ) <EOL> def expand_path ( path ) : <EOL> """<STR_LIT>""" <EOL> if path is None : <EOL> return None <EOL> else : <EOL> return os . path . expanduser ( os . path . expandvars ( path ) ) <EOL> def file_ext ( filename ) : <EOL> """<STR_LIT>""" <EOL> dot_index = filename . find ( '<STR_LIT:.>' ) <EOL> if dot_index == - <NUM_LIT:1> : <EOL> return '<STR_LIT>' <EOL> return filename [ dot_index : ] <EOL> def gunzip_stream ( fileobj , bufsize = <NUM_LIT> ) : <EOL> """<STR_LIT>""" <EOL> READ_GZIP_DATA = <NUM_LIT:16> <EOL> d = zlib . decompressobj ( READ_GZIP_DATA | zlib . MAX_WBITS ) <EOL> while True : <EOL> chunk = fileobj . read ( bufsize ) <EOL> if not chunk : <EOL> return <EOL> data = d . decompress ( chunk ) <EOL> if data : <EOL> yield data <EOL> def log_to_null ( name = None ) : <EOL> """<STR_LIT>""" <EOL> logger = logging . getLogger ( name ) <EOL> logger . addHandler ( NullHandler ( ) ) <EOL> def log_to_stream ( name = None , stream = None , format = None , level = None , <EOL> debug = False ) : <EOL> """<STR_LIT>""" <EOL> if level is None : <EOL> level = logging . DEBUG if debug else logging . INFO <EOL> if format is None : <EOL> format = '<STR_LIT>' <EOL> if stream is None : <EOL> stream = sys . stderr <EOL> handler = logging . StreamHandler ( stream ) <EOL> handler . setLevel ( level ) <EOL> handler . setFormatter ( logging . Formatter ( format ) ) <EOL> logger = logging . getLogger ( name ) <EOL> logger . setLevel ( level ) <EOL> logger . addHandler ( handler ) <EOL> def _process_long_opt ( option_parser , rargs , values , dests ) : <EOL> """<STR_LIT>""" <EOL> arg = rargs . pop ( <NUM_LIT:0> ) <EOL> if "<STR_LIT:=>" in arg : <EOL> ( opt , next_arg ) = arg . split ( "<STR_LIT:=>" , <NUM_LIT:1> ) <EOL> rargs . insert ( <NUM_LIT:0> , next_arg ) <EOL> else : <EOL> opt = arg <EOL> opt = option_parser . _match_long_opt ( opt ) <EOL> option = option_parser . _long_opt [ opt ] <EOL> rargs_before_processing = [ x for x in rargs ] <EOL> if option . takes_value ( ) : <EOL> nargs = option . nargs <EOL> if nargs == <NUM_LIT:1> : <EOL> value = rargs . pop ( <NUM_LIT:0> ) <EOL> else : <EOL> value = tuple ( rargs [ <NUM_LIT:0> : nargs ] ) <EOL> del rargs [ <NUM_LIT:0> : nargs ] <EOL> else : <EOL> value = None <EOL> option . process ( opt , value , values , option_parser ) <EOL> if dests is None or option . dest in dests : <EOL> length_difference = len ( rargs_before_processing ) - len ( rargs ) <EOL> for item in [ opt ] + rargs_before_processing [ : length_difference ] : <EOL> yield option . dest , item <EOL> def parse_and_save_options ( option_parser , args ) : <EOL> """<STR_LIT>""" <EOL> arg_map = defaultdict ( list ) <EOL> for dest , value in _args_for_opt_dest_subset ( option_parser , args , None ) : <EOL> arg_map [ dest ] . append ( value ) <EOL> return arg_map <EOL> def populate_option_groups_with_options ( assignments , indexed_options ) : <EOL> """<STR_LIT>""" <EOL> for opt_group , opt_dest_list in assignments . items ( ) : <EOL> new_options = [ ] <EOL> for option_dest in assignments [ opt_group ] : <EOL> for option in indexed_options [ option_dest ] : <EOL> new_options . append ( option ) <EOL> opt_group . add_options ( new_options ) <EOL> opt_group . option_list = sorted ( opt_group . option_list , <EOL> key = lambda item : item . get_opt_string ( ) ) <EOL> def random_identifier ( ) : <EOL> """<STR_LIT>""" <EOL> return '<STR_LIT>' % random . randint ( <NUM_LIT:0> , <NUM_LIT:2> ** <NUM_LIT:64> - <NUM_LIT:1> ) <EOL> def read_file ( path , fileobj = None , yields_lines = True , cleanup = None ) : <EOL> """<STR_LIT>""" <EOL> f = None <EOL> try : <EOL> if fileobj is None : <EOL> f = open ( path , '<STR_LIT:rb>' ) <EOL> else : <EOL> f = fileobj <EOL> if path . endswith ( '<STR_LIT>' ) : <EOL> lines = to_lines ( gunzip_stream ( f ) ) <EOL> elif path . endswith ( '<STR_LIT>' ) : <EOL> if bz2 is None : <EOL> raise Exception ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> else : <EOL> lines = to_lines ( bunzip2_stream ( f ) ) <EOL> else : <EOL> if yields_lines : <EOL> lines = f <EOL> else : <EOL> lines = to_lines ( f ) <EOL> for line in lines : <EOL> yield line <EOL> finally : <EOL> try : <EOL> if f and f is not fileobj : <EOL> f . close ( ) <EOL> finally : <EOL> if cleanup : <EOL> cleanup ( ) <EOL> def read_input ( path , stdin = None ) : <EOL> """<STR_LIT>""" <EOL> if stdin is None : <EOL> stdin = sys . stdin <EOL> if path == '<STR_LIT:->' : <EOL> for line in stdin : <EOL> yield line <EOL> return <EOL> paths = glob . glob ( path ) <EOL> if not paths : <EOL> raise IOError ( <NUM_LIT:2> , '<STR_LIT>' % path ) <EOL> elif len ( paths ) > <NUM_LIT:1> : <EOL> for path in paths : <EOL> for line in read_input ( path , stdin = stdin ) : <EOL> yield line <EOL> return <EOL> else : <EOL> path = paths [ <NUM_LIT:0> ] <EOL> if os . path . isdir ( path ) : <EOL> for dirname , _ , filenames in os . walk ( path , followlinks = True ) : <EOL> for filename in filenames : <EOL> for line in read_input ( os . path . join ( dirname , filename ) , <EOL> stdin = stdin ) : <EOL> yield line <EOL> return <EOL> for line in read_file ( path ) : <EOL> yield line <EOL> def safeeval ( expr , globals = None , locals = None ) : <EOL> """<STR_LIT>""" <EOL> safe_globals = { <EOL> '<STR_LIT:False>' : False , <EOL> '<STR_LIT:None>' : None , <EOL> '<STR_LIT:True>' : True , <EOL> '<STR_LIT>' : None , <EOL> '<STR_LIT>' : None , <EOL> '<STR_LIT>' : set <EOL> } <EOL> if PY2 : <EOL> safe_globals [ '<STR_LIT>' ] = xrange <EOL> else : <EOL> safe_globals [ '<STR_LIT>' ] = range <EOL> def open ( * args , ** kwargs ) : <EOL> raise NameError ( "<STR_LIT>" ) <EOL> safe_globals [ '<STR_LIT>' ] = open <EOL> if globals : <EOL> safe_globals . update ( globals ) <EOL> return eval ( expr , safe_globals , locals ) <EOL> @ contextlib . contextmanager <EOL> def save_current_environment ( ) : <EOL> """<STR_LIT>""" <EOL> original_environ = os . environ . copy ( ) <EOL> try : <EOL> yield <EOL> finally : <EOL> os . environ . clear ( ) <EOL> os . environ . update ( original_environ ) <EOL> @ contextlib . contextmanager <EOL> def save_cwd ( ) : <EOL> """<STR_LIT>""" <EOL> original_cwd = os . getcwd ( ) <EOL> try : <EOL> yield <EOL> finally : <EOL> os . chdir ( original_cwd ) <EOL> def scrape_options_and_index_by_dest ( * parsers_and_groups ) : <EOL> """<STR_LIT>""" <EOL> all_options = { } <EOL> job_option_lists = [ g . option_list for g in parsers_and_groups ] <EOL> for option in itertools . chain ( * job_option_lists ) : <EOL> other_options = all_options . get ( option . dest , [ ] ) <EOL> other_options . append ( option ) <EOL> all_options [ option . dest ] = other_options <EOL> return all_options <EOL> def scrape_options_into_new_groups ( source_groups , assignments ) : <EOL> """<STR_LIT>""" <EOL> all_options = scrape_options_and_index_by_dest ( * source_groups ) <EOL> return populate_option_groups_with_options ( assignments , all_options ) <EOL> def shlex_split ( s ) : <EOL> """<STR_LIT>""" <EOL> if sys . version_info < ( <NUM_LIT:2> , <NUM_LIT:7> , <NUM_LIT:3> ) : <EOL> return shlex . split ( str ( s ) ) <EOL> else : <EOL> return shlex . split ( s ) <EOL> def strip_microseconds ( delta ) : <EOL> """<STR_LIT>""" <EOL> return timedelta ( delta . days , delta . seconds ) <EOL> def tar_and_gzip ( dir , out_path , filter = None , prefix = '<STR_LIT>' ) : <EOL> """<STR_LIT>""" <EOL> if not os . path . isdir ( dir ) : <EOL> raise IOError ( '<STR_LIT>' % ( dir , ) ) <EOL> if not filter : <EOL> filter = lambda path : True <EOL> tar_gz = tarfile . open ( out_path , mode = '<STR_LIT>' ) <EOL> for dirpath , dirnames , filenames in os . walk ( dir , followlinks = True ) : <EOL> for filename in filenames : <EOL> path = os . path . join ( dirpath , filename ) <EOL> rel_path = path [ len ( os . path . join ( dir , '<STR_LIT>' ) ) : ] <EOL> if filter ( rel_path ) : <EOL> real_path = os . path . realpath ( path ) <EOL> path_in_tar_gz = os . path . join ( prefix , rel_path ) <EOL> tar_gz . add ( real_path , arcname = path_in_tar_gz , recursive = False ) <EOL> tar_gz . close ( ) <EOL> def to_lines ( chunks ) : <EOL> """<STR_LIT>""" <EOL> leftovers = [ ] <EOL> for chunk in chunks : <EOL> start = <NUM_LIT:0> <EOL> while start < len ( chunk ) : <EOL> end = chunk . find ( b'<STR_LIT:\n>' , start ) + <NUM_LIT:1> <EOL> if end == <NUM_LIT:0> : <EOL> leftovers . append ( chunk [ start : ] ) <EOL> break <EOL> if leftovers : <EOL> leftovers . append ( chunk [ start : end ] ) <EOL> yield b'<STR_LIT>' . join ( leftovers ) <EOL> leftovers = [ ] <EOL> else : <EOL> yield chunk [ start : end ] <EOL> start = end <EOL> if leftovers : <EOL> yield b'<STR_LIT>' . join ( leftovers ) <EOL> def unique ( items ) : <EOL> """<STR_LIT>""" <EOL> seen = set ( ) <EOL> for item in items : <EOL> if item in seen : <EOL> continue <EOL> else : <EOL> yield item <EOL> seen . add ( item ) <EOL> def unarchive ( archive_path , dest ) : <EOL> """<STR_LIT>""" <EOL> if tarfile . is_tarfile ( archive_path ) : <EOL> with contextlib . closing ( tarfile . open ( archive_path , '<STR_LIT:r>' ) ) as archive : <EOL> archive . extractall ( dest ) <EOL> elif zipfile . is_zipfile ( archive_path ) : <EOL> with contextlib . closing ( zipfile . ZipFile ( archive_path , '<STR_LIT:r>' ) ) as archive : <EOL> for name in archive . namelist ( ) : <EOL> dest_path = os . path . join ( dest , * name . split ( '<STR_LIT:/>' ) ) <EOL> dirname , filename = os . path . split ( dest_path ) <EOL> if dirname and not os . path . exists ( dirname ) : <EOL> os . makedirs ( dirname ) <EOL> if filename : <EOL> with open ( dest_path , '<STR_LIT:wb>' ) as dest_file : <EOL> dest_file . write ( archive . read ( name ) ) <EOL> else : <EOL> raise IOError ( '<STR_LIT>' % ( archive_path , ) ) <EOL> def which ( cmd , path = None ) : <EOL> """<STR_LIT>""" <EOL> if hasattr ( shutil , '<STR_LIT>' ) : <EOL> return shutil . which ( cmd , path = path ) <EOL> elif path is None and os . environ . get ( '<STR_LIT>' ) is None : <EOL> return None <EOL> else : <EOL> return find_executable ( cmd , path = path ) </s>
