<s> import datetime <EOL> import numpy as np <EOL> import logging <EOL> import cPickle <EOL> import os <EOL> import shutil <EOL> logger = logging . getLogger ( __name__ ) <EOL> import numpy <EOL> import pyll <EOL> def import_tokens ( tokens ) : <EOL> rval = None <EOL> for i in range ( len ( tokens ) ) : <EOL> modname = '<STR_LIT:.>' . join ( tokens [ : i + <NUM_LIT:1> ] ) <EOL> try : <EOL> logger . info ( '<STR_LIT>' % modname ) <EOL> exec "<STR_LIT>" % modname <EOL> exec "<STR_LIT>" % modname <EOL> except ImportError , e : <EOL> logger . info ( '<STR_LIT>' % modname ) <EOL> logger . info ( '<STR_LIT>' % str ( e ) ) <EOL> break <EOL> return rval , tokens [ i : ] <EOL> def load_tokens ( tokens ) : <EOL> logger . info ( '<STR_LIT>' % str ( tokens ) ) <EOL> symbol , remainder = import_tokens ( tokens ) <EOL> for attr in remainder : <EOL> symbol = getattr ( symbol , attr ) <EOL> return symbol <EOL> def json_lookup ( json ) : <EOL> symbol = load_tokens ( json . split ( '<STR_LIT:.>' ) ) <EOL> return symbol <EOL> def json_call ( json , args = ( ) , kwargs = None ) : <EOL> """<STR_LIT>""" <EOL> if kwargs is None : <EOL> kwargs = { } <EOL> if isinstance ( json , basestring ) : <EOL> symbol = json_lookup ( json ) <EOL> return symbol ( * args , ** kwargs ) <EOL> elif isinstance ( json , dict ) : <EOL> raise NotImplementedError ( '<STR_LIT>' , json ) <EOL> elif isinstance ( json , ( tuple , list ) ) : <EOL> raise NotImplementedError ( '<STR_LIT>' , json ) <EOL> else : <EOL> raise TypeError ( json ) <EOL> def get_obj ( f , argfile = None , argstr = None , args = ( ) , kwargs = None ) : <EOL> """<STR_LIT>""" <EOL> if kwargs is None : <EOL> kwargs = { } <EOL> if argfile is not None : <EOL> argstr = open ( argfile ) . read ( ) <EOL> if argstr is not None : <EOL> argd = cPickle . loads ( argstr ) <EOL> else : <EOL> argd = { } <EOL> args = args + argd . get ( '<STR_LIT:args>' , ( ) ) <EOL> kwargs . update ( argd . get ( '<STR_LIT>' , { } ) ) <EOL> return json_call ( f , args = args , kwargs = kwargs ) <EOL> def pmin_sampled ( mean , var , n_samples = <NUM_LIT:1000> , rng = None ) : <EOL> """<STR_LIT>""" <EOL> if rng is None : <EOL> rng = numpy . random . RandomState ( <NUM_LIT> ) <EOL> samples = rng . randn ( n_samples , len ( mean ) ) * numpy . sqrt ( var ) + mean <EOL> winners = ( samples . T == samples . min ( axis = <NUM_LIT:1> ) ) . T <EOL> wincounts = winners . sum ( axis = <NUM_LIT:0> ) <EOL> assert wincounts . shape == mean . shape <EOL> return wincounts . astype ( '<STR_LIT>' ) / wincounts . sum ( ) <EOL> def fast_isin ( X , Y ) : <EOL> """<STR_LIT>""" <EOL> if len ( Y ) > <NUM_LIT:0> : <EOL> T = Y . copy ( ) <EOL> T . sort ( ) <EOL> D = T . searchsorted ( X ) <EOL> T = np . append ( T , np . array ( [ <NUM_LIT:0> ] ) ) <EOL> W = ( T [ D ] == X ) <EOL> if isinstance ( W , bool ) : <EOL> return np . zeros ( ( len ( X ) , ) , bool ) <EOL> else : <EOL> return ( T [ D ] == X ) <EOL> else : <EOL> return np . zeros ( ( len ( X ) , ) , bool ) <EOL> def get_most_recent_inds ( obj ) : <EOL> data = numpy . rec . array ( [ ( x [ '<STR_LIT>' ] , int ( x [ '<STR_LIT:version>' ] ) ) <EOL> for x in obj ] , <EOL> names = [ '<STR_LIT>' , '<STR_LIT:version>' ] ) <EOL> s = data . argsort ( order = [ '<STR_LIT>' , '<STR_LIT:version>' ] ) <EOL> data = data [ s ] <EOL> recent = ( data [ '<STR_LIT>' ] [ <NUM_LIT:1> : ] != data [ '<STR_LIT>' ] [ : - <NUM_LIT:1> ] ) . nonzero ( ) [ <NUM_LIT:0> ] <EOL> recent = numpy . append ( recent , [ len ( data ) - <NUM_LIT:1> ] ) <EOL> return s [ recent ] <EOL> def use_obj_for_literal_in_memo ( expr , obj , lit , memo ) : <EOL> """<STR_LIT>""" <EOL> for node in pyll . dfs ( expr ) : <EOL> try : <EOL> if node . obj == lit : <EOL> memo [ node ] = obj <EOL> except AttributeError : <EOL> pass <EOL> return memo <EOL> def coarse_utcnow ( ) : <EOL> """<STR_LIT>""" <EOL> now = datetime . datetime . utcnow ( ) <EOL> microsec = ( now . microsecond // <NUM_LIT:10> ** <NUM_LIT:3> ) * ( <NUM_LIT:10> ** <NUM_LIT:3> ) <EOL> return datetime . datetime ( now . year , now . month , now . day , now . hour , <EOL> now . minute , now . second , microsec ) <EOL> from contextlib import contextmanager <EOL> @ contextmanager <EOL> def working_dir ( dir ) : <EOL> cwd = os . getcwd ( ) <EOL> os . chdir ( dir ) <EOL> yield <EOL> os . chdir ( cwd ) <EOL> def path_split_all ( path ) : <EOL> """<STR_LIT>""" <EOL> parts = [ ] <EOL> while True : <EOL> path , fn = os . path . split ( path ) <EOL> if len ( fn ) == <NUM_LIT:0> : <EOL> break <EOL> parts . append ( fn ) <EOL> return reversed ( parts ) <EOL> def get_closest_dir ( workdir ) : <EOL> """<STR_LIT>""" <EOL> closest_dir = '<STR_LIT>' <EOL> for wdi in path_split_all ( workdir ) : <EOL> if os . path . isdir ( os . path . join ( closest_dir , wdi ) ) : <EOL> closest_dir = os . path . join ( closest_dir , wdi ) <EOL> else : <EOL> break <EOL> assert closest_dir != workdir <EOL> return closest_dir , wdi <EOL> @ contextmanager <EOL> def temp_dir ( dir , erase_after = False , with_sentinel = True ) : <EOL> created_by_me = False <EOL> if not os . path . exists ( dir ) : <EOL> if os . pardir in dir : <EOL> raise RuntimeError ( "<STR_LIT>" ) <EOL> if erase_after and with_sentinel : <EOL> closest_dir , fn = get_closest_dir ( dir ) <EOL> sentinel = os . path . join ( closest_dir , fn + "<STR_LIT>" ) <EOL> open ( sentinel , '<STR_LIT:w>' ) . close ( ) <EOL> os . makedirs ( dir ) <EOL> created_by_me = True <EOL> else : <EOL> assert os . path . isdir ( dir ) <EOL> yield <EOL> if erase_after and created_by_me : <EOL> shutil . rmtree ( dir ) <EOL> if with_sentinel : <EOL> os . mkdir ( dir ) <EOL> os . removedirs ( dir ) <EOL> os . remove ( sentinel ) </s>
