<s> __all__ = [ "<STR_LIT>" , "<STR_LIT>" ] <EOL> import sys <EOL> import __future__ <EOL> import inspect <EOL> import tokenize <EOL> import ast <EOL> import numbers <EOL> import six <EOL> from patsy import PatsyError <EOL> from patsy . util import PushbackAdapter , no_pickling , assert_no_pickling <EOL> from patsy . tokens import ( pretty_untokenize , normalize_token_spacing , <EOL> python_tokenize ) <EOL> from patsy . compat import call_and_wrap_exc <EOL> def _all_future_flags ( ) : <EOL> flags = <NUM_LIT:0> <EOL> for feature_name in __future__ . all_feature_names : <EOL> feature = getattr ( __future__ , feature_name ) <EOL> if feature . getMandatoryRelease ( ) > sys . version_info : <EOL> flags |= feature . compiler_flag <EOL> return flags <EOL> _ALL_FUTURE_FLAGS = _all_future_flags ( ) <EOL> class VarLookupDict ( object ) : <EOL> def __init__ ( self , dicts ) : <EOL> self . _dicts = [ { } ] + list ( dicts ) <EOL> def __getitem__ ( self , key ) : <EOL> for d in self . _dicts : <EOL> try : <EOL> return d [ key ] <EOL> except KeyError : <EOL> pass <EOL> raise KeyError ( key ) <EOL> def __setitem__ ( self , key , value ) : <EOL> self . _dicts [ <NUM_LIT:0> ] [ key ] = value <EOL> def __contains__ ( self , key ) : <EOL> try : <EOL> self [ key ] <EOL> except KeyError : <EOL> return False <EOL> else : <EOL> return True <EOL> def get ( self , key , default = None ) : <EOL> try : <EOL> return self [ key ] <EOL> except KeyError : <EOL> return default <EOL> def __repr__ ( self ) : <EOL> return "<STR_LIT>" % ( self . __class__ . __name__ , self . _dicts ) <EOL> __getstate__ = no_pickling <EOL> def test_VarLookupDict ( ) : <EOL> d1 = { "<STR_LIT:a>" : <NUM_LIT:1> } <EOL> d2 = { "<STR_LIT:a>" : <NUM_LIT:2> , "<STR_LIT:b>" : <NUM_LIT:3> } <EOL> ds = VarLookupDict ( [ d1 , d2 ] ) <EOL> assert ds [ "<STR_LIT:a>" ] == <NUM_LIT:1> <EOL> assert ds [ "<STR_LIT:b>" ] == <NUM_LIT:3> <EOL> assert "<STR_LIT:a>" in ds <EOL> assert "<STR_LIT:c>" not in ds <EOL> from nose . tools import assert_raises <EOL> assert_raises ( KeyError , ds . __getitem__ , "<STR_LIT:c>" ) <EOL> ds [ "<STR_LIT:a>" ] = <NUM_LIT:10> <EOL> assert ds [ "<STR_LIT:a>" ] == <NUM_LIT:10> <EOL> assert d1 [ "<STR_LIT:a>" ] == <NUM_LIT:1> <EOL> assert ds . get ( "<STR_LIT:c>" ) is None <EOL> assert isinstance ( repr ( ds ) , six . string_types ) <EOL> assert_no_pickling ( ds ) <EOL> def ast_names ( code ) : <EOL> """<STR_LIT>""" <EOL> disallowed_ast_nodes = ( ast . Lambda , ast . ListComp , ast . GeneratorExp ) <EOL> if sys . version_info >= ( <NUM_LIT:2> , <NUM_LIT:7> ) : <EOL> disallowed_ast_nodes += ( ast . DictComp , ast . SetComp ) <EOL> for node in ast . walk ( ast . parse ( code ) ) : <EOL> if isinstance ( node , disallowed_ast_nodes ) : <EOL> raise PatsyError ( "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> if isinstance ( node , ast . Name ) : <EOL> yield node . id <EOL> def test_ast_names ( ) : <EOL> test_data = [ ( '<STR_LIT>' , [ '<STR_LIT>' , '<STR_LIT:x>' ] ) , <EOL> ( '<STR_LIT:x>' , [ '<STR_LIT:x>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' , '<STR_LIT:x>' ] ) , <EOL> ( '<STR_LIT>' , [ '<STR_LIT>' ] ) ] <EOL> for code , expected in test_data : <EOL> assert set ( ast_names ( code ) ) == set ( expected ) <EOL> def test_ast_names_disallowed_nodes ( ) : <EOL> from nose . tools import assert_raises <EOL> def list_ast_names ( code ) : <EOL> return list ( ast_names ( code ) ) <EOL> assert_raises ( PatsyError , list_ast_names , "<STR_LIT>" ) <EOL> assert_raises ( PatsyError , list_ast_names , "<STR_LIT>" ) <EOL> assert_raises ( PatsyError , list_ast_names , "<STR_LIT>" ) <EOL> if sys . version_info >= ( <NUM_LIT:2> , <NUM_LIT:7> ) : <EOL> assert_raises ( PatsyError , list_ast_names , "<STR_LIT>" ) <EOL> assert_raises ( PatsyError , list_ast_names , "<STR_LIT>" ) <EOL> class EvalEnvironment ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , namespaces , flags = <NUM_LIT:0> ) : <EOL> assert not flags & ~ _ALL_FUTURE_FLAGS <EOL> self . _namespaces = list ( namespaces ) <EOL> self . flags = flags <EOL> @ property <EOL> def namespace ( self ) : <EOL> """<STR_LIT>""" <EOL> return VarLookupDict ( self . _namespaces ) <EOL> def with_outer_namespace ( self , outer_namespace ) : <EOL> """<STR_LIT>""" <EOL> return self . __class__ ( self . _namespaces + [ outer_namespace ] , <EOL> self . flags ) <EOL> def eval ( self , expr , source_name = "<STR_LIT>" , inner_namespace = { } ) : <EOL> """<STR_LIT>""" <EOL> code = compile ( expr , source_name , "<STR_LIT>" , self . flags , False ) <EOL> return eval ( code , { } , VarLookupDict ( [ inner_namespace ] <EOL> + self . _namespaces ) ) <EOL> @ classmethod <EOL> def capture ( cls , eval_env = <NUM_LIT:0> , reference = <NUM_LIT:0> ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( eval_env , cls ) : <EOL> return eval_env <EOL> elif isinstance ( eval_env , numbers . Integral ) : <EOL> depth = eval_env + reference <EOL> else : <EOL> raise TypeError ( "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> frame = inspect . currentframe ( ) <EOL> try : <EOL> for i in range ( depth + <NUM_LIT:1> ) : <EOL> if frame is None : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> frame = frame . f_back <EOL> return cls ( [ frame . f_locals , frame . f_globals ] , <EOL> frame . f_code . co_flags & _ALL_FUTURE_FLAGS ) <EOL> finally : <EOL> del frame <EOL> def subset ( self , names ) : <EOL> """<STR_LIT>""" <EOL> vld = VarLookupDict ( self . _namespaces ) <EOL> new_ns = dict ( ( name , vld [ name ] ) for name in names ) <EOL> return EvalEnvironment ( [ new_ns ] , self . flags ) <EOL> def _namespace_ids ( self ) : <EOL> return [ id ( n ) for n in self . _namespaces ] <EOL> def __eq__ ( self , other ) : <EOL> return ( isinstance ( other , EvalEnvironment ) <EOL> and self . flags == other . flags <EOL> and self . _namespace_ids ( ) == other . _namespace_ids ( ) ) <EOL> def __ne__ ( self , other ) : <EOL> return not self == other <EOL> def __hash__ ( self ) : <EOL> return hash ( ( EvalEnvironment , <EOL> self . flags , <EOL> tuple ( self . _namespace_ids ( ) ) ) ) <EOL> __getstate__ = no_pickling <EOL> def _a ( ) : <EOL> _a = <NUM_LIT:1> <EOL> return _b ( ) <EOL> def _b ( ) : <EOL> _b = <NUM_LIT:1> <EOL> return _c ( ) <EOL> def _c ( ) : <EOL> _c = <NUM_LIT:1> <EOL> return [ EvalEnvironment . capture ( ) , <EOL> EvalEnvironment . capture ( <NUM_LIT:0> ) , <EOL> EvalEnvironment . capture ( <NUM_LIT:1> ) , <EOL> EvalEnvironment . capture ( <NUM_LIT:0> , reference = <NUM_LIT:1> ) , <EOL> EvalEnvironment . capture ( <NUM_LIT:2> ) , <EOL> EvalEnvironment . capture ( <NUM_LIT:0> , <NUM_LIT:2> ) , <EOL> ] <EOL> def test_EvalEnvironment_capture_namespace ( ) : <EOL> c0 , c , b1 , b2 , a1 , a2 = _a ( ) <EOL> assert "<STR_LIT>" in c0 . namespace <EOL> assert "<STR_LIT>" in c . namespace <EOL> assert "<STR_LIT>" in b1 . namespace <EOL> assert "<STR_LIT>" in b2 . namespace <EOL> assert "<STR_LIT>" in a1 . namespace <EOL> assert "<STR_LIT>" in a2 . namespace <EOL> assert c0 . namespace [ "<STR_LIT>" ] == <NUM_LIT:1> <EOL> assert c . namespace [ "<STR_LIT>" ] == <NUM_LIT:1> <EOL> assert b1 . namespace [ "<STR_LIT>" ] == <NUM_LIT:1> <EOL> assert b2 . namespace [ "<STR_LIT>" ] == <NUM_LIT:1> <EOL> assert a1 . namespace [ "<STR_LIT>" ] == <NUM_LIT:1> <EOL> assert a2 . namespace [ "<STR_LIT>" ] == <NUM_LIT:1> <EOL> assert b1 . namespace [ "<STR_LIT>" ] is _c <EOL> assert b2 . namespace [ "<STR_LIT>" ] is _c <EOL> from nose . tools import assert_raises <EOL> assert_raises ( ValueError , EvalEnvironment . capture , <NUM_LIT:10> ** <NUM_LIT:6> ) <EOL> assert EvalEnvironment . capture ( b1 ) is b1 <EOL> assert_raises ( TypeError , EvalEnvironment . capture , <NUM_LIT> ) <EOL> assert_no_pickling ( EvalEnvironment . capture ( ) ) <EOL> def test_EvalEnvironment_capture_flags ( ) : <EOL> if sys . version_info >= ( <NUM_LIT:3> , ) : <EOL> TEST_FEATURE = "<STR_LIT>" <EOL> else : <EOL> TEST_FEATURE = "<STR_LIT>" <EOL> test_flag = getattr ( __future__ , TEST_FEATURE ) . compiler_flag <EOL> assert test_flag & _ALL_FUTURE_FLAGS <EOL> source = ( "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> code = compile ( source , "<STR_LIT>" , "<STR_LIT>" , <NUM_LIT:0> , <NUM_LIT:1> ) <EOL> env = { "<STR_LIT>" : EvalEnvironment , <EOL> "<STR_LIT>" : lambda : EvalEnvironment . capture ( <NUM_LIT:0> ) , <EOL> "<STR_LIT>" : lambda : EvalEnvironment . capture ( <NUM_LIT:1> ) , <EOL> } <EOL> env2 = dict ( env ) <EOL> six . exec_ ( code , env ) <EOL> assert env [ "<STR_LIT>" ] . namespace [ "<STR_LIT>" ] == "<STR_LIT>" <EOL> assert env [ "<STR_LIT>" ] . namespace [ "<STR_LIT>" ] == "<STR_LIT>" <EOL> assert "<STR_LIT>" not in env [ "<STR_LIT>" ] . namespace <EOL> assert env [ "<STR_LIT>" ] . flags & _ALL_FUTURE_FLAGS == <NUM_LIT:0> <EOL> assert env [ "<STR_LIT>" ] . flags & _ALL_FUTURE_FLAGS == <NUM_LIT:0> <EOL> assert env [ "<STR_LIT>" ] . flags & _ALL_FUTURE_FLAGS == <NUM_LIT:0> <EOL> code2 = compile ( ( "<STR_LIT>" % ( TEST_FEATURE , ) ) <EOL> + source , <EOL> "<STR_LIT>" , "<STR_LIT>" , <NUM_LIT:0> , <NUM_LIT:1> ) <EOL> six . exec_ ( code2 , env2 ) <EOL> assert env2 [ "<STR_LIT>" ] . namespace [ "<STR_LIT>" ] == "<STR_LIT>" <EOL> assert env2 [ "<STR_LIT>" ] . namespace [ "<STR_LIT>" ] == "<STR_LIT>" <EOL> assert "<STR_LIT>" not in env2 [ "<STR_LIT>" ] . namespace <EOL> assert env2 [ "<STR_LIT>" ] . flags & _ALL_FUTURE_FLAGS == test_flag <EOL> assert env2 [ "<STR_LIT>" ] . flags & _ALL_FUTURE_FLAGS == <NUM_LIT:0> <EOL> assert env2 [ "<STR_LIT>" ] . flags & _ALL_FUTURE_FLAGS == test_flag <EOL> def test_EvalEnvironment_eval_namespace ( ) : <EOL> env = EvalEnvironment ( [ { "<STR_LIT:a>" : <NUM_LIT:1> } ] ) <EOL> assert env . eval ( "<STR_LIT>" ) == <NUM_LIT:2> <EOL> assert env . eval ( "<STR_LIT>" , inner_namespace = { "<STR_LIT:a>" : <NUM_LIT:2> } ) == <NUM_LIT:4> <EOL> from nose . tools import assert_raises <EOL> assert_raises ( NameError , env . eval , "<STR_LIT>" ) <EOL> a = <NUM_LIT:3> <EOL> env2 = EvalEnvironment . capture ( <NUM_LIT:0> ) <EOL> assert env2 . eval ( "<STR_LIT>" ) == <NUM_LIT:6> <EOL> env3 = env . with_outer_namespace ( { "<STR_LIT:a>" : <NUM_LIT:10> , "<STR_LIT:b>" : <NUM_LIT:3> } ) <EOL> assert env3 . eval ( "<STR_LIT>" ) == <NUM_LIT:2> <EOL> assert env3 . eval ( "<STR_LIT>" ) == <NUM_LIT:6> <EOL> def test_EvalEnvironment_eval_flags ( ) : <EOL> from nose . tools import assert_raises <EOL> if sys . version_info >= ( <NUM_LIT:3> , ) : <EOL> test_flag = __future__ . barry_as_FLUFL . compiler_flag <EOL> assert test_flag & _ALL_FUTURE_FLAGS <EOL> env = EvalEnvironment ( [ { "<STR_LIT:a>" : <NUM_LIT:11> } ] , flags = <NUM_LIT:0> ) <EOL> assert env . eval ( "<STR_LIT>" ) == True <EOL> assert_raises ( SyntaxError , env . eval , "<STR_LIT>" ) <EOL> assert env . subset ( [ "<STR_LIT:a>" ] ) . flags == <NUM_LIT:0> <EOL> assert env . with_outer_namespace ( { "<STR_LIT:b>" : <NUM_LIT:10> } ) . flags == <NUM_LIT:0> <EOL> env2 = EvalEnvironment ( [ { "<STR_LIT:a>" : <NUM_LIT:11> } ] , flags = test_flag ) <EOL> assert env2 . eval ( "<STR_LIT>" ) == True <EOL> assert_raises ( SyntaxError , env2 . eval , "<STR_LIT>" ) <EOL> assert env2 . subset ( [ "<STR_LIT:a>" ] ) . flags == test_flag <EOL> assert env2 . with_outer_namespace ( { "<STR_LIT:b>" : <NUM_LIT:10> } ) . flags == test_flag <EOL> else : <EOL> test_flag = __future__ . division . compiler_flag <EOL> assert test_flag & _ALL_FUTURE_FLAGS <EOL> env = EvalEnvironment ( [ { "<STR_LIT:a>" : <NUM_LIT:11> } ] , flags = <NUM_LIT:0> ) <EOL> assert env . eval ( "<STR_LIT>" ) == <NUM_LIT:11> // <NUM_LIT:2> == <NUM_LIT:5> <EOL> assert env . subset ( [ "<STR_LIT:a>" ] ) . flags == <NUM_LIT:0> <EOL> assert env . with_outer_namespace ( { "<STR_LIT:b>" : <NUM_LIT:10> } ) . flags == <NUM_LIT:0> <EOL> env2 = EvalEnvironment ( [ { "<STR_LIT:a>" : <NUM_LIT:11> } ] , flags = test_flag ) <EOL> assert env2 . eval ( "<STR_LIT>" ) == <NUM_LIT:11> * <NUM_LIT:1.> / <NUM_LIT:2> != <NUM_LIT:5> <EOL> env2 . subset ( [ "<STR_LIT:a>" ] ) . flags == test_flag <EOL> assert env2 . with_outer_namespace ( { "<STR_LIT:b>" : <NUM_LIT:10> } ) . flags == test_flag <EOL> def test_EvalEnvironment_subset ( ) : <EOL> env = EvalEnvironment ( [ { "<STR_LIT:a>" : <NUM_LIT:1> } , { "<STR_LIT:b>" : <NUM_LIT:2> } , { "<STR_LIT:c>" : <NUM_LIT:3> } ] ) <EOL> subset_a = env . subset ( [ "<STR_LIT:a>" ] ) <EOL> assert subset_a . eval ( "<STR_LIT:a>" ) == <NUM_LIT:1> <EOL> from nose . tools import assert_raises <EOL> assert_raises ( NameError , subset_a . eval , "<STR_LIT:b>" ) <EOL> assert_raises ( NameError , subset_a . eval , "<STR_LIT:c>" ) <EOL> subset_bc = env . subset ( [ "<STR_LIT:b>" , "<STR_LIT:c>" ] ) <EOL> assert subset_bc . eval ( "<STR_LIT>" ) == <NUM_LIT:6> <EOL> assert_raises ( NameError , subset_bc . eval , "<STR_LIT:a>" ) <EOL> def test_EvalEnvironment_eq ( ) : <EOL> env1 = EvalEnvironment . capture ( <NUM_LIT:0> ) <EOL> env2 = EvalEnvironment . capture ( <NUM_LIT:0> ) <EOL> assert env1 == env2 <EOL> assert hash ( env1 ) == hash ( env2 ) <EOL> capture_local_env = lambda : EvalEnvironment . capture ( <NUM_LIT:0> ) <EOL> env3 = capture_local_env ( ) <EOL> env4 = capture_local_env ( ) <EOL> assert env3 != env4 <EOL> _builtins_dict = { } <EOL> six . exec_ ( "<STR_LIT>" , { } , _builtins_dict ) <EOL> import patsy . builtins <EOL> class EvalFactor ( object ) : <EOL> def __init__ ( self , code , origin = None ) : <EOL> """<STR_LIT>""" <EOL> self . code = normalize_token_spacing ( code ) <EOL> self . origin = origin <EOL> def name ( self ) : <EOL> return self . code <EOL> def __repr__ ( self ) : <EOL> return "<STR_LIT>" % ( self . __class__ . __name__ , self . code ) <EOL> def __eq__ ( self , other ) : <EOL> return ( isinstance ( other , EvalFactor ) <EOL> and self . code == other . code ) <EOL> def __ne__ ( self , other ) : <EOL> return not self == other <EOL> def __hash__ ( self ) : <EOL> return hash ( ( EvalFactor , self . code ) ) <EOL> def memorize_passes_needed ( self , state , eval_env ) : <EOL> state [ "<STR_LIT>" ] = { } <EOL> eval_env = eval_env . with_outer_namespace ( _builtins_dict ) <EOL> env_namespace = eval_env . namespace <EOL> subset_names = [ name for name in ast_names ( self . code ) <EOL> if name in env_namespace ] <EOL> eval_env = eval_env . subset ( subset_names ) <EOL> state [ "<STR_LIT>" ] = eval_env <EOL> i = [ <NUM_LIT:0> ] <EOL> def new_name_maker ( token ) : <EOL> value = eval_env . namespace . get ( token ) <EOL> if hasattr ( value , "<STR_LIT>" ) : <EOL> obj_name = "<STR_LIT>" % ( i [ <NUM_LIT:0> ] , token ) <EOL> i [ <NUM_LIT:0> ] += <NUM_LIT:1> <EOL> obj = value . __patsy_stateful_transform__ ( ) <EOL> state [ "<STR_LIT>" ] [ obj_name ] = obj <EOL> return obj_name + "<STR_LIT>" <EOL> else : <EOL> return token <EOL> eval_code = replace_bare_funcalls ( self . code , new_name_maker ) <EOL> state [ "<STR_LIT>" ] = eval_code <EOL> if has_bare_variable_reference ( state [ "<STR_LIT>" ] , self . code ) : <EOL> raise PatsyError ( "<STR_LIT>" <EOL> "<STR_LIT>" % ( token , ) , token . origin ) <EOL> state [ "<STR_LIT>" ] = { } <EOL> for obj_name in state [ "<STR_LIT>" ] : <EOL> transform_calls = capture_obj_method_calls ( obj_name , eval_code ) <EOL> assert len ( transform_calls ) == <NUM_LIT:1> <EOL> transform_call = transform_calls [ <NUM_LIT:0> ] <EOL> transform_call_name , transform_call_code = transform_call <EOL> assert transform_call_name == obj_name + "<STR_LIT>" <EOL> assert transform_call_code . startswith ( transform_call_name + "<STR_LIT:(>" ) <EOL> memorize_code = ( obj_name <EOL> + "<STR_LIT>" <EOL> + transform_call_code [ len ( transform_call_name ) : ] ) <EOL> state [ "<STR_LIT>" ] [ obj_name ] = memorize_code <EOL> pass_bins = [ ] <EOL> unsorted = set ( state [ "<STR_LIT>" ] ) <EOL> while unsorted : <EOL> pass_bin = set ( ) <EOL> for obj_name in unsorted : <EOL> other_objs = unsorted . difference ( [ obj_name ] ) <EOL> memorize_code = state [ "<STR_LIT>" ] [ obj_name ] <EOL> if not has_bare_variable_reference ( other_objs , memorize_code ) : <EOL> pass_bin . add ( obj_name ) <EOL> assert pass_bin <EOL> unsorted . difference_update ( pass_bin ) <EOL> pass_bins . append ( pass_bin ) <EOL> state [ "<STR_LIT>" ] = pass_bins <EOL> return len ( pass_bins ) <EOL> def _eval ( self , code , memorize_state , data ) : <EOL> inner_namespace = VarLookupDict ( [ data , memorize_state [ "<STR_LIT>" ] ] ) <EOL> return call_and_wrap_exc ( "<STR_LIT>" , <EOL> self , <EOL> memorize_state [ "<STR_LIT>" ] . eval , <EOL> code , <EOL> inner_namespace = inner_namespace ) <EOL> def memorize_chunk ( self , state , which_pass , data ) : <EOL> for obj_name in state [ "<STR_LIT>" ] [ which_pass ] : <EOL> self . _eval ( state [ "<STR_LIT>" ] [ obj_name ] , <EOL> state , <EOL> data ) <EOL> def memorize_finish ( self , state , which_pass ) : <EOL> for obj_name in state [ "<STR_LIT>" ] [ which_pass ] : <EOL> state [ "<STR_LIT>" ] [ obj_name ] . memorize_finish ( ) <EOL> def eval ( self , memorize_state , data ) : <EOL> return self . _eval ( memorize_state [ "<STR_LIT>" ] , <EOL> memorize_state , <EOL> data ) <EOL> __getstate__ = no_pickling <EOL> def test_EvalFactor_basics ( ) : <EOL> e = EvalFactor ( "<STR_LIT>" ) <EOL> assert e . code == "<STR_LIT>" <EOL> assert e . name ( ) == "<STR_LIT>" <EOL> e2 = EvalFactor ( "<STR_LIT>" , origin = "<STR_LIT>" ) <EOL> assert e == e2 <EOL> assert hash ( e ) == hash ( e2 ) <EOL> assert e . origin is None <EOL> assert e2 . origin == "<STR_LIT>" <EOL> assert_no_pickling ( e ) <EOL> def test_EvalFactor_memorize_passes_needed ( ) : <EOL> from patsy . state import stateful_transform <EOL> foo = stateful_transform ( lambda : "<STR_LIT>" ) <EOL> bar = stateful_transform ( lambda : "<STR_LIT>" ) <EOL> quux = stateful_transform ( lambda : "<STR_LIT>" ) <EOL> e = EvalFactor ( "<STR_LIT>" ) <EOL> state = { } <EOL> eval_env = EvalEnvironment . capture ( <NUM_LIT:0> ) <EOL> passes = e . memorize_passes_needed ( state , eval_env ) <EOL> print ( passes ) <EOL> print ( state ) <EOL> assert passes == <NUM_LIT:2> <EOL> for name in [ "<STR_LIT:foo>" , "<STR_LIT:bar>" , "<STR_LIT>" ] : <EOL> assert state [ "<STR_LIT>" ] . namespace [ name ] is locals ( ) [ name ] <EOL> for name in [ "<STR_LIT:w>" , "<STR_LIT:x>" , "<STR_LIT:y>" , "<STR_LIT:z>" , "<STR_LIT:e>" , "<STR_LIT:state>" ] : <EOL> assert name not in state [ "<STR_LIT>" ] . namespace <EOL> assert state [ "<STR_LIT>" ] == { "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT>" } <EOL> assert ( state [ "<STR_LIT>" ] <EOL> == "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> assert ( state [ "<STR_LIT>" ] <EOL> == { "<STR_LIT>" : <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" : <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" : <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" : <EOL> "<STR_LIT>" , <EOL> } ) <EOL> assert state [ "<STR_LIT>" ] == [ set ( [ "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" ] ) , <EOL> set ( [ "<STR_LIT>" ] ) ] <EOL> class _MockTransform ( object ) : <EOL> def __init__ ( self ) : <EOL> self . _sum = <NUM_LIT:0> <EOL> self . _memorize_chunk_called = <NUM_LIT:0> <EOL> self . _memorize_finish_called = <NUM_LIT:0> <EOL> def memorize_chunk ( self , data ) : <EOL> self . _memorize_chunk_called += <NUM_LIT:1> <EOL> import numpy as np <EOL> self . _sum += np . sum ( data ) <EOL> def memorize_finish ( self ) : <EOL> self . _memorize_finish_called += <NUM_LIT:1> <EOL> def transform ( self , data ) : <EOL> return data - self . _sum <EOL> def test_EvalFactor_end_to_end ( ) : <EOL> from patsy . state import stateful_transform <EOL> foo = stateful_transform ( _MockTransform ) <EOL> e = EvalFactor ( "<STR_LIT>" ) <EOL> state = { } <EOL> eval_env = EvalEnvironment . capture ( <NUM_LIT:0> ) <EOL> passes = e . memorize_passes_needed ( state , eval_env ) <EOL> print ( passes ) <EOL> print ( state ) <EOL> assert passes == <NUM_LIT:2> <EOL> assert state [ "<STR_LIT>" ] . namespace [ "<STR_LIT:foo>" ] is foo <EOL> for name in [ "<STR_LIT:x>" , "<STR_LIT:y>" , "<STR_LIT:e>" , "<STR_LIT:state>" ] : <EOL> assert name not in state [ "<STR_LIT>" ] . namespace <EOL> import numpy as np <EOL> e . memorize_chunk ( state , <NUM_LIT:0> , <EOL> { "<STR_LIT:x>" : np . array ( [ <NUM_LIT:1> , <NUM_LIT:2> ] ) , <EOL> "<STR_LIT:y>" : np . array ( [ <NUM_LIT:10> , <NUM_LIT:11> ] ) } ) <EOL> assert state [ "<STR_LIT>" ] [ "<STR_LIT>" ] . _memorize_chunk_called == <NUM_LIT:1> <EOL> assert state [ "<STR_LIT>" ] [ "<STR_LIT>" ] . _memorize_chunk_called == <NUM_LIT:1> <EOL> e . memorize_chunk ( state , <NUM_LIT:0> , { "<STR_LIT:x>" : np . array ( [ <NUM_LIT:12> , - <NUM_LIT:10> ] ) , <EOL> "<STR_LIT:y>" : np . array ( [ <NUM_LIT:100> , <NUM_LIT:3> ] ) } ) <EOL> assert state [ "<STR_LIT>" ] [ "<STR_LIT>" ] . _memorize_chunk_called == <NUM_LIT:2> <EOL> assert state [ "<STR_LIT>" ] [ "<STR_LIT>" ] . _memorize_chunk_called == <NUM_LIT:2> <EOL> assert state [ "<STR_LIT>" ] [ "<STR_LIT>" ] . _memorize_finish_called == <NUM_LIT:0> <EOL> assert state [ "<STR_LIT>" ] [ "<STR_LIT>" ] . _memorize_finish_called == <NUM_LIT:0> <EOL> e . memorize_finish ( state , <NUM_LIT:0> ) <EOL> assert state [ "<STR_LIT>" ] [ "<STR_LIT>" ] . _memorize_finish_called == <NUM_LIT:1> <EOL> assert state [ "<STR_LIT>" ] [ "<STR_LIT>" ] . _memorize_finish_called == <NUM_LIT:1> <EOL> assert state [ "<STR_LIT>" ] [ "<STR_LIT>" ] . _memorize_chunk_called == <NUM_LIT:0> <EOL> assert state [ "<STR_LIT>" ] [ "<STR_LIT>" ] . _memorize_finish_called == <NUM_LIT:0> <EOL> e . memorize_chunk ( state , <NUM_LIT:1> , { "<STR_LIT:x>" : np . array ( [ <NUM_LIT:1> , <NUM_LIT:2> ] ) , <EOL> "<STR_LIT:y>" : np . array ( [ <NUM_LIT:10> , <NUM_LIT:11> ] ) } ) <EOL> e . memorize_chunk ( state , <NUM_LIT:1> , { "<STR_LIT:x>" : np . array ( [ <NUM_LIT:12> , - <NUM_LIT:10> ] ) , <EOL> "<STR_LIT:y>" : np . array ( [ <NUM_LIT:100> , <NUM_LIT:3> ] ) } ) <EOL> e . memorize_finish ( state , <NUM_LIT:1> ) <EOL> for transform in six . itervalues ( state [ "<STR_LIT>" ] ) : <EOL> assert transform . _memorize_chunk_called == <NUM_LIT:2> <EOL> assert transform . _memorize_finish_called == <NUM_LIT:1> <EOL> assert np . all ( e . eval ( state , <EOL> { "<STR_LIT:x>" : np . array ( [ <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:12> , - <NUM_LIT:10> ] ) , <EOL> "<STR_LIT:y>" : np . array ( [ <NUM_LIT:10> , <NUM_LIT:11> , <NUM_LIT:100> , <NUM_LIT:3> ] ) } ) <EOL> == [ <NUM_LIT> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> ] ) <EOL> def annotated_tokens ( code ) : <EOL> prev_was_dot = False <EOL> it = PushbackAdapter ( python_tokenize ( code ) ) <EOL> for ( token_type , token , origin ) in it : <EOL> props = { } <EOL> props [ "<STR_LIT>" ] = ( not prev_was_dot and token_type == tokenize . NAME ) <EOL> props [ "<STR_LIT>" ] = ( props [ "<STR_LIT>" ] <EOL> and it . has_more ( ) and it . peek ( ) [ <NUM_LIT:1> ] == "<STR_LIT:(>" ) <EOL> yield ( token_type , token , origin , props ) <EOL> prev_was_dot = ( token == "<STR_LIT:.>" ) <EOL> def test_annotated_tokens ( ) : <EOL> tokens_without_origins = [ ( token_type , token , props ) <EOL> for ( token_type , token , origin , props ) <EOL> in ( annotated_tokens ( "<STR_LIT>" ) ) ] <EOL> assert ( tokens_without_origins <EOL> == [ ( tokenize . NAME , "<STR_LIT:a>" , { "<STR_LIT>" : True , "<STR_LIT>" : True } ) , <EOL> ( tokenize . OP , "<STR_LIT:(>" , { "<STR_LIT>" : False , "<STR_LIT>" : False } ) , <EOL> ( tokenize . NAME , "<STR_LIT:b>" , { "<STR_LIT>" : True , "<STR_LIT>" : False } ) , <EOL> ( tokenize . OP , "<STR_LIT:)>" , { "<STR_LIT>" : False , "<STR_LIT>" : False } ) , <EOL> ( tokenize . OP , "<STR_LIT:+>" , { "<STR_LIT>" : False , "<STR_LIT>" : False } ) , <EOL> ( tokenize . NAME , "<STR_LIT:c>" , { "<STR_LIT>" : True , "<STR_LIT>" : False } ) , <EOL> ( tokenize . OP , "<STR_LIT:.>" , { "<STR_LIT>" : False , "<STR_LIT>" : False } ) , <EOL> ( tokenize . NAME , "<STR_LIT:d>" , <EOL> { "<STR_LIT>" : False , "<STR_LIT>" : False } ) , <EOL> ] ) <EOL> assert len ( list ( annotated_tokens ( "<STR_LIT:x>" ) ) ) == <NUM_LIT:1> <EOL> def has_bare_variable_reference ( names , code ) : <EOL> for ( _ , token , _ , props ) in annotated_tokens ( code ) : <EOL> if props [ "<STR_LIT>" ] and token in names : <EOL> return True <EOL> return False <EOL> def replace_bare_funcalls ( code , replacer ) : <EOL> tokens = [ ] <EOL> for ( token_type , token , origin , props ) in annotated_tokens ( code ) : <EOL> if props [ "<STR_LIT>" ] and props [ "<STR_LIT>" ] : <EOL> token = replacer ( token ) <EOL> tokens . append ( ( token_type , token ) ) <EOL> return pretty_untokenize ( tokens ) <EOL> def test_replace_bare_funcalls ( ) : <EOL> def replacer1 ( token ) : <EOL> return { "<STR_LIT:a>" : "<STR_LIT:b>" , "<STR_LIT:foo>" : "<STR_LIT>" } . get ( token , token ) <EOL> def t1 ( code , expected ) : <EOL> replaced = replace_bare_funcalls ( code , replacer1 ) <EOL> print ( "<STR_LIT>" % ( code , replaced ) ) <EOL> print ( "<STR_LIT>" % ( expected , ) ) <EOL> assert replaced == expected <EOL> t1 ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> t1 ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> t1 ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> t1 ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> t1 ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> t1 ( "<STR_LIT>" , <EOL> "<STR_LIT>" ) <EOL> class _FuncallCapturer ( object ) : <EOL> def __init__ ( self , start_token_type , start_token ) : <EOL> self . func = [ start_token ] <EOL> self . tokens = [ ( start_token_type , start_token ) ] <EOL> self . paren_depth = <NUM_LIT:0> <EOL> self . started = False <EOL> self . done = False <EOL> def add_token ( self , token_type , token ) : <EOL> if self . done : <EOL> return <EOL> self . tokens . append ( ( token_type , token ) ) <EOL> if token in [ "<STR_LIT:(>" , "<STR_LIT:{>" , "<STR_LIT:[>" ] : <EOL> self . paren_depth += <NUM_LIT:1> <EOL> if token in [ "<STR_LIT:)>" , "<STR_LIT:}>" , "<STR_LIT:]>" ] : <EOL> self . paren_depth -= <NUM_LIT:1> <EOL> assert self . paren_depth >= <NUM_LIT:0> <EOL> if not self . started : <EOL> if token == "<STR_LIT:(>" : <EOL> self . started = True <EOL> else : <EOL> assert token_type == tokenize . NAME or token == "<STR_LIT:.>" <EOL> self . func . append ( token ) <EOL> if self . started and self . paren_depth == <NUM_LIT:0> : <EOL> self . done = True <EOL> def capture_obj_method_calls ( obj_name , code ) : <EOL> capturers = [ ] <EOL> for ( token_type , token , origin , props ) in annotated_tokens ( code ) : <EOL> for capturer in capturers : <EOL> capturer . add_token ( token_type , token ) <EOL> if props [ "<STR_LIT>" ] and token == obj_name : <EOL> capturers . append ( _FuncallCapturer ( token_type , token ) ) <EOL> return [ ( "<STR_LIT>" . join ( capturer . func ) , pretty_untokenize ( capturer . tokens ) ) <EOL> for capturer in capturers ] <EOL> def test_capture_obj_method_calls ( ) : <EOL> assert ( capture_obj_method_calls ( "<STR_LIT:foo>" , "<STR_LIT>" ) <EOL> == [ ( "<STR_LIT>" , "<STR_LIT>" ) ] ) <EOL> assert ( capture_obj_method_calls ( "<STR_LIT:b>" , "<STR_LIT>" ) <EOL> == [ ( "<STR_LIT>" , "<STR_LIT>" ) ] ) <EOL> assert ( capture_obj_method_calls ( "<STR_LIT:foo>" , "<STR_LIT>" ) <EOL> == [ ( "<STR_LIT>" , "<STR_LIT>" ) , <EOL> ( "<STR_LIT>" , "<STR_LIT>" ) ] ) <EOL> assert ( capture_obj_method_calls ( "<STR_LIT:bar>" , "<STR_LIT>" ) <EOL> == [ ( "<STR_LIT>" , "<STR_LIT>" ) ] ) </s>
