<s> """<STR_LIT>""" <EOL> from neon . backends . backend import OpTreeNode , Tensor <EOL> import numpy as np <EOL> from functools import wraps <EOL> _scalar_types = { int , float , np . float16 , np . float32 , np . uint8 , np . int8 , <EOL> np . uint16 , np . int16 , np . uint32 , np . int32 } <EOL> class GradUtil ( ) : <EOL> """<STR_LIT>""" <EOL> @ staticmethod <EOL> def get_grad_back ( grad_node ) : <EOL> """<STR_LIT>""" <EOL> if not grad_node : <EOL> return None <EOL> x = grad_node . left . op_tree if grad_node . left else None <EOL> y = grad_node . right . op_tree if grad_node . right else None <EOL> z = grad_node . op_tree <EOL> dz = grad_node . grad_op_tree <EOL> op_dict = z [ <NUM_LIT:0> ] <EOL> be = grad_node . ad . be <EOL> op = grad_node . op_tree [ <NUM_LIT:0> ] [ '<STR_LIT>' ] <EOL> grad_increments = grad_map [ op ] ( x , y , z , dz , op_dict , be ) <EOL> left_increment = GradUtil . _unbroadcast ( grad_increments [ <NUM_LIT:0> ] , x , be ) <EOL> right_increment = GradUtil . _unbroadcast ( grad_increments [ <NUM_LIT:1> ] , y , be ) <EOL> return ( left_increment , right_increment ) <EOL> @ staticmethod <EOL> def _unbroadcast ( grad_op_tree , x , be ) : <EOL> """<STR_LIT>""" <EOL> if ( not grad_op_tree ) or ( not x ) : <EOL> return grad_op_tree <EOL> if type ( x ) in _scalar_types : <EOL> return <NUM_LIT:0.> <EOL> in_shape = x . shape <EOL> out_shape = grad_op_tree . shape <EOL> if in_shape == out_shape : <EOL> return grad_op_tree <EOL> elif len ( in_shape ) == <NUM_LIT:2> and len ( out_shape ) == <NUM_LIT:2> : <EOL> if in_shape == ( <NUM_LIT:1> , <NUM_LIT:1> ) : <EOL> return be . sum ( grad_op_tree ) <EOL> elif in_shape [ <NUM_LIT:0> ] == out_shape [ <NUM_LIT:0> ] and in_shape [ <NUM_LIT:1> ] == <NUM_LIT:1> : <EOL> return be . sum ( grad_op_tree , axis = <NUM_LIT:1> ) <EOL> elif in_shape [ <NUM_LIT:0> ] == <NUM_LIT:1> and in_shape [ <NUM_LIT:1> ] == out_shape [ <NUM_LIT:1> ] : <EOL> return be . sum ( grad_op_tree , axis = <NUM_LIT:0> ) <EOL> elif ( ( out_shape [ <NUM_LIT:0> ] == in_shape [ <NUM_LIT:0> ] and out_shape [ <NUM_LIT:1> ] == <NUM_LIT:1> ) or <EOL> ( out_shape [ <NUM_LIT:0> ] == <NUM_LIT:1> and out_shape [ <NUM_LIT:1> ] == in_shape [ <NUM_LIT:1> ] ) ) : <EOL> return <NUM_LIT:0> * x + grad_op_tree <EOL> else : <EOL> return NotImplemented <EOL> else : <EOL> return NotImplemented <EOL> @ staticmethod <EOL> def is_invalid ( grad_op_tree , be ) : <EOL> """<STR_LIT>""" <EOL> grad_op_tree_val = be . empty ( grad_op_tree . shape ) <EOL> grad_op_tree_val [ : ] = grad_op_tree <EOL> grad_op_tree_val_np = grad_op_tree_val . get ( ) . reshape ( - <NUM_LIT:1> , ) <EOL> for val in grad_op_tree_val_np : <EOL> if not ( - <NUM_LIT> < val < <NUM_LIT> ) : <EOL> return True <EOL> else : <EOL> return False <EOL> """<STR_LIT>""" <EOL> @ staticmethod <EOL> def _zero_grad_unary ( x , y , z , dz , op_dict , be ) : <EOL> return ( dz * <NUM_LIT:0.> , None ) <EOL> @ staticmethod <EOL> def _zero_grad_binary ( x , y , z , dz , op_dict , be ) : <EOL> return ( dz * <NUM_LIT:0.> , dz * <NUM_LIT:0.> ) <EOL> @ staticmethod <EOL> def _add_grad ( x , y , z , dz , op_dict , be ) : <EOL> return ( dz , dz ) <EOL> @ staticmethod <EOL> def _mul_grad ( x , y , z , dz , op_dict , be ) : <EOL> return ( dz * y , dz * x ) <EOL> @ staticmethod <EOL> def _sub_grad ( x , y , z , dz , op_dict , be ) : <EOL> return ( dz , - dz ) <EOL> @ staticmethod <EOL> def _neg_grad ( x , y , z , dz , op_dict , be ) : <EOL> return ( - dz , None ) <EOL> @ staticmethod <EOL> def _pow_grad ( x , y , z , dz , op_dict , be ) : <EOL> return ( dz * y * x ** ( y - <NUM_LIT:1.> ) , dz * z * be . log ( x ) ) <EOL> @ staticmethod <EOL> def _div_grad ( x , y , z , dz , op_dict , be ) : <EOL> return ( dz / y , - dz * x / be . square ( y ) ) <EOL> @ staticmethod <EOL> def _dot_grad ( x , y , z , dz , op_dict , be ) : <EOL> return ( be . dot ( dz , y . T ) , be . dot ( x . T , dz ) ) <EOL> @ staticmethod <EOL> def _abs_grad ( x , y , z , dz , op_dict , be ) : <EOL> return ( dz * be . sgn ( x ) , None ) <EOL> @ staticmethod <EOL> def _sqrt_grad ( x , y , z , dz , op_dict , be ) : <EOL> return ( dz * <NUM_LIT:0.5> / z , None ) <EOL> @ staticmethod <EOL> def _sqr_grad ( x , y , z , dz , op_dict , be ) : <EOL> return ( dz * <NUM_LIT> * x , None ) <EOL> @ staticmethod <EOL> def _exp_grad ( x , y , z , dz , op_dict , be ) : <EOL> return ( dz * z , None ) <EOL> @ staticmethod <EOL> def _exp2_grad ( x , y , z , dz , op_dict , be ) : <EOL> return ( dz * z * be . log ( <NUM_LIT> ) , None ) <EOL> @ staticmethod <EOL> def _log_grad ( x , y , z , dz , op_dict , be ) : <EOL> return ( dz / x , None ) <EOL> @ staticmethod <EOL> def _log2_grad ( x , y , z , dz , op_dict , be ) : <EOL> return ( dz / x / be . log ( <NUM_LIT> ) , None ) <EOL> @ staticmethod <EOL> def _sig_grad ( x , y , z , dz , op_dict , be ) : <EOL> return ( dz * z * ( <NUM_LIT:1.> - z ) , None ) <EOL> @ staticmethod <EOL> def _sig2_grad ( x , y , z , dz , op_dict , be ) : <EOL> return ( dz * z * ( <NUM_LIT:1.> - z ) * be . log ( <NUM_LIT> ) , None ) <EOL> @ staticmethod <EOL> def _tanh_grad ( x , y , z , dz , op_dict , be ) : <EOL> return ( dz * ( <NUM_LIT:1.> - be . square ( z ) ) , None ) <EOL> @ staticmethod <EOL> def _tanh2_grad ( x , y , z , dz , op_dict , be ) : <EOL> return ( dz * ( <NUM_LIT:1.> - be . square ( z ) ) * be . log ( <NUM_LIT> ) , None ) <EOL> @ staticmethod <EOL> def _max_grad ( x , y , z , dz , op_dict , be ) : <EOL> return ( dz * ( x == z ) , None ) <EOL> @ staticmethod <EOL> def _min_grad ( x , y , z , dz , op_dict , be ) : <EOL> return ( dz * ( x == z ) , None ) <EOL> @ staticmethod <EOL> def _maximum_grad ( x , y , z , dz , op_dict , be ) : <EOL> return ( dz * be . greater_equal ( x , y ) , dz * be . greater_equal ( y , x ) ) <EOL> @ staticmethod <EOL> def _minimum_grad ( x , y , z , dz , op_dict , be ) : <EOL> return ( dz * be . less_equal ( x , y ) , dz * be . less_equal ( y , x ) ) <EOL> @ staticmethod <EOL> def _sum_grad ( x , y , z , dz , op_dict , be ) : <EOL> assert ( '<STR_LIT>' in op_dict and ( op_dict [ '<STR_LIT>' ] in ( <NUM_LIT:0> , <NUM_LIT:1> ) ) ) <EOL> return ( dz , None ) <EOL> @ staticmethod <EOL> def _transpose_grad ( x , y , z , dz , op_dict , be ) : <EOL> return ( dz . T , None ) <EOL> grad_map = { <EOL> '<STR_LIT>' : GradUtil . _zero_grad_binary , <EOL> '<STR_LIT>' : GradUtil . _zero_grad_binary , <EOL> '<STR_LIT>' : GradUtil . _zero_grad_binary , <EOL> '<STR_LIT>' : GradUtil . _zero_grad_binary , <EOL> '<STR_LIT>' : GradUtil . _zero_grad_binary , <EOL> '<STR_LIT>' : GradUtil . _zero_grad_unary , <EOL> '<STR_LIT>' : GradUtil . _zero_grad_unary , <EOL> '<STR_LIT>' : GradUtil . _zero_grad_unary , <EOL> '<STR_LIT>' : GradUtil . _zero_grad_unary , <EOL> '<STR_LIT>' : GradUtil . _add_grad , <EOL> '<STR_LIT>' : GradUtil . _mul_grad , <EOL> '<STR_LIT>' : GradUtil . _sub_grad , <EOL> '<STR_LIT>' : GradUtil . _pow_grad , <EOL> '<STR_LIT>' : GradUtil . _div_grad , <EOL> '<STR_LIT>' : GradUtil . _dot_grad , <EOL> '<STR_LIT>' : GradUtil . _neg_grad , <EOL> '<STR_LIT>' : GradUtil . _abs_grad , <EOL> '<STR_LIT>' : GradUtil . _sqrt_grad , <EOL> '<STR_LIT>' : GradUtil . _sqr_grad , <EOL> '<STR_LIT>' : GradUtil . _exp_grad , <EOL> '<STR_LIT>' : GradUtil . _exp2_grad , <EOL> '<STR_LIT>' : GradUtil . _log_grad , <EOL> '<STR_LIT>' : GradUtil . _log2_grad , <EOL> '<STR_LIT>' : GradUtil . _sig_grad , <EOL> '<STR_LIT>' : GradUtil . _sig2_grad , <EOL> '<STR_LIT>' : GradUtil . _tanh_grad , <EOL> '<STR_LIT>' : GradUtil . _tanh2_grad , <EOL> '<STR_LIT>' : GradUtil . _max_grad , <EOL> '<STR_LIT>' : GradUtil . _min_grad , <EOL> '<STR_LIT>' : GradUtil . _maximum_grad , <EOL> '<STR_LIT>' : GradUtil . _minimum_grad , <EOL> '<STR_LIT>' : GradUtil . _sum_grad , <EOL> '<STR_LIT>' : GradUtil . _transpose_grad <EOL> } <EOL> def memoize_autodiff ( func ) : <EOL> """<STR_LIT>""" <EOL> cache = { } <EOL> @ wraps ( func ) <EOL> def memoizer ( op_tree , be , next_error = None ) : <EOL> """<STR_LIT>""" <EOL> key = ( op_tree . key ( ) , be , next_error ) <EOL> if key not in cache : <EOL> cache [ key ] = func ( op_tree , be , next_error ) <EOL> return cache [ key ] <EOL> return memoizer <EOL> @ memoize_autodiff <EOL> class Autodiff ( object ) : <EOL> """<STR_LIT>""" <EOL> __slots__ = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' ] <EOL> def __init__ ( self , op_tree , be , next_error = None ) : <EOL> assert ( type ( op_tree ) in _scalar_types or type ( op_tree ) == OpTreeNode or <EOL> isinstance ( op_tree , Tensor ) ) , "<STR_LIT>" <EOL> assert be is not None <EOL> self . op_tree = op_tree <EOL> self . be = be <EOL> self . dtype = be . default_dtype <EOL> if next_error is not None : <EOL> assert next_error . shape == op_tree . shape , ( <EOL> "<STR_LIT>" <EOL> % ( next_error . shape , op_tree . shape ) ) <EOL> self . next_error = next_error <EOL> else : <EOL> self . next_error = self . be . ones ( op_tree . shape ) <EOL> self . map_tensor_grad_node = { } <EOL> self . map_tensor_grad_op_tree = { } <EOL> self . grad_node = GradNode ( op_tree , self ) <EOL> if self . next_error : <EOL> self . grad_node . grad_op_tree = self . next_error <EOL> else : <EOL> self . grad_node . grad_op_tree = self . be . ones ( self . op_tree . shape ) <EOL> self . grad_node . build_grad ( ) <EOL> def __del__ ( self ) : <EOL> self . cleanup ( ) <EOL> def cleanup ( self ) : <EOL> if self . grad_node is not None : <EOL> self . grad_node . cleanup ( ) <EOL> self . grad_node = None <EOL> self . dtype = None <EOL> self . next_error = None <EOL> self . op_tree = None <EOL> self . be = None <EOL> def back_prop_grad ( self , tensors , gradients ) : <EOL> """<STR_LIT>""" <EOL> for grad_buffer in gradients : <EOL> assert ( grad_buffer . _original_base not in self . map_tensor_grad_op_tree ) <EOL> skipped_tensor = None <EOL> for tensor , grad_buffer in zip ( tensors , gradients ) : <EOL> if grad_buffer is self . next_error : <EOL> skipped_tensor = tensor <EOL> else : <EOL> grad_buffer [ : ] = self . map_tensor_grad_op_tree . get ( <EOL> tensor . _original_base , grad_buffer * <NUM_LIT:0.> ) <EOL> if skipped_tensor : <EOL> self . next_error [ : ] = self . map_tensor_grad_op_tree . get ( <EOL> skipped_tensor . _original_base , self . next_error * <NUM_LIT:0.> ) <EOL> def get_grad_op_tree ( self , tensors ) : <EOL> """<STR_LIT>""" <EOL> grad_op_trees = [ ] <EOL> for tensor in tensors : <EOL> grad_op_trees . append ( <EOL> self . map_tensor_grad_op_tree . get ( tensor . _original_base , tensor * <NUM_LIT:0.> ) ) <EOL> return grad_op_trees <EOL> def get_grad_tensor ( self , tensors ) : <EOL> """<STR_LIT>""" <EOL> grad_op_trees = self . get_grad_op_tree ( tensors ) <EOL> grad_vals = [ ] <EOL> for grad_op_tree in grad_op_trees : <EOL> grad_val = self . be . empty ( grad_op_tree . shape ) <EOL> grad_val [ : ] = grad_op_tree <EOL> grad_vals . append ( grad_val ) <EOL> return grad_vals <EOL> def get_grad_asnumpyarray ( self , tensors ) : <EOL> """<STR_LIT>""" <EOL> grad_vals = self . get_grad_tensor ( tensors ) <EOL> for i in range ( len ( grad_vals ) ) : <EOL> grad_vals [ i ] = grad_vals [ i ] . get ( ) . astype ( self . dtype ) <EOL> return grad_vals <EOL> class GradNode ( object ) : <EOL> """<STR_LIT>""" <EOL> __slots__ = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:left>' , '<STR_LIT:right>' ] <EOL> def __init__ ( self , op_tree , ad ) : <EOL> """<STR_LIT>""" <EOL> assert op_tree is not None <EOL> self . op_tree = op_tree <EOL> self . grad_op_tree = None <EOL> self . ad = ad <EOL> self . left = None <EOL> self . right = None <EOL> if isinstance ( op_tree , Tensor ) : <EOL> if op_tree . _original_base not in ad . map_tensor_grad_node : <EOL> ad . map_tensor_grad_node [ op_tree . _original_base ] = self <EOL> elif type ( op_tree ) == OpTreeNode : <EOL> if op_tree [ <NUM_LIT:1> ] is not None : <EOL> if ( isinstance ( op_tree [ <NUM_LIT:1> ] , Tensor ) and <EOL> op_tree [ <NUM_LIT:1> ] . _original_base in ad . map_tensor_grad_node ) : <EOL> self . left = ad . map_tensor_grad_node [ op_tree [ <NUM_LIT:1> ] . _original_base ] <EOL> else : <EOL> self . left = GradNode ( op_tree [ <NUM_LIT:1> ] , ad ) <EOL> if op_tree [ <NUM_LIT:2> ] is not None : <EOL> if ( isinstance ( op_tree [ <NUM_LIT:2> ] , Tensor ) and <EOL> op_tree [ <NUM_LIT:2> ] . _original_base in ad . map_tensor_grad_node ) : <EOL> self . right = ad . map_tensor_grad_node [ op_tree [ <NUM_LIT:2> ] . _original_base ] <EOL> else : <EOL> self . right = GradNode ( op_tree [ <NUM_LIT:2> ] , ad ) <EOL> def __del__ ( self ) : <EOL> self . cleanup ( ) <EOL> def cleanup ( self ) : <EOL> self . op_tree = None <EOL> self . grad_op_tree = None <EOL> self . ad = None <EOL> if self . left is not None : <EOL> self . left . cleanup ( ) <EOL> self . left = None <EOL> if self . right is not None : <EOL> self . right . cleanup ( ) <EOL> self . right = None <EOL> def build_grad ( self ) : <EOL> """<STR_LIT>""" <EOL> assert self . grad_op_tree is not None <EOL> if type ( self . op_tree ) == OpTreeNode : <EOL> ( left_increment , right_increment ) = GradUtil . get_grad_back ( self ) <EOL> if self . left . grad_op_tree is None : <EOL> self . left . grad_op_tree = left_increment <EOL> else : <EOL> self . left . grad_op_tree = self . left . grad_op_tree + left_increment <EOL> self . left . build_grad ( ) <EOL> if right_increment is None : <EOL> return <EOL> if self . right . grad_op_tree is None : <EOL> self . right . grad_op_tree = right_increment <EOL> else : <EOL> self . right . grad_op_tree = self . right . grad_op_tree + right_increment <EOL> self . right . build_grad ( ) <EOL> elif isinstance ( self . op_tree , Tensor ) : <EOL> self . ad . map_tensor_grad_op_tree [ self . op_tree . _original_base ] = self . grad_op_tree </s>
