<s> import numpy as np <EOL> import pprint <EOL> import pytest <EOL> from neon import NervanaObject <EOL> from neon . backends . autodiff import Autodiff <EOL> class CustomFunc : <EOL> @ staticmethod <EOL> def sig ( x ) : <EOL> return <NUM_LIT:1.> / ( <NUM_LIT:1.> + np . exp ( - x ) ) <EOL> @ staticmethod <EOL> def sig2 ( x ) : <EOL> return <NUM_LIT:1.> / ( <NUM_LIT:1.> + np . exp2 ( - x ) ) <EOL> @ staticmethod <EOL> def tanh2 ( x ) : <EOL> return ( np . exp2 ( <NUM_LIT> * x ) - <NUM_LIT:1.0> ) / ( np . exp2 ( <NUM_LIT> * x ) + <NUM_LIT:1.0> ) <EOL> @ staticmethod <EOL> def argmax ( x , axis = <NUM_LIT:1> , keepdims = True ) : <EOL> """<STR_LIT>""" <EOL> new_shape = list ( x . shape ) <EOL> new_shape [ axis ] = <NUM_LIT:1> <EOL> new_shape = tuple ( new_shape ) <EOL> return np . argmax ( x , axis = axis ) . reshape ( new_shape ) <EOL> @ staticmethod <EOL> def argmin ( x , axis = <NUM_LIT:1> , keepdims = True ) : <EOL> """<STR_LIT>""" <EOL> new_shape = list ( x . shape ) <EOL> new_shape [ axis ] = <NUM_LIT:1> <EOL> new_shape = tuple ( new_shape ) <EOL> return np . argmin ( x , axis = axis ) . reshape ( new_shape ) <EOL> @ pytest . mark . usefixtures ( "<STR_LIT>" ) <EOL> class TestAutodiff : <EOL> def setup ( self ) : <EOL> self . m = <NUM_LIT:2> <EOL> self . n = <NUM_LIT:2> <EOL> self . be = NervanaObject . be <EOL> self . dtype = self . be . default_dtype <EOL> self . test_epoch = <NUM_LIT:1> <EOL> self . delta = <NUM_LIT> <EOL> def _rand_gen ( self , * flags ) : <EOL> '''<STR_LIT>''' <EOL> val = None <EOL> m = self . m <EOL> n = self . n <EOL> if '<STR_LIT>' in flags : <EOL> m = <NUM_LIT:1.> <EOL> n = <NUM_LIT:1.> <EOL> if '<STR_LIT>' in flags : <EOL> m = <NUM_LIT:1.> <EOL> if '<STR_LIT>' in flags : <EOL> n = <NUM_LIT:1.> <EOL> if '<STR_LIT:int>' in flags : <EOL> if '<STR_LIT>' in flags : <EOL> val = np . random . randint ( <NUM_LIT> , size = ( m , n ) ) . astype ( float ) + <NUM_LIT:1.> <EOL> else : <EOL> val = np . random . randint ( <NUM_LIT> , size = ( m , n ) ) . astype ( float ) - <NUM_LIT> <EOL> else : <EOL> if '<STR_LIT>' in flags : <EOL> val = np . absolute ( np . random . randn ( m , n ) ) + <NUM_LIT:0.1> <EOL> else : <EOL> val = np . random . randn ( m , n ) <EOL> val [ val > <NUM_LIT> ] = <NUM_LIT> <EOL> val [ val < - <NUM_LIT> ] = - <NUM_LIT> <EOL> return val <EOL> @ staticmethod <EOL> def _numpy_call ( f_str , tensors ) : <EOL> '''<STR_LIT>''' <EOL> f_str = f_str . replace ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> f_str = f_str . replace ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> f_str = f_str . replace ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> f_str = f_str . replace ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> f_str = f_str . replace ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> f_str = f_str . replace ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> f_str = f_str . replace ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> f_str = f_str . replace ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> count = <NUM_LIT:0> <EOL> for tensor in tensors : <EOL> exec ( '<STR_LIT>' % count ) in globals ( ) , locals ( ) <EOL> count += <NUM_LIT:1> <EOL> result = None <EOL> exec ( '<STR_LIT>' + f_str ) <EOL> return result <EOL> def _get_autodiff_grads_and_val ( self , f_str , tensor_vals , get_op_tree = False , <EOL> next_error = None ) : <EOL> '''<STR_LIT>''' <EOL> be = self . be <EOL> count = <NUM_LIT:0> <EOL> tensors = [ ] <EOL> for tensor_val in tensor_vals : <EOL> exec ( '<STR_LIT>' <EOL> % ( count , count ) ) <EOL> exec ( '<STR_LIT>' % count ) <EOL> count += <NUM_LIT:1> <EOL> f = None <EOL> exec ( '<STR_LIT>' % f_str ) <EOL> f_val = be . empty ( f . shape ) <EOL> f_val [ : ] = f <EOL> if next_error is not None : <EOL> next_error = self . be . array ( next_error ) <EOL> ad = Autodiff ( f , be , next_error = next_error ) <EOL> if get_op_tree : <EOL> gradients = list ( ad . get_grad_op_tree ( tensors ) ) <EOL> else : <EOL> gradients = list ( ad . get_grad_asnumpyarray ( tensors ) ) <EOL> return [ gradients , f_val . get ( ) ] <EOL> def _get_numerical_grads_and_val ( self , f_str , tensors , next_error = None ) : <EOL> '''<STR_LIT>''' <EOL> gradients = [ ] <EOL> for tensor in tensors : <EOL> gradients . append ( np . zeros ( tensor . shape ) ) <EOL> f_val = TestAutodiff . _numpy_call ( f_str , tensors ) <EOL> if next_error is None : <EOL> next_error = np . ones_like ( f_val ) <EOL> for tensor , gradient in zip ( tensors , gradients ) : <EOL> gradient_flat = np . copy ( gradient . reshape ( ( - <NUM_LIT:1> , ) ) ) <EOL> ind = <NUM_LIT:0> <EOL> for x in np . nditer ( tensor , op_flags = [ '<STR_LIT>' ] ) : <EOL> x_backup = np . copy ( x ) <EOL> x [ ... ] = x + self . delta <EOL> f_inc = np . sum ( TestAutodiff . _numpy_call ( f_str , tensors ) * next_error ) <EOL> x [ ... ] = x_backup <EOL> x [ ... ] = x - self . delta <EOL> f_dec = np . sum ( TestAutodiff . _numpy_call ( f_str , tensors ) * next_error ) <EOL> x [ ... ] = x_backup <EOL> gradient_flat [ ind ] = ( f_inc - f_dec ) / ( <NUM_LIT> * self . delta ) <EOL> ind += <NUM_LIT:1> <EOL> gradient [ : ] = gradient_flat . reshape ( gradient . shape ) <EOL> return [ gradients , f_val ] <EOL> def _assert_grad_equal ( self , f_str , tensors , rtol = <NUM_LIT> , atol = <NUM_LIT> , next_error = None ) : <EOL> def debug_msg ( count ) : <EOL> msg = '<STR_LIT>' <EOL> msg += '<STR_LIT>' % ( count , ) + '<STR_LIT:\n>' <EOL> msg += pprint . pformat ( tensors ) + '<STR_LIT:\n>' <EOL> grad_op_trees = self . _get_autodiff_grads_and_val ( f_str , tensors , <EOL> get_op_tree = True ) <EOL> grad_op_tree = grad_op_trees [ <NUM_LIT:0> ] [ count - <NUM_LIT:1> ] <EOL> msg += grad_op_tree . pp ( ) + '<STR_LIT:\n>' <EOL> return msg <EOL> autodiff_grads_and_val = self . _get_autodiff_grads_and_val ( <EOL> f_str , tensors , next_error = next_error ) <EOL> numerical_grads_and_val = self . _get_numerical_grads_and_val ( <EOL> f_str , tensors , next_error = next_error ) <EOL> assert ( len ( autodiff_grads_and_val ) == len ( numerical_grads_and_val ) ) <EOL> numerical_grads_and_val [ <NUM_LIT:1> ] = numerical_grads_and_val [ <EOL> <NUM_LIT:1> ] . reshape ( autodiff_grads_and_val [ <NUM_LIT:1> ] . shape ) <EOL> assert np . allclose ( autodiff_grads_and_val [ <NUM_LIT:1> ] . astype ( self . dtype ) , <EOL> numerical_grads_and_val [ <NUM_LIT:1> ] . astype ( self . dtype ) , <EOL> rtol = rtol , atol = atol ) <EOL> count = <NUM_LIT:0> <EOL> for autodiff_grad , numerical_grad in zip ( autodiff_grads_and_val [ <NUM_LIT:0> ] , <EOL> numerical_grads_and_val [ <NUM_LIT:0> ] ) : <EOL> count += <NUM_LIT:1> <EOL> if not np . allclose ( autodiff_grad . astype ( self . dtype ) , numerical_grad . astype ( self . dtype ) , <EOL> rtol = rtol , atol = atol ) : <EOL> raise ValueError ( debug_msg ( count ) ) <EOL> def test_reduction_shape ( self ) : <EOL> be = self . be <EOL> x0 = be . array ( np . array ( [ [ <NUM_LIT:1> , <NUM_LIT:2> ] , [ <NUM_LIT:4> , <NUM_LIT:5> ] ] ) , name = '<STR_LIT>' ) <EOL> f = be . sum ( x0 , axis = <NUM_LIT:0> ) <EOL> assert ( f . shape == ( <NUM_LIT:1> , <NUM_LIT:2> ) ) <EOL> f = be . sum ( x0 , axis = <NUM_LIT:1> ) <EOL> assert ( f . shape == ( <NUM_LIT:2> , <NUM_LIT:1> ) ) <EOL> f = be . sum ( x0 ) <EOL> assert ( f . shape == ( <NUM_LIT:1> , <NUM_LIT:1> ) ) <EOL> def test_reduction ( self ) : <EOL> for _ in range ( self . test_epoch ) : <EOL> x0 , x1 , x2 , x3 = [ self . _rand_gen ( ) for _ in range ( <NUM_LIT:4> ) ] <EOL> f_str = ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> self . _assert_grad_equal ( f_str , [ x0 , x1 , x2 , x3 ] , rtol = <NUM_LIT> ) <EOL> def test_batchnorm ( self ) : <EOL> for _ in range ( self . test_epoch ) : <EOL> x0 = np . random . randn ( <NUM_LIT:10> , <NUM_LIT:64> ) <EOL> x1 = np . random . randn ( <NUM_LIT:10> , <NUM_LIT:1> ) <EOL> x2 = np . random . randn ( <NUM_LIT:10> , <NUM_LIT:1> ) <EOL> next_error = np . random . randn ( <NUM_LIT:10> , <NUM_LIT:64> ) / <NUM_LIT> <EOL> f_str = '<STR_LIT>' <EOL> self . _assert_grad_equal ( f_str , [ x0 , x1 , x2 ] , rtol = <NUM_LIT> , atol = <NUM_LIT> , <EOL> next_error = next_error ) <EOL> def test_positive ( self ) : <EOL> for _ in range ( self . test_epoch ) : <EOL> x0 , x1 , x2 , x3 = [ self . _rand_gen ( '<STR_LIT>' ) for _ in range ( <NUM_LIT:4> ) ] <EOL> f_str = ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> self . _assert_grad_equal ( f_str , [ x0 , x1 , x2 , x3 ] , rtol = <NUM_LIT> ) <EOL> def test_real ( self ) : <EOL> for _ in range ( self . test_epoch ) : <EOL> x0 , x1 , x2 , x3 = [ self . _rand_gen ( ) for _ in range ( <NUM_LIT:4> ) ] <EOL> f_str = ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> self . _assert_grad_equal ( f_str , [ x0 , x1 , x2 , x3 ] , rtol = <NUM_LIT> ) <EOL> def test_unbroadcast ( self ) : <EOL> for _ in range ( self . test_epoch ) : <EOL> x0 = self . _rand_gen ( '<STR_LIT>' ) <EOL> x1 = self . _rand_gen ( ) <EOL> f_str = ( '<STR_LIT>' ) <EOL> self . _assert_grad_equal ( f_str , [ x0 , x1 ] , rtol = <NUM_LIT> ) <EOL> x0 = self . _rand_gen ( '<STR_LIT>' ) <EOL> x1 = self . _rand_gen ( ) <EOL> f_str = ( '<STR_LIT>' ) <EOL> self . _assert_grad_equal ( f_str , [ x0 , x1 ] , rtol = <NUM_LIT> ) <EOL> x0 = self . _rand_gen ( '<STR_LIT>' ) <EOL> x1 = self . _rand_gen ( ) <EOL> f_str = ( '<STR_LIT>' ) <EOL> self . _assert_grad_equal ( f_str , [ x0 , x1 ] , rtol = <NUM_LIT> ) <EOL> x0 = self . _rand_gen ( '<STR_LIT>' ) <EOL> x1 = self . _rand_gen ( '<STR_LIT>' ) <EOL> x2 = self . _rand_gen ( '<STR_LIT>' ) <EOL> x3 = self . _rand_gen ( ) <EOL> f_str = ( '<STR_LIT>' ) <EOL> self . _assert_grad_equal ( f_str , [ x0 , x1 , x2 , x3 ] , rtol = <NUM_LIT> ) <EOL> def test_hard_coded ( self ) : <EOL> """<STR_LIT>""" <EOL> be = self . be <EOL> x0 = be . array ( np . ones ( ( <NUM_LIT:3> , <NUM_LIT:3> ) ) * <NUM_LIT:1> , name = '<STR_LIT>' , dtype = self . dtype ) <EOL> x1 = be . array ( np . ones ( ( <NUM_LIT:3> , <NUM_LIT:3> ) ) * <NUM_LIT:2> , name = '<STR_LIT>' , dtype = self . dtype ) <EOL> x2 = be . array ( np . ones ( ( <NUM_LIT:3> , <NUM_LIT:3> ) ) * <NUM_LIT:3> , name = '<STR_LIT>' , dtype = self . dtype ) <EOL> x3 = be . array ( np . ones ( ( <NUM_LIT:3> , <NUM_LIT:3> ) ) * <NUM_LIT:5> , name = '<STR_LIT>' , dtype = self . dtype ) <EOL> f = x0 * x0 - x1 * x0 + x0 * x2 - x2 * x1 * x0 + x3 * x3 * x3 <EOL> ad = Autodiff ( f , be ) <EOL> x0_grad = be . array ( np . ones ( ( <NUM_LIT:3> , <NUM_LIT:3> ) ) * - <NUM_LIT:3> , dtype = self . dtype ) <EOL> x1_grad = be . array ( np . ones ( ( <NUM_LIT:3> , <NUM_LIT:3> ) ) * - <NUM_LIT:4> , dtype = self . dtype ) <EOL> x2_grad = be . array ( np . ones ( ( <NUM_LIT:3> , <NUM_LIT:3> ) ) * - <NUM_LIT:1> , dtype = self . dtype ) <EOL> x3_grad = be . array ( np . ones ( ( <NUM_LIT:3> , <NUM_LIT:3> ) ) * <NUM_LIT> , dtype = self . dtype ) <EOL> assert np . allclose ( ad . get_grad_asnumpyarray ( [ x0 ] ) [ <NUM_LIT:0> ] , x0_grad . get ( ) , atol = <NUM_LIT> ) <EOL> assert np . allclose ( ad . get_grad_asnumpyarray ( [ x1 ] ) [ <NUM_LIT:0> ] , x1_grad . get ( ) , atol = <NUM_LIT> ) <EOL> assert np . allclose ( ad . get_grad_asnumpyarray ( [ x2 ] ) [ <NUM_LIT:0> ] , x2_grad . get ( ) , atol = <NUM_LIT> ) <EOL> assert np . allclose ( ad . get_grad_asnumpyarray ( [ x3 ] ) [ <NUM_LIT:0> ] , x3_grad . get ( ) , atol = <NUM_LIT> ) </s>
