<s> from __future__ import print_function <EOL> import logging <EOL> import numpy <EOL> from six . moves import xrange <EOL> import theano <EOL> from theano . tensor import as_tensor_variable <EOL> from theano . gof import Op , Apply <EOL> from theano . gradient import DisconnectedType <EOL> from theano . tensor import basic as tensor <EOL> logger = logging . getLogger ( __name__ ) <EOL> class MatrixPinv ( Op ) : <EOL> """<STR_LIT>""" <EOL> __props__ = ( ) <EOL> def __init__ ( self ) : <EOL> pass <EOL> def make_node ( self , x ) : <EOL> x = as_tensor_variable ( x ) <EOL> assert x . ndim == <NUM_LIT:2> <EOL> return Apply ( self , [ x ] , [ x . type ( ) ] ) <EOL> def perform ( self , node , inputs , outputs ) : <EOL> ( x , ) = inputs <EOL> ( z , ) = outputs <EOL> z [ <NUM_LIT:0> ] = numpy . linalg . pinv ( x ) . astype ( x . dtype ) <EOL> pinv = MatrixPinv ( ) <EOL> class MatrixInverse ( Op ) : <EOL> """<STR_LIT>""" <EOL> __props__ = ( ) <EOL> def __init__ ( self ) : <EOL> pass <EOL> def make_node ( self , x ) : <EOL> x = as_tensor_variable ( x ) <EOL> assert x . ndim == <NUM_LIT:2> <EOL> return Apply ( self , [ x ] , [ x . type ( ) ] ) <EOL> def perform ( self , node , inputs , outputs ) : <EOL> ( x , ) = inputs <EOL> ( z , ) = outputs <EOL> z [ <NUM_LIT:0> ] = numpy . linalg . inv ( x ) . astype ( x . dtype ) <EOL> def grad ( self , inputs , g_outputs ) : <EOL> r"""<STR_LIT>""" <EOL> x , = inputs <EOL> xi = self ( x ) <EOL> gz , = g_outputs <EOL> return [ - matrix_dot ( xi , gz . T , xi ) . T ] <EOL> def R_op ( self , inputs , eval_points ) : <EOL> r"""<STR_LIT>""" <EOL> x , = inputs <EOL> xi = self ( x ) <EOL> ev , = eval_points <EOL> if ev is None : <EOL> return [ None ] <EOL> return [ - matrix_dot ( xi , ev , xi ) ] <EOL> def infer_shape ( self , node , shapes ) : <EOL> return shapes <EOL> matrix_inverse = MatrixInverse ( ) <EOL> def matrix_dot ( * args ) : <EOL> """<STR_LIT>""" <EOL> rval = args [ <NUM_LIT:0> ] <EOL> for a in args [ <NUM_LIT:1> : ] : <EOL> rval = theano . tensor . dot ( rval , a ) <EOL> return rval <EOL> class AllocDiag ( Op ) : <EOL> """<STR_LIT>""" <EOL> __props__ = ( ) <EOL> def make_node ( self , _x ) : <EOL> x = as_tensor_variable ( _x ) <EOL> if x . type . ndim != <NUM_LIT:1> : <EOL> raise TypeError ( '<STR_LIT>' , _x ) <EOL> return Apply ( self , [ x ] , [ theano . tensor . matrix ( dtype = x . type . dtype ) ] ) <EOL> def grad ( self , inputs , g_outputs ) : <EOL> return [ extract_diag ( g_outputs [ <NUM_LIT:0> ] ) ] <EOL> def perform ( self , node , inputs , outputs ) : <EOL> ( x , ) = inputs <EOL> ( z , ) = outputs <EOL> if x . ndim != <NUM_LIT:1> : <EOL> raise TypeError ( x ) <EOL> z [ <NUM_LIT:0> ] = numpy . diag ( x ) <EOL> def infer_shape ( self , node , shapes ) : <EOL> x_s , = shapes <EOL> return [ ( x_s [ <NUM_LIT:0> ] , x_s [ <NUM_LIT:0> ] ) ] <EOL> alloc_diag = AllocDiag ( ) <EOL> class ExtractDiag ( Op ) : <EOL> """<STR_LIT>""" <EOL> __props__ = ( "<STR_LIT>" , ) <EOL> def __init__ ( self , view = False ) : <EOL> self . view = view <EOL> if self . view : <EOL> self . view_map = { <NUM_LIT:0> : [ <NUM_LIT:0> ] } <EOL> def make_node ( self , _x ) : <EOL> if not isinstance ( _x , theano . Variable ) : <EOL> x = as_tensor_variable ( _x ) <EOL> else : <EOL> x = _x <EOL> if x . type . ndim != <NUM_LIT:2> : <EOL> raise TypeError ( '<STR_LIT>' , _x ) <EOL> return Apply ( self , [ x ] , [ x . type . __class__ ( broadcastable = ( False , ) , <EOL> dtype = x . type . dtype ) ( ) ] ) <EOL> def perform ( self , node , ins , outs ) : <EOL> """<STR_LIT>""" <EOL> x , = ins <EOL> z , = outs <EOL> if x . shape [ <NUM_LIT:0> ] == <NUM_LIT:0> or x . shape [ <NUM_LIT:1> ] == <NUM_LIT:0> : <EOL> z [ <NUM_LIT:0> ] = node . outputs [ <NUM_LIT:0> ] . type . value_zeros ( ( <NUM_LIT:0> , ) ) <EOL> return <EOL> if x . shape [ <NUM_LIT:0> ] < x . shape [ <NUM_LIT:1> ] : <EOL> rval = x [ : , <NUM_LIT:0> ] <EOL> else : <EOL> rval = x [ <NUM_LIT:0> ] <EOL> rval . strides = ( x . strides [ <NUM_LIT:0> ] + x . strides [ <NUM_LIT:1> ] , ) <EOL> if self . view : <EOL> z [ <NUM_LIT:0> ] = rval <EOL> else : <EOL> z [ <NUM_LIT:0> ] = rval . copy ( ) <EOL> def __str__ ( self ) : <EOL> return '<STR_LIT>' % self . view <EOL> def grad ( self , inputs , g_outputs ) : <EOL> x = theano . tensor . zeros_like ( inputs [ <NUM_LIT:0> ] ) <EOL> xdiag = alloc_diag ( g_outputs [ <NUM_LIT:0> ] ) <EOL> return [ theano . tensor . set_subtensor ( <EOL> x [ : xdiag . shape [ <NUM_LIT:0> ] , : xdiag . shape [ <NUM_LIT:1> ] ] , <EOL> xdiag ) ] <EOL> def infer_shape ( self , node , shapes ) : <EOL> x_s , = shapes <EOL> shp = theano . tensor . min ( node . inputs [ <NUM_LIT:0> ] . shape ) <EOL> return [ ( shp , ) ] <EOL> extract_diag = ExtractDiag ( ) <EOL> def diag ( x ) : <EOL> """<STR_LIT>""" <EOL> xx = as_tensor_variable ( x ) <EOL> if xx . type . ndim == <NUM_LIT:1> : <EOL> return alloc_diag ( xx ) <EOL> elif xx . type . ndim == <NUM_LIT:2> : <EOL> return extract_diag ( xx ) <EOL> else : <EOL> raise TypeError ( '<STR_LIT>' , x ) <EOL> def trace ( X ) : <EOL> """<STR_LIT>""" <EOL> return extract_diag ( X ) . sum ( ) <EOL> class Det ( Op ) : <EOL> """<STR_LIT>""" <EOL> __props__ = ( ) <EOL> def make_node ( self , x ) : <EOL> x = as_tensor_variable ( x ) <EOL> assert x . ndim == <NUM_LIT:2> <EOL> o = theano . tensor . scalar ( dtype = x . dtype ) <EOL> return Apply ( self , [ x ] , [ o ] ) <EOL> def perform ( self , node , inputs , outputs ) : <EOL> ( x , ) = inputs <EOL> ( z , ) = outputs <EOL> try : <EOL> z [ <NUM_LIT:0> ] = numpy . asarray ( numpy . linalg . det ( x ) , dtype = x . dtype ) <EOL> except Exception : <EOL> print ( '<STR_LIT>' , x ) <EOL> raise <EOL> def grad ( self , inputs , g_outputs ) : <EOL> gz , = g_outputs <EOL> x , = inputs <EOL> return [ gz * self ( x ) * matrix_inverse ( x ) . T ] <EOL> def infer_shape ( self , node , shapes ) : <EOL> return [ ( ) ] <EOL> def __str__ ( self ) : <EOL> return "<STR_LIT>" <EOL> det = Det ( ) <EOL> class Eig ( Op ) : <EOL> """<STR_LIT>""" <EOL> _numop = staticmethod ( numpy . linalg . eig ) <EOL> __props__ = ( ) <EOL> def make_node ( self , x ) : <EOL> x = as_tensor_variable ( x ) <EOL> assert x . ndim == <NUM_LIT:2> <EOL> w = theano . tensor . vector ( dtype = x . dtype ) <EOL> v = theano . tensor . matrix ( dtype = x . dtype ) <EOL> return Apply ( self , [ x ] , [ w , v ] ) <EOL> def perform ( self , node , inputs , outputs ) : <EOL> ( x , ) = inputs <EOL> ( w , v ) = outputs <EOL> w [ <NUM_LIT:0> ] , v [ <NUM_LIT:0> ] = [ z . astype ( x . dtype ) for z in self . _numop ( x ) ] <EOL> def infer_shape ( self , node , shapes ) : <EOL> n = shapes [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] <EOL> return [ ( n , ) , ( n , n ) ] <EOL> eig = Eig ( ) <EOL> class Eigh ( Eig ) : <EOL> """<STR_LIT>""" <EOL> _numop = staticmethod ( numpy . linalg . eigh ) <EOL> __props__ = ( '<STR_LIT>' , ) <EOL> def __init__ ( self , UPLO = '<STR_LIT:L>' ) : <EOL> assert UPLO in [ '<STR_LIT:L>' , '<STR_LIT>' ] <EOL> self . UPLO = UPLO <EOL> def make_node ( self , x ) : <EOL> x = as_tensor_variable ( x ) <EOL> assert x . ndim == <NUM_LIT:2> <EOL> w_dtype = self . _numop ( [ [ numpy . dtype ( x . dtype ) . type ( ) ] ] ) [ <NUM_LIT:0> ] . dtype . name <EOL> w = theano . tensor . vector ( dtype = w_dtype ) <EOL> v = theano . tensor . matrix ( dtype = x . dtype ) <EOL> return Apply ( self , [ x ] , [ w , v ] ) <EOL> def perform ( self , node , inputs , outputs ) : <EOL> ( x , ) = inputs <EOL> ( w , v ) = outputs <EOL> w [ <NUM_LIT:0> ] , v [ <NUM_LIT:0> ] = self . _numop ( x , self . UPLO ) <EOL> def grad ( self , inputs , g_outputs ) : <EOL> r"""<STR_LIT>""" <EOL> x , = inputs <EOL> w , v = self ( x ) <EOL> gw , gv = _zero_disconnected ( [ w , v ] , g_outputs ) <EOL> return [ EighGrad ( self . UPLO ) ( x , w , v , gw , gv ) ] <EOL> def _zero_disconnected ( outputs , grads ) : <EOL> l = [ ] <EOL> for o , g in zip ( outputs , grads ) : <EOL> if isinstance ( g . type , DisconnectedType ) : <EOL> l . append ( o . zeros_like ( ) ) <EOL> else : <EOL> l . append ( g ) <EOL> return l <EOL> class EighGrad ( Op ) : <EOL> """<STR_LIT>""" <EOL> __props__ = ( '<STR_LIT>' , ) <EOL> def __init__ ( self , UPLO = '<STR_LIT:L>' ) : <EOL> assert UPLO in [ '<STR_LIT:L>' , '<STR_LIT>' ] <EOL> self . UPLO = UPLO <EOL> if UPLO == '<STR_LIT:L>' : <EOL> self . tri0 = numpy . tril <EOL> self . tri1 = lambda a : numpy . triu ( a , <NUM_LIT:1> ) <EOL> else : <EOL> self . tri0 = numpy . triu <EOL> self . tri1 = lambda a : numpy . tril ( a , - <NUM_LIT:1> ) <EOL> def make_node ( self , x , w , v , gw , gv ) : <EOL> x , w , v , gw , gv = map ( as_tensor_variable , ( x , w , v , gw , gv ) ) <EOL> assert x . ndim == <NUM_LIT:2> <EOL> assert w . ndim == <NUM_LIT:1> <EOL> assert v . ndim == <NUM_LIT:2> <EOL> assert gw . ndim == <NUM_LIT:1> <EOL> assert gv . ndim == <NUM_LIT:2> <EOL> out_dtype = theano . scalar . upcast ( x . dtype , w . dtype , v . dtype , <EOL> gw . dtype , gv . dtype ) <EOL> out = theano . tensor . matrix ( dtype = out_dtype ) <EOL> return Apply ( self , [ x , w , v , gw , gv ] , [ out ] ) <EOL> def perform ( self , node , inputs , outputs ) : <EOL> """<STR_LIT>""" <EOL> x , w , v , W , V = inputs <EOL> N = x . shape [ <NUM_LIT:0> ] <EOL> outer = numpy . outer <EOL> def G ( n ) : <EOL> return sum ( v [ : , m ] * V . T [ n ] . dot ( v [ : , m ] ) / ( w [ n ] - w [ m ] ) <EOL> for m in xrange ( N ) if m != n ) <EOL> g = sum ( outer ( v [ : , n ] , v [ : , n ] * W [ n ] + G ( n ) ) <EOL> for n in xrange ( N ) ) <EOL> out = self . tri0 ( g ) + self . tri1 ( g ) . T <EOL> outputs [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] = numpy . asarray ( out , dtype = node . outputs [ <NUM_LIT:0> ] . dtype ) <EOL> def infer_shape ( self , node , shapes ) : <EOL> return [ shapes [ <NUM_LIT:0> ] ] <EOL> def eigh ( a , UPLO = '<STR_LIT:L>' ) : <EOL> return Eigh ( UPLO ) ( a ) <EOL> class QRFull ( Op ) : <EOL> """<STR_LIT>""" <EOL> _numop = staticmethod ( numpy . linalg . qr ) <EOL> __props__ = ( '<STR_LIT>' , ) <EOL> def __init__ ( self , mode ) : <EOL> self . mode = mode <EOL> def make_node ( self , x ) : <EOL> x = as_tensor_variable ( x ) <EOL> assert x . ndim == <NUM_LIT:2> , "<STR_LIT>" <EOL> q = theano . tensor . matrix ( dtype = x . dtype ) <EOL> if self . mode != '<STR_LIT>' : <EOL> r = theano . tensor . matrix ( dtype = x . dtype ) <EOL> else : <EOL> r = theano . tensor . vector ( dtype = x . dtype ) <EOL> return Apply ( self , [ x ] , [ q , r ] ) <EOL> def perform ( self , node , inputs , outputs ) : <EOL> ( x , ) = inputs <EOL> ( q , r ) = outputs <EOL> assert x . ndim == <NUM_LIT:2> , "<STR_LIT>" <EOL> q [ <NUM_LIT:0> ] , r [ <NUM_LIT:0> ] = self . _numop ( x , self . mode ) <EOL> class QRIncomplete ( Op ) : <EOL> """<STR_LIT>""" <EOL> _numop = staticmethod ( numpy . linalg . qr ) <EOL> __props__ = ( '<STR_LIT>' , ) <EOL> def __init__ ( self , mode ) : <EOL> self . mode = mode <EOL> def make_node ( self , x ) : <EOL> x = as_tensor_variable ( x ) <EOL> assert x . ndim == <NUM_LIT:2> , "<STR_LIT>" <EOL> q = theano . tensor . matrix ( dtype = x . dtype ) <EOL> return Apply ( self , [ x ] , [ q ] ) <EOL> def perform ( self , node , inputs , outputs ) : <EOL> ( x , ) = inputs <EOL> ( q , ) = outputs <EOL> assert x . ndim == <NUM_LIT:2> , "<STR_LIT>" <EOL> q [ <NUM_LIT:0> ] = self . _numop ( x , <EOL> self . mode ) <EOL> def qr ( a , mode = "<STR_LIT>" ) : <EOL> """<STR_LIT>""" <EOL> x = [ [ <NUM_LIT:2> , <NUM_LIT:1> ] , [ <NUM_LIT:3> , <NUM_LIT:4> ] ] <EOL> if isinstance ( numpy . linalg . qr ( x , mode ) , tuple ) : <EOL> return QRFull ( mode ) ( a ) <EOL> else : <EOL> return QRIncomplete ( mode ) ( a ) <EOL> class SVD ( Op ) : <EOL> """<STR_LIT>""" <EOL> _numop = staticmethod ( numpy . linalg . svd ) <EOL> __props__ = ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> def __init__ ( self , full_matrices = True , compute_uv = True ) : <EOL> self . full_matrices = full_matrices <EOL> self . compute_uv = compute_uv <EOL> def make_node ( self , x ) : <EOL> x = as_tensor_variable ( x ) <EOL> assert x . ndim == <NUM_LIT:2> , "<STR_LIT>" <EOL> w = theano . tensor . matrix ( dtype = x . dtype ) <EOL> u = theano . tensor . vector ( dtype = x . dtype ) <EOL> v = theano . tensor . matrix ( dtype = x . dtype ) <EOL> return Apply ( self , [ x ] , [ w , u , v ] ) <EOL> def perform ( self , node , inputs , outputs ) : <EOL> ( x , ) = inputs <EOL> ( w , u , v ) = outputs <EOL> assert x . ndim == <NUM_LIT:2> , "<STR_LIT>" <EOL> w [ <NUM_LIT:0> ] , u [ <NUM_LIT:0> ] , v [ <NUM_LIT:0> ] = self . _numop ( x , <EOL> self . full_matrices , <EOL> self . compute_uv ) <EOL> def svd ( a , full_matrices = <NUM_LIT:1> , compute_uv = <NUM_LIT:1> ) : <EOL> """<STR_LIT>""" <EOL> return SVD ( full_matrices , compute_uv ) ( a ) <EOL> class lstsq ( Op ) : <EOL> __props__ = ( ) <EOL> def make_node ( self , x , y , rcond ) : <EOL> x = theano . tensor . as_tensor_variable ( x ) <EOL> y = theano . tensor . as_tensor_variable ( y ) <EOL> rcond = theano . tensor . as_tensor_variable ( rcond ) <EOL> return theano . Apply ( self , [ x , y , rcond ] , <EOL> [ theano . tensor . matrix ( ) , theano . tensor . dvector ( ) , <EOL> theano . tensor . lscalar ( ) , theano . tensor . dvector ( ) ] ) <EOL> def perform ( self , node , inputs , outputs ) : <EOL> zz = numpy . linalg . lstsq ( inputs [ <NUM_LIT:0> ] , inputs [ <NUM_LIT:1> ] , inputs [ <NUM_LIT:2> ] ) <EOL> outputs [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] = zz [ <NUM_LIT:0> ] <EOL> outputs [ <NUM_LIT:1> ] [ <NUM_LIT:0> ] = zz [ <NUM_LIT:1> ] <EOL> outputs [ <NUM_LIT:2> ] [ <NUM_LIT:0> ] = numpy . array ( zz [ <NUM_LIT:2> ] ) <EOL> outputs [ <NUM_LIT:3> ] [ <NUM_LIT:0> ] = zz [ <NUM_LIT:3> ] <EOL> def matrix_power ( M , n ) : <EOL> result = <NUM_LIT:1> <EOL> for i in xrange ( n ) : <EOL> result = theano . dot ( result , M ) <EOL> return result <EOL> def norm ( x , ord ) : <EOL> x = as_tensor_variable ( x ) <EOL> ndim = x . ndim <EOL> if ndim == <NUM_LIT:0> : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> elif ndim == <NUM_LIT:1> : <EOL> if ord is None : <EOL> return tensor . sum ( x ** <NUM_LIT:2> ) ** <NUM_LIT:0.5> <EOL> elif ord == '<STR_LIT>' : <EOL> return tensor . max ( abs ( x ) ) <EOL> elif ord == '<STR_LIT>' : <EOL> return tensor . min ( abs ( x ) ) <EOL> elif ord == <NUM_LIT:0> : <EOL> return x [ x . nonzero ( ) ] . shape [ <NUM_LIT:0> ] <EOL> else : <EOL> try : <EOL> z = tensor . sum ( abs ( x ** ord ) ) ** ( <NUM_LIT:1.> / ord ) <EOL> except TypeError : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> return z <EOL> elif ndim == <NUM_LIT:2> : <EOL> if ord is None or ord == '<STR_LIT>' : <EOL> return tensor . sum ( abs ( x ** <NUM_LIT:2> ) ) ** ( <NUM_LIT:0.5> ) <EOL> elif ord == '<STR_LIT>' : <EOL> return tensor . max ( tensor . sum ( abs ( x ) , <NUM_LIT:1> ) ) <EOL> elif ord == '<STR_LIT>' : <EOL> return tensor . min ( tensor . sum ( abs ( x ) , <NUM_LIT:1> ) ) <EOL> elif ord == <NUM_LIT:1> : <EOL> return tensor . max ( tensor . sum ( abs ( x ) , <NUM_LIT:0> ) ) <EOL> elif ord == - <NUM_LIT:1> : <EOL> return tensor . min ( tensor . sum ( abs ( x ) , <NUM_LIT:0> ) ) <EOL> else : <EOL> raise ValueError ( <NUM_LIT:0> ) <EOL> elif ndim > <NUM_LIT:2> : <EOL> raise NotImplementedError ( "<STR_LIT>" ) </s>
