<s> import collections <EOL> import numpy as np <EOL> import numpy . linalg <EOL> import collections as C <EOL> import gzip <EOL> import time <EOL> import os <EOL> import tarfile <EOL> import theano <EOL> def getCols ( d , ifrom , ito ) : <EOL> result = { } <EOL> for i in d : <EOL> result [ i ] = d [ i ] [ : , ifrom : ito ] <EOL> return result <EOL> def getColsFromIndices ( d , colIndices ) : <EOL> result = { } <EOL> for i in d : <EOL> result [ i ] = d [ i ] [ : , colIndices ] <EOL> return result <EOL> def sum ( ds ) : <EOL> d = ds [ <NUM_LIT:0> ] <EOL> for i in range ( <NUM_LIT:1> , len ( ds ) ) : <EOL> for j in ds [ i ] : <EOL> d [ j ] += ds [ i ] [ j ] <EOL> return d <EOL> def getcols_multiple ( ds , ifrom , ito ) : <EOL> result = [ ] <EOL> for d in ds : <EOL> result . append ( getCols ( d , ifrom , ito ) ) <EOL> return result <EOL> def setCols ( d , ifrom , ito , x ) : <EOL> for i in d : <EOL> d [ i ] [ : , ifrom : ito ] = x [ i ] <EOL> def mapCols ( d , func , ifrom , ito ) : <EOL> x = getCols ( d , ifrom , ito ) <EOL> func ( x ) <EOL> setCols ( d , ifrom , ito , x ) <EOL> def size ( d ) : <EOL> result = <NUM_LIT:0> <EOL> for i in d : <EOL> result += d [ i ] . size <EOL> return result <EOL> def cloneZeros ( d ) : <EOL> result = { } <EOL> for i in d : <EOL> result [ i ] = np . zeros ( d [ i ] . shape ) <EOL> return result <EOL> def cloneOnes ( d ) : <EOL> result = { } <EOL> for i in d : <EOL> result [ i ] = np . ones ( d [ i ] . shape ) <EOL> return result <EOL> def clone ( d ) : <EOL> result = { } <EOL> for i in d : <EOL> result [ i ] = d [ i ] . copy ( ) <EOL> return result <EOL> def flatten ( d ) : <EOL> d = ordered ( d ) <EOL> result = np . zeros ( ( size ( d ) ) ) <EOL> pointer = <NUM_LIT:0> <EOL> for i in d : <EOL> result [ pointer : pointer + d [ i ] . size ] = d [ i ] . reshape ( ( d [ i ] . size ) ) <EOL> pointer += d [ i ] . size <EOL> return result <EOL> def flatten_multiple ( ds ) : <EOL> result = [ ] <EOL> for d in ds : <EOL> result . append ( flatten ( d ) ) <EOL> return np . concatenate ( result ) <EOL> def unflatten ( flat , d ) : <EOL> d = ordered ( d ) <EOL> result = { } <EOL> pointer = <NUM_LIT:0> <EOL> for i in d : <EOL> result [ i ] = flat [ pointer : pointer + d [ i ] . size ] . reshape ( d [ i ] . shape ) <EOL> pointer += d [ i ] . size <EOL> return result <EOL> def unflatten_multiple ( flat , ds ) : <EOL> result = [ ] <EOL> pointer = <NUM_LIT:0> <EOL> for d in ds : <EOL> result . append ( unflatten ( flat [ pointer : pointer + size ( d ) ] , d ) ) <EOL> pointer += size ( d ) <EOL> return result <EOL> def merge ( ds , prefixes ) : <EOL> result = { } <EOL> for i in range ( len ( ds ) ) : <EOL> for j in ds [ i ] : <EOL> result [ prefixes [ i ] + j ] = ds [ i ] [ j ] <EOL> return result <EOL> def unmerge ( d , prefixes ) : <EOL> results = [ { } for _ in range ( len ( prefixes ) ) ] <EOL> for key in d : <EOL> for j in range ( len ( prefixes ) ) : <EOL> prefix = prefixes [ j ] <EOL> if key [ : len ( prefix ) ] == prefix : <EOL> results [ j ] [ key [ len ( prefix ) : ] ] = d [ key ] <EOL> return results <EOL> def getShapes ( d ) : <EOL> shapes = { } <EOL> for i in d : <EOL> shapes [ i ] = d [ i ] . shape <EOL> return shapes <EOL> def setShapes ( d , shapes ) : <EOL> result = { } <EOL> for i in d : <EOL> result [ i ] = d [ i ] . reshape ( shapes [ i ] ) <EOL> return result <EOL> def p ( d ) : <EOL> for i in d : print i + '<STR_LIT:\n>' , d [ i ] <EOL> def pNorm ( d ) : <EOL> for i in d : print i , numpy . linalg . norm ( d [ i ] ) <EOL> def pShape ( d ) : <EOL> for i in d : print i , d [ i ] . shape <EOL> def hasNaN ( d ) : <EOL> result = False <EOL> for i in d : result = result or np . isnan ( d [ i ] ) . any ( ) <EOL> return result <EOL> def astype ( d , _type ) : <EOL> return { i : d [ i ] . astype ( _type ) for i in d } <EOL> def get_value ( d ) : <EOL> return { i : d [ i ] . get_value ( ) for i in d } <EOL> def set_value ( d , d2 ) : <EOL> return { i : d [ i ] . set_value ( d2 [ i ] ) for i in d } <EOL> def savetext ( d , name ) : <EOL> for i in d : np . savetxt ( file ( '<STR_LIT>' + name + '<STR_LIT>' , '<STR_LIT:w>' ) , d [ i ] ) <EOL> def ordered ( d ) : <EOL> return C . OrderedDict ( sorted ( d . items ( ) ) ) <EOL> def ordereddicts ( ds ) : <EOL> return [ ordered ( d ) for d in ds ] <EOL> def orderedvals ( ds ) : <EOL> vals = [ ] <EOL> for d in ds : <EOL> vals += ordered ( d ) . values ( ) <EOL> return vals <EOL> def shuffleCols ( d ) : <EOL> n_cols = d . itervalues ( ) . next ( ) . shape [ <NUM_LIT:1> ] <EOL> idx = np . arange ( n_cols ) <EOL> np . random . shuffle ( idx ) <EOL> for i in d : <EOL> d [ i ] = d [ i ] [ : , idx ] <EOL> def savez ( d , filename , addext = True ) : <EOL> if addext : <EOL> filename = filename + '<STR_LIT>' <EOL> fname1 = '<STR_LIT>' <EOL> fname2 = '<STR_LIT>' <EOL> _d = ordered ( d ) <EOL> np . savez ( filename + '<STR_LIT:.>' + fname1 , * _d . values ( ) ) <EOL> with open ( filename + '<STR_LIT:.>' + fname2 , '<STR_LIT:w>' ) as thefile : <EOL> for key in _d . keys ( ) : thefile . write ( "<STR_LIT>" % key ) <EOL> tar = tarfile . open ( filename , "<STR_LIT>" ) <EOL> for fname in [ fname1 , fname2 ] : <EOL> tar . add ( filename + '<STR_LIT:.>' + fname , fname ) <EOL> os . remove ( filename + '<STR_LIT:.>' + fname ) <EOL> tar . close ( ) <EOL> def loadz ( filename ) : <EOL> with tarfile . open ( filename , '<STR_LIT>' ) as tar : <EOL> members = tar . getmembers ( ) <EOL> arrays = np . load ( tar . extractfile ( members [ <NUM_LIT:0> ] ) ) <EOL> names = tar . extractfile ( members [ <NUM_LIT:1> ] ) . readlines ( ) <EOL> result = { names [ i ] [ : - <NUM_LIT:1> ] : arrays [ '<STR_LIT>' + str ( i ) ] for i in range ( len ( names ) ) } <EOL> return ordered ( result ) <EOL> '''<STR_LIT>''' <EOL> def cloneZerosShared ( d ) : <EOL> result = { } <EOL> for i in d : <EOL> result [ i ] = theano . shared ( d [ i ] . get_value ( ) * <NUM_LIT:0.> ) <EOL> return result </s>
