<s> import numpy as np <EOL> import urllib2 <EOL> import tempfile <EOL> import h5py <EOL> import csv <EOL> import cStringIO <EOL> import collections <EOL> import re <EOL> import annotation <EOL> from ocpcaerror import OCPCAError <EOL> import logging <EOL> logger = logging . getLogger ( "<STR_LIT>" ) <EOL> """<STR_LIT>""" <EOL> class H5Annotation : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , annotype , annoid , h5fh ) : <EOL> """<STR_LIT>""" <EOL> self . h5fh = h5fh <EOL> self . idgrp = self . h5fh . create_group ( str ( annoid ) ) <EOL> self . idgrp . create_dataset ( "<STR_LIT>" , ( <NUM_LIT:1> , ) , np . uint32 , data = annotype ) <EOL> self . mdgrp = self . idgrp . create_group ( "<STR_LIT>" ) <EOL> def addVoxels ( self , resolution , voxlist ) : <EOL> """<STR_LIT>""" <EOL> self . idgrp . create_dataset ( "<STR_LIT>" , ( <NUM_LIT:1> , ) , np . uint32 , data = resolution ) <EOL> if len ( voxlist ) != <NUM_LIT:0> : <EOL> self . idgrp . create_dataset ( "<STR_LIT>" , ( len ( voxlist ) , <NUM_LIT:3> ) , np . uint32 , data = voxlist ) <EOL> def addCutout ( self , resolution , corner , volume ) : <EOL> """<STR_LIT>""" <EOL> self . idgrp . create_dataset ( "<STR_LIT>" , ( <NUM_LIT:1> , ) , np . uint32 , data = resolution ) <EOL> self . idgrp . create_dataset ( "<STR_LIT>" , ( <NUM_LIT:3> , ) , np . uint32 , data = corner ) <EOL> if volume != None : <EOL> self . idgrp . create_dataset ( "<STR_LIT>" , volume . shape , volume . dtype , data = volume ) <EOL> def mkCuboidGroup ( self , resolution ) : <EOL> """<STR_LIT>""" <EOL> self . idgrp . create_dataset ( "<STR_LIT>" , ( <NUM_LIT:1> , ) , np . uint32 , data = resolution ) <EOL> self . cbgrp = self . idgrp . create_group ( "<STR_LIT>" ) <EOL> def addCuboid ( self , offset , cbdata ) : <EOL> """<STR_LIT>""" <EOL> offgrp = self . cbgrp . create_group ( '<STR_LIT:{}>' . format ( offset ) ) <EOL> offgrp . create_dataset ( "<STR_LIT>" , ( <NUM_LIT:3> , ) , np . uint32 , data = offset ) <EOL> offgrp . create_dataset ( "<STR_LIT>" , cbdata . shape , cbdata . dtype , data = cbdata ) <EOL> def addBoundingBox ( self , resolution , corner , dim ) : <EOL> """<STR_LIT>""" <EOL> self . idgrp . create_dataset ( "<STR_LIT>" , ( <NUM_LIT:1> , ) , np . uint32 , data = resolution ) <EOL> self . idgrp . create_dataset ( "<STR_LIT>" , ( <NUM_LIT:3> , ) , np . uint32 , data = corner ) <EOL> self . idgrp . create_dataset ( "<STR_LIT>" , ( <NUM_LIT:3> , ) , np . uint32 , data = dim ) <EOL> def H5toAnnotation ( key , idgrp , annodb ) : <EOL> """<STR_LIT>""" <EOL> if '<STR_LIT>' in idgrp : <EOL> annotype = idgrp [ '<STR_LIT>' ] [ <NUM_LIT:0> ] <EOL> else : <EOL> annotype = annotation . ANNO_ANNOTATION <EOL> mdgrp = idgrp . get ( '<STR_LIT>' ) <EOL> if annotype == annotation . ANNO_SEED : <EOL> anno = annotation . AnnSeed ( annodb ) <EOL> if mdgrp : <EOL> if '<STR_LIT>' in mdgrp : <EOL> anno . parent = mdgrp [ '<STR_LIT>' ] [ <NUM_LIT:0> ] <EOL> if '<STR_LIT>' in mdgrp : <EOL> anno . position = mdgrp [ '<STR_LIT>' ] [ : ] <EOL> if '<STR_LIT>' in mdgrp : <EOL> anno . cubelocation = mdgrp [ '<STR_LIT>' ] [ <NUM_LIT:0> ] <EOL> if '<STR_LIT>' in mdgrp : <EOL> anno . source = mdgrp [ '<STR_LIT>' ] [ <NUM_LIT:0> ] <EOL> elif annotype == annotation . ANNO_SYNAPSE : <EOL> anno = annotation . AnnSynapse ( annodb ) <EOL> if mdgrp : <EOL> if '<STR_LIT>' in mdgrp : <EOL> anno . synapse_type = mdgrp [ '<STR_LIT>' ] [ <NUM_LIT:0> ] <EOL> if '<STR_LIT>' in mdgrp : <EOL> anno . weight = mdgrp [ '<STR_LIT>' ] [ <NUM_LIT:0> ] <EOL> if '<STR_LIT>' in mdgrp : <EOL> anno . seeds = mdgrp [ '<STR_LIT>' ] [ : ] <EOL> if '<STR_LIT>' in mdgrp : <EOL> anno . segments = mdgrp [ '<STR_LIT>' ] [ : ] <EOL> elif annotype == annotation . ANNO_SEGMENT : <EOL> anno = annotation . AnnSegment ( annodb ) <EOL> if mdgrp : <EOL> if mdgrp . get ( '<STR_LIT>' ) : <EOL> anno . parentseed = mdgrp [ '<STR_LIT>' ] [ <NUM_LIT:0> ] <EOL> if mdgrp . get ( '<STR_LIT>' ) : <EOL> anno . segmentclass = mdgrp [ '<STR_LIT>' ] [ <NUM_LIT:0> ] <EOL> if mdgrp . get ( '<STR_LIT>' ) : <EOL> anno . neuron = mdgrp [ '<STR_LIT>' ] [ <NUM_LIT:0> ] <EOL> if mdgrp . get ( '<STR_LIT>' ) and len ( mdgrp [ '<STR_LIT>' ] ) != <NUM_LIT:0> : <EOL> anno . synapses = mdgrp [ '<STR_LIT>' ] [ : ] <EOL> if mdgrp . get ( '<STR_LIT>' ) and len ( mdgrp [ '<STR_LIT>' ] ) != <NUM_LIT:0> : <EOL> anno . organelles = mdgrp [ '<STR_LIT>' ] [ : ] <EOL> elif annotype == annotation . ANNO_NEURON : <EOL> anno = annotation . AnnNeuron ( annodb ) <EOL> if mdgrp : <EOL> if mdgrp . get ( '<STR_LIT>' ) and len ( mdgrp [ '<STR_LIT>' ] ) != <NUM_LIT:0> : <EOL> anno . segments = mdgrp [ '<STR_LIT>' ] [ : ] <EOL> elif annotype == annotation . ANNO_ORGANELLE : <EOL> anno = annotation . AnnOrganelle ( annodb ) <EOL> if mdgrp : <EOL> if mdgrp . get ( '<STR_LIT>' ) : <EOL> anno . parentseed = mdgrp [ '<STR_LIT>' ] [ <NUM_LIT:0> ] <EOL> if mdgrp . get ( '<STR_LIT>' ) : <EOL> anno . organelleclass = mdgrp [ '<STR_LIT>' ] [ <NUM_LIT:0> ] <EOL> if mdgrp . get ( '<STR_LIT>' ) and len ( mdgrp [ '<STR_LIT>' ] ) != <NUM_LIT:0> : <EOL> anno . seeds = mdgrp [ '<STR_LIT>' ] [ : ] <EOL> if mdgrp . get ( '<STR_LIT>' ) : <EOL> anno . centroid = mdgrp [ '<STR_LIT>' ] [ : ] <EOL> elif annotype == annotation . ANNO_NODE : <EOL> anno = annotation . AnnNode ( annodb ) <EOL> if mdgrp : <EOL> if '<STR_LIT>' in mdgrp : <EOL> anno . nodetype = mdgrp [ '<STR_LIT>' ] [ <NUM_LIT:0> ] <EOL> if '<STR_LIT>' in mdgrp : <EOL> anno . parentid = mdgrp [ '<STR_LIT>' ] [ <NUM_LIT:0> ] <EOL> if '<STR_LIT>' in mdgrp : <EOL> anno . skeletonid = mdgrp [ '<STR_LIT>' ] [ <NUM_LIT:0> ] <EOL> if '<STR_LIT>' in mdgrp : <EOL> anno . radius = mdgrp [ '<STR_LIT>' ] [ <NUM_LIT:0> ] <EOL> if mdgrp . get ( '<STR_LIT>' ) and len ( mdgrp [ '<STR_LIT>' ] ) != <NUM_LIT:0> : <EOL> anno . children = mdgrp [ '<STR_LIT>' ] [ : ] <EOL> if mdgrp . get ( '<STR_LIT>' ) : <EOL> anno . location = mdgrp [ '<STR_LIT>' ] [ : ] <EOL> elif annotype == annotation . ANNO_SKELETON : <EOL> anno = annotation . AnnSkeleton ( annodb ) <EOL> if mdgrp : <EOL> if '<STR_LIT>' in mdgrp : <EOL> anno . skeletontype = mdgrp [ '<STR_LIT>' ] [ <NUM_LIT:0> ] <EOL> if '<STR_LIT>' in mdgrp : <EOL> anno . rootnode = mdgrp [ '<STR_LIT>' ] [ <NUM_LIT:0> ] <EOL> elif annotype == annotation . ANNO_ANNOTATION : <EOL> anno = annotation . Annotation ( annodb ) <EOL> else : <EOL> logger . warning ( "<STR_LIT>" % annotype ) <EOL> raise OCPCAError ( "<STR_LIT>" % annotype ) <EOL> if re . match ( "<STR_LIT>" , key ) : <EOL> anno . annid = int ( key ) <EOL> else : <EOL> anno . annid = <NUM_LIT:0> <EOL> if mdgrp : <EOL> if mdgrp . get ( '<STR_LIT>' ) : <EOL> anno . status = mdgrp [ '<STR_LIT>' ] [ <NUM_LIT:0> ] <EOL> if mdgrp . get ( '<STR_LIT>' ) : <EOL> anno . confidence = mdgrp [ '<STR_LIT>' ] [ <NUM_LIT:0> ] <EOL> if mdgrp . get ( '<STR_LIT>' ) : <EOL> anno . author = mdgrp [ '<STR_LIT>' ] [ <NUM_LIT:0> ] <EOL> if mdgrp . get ( '<STR_LIT>' ) : <EOL> fstring = cStringIO . StringIO ( mdgrp [ '<STR_LIT>' ] [ <NUM_LIT:0> ] ) <EOL> csvr = csv . reader ( fstring , delimiter = '<STR_LIT:U+002C>' ) <EOL> for r in csvr : <EOL> anno . kvpairs [ r [ <NUM_LIT:0> ] ] = r [ <NUM_LIT:1> ] <EOL> return anno <EOL> def H5GetVoxels ( h5fh ) : <EOL> """<STR_LIT>""" <EOL> keys = h5fh . keys ( ) <EOL> idgrp = h5fh . get ( keys [ <NUM_LIT:0> ] ) <EOL> if idgrp . get ( '<STR_LIT>' ) : <EOL> return idgrp [ '<STR_LIT>' ] <EOL> else : <EOL> return None <EOL> def H5GetVolume ( h5fh ) : <EOL> """<STR_LIT>""" <EOL> keys = h5fh . keys ( ) <EOL> idgrp = h5fh . get ( keys [ <NUM_LIT:0> ] ) <EOL> if idgrp . get ( '<STR_LIT>' ) : <EOL> if idgrp . get ( '<STR_LIT>' ) : <EOL> return ( idgrp [ '<STR_LIT>' ] , idgrp [ '<STR_LIT>' ] ) <EOL> else : <EOL> logger . warning ( "<STR_LIT>" ) <EOL> raise OCPCAError ( "<STR_LIT>" ) <EOL> else : <EOL> return None <EOL> def H5getCuboids ( idgrp ) : <EOL> """<STR_LIT>""" <EOL> cuboidsgrp = idgrp . get ( "<STR_LIT>" ) <EOL> for k in cuboidsgrp : <EOL> corner = np . array ( cuboidsgrp [ k ] [ "<STR_LIT>" ] ) <EOL> cuboiddata = np . array ( cuboidsgrp [ k ] [ '<STR_LIT>' ] ) <EOL> yield ( corner , cuboiddata ) <EOL> def BasetoH5 ( anno , annotype , h5fh ) : <EOL> """<STR_LIT>""" <EOL> h5anno = H5Annotation ( annotype , anno . annid , h5fh ) <EOL> h5anno . mdgrp . create_dataset ( "<STR_LIT>" , ( <NUM_LIT:1> , ) , np . uint32 , data = anno . status ) <EOL> h5anno . mdgrp . create_dataset ( "<STR_LIT>" , ( <NUM_LIT:1> , ) , np . float , data = anno . confidence ) <EOL> h5anno . mdgrp . create_dataset ( "<STR_LIT>" , ( <NUM_LIT:1> , ) , dtype = h5py . special_dtype ( vlen = str ) , data = anno . author ) <EOL> fstring = cStringIO . StringIO ( ) <EOL> csvw = csv . writer ( fstring , delimiter = '<STR_LIT:U+002C>' ) <EOL> csvw . writerows ( [ r for r in anno . kvpairs . iteritems ( ) ] ) <EOL> h5anno . mdgrp . create_dataset ( "<STR_LIT>" , ( <NUM_LIT:1> , ) , dtype = h5py . special_dtype ( vlen = str ) , data = fstring . getvalue ( ) ) <EOL> return h5anno <EOL> def SynapsetoH5 ( synapse , h5fh ) : <EOL> """<STR_LIT>""" <EOL> h5synapse = BasetoH5 ( synapse , annotation . ANNO_SYNAPSE , h5fh ) <EOL> h5synapse . mdgrp . create_dataset ( "<STR_LIT>" , ( <NUM_LIT:1> , ) , np . float , data = synapse . weight ) <EOL> h5synapse . mdgrp . create_dataset ( "<STR_LIT>" , ( <NUM_LIT:1> , ) , np . uint32 , data = synapse . synapse_type ) <EOL> if ( synapse . seeds != [ ] ) : <EOL> h5synapse . mdgrp . create_dataset ( "<STR_LIT>" , ( len ( synapse . seeds ) , ) , np . uint32 , synapse . seeds ) <EOL> if ( synapse . segments != [ ] ) : <EOL> h5synapse . mdgrp . create_dataset ( "<STR_LIT>" , ( len ( synapse . segments ) , <NUM_LIT:2> ) , np . uint32 , data = synapse . segments ) <EOL> return h5synapse <EOL> def SeedtoH5 ( seed , h5fh ) : <EOL> """<STR_LIT>""" <EOL> h5seed = BasetoH5 ( seed , annotation . ANNO_SEED , h5fh ) <EOL> h5seed . mdgrp . create_dataset ( "<STR_LIT>" , ( <NUM_LIT:1> , ) , np . uint32 , data = seed . parent ) <EOL> h5seed . mdgrp . create_dataset ( "<STR_LIT>" , ( <NUM_LIT:1> , ) , np . uint32 , data = seed . cubelocation ) <EOL> h5seed . mdgrp . create_dataset ( "<STR_LIT>" , ( <NUM_LIT:1> , ) , np . uint32 , data = seed . source ) <EOL> if seed . position != [ None , None , None ] : <EOL> h5seed . mdgrp . create_dataset ( "<STR_LIT>" , ( <NUM_LIT:3> , ) , np . uint32 , data = seed . position ) <EOL> return h5seed <EOL> def SegmenttoH5 ( segment , h5fh ) : <EOL> """<STR_LIT>""" <EOL> h5segment = BasetoH5 ( segment , annotation . ANNO_SEGMENT , h5fh ) <EOL> h5segment . mdgrp . create_dataset ( "<STR_LIT>" , ( <NUM_LIT:1> , ) , np . uint32 , data = segment . segmentclass ) <EOL> h5segment . mdgrp . create_dataset ( "<STR_LIT>" , ( <NUM_LIT:1> , ) , np . uint32 , data = segment . parentseed ) <EOL> h5segment . mdgrp . create_dataset ( "<STR_LIT>" , ( <NUM_LIT:1> , ) , np . uint32 , data = segment . neuron ) <EOL> if ( segment . synapses != [ ] ) : <EOL> h5segment . mdgrp . create_dataset ( "<STR_LIT>" , ( len ( segment . synapses ) , ) , np . uint32 , segment . synapses ) <EOL> if ( segment . organelles != [ ] ) : <EOL> h5segment . mdgrp . create_dataset ( "<STR_LIT>" , ( len ( segment . organelles ) , ) , np . uint32 , segment . organelles ) <EOL> return h5segment <EOL> def NeurontoH5 ( neuron , h5fh ) : <EOL> """<STR_LIT>""" <EOL> h5neuron = BasetoH5 ( neuron , annotation . ANNO_NEURON , h5fh ) <EOL> if ( neuron . segments != [ ] ) : <EOL> h5neuron . mdgrp . create_dataset ( "<STR_LIT>" , ( len ( neuron . segments ) , ) , np . uint32 , neuron . segments ) <EOL> return h5neuron <EOL> def OrganelletoH5 ( organelle , h5fh ) : <EOL> """<STR_LIT>""" <EOL> h5organelle = BasetoH5 ( organelle , annotation . ANNO_ORGANELLE , h5fh ) <EOL> h5organelle . mdgrp . create_dataset ( "<STR_LIT>" , ( <NUM_LIT:1> , ) , np . uint32 , data = organelle . organelleclass ) <EOL> h5organelle . mdgrp . create_dataset ( "<STR_LIT>" , ( <NUM_LIT:1> , ) , np . uint32 , data = organelle . parentseed ) <EOL> if ( organelle . seeds != [ ] ) : <EOL> h5organelle . mdgrp . create_dataset ( "<STR_LIT>" , ( len ( organelle . seeds ) , ) , np . uint32 , organelle . seeds ) <EOL> if organelle . centroid != [ None , None , None ] : <EOL> h5organelle . mdgrp . create_dataset ( "<STR_LIT>" , ( <NUM_LIT:3> , ) , np . uint32 , data = organelle . centroid ) <EOL> return h5organelle <EOL> def NodetoH5 ( node , h5fh ) : <EOL> """<STR_LIT>""" <EOL> h5node = BasetoH5 ( node , annotation . ANNO_NODE , h5fh ) <EOL> h5node . mdgrp . create_dataset ( "<STR_LIT>" , ( <NUM_LIT:1> , ) , np . uint32 , data = node . nodetype ) <EOL> h5node . mdgrp . create_dataset ( "<STR_LIT>" , ( <NUM_LIT:1> , ) , np . int32 , data = node . parentid ) <EOL> h5node . mdgrp . create_dataset ( "<STR_LIT>" , ( <NUM_LIT:1> , ) , np . uint32 , data = node . skeletonid ) <EOL> h5node . mdgrp . create_dataset ( "<STR_LIT>" , ( <NUM_LIT:1> , ) , np . float , data = node . radius ) <EOL> if ( node . children != [ ] ) : <EOL> h5node . mdgrp . create_dataset ( "<STR_LIT>" , ( len ( node . children ) , ) , np . uint32 , node . children ) <EOL> if node . location != [ None , None , None ] : <EOL> h5node . mdgrp . create_dataset ( "<STR_LIT>" , ( <NUM_LIT:3> , ) , np . uint32 , data = node . location ) <EOL> return h5node <EOL> def SkeletontoH5 ( skeleton , h5fh ) : <EOL> """<STR_LIT>""" <EOL> h5skeleton = BasetoH5 ( skeleton , annotation . ANNO_SKELETON , h5fh ) <EOL> h5skeleton . mdgrp . create_dataset ( "<STR_LIT>" , ( <NUM_LIT:1> , ) , np . uint32 , data = skeleton . skeletontype ) <EOL> h5skeleton . mdgrp . create_dataset ( "<STR_LIT>" , ( <NUM_LIT:1> , ) , np . uint32 , data = skeleton . rootnode ) <EOL> return h5skeleton <EOL> def AnnotationtoH5 ( anno , h5fh ) : <EOL> """<STR_LIT>""" <EOL> if anno . __class__ == annotation . AnnSynapse : <EOL> return SynapsetoH5 ( anno , h5fh ) <EOL> elif anno . __class__ == annotation . AnnSeed : <EOL> return SeedtoH5 ( anno , h5fh ) <EOL> if anno . __class__ == annotation . AnnSegment : <EOL> return SegmenttoH5 ( anno , h5fh ) <EOL> if anno . __class__ == annotation . AnnNeuron : <EOL> return NeurontoH5 ( anno , h5fh ) <EOL> if anno . __class__ == annotation . AnnOrganelle : <EOL> return OrganelletoH5 ( anno , h5fh ) <EOL> if anno . __class__ == annotation . AnnNode : <EOL> return NodetoH5 ( anno , h5fh ) <EOL> if anno . __class__ == annotation . AnnSkeleton : <EOL> return SkeletontoH5 ( anno , h5fh ) <EOL> elif anno . __class__ == annotation . Annotation : <EOL> return BasetoH5 ( anno , annotation . ANNO_ANNOTATION , h5fh ) <EOL> else : <EOL> logger . warning ( "<STR_LIT>" % anno . __class__ ) <EOL> raise OCPCAError ( "<STR_LIT>" % anno . __class__ ) <EOL> def PackageIDs ( annoids ) : <EOL> """<STR_LIT>""" <EOL> tmpfile = tempfile . NamedTemporaryFile ( ) <EOL> h5fh = h5py . File ( tmpfile . name ) <EOL> if len ( annoids ) != <NUM_LIT:0> : <EOL> h5fh . create_dataset ( "<STR_LIT>" , annoids . shape , np . uint32 , data = annoids ) <EOL> h5fh . flush ( ) <EOL> tmpfile . seek ( <NUM_LIT:0> ) <EOL> return tmpfile . read ( ) <EOL> def h5toCSV ( h5f ) : <EOL> """<STR_LIT>""" <EOL> fstring = cStringIO . StringIO ( ) <EOL> csvw = csv . writer ( fstring , delimiter = '<STR_LIT:U+002C>' ) <EOL> keys = h5f . keys ( ) <EOL> idgrp = h5f . get ( keys [ <NUM_LIT:0> ] ) <EOL> for k in idgrp . keys ( ) : <EOL> if k == '<STR_LIT>' : <EOL> mdgrp = idgrp . get ( '<STR_LIT>' ) <EOL> for m in mdgrp . keys ( ) : <EOL> if len ( mdgrp [ m ] ) == <NUM_LIT:1> : <EOL> csvw . writerow ( [ m , mdgrp [ m ] [ <NUM_LIT:0> ] ] ) <EOL> else : <EOL> csvw . writerow ( [ m , mdgrp [ m ] [ : ] ] ) <EOL> else : <EOL> if len ( idgrp [ k ] ) == <NUM_LIT:1> : <EOL> csvw . writerow ( [ k , idgrp [ k ] [ <NUM_LIT:0> ] ] ) <EOL> else : <EOL> csvw . writerow ( [ k , idgrp [ k ] [ : ] ] ) <EOL> return fstring . getvalue ( ) </s>
