<s> import synapseclient <EOL> from synapseclient import File , Project , Folder , Table , Schema , Link , Wiki , Activity , exceptions <EOL> import time <EOL> from synapseclient . exceptions import * <EOL> import tempfile <EOL> import re <EOL> def copy ( syn , entity , destinationId = None , copyWikiPage = True , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> mapping = kwargs . get ( '<STR_LIT>' , dict ( ) ) <EOL> updateLinks = kwargs . get ( '<STR_LIT>' , True ) <EOL> updateSynIds = kwargs . get ( '<STR_LIT>' , True ) <EOL> entitySubPageId = kwargs . get ( '<STR_LIT>' , None ) <EOL> destinationSubPageId = kwargs . get ( '<STR_LIT>' , None ) <EOL> mapping = _copyRecursive ( syn , entity , destinationId , mapping = mapping , ** kwargs ) <EOL> if copyWikiPage : <EOL> for oldEnt in mapping : <EOL> newWikig = copyWiki ( syn , oldEnt , mapping [ oldEnt ] , entitySubPageId = entitySubPageId , destinationSubPageId = destinationSubPageId , updateLinks = updateLinks , updateSynIds = updateSynIds , entityMap = mapping ) <EOL> return ( mapping ) <EOL> def _copyRecursive ( syn , entity , destinationId , mapping = dict ( ) , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> version = kwargs . get ( '<STR_LIT:version>' , None ) <EOL> setProvenance = kwargs . get ( '<STR_LIT>' , "<STR_LIT>" ) <EOL> recursive = kwargs . get ( '<STR_LIT>' , True ) <EOL> copyTable = kwargs . get ( '<STR_LIT>' , True ) <EOL> replace = kwargs . get ( '<STR_LIT:replace>' , False ) <EOL> ent = syn . get ( entity , downloadFile = False ) <EOL> if isinstance ( ent , Project ) : <EOL> if destinationId is None : <EOL> newProject = syn . store ( Project ( "<STR_LIT>" % ( time . time ( ) , ent . name ) ) ) <EOL> destinationId = newProject . id <EOL> elif not isinstance ( syn . get ( destinationId ) , Project ) : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> copiedId = destinationId <EOL> entities = syn . chunkedQuery ( '<STR_LIT>' % ent . id ) <EOL> for i in entities : <EOL> mapping = _copyRecursive ( syn , i [ '<STR_LIT>' ] , destinationId , mapping = mapping , ** kwargs ) <EOL> else : <EOL> if destinationId is None : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> elif isinstance ( ent , Folder ) : <EOL> copiedId = _copyFolder ( syn , ent . id , destinationId , mapping = mapping , ** kwargs ) <EOL> elif isinstance ( ent , File ) : <EOL> copiedId = _copyFile ( syn , ent . id , destinationId , version = version , replace = replace , setProvenance = setProvenance ) <EOL> elif isinstance ( ent , Link ) : <EOL> copiedId = _copyLink ( syn , ent . id , destinationId ) <EOL> elif isinstance ( ent , Schema ) and copyTable : <EOL> copiedId = _copyTable ( syn , ent . id , destinationId ) <EOL> else : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> print ( "<STR_LIT>" % ( ent . id , copiedId ) ) <EOL> mapping [ ent . id ] = copiedId <EOL> return ( mapping ) <EOL> def _copyFolder ( syn , entity , destinationId , mapping = dict ( ) , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> oldFolder = syn . get ( entity ) <EOL> recursive = kwargs . get ( '<STR_LIT>' , True ) <EOL> search = syn . query ( '<STR_LIT>' % destinationId ) <EOL> for i in search [ '<STR_LIT>' ] : <EOL> if i [ '<STR_LIT>' ] == oldFolder . name : <EOL> raise ValueError ( '<STR_LIT>' % oldFolder . name ) <EOL> newFolder = Folder ( name = oldFolder . name , parent = destinationId ) <EOL> newFolder . annotations = oldFolder . annotations <EOL> newFolder = syn . store ( newFolder ) <EOL> if recursive : <EOL> entities = syn . chunkedQuery ( '<STR_LIT>' % entity ) <EOL> for ent in entities : <EOL> copied = _copyRecursive ( syn , ent [ '<STR_LIT>' ] , newFolder . id , mapping , ** kwargs ) <EOL> return ( newFolder . id ) <EOL> def _copyFile ( syn , entity , destinationId , version = None , replace = False , setProvenance = "<STR_LIT>" ) : <EOL> """<STR_LIT>""" <EOL> ent = syn . get ( entity , downloadFile = False , version = version , followLink = False ) <EOL> if not replace : <EOL> search = syn . query ( '<STR_LIT>' % destinationId ) <EOL> for i in search [ '<STR_LIT>' ] : <EOL> if i [ '<STR_LIT>' ] == ent . name : <EOL> raise ValueError ( '<STR_LIT>' % ent . name ) <EOL> profile = syn . getUserProfile ( ) <EOL> if setProvenance == "<STR_LIT>" : <EOL> act = Activity ( "<STR_LIT>" , used = ent ) <EOL> elif setProvenance == "<STR_LIT>" : <EOL> try : <EOL> act = syn . getProvenance ( ent . id ) <EOL> except Exception as e : <EOL> if e . message . find ( '<STR_LIT>' ) >= <NUM_LIT:0> : <EOL> act = None <EOL> else : <EOL> raise e <EOL> elif setProvenance is None or setProvenance . lower ( ) == '<STR_LIT:none>' : <EOL> act = None <EOL> else : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> fileHandleList = syn . restGET ( '<STR_LIT>' % ( ent . id , ent . versionNumber ) ) <EOL> for fileHandle in fileHandleList [ '<STR_LIT:list>' ] : <EOL> if fileHandle [ '<STR_LIT:id>' ] == ent . dataFileHandleId : <EOL> createdBy = fileHandle [ '<STR_LIT>' ] <EOL> break <EOL> else : <EOL> createdBy = None <EOL> if profile . ownerId == createdBy : <EOL> new_ent = File ( name = ent . name , parentId = destinationId ) <EOL> new_ent . dataFileHandleId = ent . dataFileHandleId <EOL> else : <EOL> if ent . externalURL is None : <EOL> store = True <EOL> ent = syn . get ( entity , downloadFile = store , version = version ) <EOL> path = ent . path <EOL> else : <EOL> store = False <EOL> ent = syn . get ( entity , downloadFile = store , version = version ) <EOL> path = ent . externalURL <EOL> new_ent = File ( path , name = ent . name , parentId = destinationId , synapseStore = store ) <EOL> new_ent . annotations = ent . annotations <EOL> if act is not None : <EOL> new_ent = syn . store ( new_ent , activity = act ) <EOL> else : <EOL> new_ent = syn . store ( new_ent ) <EOL> return new_ent [ '<STR_LIT:id>' ] <EOL> def _copyTable ( syn , entity , destinationId , setAnnotations = False ) : <EOL> """<STR_LIT>""" <EOL> print ( "<STR_LIT>" % entity ) <EOL> myTableSchema = syn . get ( entity ) <EOL> search = syn . query ( '<STR_LIT>' % destinationId ) <EOL> for i in search [ '<STR_LIT>' ] : <EOL> if i [ '<STR_LIT>' ] == myTableSchema . name : <EOL> raise ValueError ( '<STR_LIT>' % myTableSchema . name ) <EOL> d = syn . tableQuery ( '<STR_LIT>' % myTableSchema . id ) <EOL> d = d . asDataFrame ( ) <EOL> d = d . reset_index ( ) <EOL> del d [ '<STR_LIT:index>' ] <EOL> colIds = myTableSchema . columnIds <EOL> newTableSchema = Schema ( name = myTableSchema . name , <EOL> parent = destinationId , <EOL> columns = colIds ) <EOL> if setAnnotations : <EOL> newTableSchema . annotations = myTableSchema . annotations <EOL> if len ( d ) > <NUM_LIT:0> : <EOL> print ( "<STR_LIT>" % newTableSchema . name ) <EOL> newTable = Table ( schema = newTableSchema , values = d ) <EOL> newTable = syn . store ( newTable ) <EOL> return ( newTable . schema . id ) <EOL> else : <EOL> print ( "<STR_LIT>" % newTableSchema . name ) <EOL> newTableSchema = syn . store ( newTableSchema ) <EOL> return ( newTableSchema . id ) <EOL> def _copyLink ( syn , entity , destinationId ) : <EOL> """<STR_LIT>""" <EOL> ent = syn . get ( entity ) <EOL> search = syn . query ( '<STR_LIT>' % destinationId ) <EOL> for i in search [ '<STR_LIT>' ] : <EOL> if i [ '<STR_LIT>' ] == ent . name : <EOL> raise ValueError ( '<STR_LIT>' % ent . name ) <EOL> newLink = Link ( ent . linksTo [ '<STR_LIT>' ] , parent = destinationId , targetVersion = ent . linksTo [ '<STR_LIT>' ] ) <EOL> newLink = syn . store ( newLink ) <EOL> return ( newLink . id ) <EOL> def _getSubWikiHeaders ( wikiHeaders , subPageId , mapping = [ ] ) : <EOL> subPageId = str ( subPageId ) <EOL> for i in wikiHeaders : <EOL> if i [ '<STR_LIT:id>' ] == subPageId and len ( mapping ) == <NUM_LIT:0> : <EOL> i . pop ( "<STR_LIT>" , None ) <EOL> mapping . append ( i ) <EOL> elif i [ '<STR_LIT:id>' ] == subPageId : <EOL> mapping . append ( i ) <EOL> elif i . get ( '<STR_LIT>' , None ) is not None : <EOL> if i [ '<STR_LIT>' ] == subPageId : <EOL> mapping = _getSubWikiHeaders ( wikiHeaders , subPageId = i [ '<STR_LIT:id>' ] , mapping = mapping ) <EOL> return ( mapping ) <EOL> def copyWiki ( syn , entity , destinationId , entitySubPageId = None , destinationSubPageId = None , updateLinks = True , updateSynIds = True , entityMap = None ) : <EOL> """<STR_LIT>""" <EOL> oldOwn = syn . get ( entity , downloadFile = False ) <EOL> try : <EOL> oldWh = syn . getWikiHeaders ( oldOwn ) <EOL> store = True <EOL> except SynapseHTTPError : <EOL> store = False <EOL> if store : <EOL> if entitySubPageId is not None : <EOL> oldWh = _getSubWikiHeaders ( oldWh , entitySubPageId , mapping = [ ] ) <EOL> newOwn = syn . get ( destinationId , downloadFile = False ) <EOL> wikiIdMap = dict ( ) <EOL> newWikis = dict ( ) <EOL> for i in oldWh : <EOL> attDir = tempfile . NamedTemporaryFile ( prefix = '<STR_LIT>' , suffix = '<STR_LIT>' ) <EOL> wiki = syn . getWiki ( oldOwn , i . id ) <EOL> print ( '<STR_LIT>' % i . id ) <EOL> if wiki [ '<STR_LIT>' ] == [ ] : <EOL> attachments = [ ] <EOL> elif wiki [ '<STR_LIT>' ] != [ ] : <EOL> uri = "<STR_LIT>" % ( wiki . ownerId , wiki . id ) <EOL> results = syn . restGET ( uri ) <EOL> file_handles = { fh [ '<STR_LIT:id>' ] : fh for fh in results [ '<STR_LIT:list>' ] } <EOL> attachments = [ ] <EOL> tempdir = tempfile . gettempdir ( ) <EOL> for fhid in wiki . attachmentFileHandleIds : <EOL> file_info = syn . _downloadWikiAttachment ( wiki . ownerId , wiki , file_handles [ fhid ] [ '<STR_LIT>' ] , destination = tempdir ) <EOL> attachments . append ( file_info [ '<STR_LIT:path>' ] ) <EOL> if hasattr ( i , '<STR_LIT>' ) : <EOL> wNew = Wiki ( owner = newOwn , title = wiki . get ( '<STR_LIT:title>' , '<STR_LIT>' ) , markdown = wiki . markdown , attachments = attachments , parentWikiId = wikiIdMap [ wiki . parentWikiId ] ) <EOL> wNew = syn . store ( wNew ) <EOL> else : <EOL> wNew = Wiki ( owner = newOwn , title = wiki . get ( '<STR_LIT:title>' , '<STR_LIT>' ) , markdown = wiki . markdown , attachments = attachments , parentWikiId = destinationSubPageId ) <EOL> wNew = syn . store ( wNew ) <EOL> parentWikiId = wNew . id <EOL> newWikis [ wNew . id ] = wNew <EOL> wikiIdMap [ wiki . id ] = wNew . id <EOL> if updateLinks : <EOL> print ( "<STR_LIT>" ) <EOL> for oldWikiId in wikiIdMap . keys ( ) : <EOL> newWikiId = wikiIdMap [ oldWikiId ] <EOL> newWiki = newWikis [ newWikiId ] <EOL> print ( "<STR_LIT>" % newWikiId ) <EOL> s = newWiki . markdown <EOL> for oldWikiId2 in wikiIdMap . keys ( ) : <EOL> oldProjectAndWikiId = "<STR_LIT>" % ( entity , oldWikiId2 ) <EOL> newProjectAndWikiId = "<STR_LIT>" % ( destinationId , wikiIdMap [ oldWikiId2 ] ) <EOL> s = re . sub ( oldProjectAndWikiId , newProjectAndWikiId , s ) <EOL> s = re . sub ( entity , destinationId , s ) <EOL> newWikis [ newWikiId ] . markdown = s <EOL> if updateSynIds and entityMap is not None : <EOL> print ( "<STR_LIT>" ) <EOL> for oldWikiId in wikiIdMap . keys ( ) : <EOL> newWikiId = wikiIdMap [ oldWikiId ] <EOL> newWiki = newWikis [ newWikiId ] <EOL> print ( '<STR_LIT>' % newWikiId ) <EOL> s = newWiki . markdown <EOL> for oldSynId in entityMap . keys ( ) : <EOL> newSynId = entityMap [ oldSynId ] <EOL> s = re . sub ( oldSynId , newSynId , s ) <EOL> print ( "<STR_LIT>" ) <EOL> newWikis [ newWikiId ] . markdown = s <EOL> print ( "<STR_LIT>" ) <EOL> for oldWikiId in wikiIdMap . keys ( ) : <EOL> newWikiId = wikiIdMap [ oldWikiId ] <EOL> newWikis [ newWikiId ] = syn . store ( newWikis [ newWikiId ] ) <EOL> print ( "<STR_LIT>" , newWikiId ) <EOL> newWh = syn . getWikiHeaders ( newOwn ) <EOL> return ( newWh ) <EOL> else : <EOL> return ( "<STR_LIT>" ) </s>
