<s> import os <EOL> import itertools <EOL> from conary import files , trove , versions , changelog , callbacks <EOL> from conary . cmds import metadata <EOL> from conary . dbstore import idtable <EOL> from conary . deps import deps <EOL> from conary . lib import util , tracelog <EOL> from conary . local import deptable <EOL> from conary . local . sqldb import VersionCache , FlavorCache <EOL> from conary . local import versiontable <EOL> from conary . repository import errors <EOL> from conary . repository . netrepos import instances , items , keytable , flavors , troveinfo , versionops , cltable , accessmap <EOL> from conary . server import schema <EOL> class TroveAdder : <EOL> def addStream ( self , fileId , fileStream = None , withContents = True ) : <EOL> existing = self . newStreamsByFileId . get ( fileId ) <EOL> if existing and existing [ <NUM_LIT:1> ] : <EOL> return <EOL> if fileStream and withContents : <EOL> sha1 = None <EOL> if ( not files . frozenFileFlags ( fileStream ) . isEncapsulatedContent ( ) <EOL> and files . frozenFileHasContents ( fileStream ) ) : <EOL> cont = files . frozenFileContentInfo ( fileStream ) <EOL> sha1 = cont . sha1 ( ) <EOL> else : <EOL> sha1 = None <EOL> self . newStreamsByFileId [ fileId ] = ( self . cu . binary ( fileId ) , <EOL> self . cu . binary ( fileStream ) , <EOL> self . cu . binary ( sha1 ) ) <EOL> def addFile ( self , pathId , path , fileId , fileVersion , <EOL> fileStream = None , withContents = True ) : <EOL> dirname , basename = os . path . split ( path ) <EOL> pathChanged = <NUM_LIT:1> <EOL> versionId = self . troveStore . getVersionId ( fileVersion ) <EOL> self . newFilesInsertList . append ( ( self . cu . binary ( pathId ) , versionId , <EOL> self . cu . binary ( fileId ) , <EOL> self . dirMap [ dirname ] , <EOL> self . baseMap [ basename ] , <EOL> pathChanged ) ) <EOL> changeInfo = self . changeMap . get ( pathId , None ) <EOL> if changeInfo : <EOL> if not changeInfo [ <NUM_LIT:1> ] : <EOL> pathChanged = <NUM_LIT:0> <EOL> if not changeInfo [ <NUM_LIT:3> ] : <EOL> versionChanged = False <EOL> else : <EOL> versionChanged = True <EOL> else : <EOL> versionChanged = ( pathId in self . newSet ) <EOL> if not versionChanged : <EOL> fileStream = None <EOL> self . addStream ( fileId , fileStream = fileStream , <EOL> withContents = withContents ) <EOL> def __init__ ( self , troveStore , cu , trv , trvCs , hidden , newSet , changeMap , <EOL> dirMap , baseMap , newStreamsByFileId ) : <EOL> self . troveStore = troveStore <EOL> self . cu = cu <EOL> self . trv = trv <EOL> self . trvCs = trvCs <EOL> self . hidden = hidden <EOL> self . newFilesInsertList = [ ] <EOL> self . newStreamsByFileId = newStreamsByFileId <EOL> self . newSet = newSet <EOL> self . changeMap = changeMap <EOL> self . dirMap = dirMap <EOL> self . baseMap = baseMap <EOL> class TroveStore : <EOL> def __init__ ( self , db , log = None ) : <EOL> self . db = db <EOL> self . items = items . Items ( self . db ) <EOL> self . flavors = flavors . Flavors ( self . db ) <EOL> self . branchTable = versionops . BranchTable ( self . db ) <EOL> self . changeLogs = cltable . ChangeLogTable ( self . db ) <EOL> self . versionTable = versiontable . VersionTable ( self . db ) <EOL> self . versionOps = versionops . SqlVersioning ( <EOL> self . db , self . versionTable , self . branchTable ) <EOL> self . instances = instances . InstanceTable ( self . db ) <EOL> self . latest = versionops . LatestTable ( self . db ) <EOL> self . ri = accessmap . RoleInstances ( self . db ) <EOL> self . keyTable = keytable . OpenPGPKeyTable ( self . db ) <EOL> self . depTables = deptable . DependencyTables ( self . db ) <EOL> self . metadataTable = metadata . MetadataTable ( self . db , create = False ) <EOL> self . troveInfoTable = troveinfo . TroveInfoTable ( self . db ) <EOL> self . needsCleanup = False <EOL> self . log = log or tracelog . getLog ( None ) <EOL> self . versionIdCache = { } <EOL> self . itemIdCache = { } <EOL> def __del__ ( self ) : <EOL> self . db = self . log = None <EOL> def getLabelId ( self , label ) : <EOL> self . versionOps . labels . getOrAddId ( label ) <EOL> def getItemId ( self , item ) : <EOL> itemId = self . itemIdCache . get ( item , None ) <EOL> if itemId is not None : <EOL> return itemId <EOL> itemId = self . items . get ( item , None ) <EOL> if itemId is None : <EOL> itemId = self . items . addId ( item ) <EOL> self . itemIdCache [ item ] = itemId <EOL> return itemId <EOL> def getInstanceId ( self , itemId , versionId , flavorId , clonedFromId , <EOL> troveType , isPresent = instances . INSTANCE_PRESENT_NORMAL ) : <EOL> theId = self . instances . get ( ( itemId , versionId , flavorId ) , None ) <EOL> if theId == None : <EOL> theId = self . instances . addId ( itemId , versionId , flavorId , <EOL> clonedFromId , <EOL> troveType , isPresent = isPresent ) <EOL> if isPresent != instances . INSTANCE_PRESENT_MISSING : <EOL> self . instances . update ( theId , isPresent = isPresent , clonedFromId = clonedFromId ) <EOL> self . items . setTroveFlag ( itemId , <NUM_LIT:1> ) <EOL> return theId <EOL> def getVersionId ( self , version ) : <EOL> theId = self . versionIdCache . get ( version , None ) <EOL> if theId : <EOL> return theId <EOL> theId = self . versionTable . get ( version , None ) <EOL> if theId == None : <EOL> theId = self . versionTable . addId ( version ) <EOL> self . versionIdCache [ version ] = theId <EOL> return theId <EOL> def getFullVersion ( self , item , version ) : <EOL> """<STR_LIT>""" <EOL> cu = self . db . cursor ( ) <EOL> cu . execute ( """<STR_LIT>""" , item , version . asString ( ) ) <EOL> timeStamps = cu . fetchone ( ) [ <NUM_LIT:0> ] <EOL> version . setTimeStamps ( [ float ( x ) for x in timeStamps . split ( "<STR_LIT::>" ) ] ) <EOL> def createTroveBranch ( self , troveName , branch ) : <EOL> itemId = self . getItemId ( troveName ) <EOL> self . versionOps . createBranch ( itemId , branch ) <EOL> def getTroveFlavors ( self , troveDict ) : <EOL> cu = self . db . cursor ( ) <EOL> vMap = { } <EOL> outD = { } <EOL> schema . resetTable ( cu , '<STR_LIT>' ) <EOL> for troveName in troveDict . keys ( ) : <EOL> outD [ troveName ] = { } <EOL> for version in troveDict [ troveName ] : <EOL> outD [ troveName ] [ version ] = [ ] <EOL> versionStr = version . asString ( ) <EOL> vMap [ versionStr ] = version <EOL> cu . execute ( "<STR_LIT>" , <EOL> ( troveName , versionStr , versionStr ) , <EOL> start_transaction = False ) <EOL> self . db . analyze ( "<STR_LIT>" ) <EOL> cu . execute ( """<STR_LIT>""" ) <EOL> for ( item , verString , flavor ) in cu : <EOL> ver = vMap [ verString ] <EOL> outD [ item ] [ ver ] . append ( flavor ) <EOL> return outD <EOL> def iterTroveNames ( self ) : <EOL> cu = self . db . cursor ( ) <EOL> cu . execute ( """<STR_LIT>""" , instances . INSTANCE_PRESENT_NORMAL ) <EOL> for ( item , ) in cu : <EOL> yield item <EOL> def presentHiddenTroves ( self ) : <EOL> cu = self . db . cursor ( ) <EOL> schema . resetTable ( cu , '<STR_LIT>' ) <EOL> cu . execute ( """<STR_LIT>""" , instances . INSTANCE_PRESENT_HIDDEN ) <EOL> cu . execute ( """<STR_LIT>""" , instances . INSTANCE_PRESENT_NORMAL ) <EOL> self . latest . updateFromNewTroves ( '<STR_LIT>' ) <EOL> def addTrove ( self , trv , trvCs , hidden = False ) : <EOL> cu = self . db . cursor ( ) <EOL> changeMap = dict ( ( x [ <NUM_LIT:0> ] , x ) for x in trvCs . getChangedFileList ( ) ) <EOL> newSet = set ( x [ <NUM_LIT:0> ] for x in trvCs . getNewFileList ( ) ) <EOL> return TroveAdder ( self , cu , trv , trvCs , hidden , newSet , changeMap , <EOL> self . dirMap , self . baseMap , self . newStreamsByFileId ) <EOL> def _addTroveNewFlavors ( self , cu , trv ) : <EOL> troveFlavor = trv . getFlavor ( ) <EOL> flavorsNeeded = { } <EOL> if troveFlavor is not None : <EOL> flavorsNeeded [ troveFlavor ] = True <EOL> for ( name , version , flavor ) in trv . iterTroveList ( strongRefs = True , <EOL> weakRefs = True ) : <EOL> if flavor is not None : <EOL> flavorsNeeded [ flavor ] = True <EOL> for ( name , branch , flavor ) in trv . iterRedirects ( ) : <EOL> if flavor is not None : <EOL> flavorsNeeded [ flavor ] = True <EOL> if len ( flavorsNeeded ) == <NUM_LIT:1> : <EOL> newFlavor = flavorsNeeded . keys ( ) [ <NUM_LIT:0> ] <EOL> i = self . flavors . get ( newFlavor , None ) <EOL> if i is None : <EOL> i = self . flavors . createFlavor ( newFlavor ) <EOL> flavors = { newFlavor : i } <EOL> else : <EOL> flavorIndex = { } <EOL> schema . resetTable ( cu , "<STR_LIT>" ) <EOL> for flavor in flavorsNeeded . iterkeys ( ) : <EOL> flavorIndex [ flavor . freeze ( ) ] = flavor <EOL> cu . execute ( "<STR_LIT>" , <EOL> flavor . freeze ( ) , start_transaction = False ) <EOL> del flavorsNeeded <EOL> self . db . analyze ( "<STR_LIT>" ) <EOL> cu . execute ( """<STR_LIT>""" ) <EOL> l = [ ] <EOL> for ( flavorStr , ) in cu : <EOL> l . append ( flavorIndex [ flavorStr ] ) <EOL> for flavor in l : <EOL> self . flavors . createFlavor ( flavor ) <EOL> flavors = { } <EOL> cu . execute ( """<STR_LIT>""" ) <EOL> for ( flavorStr , flavorId ) in cu : <EOL> flavors [ flavorIndex [ flavorStr ] ] = flavorId <EOL> del flavorIndex <EOL> if troveFlavor is not None : <EOL> troveFlavorId = flavors [ troveFlavor ] <EOL> else : <EOL> troveFlavorId = <NUM_LIT:0> <EOL> return troveFlavorId <EOL> def _mergeTroveNewFiles ( self , cu ) : <EOL> self . db . analyze ( "<STR_LIT>" ) <EOL> if len ( self . newStreamsByFileId ) : <EOL> self . db . bulkload ( "<STR_LIT>" , self . newStreamsByFileId . values ( ) , <EOL> [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] ) <EOL> self . db . analyze ( "<STR_LIT>" ) <EOL> cu . execute ( """<STR_LIT>""" ) <EOL> for ( fileId , stream , sha1 ) in cu . fetchall ( ) : <EOL> cu . execute ( "<STR_LIT>" , <EOL> ( cu . binary ( stream ) , cu . binary ( fileId ) ) ) <EOL> if sha1 : <EOL> cu . execute ( "<STR_LIT>" <EOL> "<STR_LIT>" , <EOL> ( cu . binary ( sha1 ) , cu . binary ( fileId ) ) ) <EOL> cu . execute ( """<STR_LIT>""" ) <EOL> cu . execute ( """<STR_LIT>""" ) <EOL> cu . execute ( """<STR_LIT>""" ) <EOL> cu . execute ( """<STR_LIT>""" ) <EOL> def _mergeIncludedTroves ( self , cu ) : <EOL> cu . execute ( """<STR_LIT>""" ) <EOL> cu . execute ( """<STR_LIT>""" ) <EOL> cu . execute ( """<STR_LIT>""" ) <EOL> cu . execute ( """<STR_LIT>""" ) <EOL> cu . execute ( """<STR_LIT>""" ) <EOL> cu . execute ( """<STR_LIT>""" ) <EOL> cu . execute ( """<STR_LIT>""" , instances . INSTANCE_PRESENT_MISSING ) <EOL> schema . resetTable ( cu , '<STR_LIT>' ) <EOL> cu . execute ( """<STR_LIT>""" ) <EOL> cu . execute ( """<STR_LIT>""" ) <EOL> def addTroveDone ( self , troveInfo , mirror = False ) : <EOL> cu = troveInfo . cu <EOL> trv = troveInfo . trv <EOL> trvCs = troveInfo . trvCs <EOL> hidden = troveInfo . hidden <EOL> newFilesInsertList = troveInfo . newFilesInsertList <EOL> self . log ( <NUM_LIT:3> , trv ) <EOL> troveVersion = trv . getVersion ( ) <EOL> troveItemId = self . getItemId ( trv . getName ( ) ) <EOL> sourceName = trv . troveInfo . sourceName ( ) <EOL> clonedFrom = trv . troveInfo . clonedFrom ( ) <EOL> clonedFromId = None <EOL> if clonedFrom : <EOL> clonedFromId = self . versionTable . get ( clonedFrom , None ) <EOL> if clonedFromId is None : <EOL> clonedFromId = self . versionTable . addId ( clonedFrom ) <EOL> isPackage = ( not trv . getName ( ) . startswith ( '<STR_LIT>' ) and <EOL> not trv . getName ( ) . startswith ( '<STR_LIT>' ) and <EOL> '<STR_LIT::>' not in trv . getName ( ) ) <EOL> troveVersionId = self . versionTable . get ( troveVersion , None ) <EOL> if troveVersionId is not None : <EOL> nodeId = self . versionOps . nodes . getRow ( <EOL> troveItemId , troveVersionId , None ) <EOL> troveFlavorId = self . _addTroveNewFlavors ( cu , trv ) <EOL> if troveVersionId is None or nodeId is None : <EOL> ( nodeId , troveVersionId ) = self . versionOps . createVersion ( <EOL> troveItemId , troveVersion , troveFlavorId , sourceName ) <EOL> if trv . getChangeLog ( ) and trv . getChangeLog ( ) . getName ( ) : <EOL> self . changeLogs . add ( nodeId , trv . getChangeLog ( ) ) <EOL> elif sourceName : <EOL> sourceItemId = self . items . getOrAddId ( sourceName ) <EOL> self . versionOps . nodes . updateSourceItemId ( nodeId , sourceItemId , mirrorMode = mirror ) <EOL> if hidden : <EOL> presence = instances . INSTANCE_PRESENT_HIDDEN <EOL> else : <EOL> presence = instances . INSTANCE_PRESENT_NORMAL <EOL> troveInstanceId = self . getInstanceId ( troveItemId , troveVersionId , <EOL> troveFlavorId , clonedFromId , trv . getType ( ) , <EOL> isPresent = presence ) <EOL> cu . execute ( "<STR_LIT>" <EOL> "<STR_LIT>" , troveInstanceId ) <EOL> assert ( len ( cu . fetchall ( ) ) == <NUM_LIT:0> ) , "<STR_LIT>" % ( <EOL> troveInstanceId , trv . getNameVersionFlavor ( ) ) <EOL> troveBranchId = self . branchTable [ troveVersion . branch ( ) ] <EOL> self . depAdder . add ( trv , troveInstanceId ) <EOL> if trvCs . getOldVersion ( ) : <EOL> oldTroveVersionId = self . versionTable . get ( trvCs . getOldVersion ( ) , <EOL> None ) <EOL> oldFlavorId = self . flavors . get ( trvCs . getOldFlavor ( ) , None ) <EOL> oldInstanceId = self . instances [ ( troveItemId , <EOL> oldTroveVersionId , <EOL> oldFlavorId ) ] <EOL> else : <EOL> oldInstanceId = None <EOL> cu . execute ( """<STR_LIT>""" , troveItemId , troveBranchId , troveFlavorId , <EOL> troveInstanceId , troveVersionId , <EOL> '<STR_LIT>' % trv . getVersion ( ) . timeStamps ( ) [ - <NUM_LIT:1> ] , <EOL> trv . getType ( ) , oldInstanceId , int ( hidden ) ) <EOL> if len ( newFilesInsertList ) : <EOL> self . db . bulkload ( "<STR_LIT>" , <EOL> [ x + ( troveInstanceId , ) for x in newFilesInsertList ] , <EOL> [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" ] ) <EOL> insertList = [ ] <EOL> for ( ( name , version , flavor ) , weakFlag ) in itertools . chain ( <EOL> itertools . izip ( trv . iterTroveList ( strongRefs = True , <EOL> weakRefs = False ) , <EOL> itertools . repeat ( <NUM_LIT:0> ) ) , <EOL> itertools . izip ( trv . iterTroveList ( strongRefs = False , <EOL> weakRefs = True ) , <EOL> itertools . repeat ( schema . TROVE_TROVES_WEAKREF ) ) ) : <EOL> flags = weakFlag <EOL> if trv . includeTroveByDefault ( name , version , flavor ) : <EOL> flags |= schema . TROVE_TROVES_BYDEFAULT <EOL> assert ( not isPackage or version == trv . getVersion ( ) ) <EOL> assert ( not isPackage or flavor == trv . getFlavor ( ) ) <EOL> insertList . append ( ( name , str ( version ) , version . freeze ( ) , <EOL> "<STR_LIT::>" . join ( [ "<STR_LIT>" % x for x in <EOL> version . timeStamps ( ) ] ) , <EOL> '<STR_LIT>' % version . timeStamps ( ) [ - <NUM_LIT:1> ] , <EOL> str ( version . branch ( ) ) , <EOL> str ( version . trailingLabel ( ) ) , <EOL> flavor . freeze ( ) , flags , troveInstanceId , <EOL> trv . getType ( ) ) ) <EOL> if len ( insertList ) : <EOL> self . db . bulkload ( "<STR_LIT>" , insertList , [ <EOL> "<STR_LIT>" , "<STR_LIT:version>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT:label>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" ] ) <EOL> self . troveInfoTable . addInfo ( cu , trv , troveInstanceId ) <EOL> if len ( list ( trv . iterRedirects ( ) ) ) : <EOL> schema . resetTable ( cu , '<STR_LIT>' ) <EOL> for ( name , branch , flavor ) in trv . iterRedirects ( ) : <EOL> if flavor is None : <EOL> frz = None <EOL> else : <EOL> frz = flavor . freeze ( ) <EOL> cu . execute ( "<STR_LIT>" <EOL> "<STR_LIT>" , ( name , str ( branch ) , frz ) , <EOL> start_transaction = False ) <EOL> self . db . analyze ( "<STR_LIT>" ) <EOL> cu . execute ( """<STR_LIT>""" ) <EOL> cu . execute ( """<STR_LIT>""" ) <EOL> cu . execute ( """<STR_LIT>""" ) <EOL> cu . execute ( """<STR_LIT>""" % troveInstanceId ) <EOL> def addTroveSetStart ( self , oldTroveInfoList , dirNames , baseNames ) : <EOL> cu = self . db . cursor ( ) <EOL> schema . resetTable ( cu , '<STR_LIT>' ) <EOL> schema . resetTable ( cu , '<STR_LIT>' ) <EOL> schema . resetTable ( cu , '<STR_LIT>' ) <EOL> schema . resetTable ( cu , '<STR_LIT>' ) <EOL> schema . resetTable ( cu , '<STR_LIT>' ) <EOL> self . depAdder = deptable . BulkDependencyLoader ( self . db , cu ) <EOL> self . newStreamsByFileId = dict ( ) <EOL> schema . resetTable ( cu , '<STR_LIT>' ) <EOL> l = [ ( cu . binary ( x ) , ) for x in dirNames ] <EOL> self . db . bulkload ( "<STR_LIT>" , l , [ "<STR_LIT:path>" ] ) <EOL> cu . execute ( """<STR_LIT>""" ) <EOL> cu . execute ( """<STR_LIT>""" ) <EOL> self . dirMap = dict ( ( cu . frombinary ( x [ <NUM_LIT:0> ] ) , x [ <NUM_LIT:1> ] ) for x in cu ) <EOL> schema . resetTable ( cu , '<STR_LIT>' ) <EOL> l = [ ( cu . binary ( x ) , ) for x in baseNames ] <EOL> self . db . bulkload ( "<STR_LIT>" , l , [ "<STR_LIT:path>" ] ) <EOL> cu . execute ( """<STR_LIT>""" ) <EOL> cu . execute ( """<STR_LIT>""" ) <EOL> self . baseMap = dict ( ( cu . frombinary ( x [ <NUM_LIT:0> ] ) , x [ <NUM_LIT:1> ] ) for x in cu ) <EOL> schema . resetTable ( cu , '<STR_LIT>' ) <EOL> self . db . bulkload ( "<STR_LIT>" , <EOL> [ ( x [ <NUM_LIT:0> ] , str ( x [ <NUM_LIT:1> ] ) , x [ <NUM_LIT:2> ] . freeze ( ) ) for x in oldTroveInfoList ] , <EOL> [ "<STR_LIT:name>" , "<STR_LIT:version>" , "<STR_LIT>" ] ) <EOL> cu . execute ( """<STR_LIT>""" ) <EOL> self . baseMap . update ( dict ( ( cu . frombinary ( x [ <NUM_LIT:0> ] ) , x [ <NUM_LIT:1> ] ) for x in cu ) ) <EOL> cu . execute ( """<STR_LIT>""" ) <EOL> self . dirMap . update ( dict ( ( cu . frombinary ( x [ <NUM_LIT:0> ] ) , x [ <NUM_LIT:1> ] ) for x in cu ) ) <EOL> def addTroveSetDone ( self , callback = None ) : <EOL> self . dirMap = None <EOL> self . baseMap = None <EOL> if not callback : <EOL> callback = callbacks . UpdateCallback ( ) <EOL> cu = self . db . cursor ( ) <EOL> self . _mergeIncludedTroves ( cu ) <EOL> self . _mergeTroveNewFiles ( cu ) <EOL> self . newStreamsByFileId = None <EOL> self . ri . addInstanceIdSet ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> self . depAdder . done ( ) <EOL> self . depAdder = None <EOL> callback . updatingDatabase ( '<STR_LIT>' , <NUM_LIT:1> , <NUM_LIT:1> ) <EOL> self . latest . updateFromNewTroves ( '<STR_LIT>' ) <EOL> def updateMetadata ( self , troveName , branch , shortDesc , longDesc , <EOL> urls , licenses , categories , source , language ) : <EOL> itemId = self . getItemId ( troveName ) <EOL> branchId = self . branchTable [ branch ] <EOL> latestVersion = self . metadataTable . getLatestVersion ( itemId , branchId ) <EOL> if language == "<STR_LIT:C>" : <EOL> if latestVersion : <EOL> version = versions . VersionFromString ( latestVersion ) . copy ( ) <EOL> version . incrementSourceCount ( ) <EOL> else : <EOL> version = versions . _VersionFromString ( "<STR_LIT>" , defaultBranch = branch ) <EOL> if not self . versionTable . get ( version , None ) : <EOL> self . versionTable . addId ( version ) <EOL> else : <EOL> if not latestVersion : <EOL> raise KeyError , troveName <EOL> version = versions . VersionFromString ( latestVersion ) <EOL> versionId = self . versionTable . get ( version , None ) <EOL> return self . metadataTable . add ( itemId , versionId , branchId , shortDesc , longDesc , <EOL> urls , licenses , categories , source , language ) <EOL> def getMetadata ( self , troveName , branch , version = None , language = "<STR_LIT:C>" ) : <EOL> itemId = self . items . get ( troveName , None ) <EOL> if not itemId : <EOL> return None <EOL> md = None <EOL> while not md : <EOL> if self . branchTable . has_key ( branch ) : <EOL> branchId = self . branchTable [ branch ] <EOL> else : <EOL> return None <EOL> if not version : <EOL> latestVersion = self . metadataTable . getLatestVersion ( itemId , branchId ) <EOL> else : <EOL> latestVersion = version . asString ( ) <EOL> cu = self . db . cursor ( ) <EOL> cu . execute ( "<STR_LIT>" , latestVersion ) <EOL> versionId = cu . fetchone ( ) <EOL> if versionId : <EOL> versionId = versionId [ <NUM_LIT:0> ] <EOL> else : <EOL> if branch . hasParentBranch ( ) : <EOL> branch = branch . parentBranch ( ) <EOL> else : <EOL> return None <EOL> md = self . metadataTable . get ( itemId , versionId , branchId , language ) <EOL> md [ "<STR_LIT:version>" ] = versions . VersionFromString ( latestVersion ) . asString ( ) <EOL> md [ "<STR_LIT>" ] = language <EOL> return metadata . Metadata ( md ) <EOL> def hasFileContents ( self , sha1iter ) : <EOL> cu = self . db . cursor ( ) <EOL> schema . resetTable ( cu , '<STR_LIT>' ) <EOL> self . db . bulkload ( "<STR_LIT>" , ( ( cu . binary ( x ) , ) for x in sha1iter ) , <EOL> [ "<STR_LIT>" ] , start_transaction = False ) <EOL> cu . execute ( """<STR_LIT>""" ) <EOL> return [ x [ <NUM_LIT:0> ] for x in cu ] <EOL> def hasTrove ( self , troveName , troveVersion = None , troveFlavor = None ) : <EOL> self . log ( <NUM_LIT:3> , troveName , troveVersion , troveFlavor ) <EOL> if not troveVersion : <EOL> return self . items . has_key ( troveName ) <EOL> assert ( troveFlavor is not None ) <EOL> troveItemId = self . items . get ( troveName , None ) <EOL> if troveItemId is None : <EOL> return False <EOL> troveVersionId = self . versionTable . get ( troveVersion , None ) <EOL> if troveVersionId is None : <EOL> return False <EOL> troveFlavorId = self . flavors . get ( troveFlavor , None ) <EOL> if troveFlavorId is None : <EOL> return False <EOL> return self . instances . isPresent ( ( troveItemId , troveVersionId , <EOL> troveFlavorId ) ) <EOL> def getTrove ( self , troveName , troveVersion , troveFlavor , withFiles = True , <EOL> hidden = False ) : <EOL> troveIter = self . iterTroves ( ( ( troveName , troveVersion , troveFlavor ) , ) , <EOL> withFiles = withFiles , hidden = hidden ) <EOL> trv = [ x for x in troveIter ] [ <NUM_LIT:0> ] <EOL> if trv is None : <EOL> raise errors . TroveMissing ( troveName , troveVersion ) <EOL> return trv <EOL> def iterTroves ( self , troveInfoList , withFiles = True , <EOL> withFileStreams = False , <EOL> hidden = False , permCheckFilter = None ) : <EOL> self . log ( <NUM_LIT:3> , troveInfoList , "<STR_LIT>" % ( <EOL> withFiles , withFileStreams , hidden ) ) <EOL> cu = self . db . cursor ( ) <EOL> schema . resetTable ( cu , '<STR_LIT>' ) <EOL> schema . resetTable ( cu , '<STR_LIT>' ) <EOL> l = [ ] <EOL> for idx , ( n , v , f ) in enumerate ( troveInfoList ) : <EOL> l . append ( ( idx , n , v . asString ( ) , f . freeze ( ) ) ) <EOL> self . db . bulkload ( "<STR_LIT>" , l , [ "<STR_LIT>" , "<STR_LIT:name>" , "<STR_LIT:version>" , "<STR_LIT>" ] , <EOL> start_transaction = False ) <EOL> del l <EOL> self . db . analyze ( "<STR_LIT>" ) <EOL> args = [ instances . INSTANCE_PRESENT_NORMAL ] <EOL> d = dict ( presence = "<STR_LIT>" ) <EOL> if hidden : <EOL> args . append ( instances . INSTANCE_PRESENT_HIDDEN ) <EOL> d = dict ( presence = "<STR_LIT>" ) <EOL> d . update ( self . db . keywords ) <EOL> cu . execute ( """<STR_LIT>""" % d , args ) <EOL> troveIdList = [ x for x in cu ] <EOL> if not troveIdList : <EOL> for i in xrange ( len ( troveInfoList ) ) : <EOL> yield None <EOL> return <EOL> self . db . bulkload ( "<STR_LIT>" , <EOL> [ x [ <NUM_LIT:0> : <NUM_LIT:2> ] for x in troveIdList ] , <EOL> [ "<STR_LIT>" , "<STR_LIT>" ] , <EOL> start_transaction = False ) <EOL> self . db . analyze ( "<STR_LIT>" ) <EOL> if permCheckFilter : <EOL> permCheckFilter ( cu , "<STR_LIT>" ) <EOL> cu . execute ( "<STR_LIT>" ) <EOL> validIndexes = set ( x [ <NUM_LIT:0> ] for x in cu ) <EOL> troveIdList = [ x for x in troveIdList if x [ <NUM_LIT:0> ] in validIndexes ] <EOL> if self . db . kind == '<STR_LIT>' : <EOL> cu . execute ( "<STR_LIT>" ) <EOL> cu . execute ( "<STR_LIT>" ) <EOL> troveTrovesCursor = self . db . cursor ( ) <EOL> troveTrovesCursor . execute ( """<STR_LIT>""" % self . db . keywords ) <EOL> troveTrovesCursor = util . PushIterator ( troveTrovesCursor ) <EOL> if self . db . kind == '<STR_LIT>' : <EOL> cu . execute ( "<STR_LIT>" ) <EOL> cu . execute ( "<STR_LIT>" ) <EOL> troveFilesCursor = iter ( ( ) ) <EOL> if withFileStreams or withFiles : <EOL> troveFilesCursor = self . db . cursor ( ) <EOL> streamSel = "<STR_LIT>" <EOL> if withFileStreams : <EOL> streamSel = "<STR_LIT>" <EOL> troveFilesCursor . execute ( """<STR_LIT>""" % ( streamSel , ) ) <EOL> troveFilesCursor = util . PushIterator ( troveFilesCursor ) <EOL> troveRedirectsCursor = self . db . cursor ( ) <EOL> troveRedirectsCursor . execute ( """<STR_LIT>""" ) <EOL> troveRedirectsCursor = util . PushIterator ( troveRedirectsCursor ) <EOL> neededIdx = <NUM_LIT:0> <EOL> versionCache = VersionCache ( ) <EOL> flavorCache = FlavorCache ( ) <EOL> while troveIdList : <EOL> ( idx , troveInstanceId , troveType , timeStamps , <EOL> clName , clVersion , clMessage ) = troveIdList . pop ( <NUM_LIT:0> ) <EOL> while neededIdx < idx : <EOL> neededIdx += <NUM_LIT:1> <EOL> yield None <EOL> neededIdx += <NUM_LIT:1> <EOL> singleTroveInfo = troveInfoList [ idx ] <EOL> if clName is not None : <EOL> changeLog = changelog . ChangeLog ( clName , clVersion , clMessage ) <EOL> else : <EOL> changeLog = None <EOL> v = singleTroveInfo [ <NUM_LIT:1> ] <EOL> key = ( v , timeStamps ) <EOL> if versionCache . has_key ( key ) : <EOL> v = versionCache ( key ) <EOL> else : <EOL> v = v . copy ( ) <EOL> v . setTimeStamps ( [ float ( x ) for x in timeStamps . split ( "<STR_LIT::>" ) ] ) <EOL> trv = trove . Trove ( singleTroveInfo [ <NUM_LIT:0> ] , v , <EOL> singleTroveInfo [ <NUM_LIT:2> ] , changeLog , <EOL> type = troveType , <EOL> setVersion = False ) <EOL> try : <EOL> next = troveTrovesCursor . next ( ) <EOL> while next [ <NUM_LIT:0> ] == idx : <EOL> idxA , name , version , flavor , flags , timeStamps = next <EOL> version = versionCache . get ( version , timeStamps ) <EOL> flavor = flavorCache . get ( flavor ) <EOL> byDefault = ( flags & schema . TROVE_TROVES_BYDEFAULT ) != <NUM_LIT:0> <EOL> weakRef = ( flags & schema . TROVE_TROVES_WEAKREF ) != <NUM_LIT:0> <EOL> trv . addTrove ( name , version , flavor , byDefault = byDefault , <EOL> weakRef = weakRef ) <EOL> next = troveTrovesCursor . next ( ) <EOL> troveTrovesCursor . push ( next ) <EOL> except StopIteration : <EOL> pass <EOL> fileContents = { } <EOL> try : <EOL> next = troveFilesCursor . next ( ) <EOL> while next [ <NUM_LIT:0> ] == idx : <EOL> ( idxA , pathId , dirname , basename , versionId , fileId , <EOL> stream ) = next <EOL> fileId = cu . frombinary ( fileId ) <EOL> path = os . path . join ( cu . frombinary ( dirname ) , <EOL> cu . frombinary ( basename ) ) <EOL> version = versions . VersionFromString ( versionId ) <EOL> trv . addFile ( cu . frombinary ( pathId ) , path , version , fileId ) <EOL> if stream is not None : <EOL> fileContents [ fileId ] = stream <EOL> next = troveFilesCursor . next ( ) <EOL> troveFilesCursor . push ( next ) <EOL> except StopIteration : <EOL> pass <EOL> try : <EOL> next = troveRedirectsCursor . next ( ) <EOL> while next [ <NUM_LIT:0> ] == idx : <EOL> idxA , targetName , targetBranch , targetFlavor = next <EOL> targetBranch = versions . VersionFromString ( targetBranch ) <EOL> if targetFlavor is not None : <EOL> targetFlavor = deps . ThawFlavor ( targetFlavor ) <EOL> trv . addRedirect ( targetName , targetBranch , targetFlavor ) <EOL> next = troveRedirectsCursor . next ( ) <EOL> troveRedirectsCursor . push ( next ) <EOL> except StopIteration : <EOL> pass <EOL> self . depTables . get ( cu , trv , troveInstanceId ) <EOL> self . troveInfoTable . getInfo ( cu , trv , troveInstanceId ) <EOL> if withFileStreams : <EOL> yield trv , fileContents <EOL> else : <EOL> yield trv <EOL> while neededIdx < len ( troveInfoList ) : <EOL> neededIdx += <NUM_LIT:1> <EOL> yield None <EOL> def findFileVersion ( self , fileId ) : <EOL> cu = self . db . cursor ( ) <EOL> cu . execute ( "<STR_LIT>" , ( fileId , ) ) <EOL> for ( stream , ) in cu : <EOL> if stream is None : <EOL> continue <EOL> return files . ThawFile ( cu . frombinary ( stream ) , cu . frombinary ( fileId ) ) <EOL> return None <EOL> def iterFilesInTrove ( self , troveName , troveVersion , troveFlavor , <EOL> sortByPath = False , withFiles = False ) : <EOL> troveInstanceId = self . instances . getInstanceId ( <EOL> troveName , troveVersion , troveFlavor ) <EOL> sort = "<STR_LIT>" <EOL> if sortByPath : <EOL> sort = "<STR_LIT>" <EOL> cu = self . db . cursor ( ) <EOL> cu . execute ( """<STR_LIT>""" % sort , troveInstanceId ) <EOL> versionCache = { } <EOL> for ( pathId , dirname , basename , fileId , versionId , stream ) in cu : <EOL> version = versionCache . get ( versionId , None ) <EOL> path = os . path . join ( cu . frombinary ( dirname ) , <EOL> cu . frombinary ( basename ) ) <EOL> if not version : <EOL> version = self . versionTable . getBareId ( versionId ) <EOL> versionCache [ versionId ] = version <EOL> if stream : <EOL> files . ThawFile ( cu . frombinary ( stream ) , cu . frombinary ( fileId ) ) <EOL> if withFiles : <EOL> yield ( cu . frombinary ( pathId ) , path , cu . frombinary ( fileId ) , <EOL> version , cu . frombinary ( stream ) ) <EOL> else : <EOL> yield ( cu . frombinary ( pathId ) , path , cu . frombinary ( fileId ) , <EOL> version ) <EOL> def getFile ( self , pathId , fileId ) : <EOL> cu = self . db . cursor ( ) <EOL> cu . execute ( "<STR_LIT>" , <EOL> cu . binary ( fileId ) ) <EOL> try : <EOL> stream = cu . next ( ) [ <NUM_LIT:0> ] <EOL> except StopIteration : <EOL> raise errors . FileStreamMissing ( fileId ) <EOL> if stream is not None : <EOL> return files . ThawFile ( cu . frombinary ( stream ) , cu . frombinary ( pathId ) ) <EOL> else : <EOL> return None <EOL> def getFiles ( self , l ) : <EOL> if not l : <EOL> return { } <EOL> retr = FileRetriever ( self . db , self . log ) <EOL> d = retr . get ( l ) <EOL> del retr <EOL> return d <EOL> def _cleanCache ( self ) : <EOL> self . versionIdCache = { } <EOL> self . itemIdCache = { } <EOL> def begin ( self , serialize = False ) : <EOL> self . _cleanCache ( ) <EOL> cu = self . db . transaction ( ) <EOL> if serialize : <EOL> schema . lockCommits ( self . db ) <EOL> return cu <EOL> def rollback ( self ) : <EOL> self . _cleanCache ( ) <EOL> return self . db . rollback ( ) <EOL> def _removeTrove ( self , name , version , flavor , markOnly = False ) : <EOL> cu = self . db . cursor ( ) <EOL> cu . execute ( """<STR_LIT>""" , ( name , version . asString ( ) , flavor . freeze ( ) ) ) <EOL> try : <EOL> instanceId , itemId , versionId , flavorId , troveType = cu . next ( ) <EOL> except StopIteration : <EOL> raise errors . TroveMissing ( name , version ) <EOL> if troveType == trove . TROVE_TYPE_REMOVED : <EOL> return [ ] <EOL> assert ( troveType == trove . TROVE_TYPE_NORMAL or <EOL> troveType == trove . TROVE_TYPE_REDIRECT ) <EOL> cu . execute ( """<STR_LIT>""" , instanceId = instanceId ) <EOL> depsToRemove = [ x [ <NUM_LIT:0> ] for x in cu ] <EOL> cu . execute ( "<STR_LIT>" , instanceId ) <EOL> cu . execute ( "<STR_LIT>" , instanceId ) <EOL> if depsToRemove : <EOL> cu . execute ( "<STR_LIT>" <EOL> % "<STR_LIT:U+002C>" . join ( [ "<STR_LIT>" % x for x in depsToRemove ] ) ) <EOL> cu . execute ( "<STR_LIT>" , instanceId ) <EOL> cu . execute ( """<STR_LIT>""" , ( instanceId , instanceId ) ) <EOL> filePathIdsToRemove = [ x [ <NUM_LIT:0> ] for x in cu ] <EOL> cu . execute ( """<STR_LIT>""" , ( instanceId , instanceId ) ) <EOL> r = cu . fetchall ( ) <EOL> candidateSha1sToRemove = [ cu . frombinary ( x [ <NUM_LIT:1> ] ) for x in r <EOL> if x [ <NUM_LIT:1> ] is not None ] <EOL> streamIdsToRemove = [ x [ <NUM_LIT:0> ] for x in r ] <EOL> cu . execute ( "<STR_LIT>" , instanceId ) <EOL> if streamIdsToRemove : <EOL> cu . execute ( "<STR_LIT>" <EOL> % "<STR_LIT:U+002C>" . join ( "<STR_LIT>" % x for x in streamIdsToRemove ) ) <EOL> if filePathIdsToRemove : <EOL> cu . execute ( "<STR_LIT>" <EOL> % "<STR_LIT:U+002C>" . join ( "<STR_LIT>" % x for x in filePathIdsToRemove ) ) <EOL> cu . execute ( """<STR_LIT>""" ) <EOL> cu . execute ( """<STR_LIT>""" ) <EOL> filesToRemove = [ ] <EOL> for sha1 in candidateSha1sToRemove : <EOL> cu . execute ( "<STR_LIT>" , cu . binary ( sha1 ) ) <EOL> if cu . next ( ) [ <NUM_LIT:0> ] == <NUM_LIT:0> : <EOL> filesToRemove . append ( sha1 ) <EOL> schema . resetTable ( cu , '<STR_LIT>' ) <EOL> cu . execute ( "<STR_LIT>" <EOL> "<STR_LIT>" , ( itemId , versionId ) ) <EOL> nodeId , branchId = cu . next ( ) <EOL> cu . execute ( "<STR_LIT>" <EOL> "<STR_LIT>" , ( itemId , versionId , flavorId , branchId ) , <EOL> start_transaction = False ) <EOL> cu . execute ( """<STR_LIT>""" , ( instanceId , instances . INSTANCE_PRESENT_MISSING , instanceId ) , <EOL> start_transaction = False ) <EOL> cu . execute ( """<STR_LIT>""" , instanceId , start_transaction = False ) <EOL> self . db . analyze ( "<STR_LIT>" ) <EOL> self . ri . deleteInstanceIds ( "<STR_LIT>" ) <EOL> cu . execute ( "<STR_LIT>" , instanceId ) <EOL> cu . execute ( "<STR_LIT>" , instanceId ) <EOL> cu . execute ( "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> if markOnly : <EOL> cu . execute ( "<STR_LIT>" , <EOL> trove . TROVE_TYPE_REMOVED , instanceId ) <EOL> self . latest . update ( cu , itemId , branchId , flavorId ) <EOL> else : <EOL> self . ri . deleteInstanceId ( instanceId ) <EOL> cu . execute ( "<STR_LIT>" , instanceId ) <EOL> schema . resetTable ( cu , '<STR_LIT>' ) <EOL> cu . execute ( """<STR_LIT>""" , start_transaction = False ) <EOL> self . db . analyze ( "<STR_LIT>" ) <EOL> cu . execute ( """<STR_LIT>""" ) <EOL> cu . execute ( """<STR_LIT>""" ) <EOL> self . latest . update ( cu , itemId , branchId , flavorId ) <EOL> cu . execute ( """<STR_LIT>""" ) <EOL> cu . execute ( """<STR_LIT>""" ) <EOL> schema . resetTable ( cu , '<STR_LIT>' ) <EOL> cu . execute ( """<STR_LIT>""" , start_transaction = False ) <EOL> self . db . analyze ( "<STR_LIT>" ) <EOL> cu . execute ( "<STR_LIT>" ) <EOL> cu . execute ( """<STR_LIT>""" ) <EOL> schema . resetTable ( cu , '<STR_LIT>' ) <EOL> cu . execute ( """<STR_LIT>""" , start_transaction = False ) <EOL> self . db . analyze ( "<STR_LIT>" ) <EOL> cu . execute ( "<STR_LIT>" ) <EOL> cu . execute ( """<STR_LIT>""" ) <EOL> cu . execute ( """<STR_LIT>""" ) <EOL> return filesToRemove <EOL> def markTroveRemoved ( self , name , version , flavor ) : <EOL> self . log ( <NUM_LIT:2> , name , version , flavor ) <EOL> return self . _removeTrove ( name , version , flavor , markOnly = True ) <EOL> def getParentTroves ( self , troveList ) : <EOL> cu = self . db . cursor ( ) <EOL> schema . resetTable ( cu , "<STR_LIT>" ) <EOL> schema . resetTable ( cu , "<STR_LIT>" ) <EOL> for ( n , v , f ) in troveList : <EOL> cu . execute ( "<STR_LIT>" , <EOL> ( n , v , f ) , start_transaction = False ) <EOL> self . db . analyze ( "<STR_LIT>" ) <EOL> cu . execute ( """<STR_LIT>""" , start_transaction = False ) <EOL> self . db . analyze ( "<STR_LIT>" ) <EOL> cu . execute ( """<STR_LIT>""" ) <EOL> return cu . fetchall ( ) <EOL> def commit ( self ) : <EOL> if self . needsCleanup : <EOL> assert ( <NUM_LIT:0> ) <EOL> self . instances . removeUnused ( ) <EOL> self . items . removeUnused ( ) <EOL> self . needsCleanup = False <EOL> if self . versionOps . needsCleanup : <EOL> assert ( <NUM_LIT:0> ) <EOL> self . versionTable . removeUnused ( ) <EOL> self . branchTable . removeUnused ( ) <EOL> self . versionOps . labelMap . removeUnused ( ) <EOL> self . versionOps . needsCleanup = False <EOL> self . db . commit ( ) <EOL> self . _cleanCache ( ) <EOL> class FileRetriever : <EOL> def __init__ ( self , db , log = None ) : <EOL> self . db = db <EOL> self . cu = db . cursor ( ) <EOL> schema . resetTable ( self . cu , '<STR_LIT>' ) <EOL> self . log = log or tracelog . getLog ( None ) <EOL> def get ( self , l ) : <EOL> lookup = range ( len ( l ) ) <EOL> insertL = [ ] <EOL> for itemId , tup in enumerate ( l ) : <EOL> ( pathId , fileId ) = tup [ : <NUM_LIT:2> ] <EOL> insertL . append ( ( itemId , self . cu . binary ( fileId ) ) ) <EOL> lookup [ itemId ] = ( pathId , fileId ) <EOL> self . db . bulkload ( "<STR_LIT>" , insertL , [ "<STR_LIT>" , "<STR_LIT>" ] , <EOL> start_transaction = False ) <EOL> self . db . analyze ( "<STR_LIT>" ) <EOL> self . cu . execute ( "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> d = { } <EOL> for itemId , stream in self . cu : <EOL> pathId , fileId = lookup [ itemId ] <EOL> if stream is not None : <EOL> f = files . ThawFile ( self . cu . frombinary ( stream ) , pathId ) <EOL> else : <EOL> f = None <EOL> d [ ( pathId , fileId ) ] = f <EOL> schema . resetTable ( self . cu , "<STR_LIT>" ) <EOL> return d </s>
