<s> """<STR_LIT>""" <EOL> import base64 <EOL> import errno <EOL> import gzip <EOL> import itertools <EOL> import os <EOL> import tempfile <EOL> from conary . lib import util <EOL> from conary . lib import digestlib <EOL> from conary . lib import sha1helper <EOL> from conary . lib . ext import digest_uncompress <EOL> from conary . repository import errors , filecontents <EOL> _cached_umask = None <EOL> class AbstractDataStore : <EOL> @ staticmethod <EOL> def _fchmod ( fd , mode = <NUM_LIT:0> <NUM_LIT> ) : <EOL> global _cached_umask <EOL> if _cached_umask is None : <EOL> _cached_umask = os . umask ( <NUM_LIT:0> <NUM_LIT> ) <EOL> os . umask ( _cached_umask ) <EOL> util . fchmod ( fd , mode & ~ _cached_umask ) <EOL> @ classmethod <EOL> def _writeFile ( cls , fileObj , outFds , precompressed , computeSha1 ) : <EOL> if precompressed and hasattr ( fileObj , '<STR_LIT>' ) : <EOL> ( fd , start , size ) = fileObj . _fdInfo ( ) <EOL> realHash = digest_uncompress . sha1Copy ( ( fd , start , size ) , outFds ) <EOL> for x in outFds : <EOL> cls . _fchmod ( x ) <EOL> os . close ( x ) <EOL> return realHash <EOL> else : <EOL> for fd in outFds : <EOL> outFileObj = os . fdopen ( fd , "<STR_LIT:w>" ) <EOL> contentSha1 = digestlib . sha1 ( ) <EOL> if precompressed and computeSha1 : <EOL> tee = Tee ( fileObj , outFileObj ) <EOL> uncompObj = gzip . GzipFile ( mode = "<STR_LIT:r>" , fileobj = tee ) <EOL> s = uncompObj . read ( <NUM_LIT> * <NUM_LIT> ) <EOL> while s : <EOL> contentSha1 . update ( s ) <EOL> s = uncompObj . read ( <NUM_LIT> * <NUM_LIT> ) <EOL> uncompObj . close ( ) <EOL> elif precompressed : <EOL> util . copyfileobj ( fileObj , outFileObj ) <EOL> else : <EOL> dest = gzip . GzipFile ( mode = "<STR_LIT:w>" , fileobj = outFileObj ) <EOL> util . copyfileobj ( fileObj , dest , digest = contentSha1 ) <EOL> dest . close ( ) <EOL> cls . _fchmod ( fd ) <EOL> outFileObj . close ( ) <EOL> fileObj . seek ( <NUM_LIT:0> ) <EOL> return contentSha1 . digest ( ) <EOL> def hasFile ( self , hash ) : <EOL> raise NotImplementedError <EOL> def addFile ( self , f , hash , precompressed = False ) : <EOL> raise NotImplementedError <EOL> def addFileReference ( self , hash ) : <EOL> raise NotImplementedError <EOL> def openFile ( self , hash , mode = "<STR_LIT:r>" ) : <EOL> raise NotImplementedError <EOL> def openRawFile ( self , hash ) : <EOL> raise NotImplementedError <EOL> def removeFile ( self , hash ) : <EOL> raise NotImplementedError <EOL> class Tee : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , inf , outf ) : <EOL> self . inf = inf <EOL> self . outf = outf <EOL> def read ( self , * args , ** kw ) : <EOL> buf = self . inf . read ( * args , ** kw ) <EOL> self . outf . write ( buf ) <EOL> return buf <EOL> def seek ( self , * args , ** kw ) : <EOL> self . outf . seek ( * args , ** kw ) <EOL> return self . inf . seek ( * args , ** kw ) <EOL> def tell ( self , * args , ** kw ) : <EOL> return self . inf . tell ( * args , ** kw ) <EOL> class DataStore ( AbstractDataStore ) : <EOL> def hashToPath ( self , hash ) : <EOL> if len ( hash ) < <NUM_LIT> : <EOL> hash = sha1helper . sha1ToString ( hash ) <EOL> if ( len ( hash ) < <NUM_LIT:5> ) : <EOL> raise KeyError , ( "<STR_LIT>" % hash ) <EOL> return os . sep . join ( ( self . top , hash [ <NUM_LIT:0> : <NUM_LIT:2> ] , hash [ <NUM_LIT:2> : <NUM_LIT:4> ] , hash [ <NUM_LIT:4> : ] ) ) <EOL> def hasFile ( self , hash ) : <EOL> path = self . hashToPath ( hash ) <EOL> return os . path . exists ( path ) <EOL> def addFileReference ( self , hash ) : <EOL> return <EOL> def makeDir ( self , path ) : <EOL> d = os . path . dirname ( path ) <EOL> shortPath = d [ : - <NUM_LIT:3> ] <EOL> for _dir in ( shortPath , d ) : <EOL> try : <EOL> os . mkdir ( _dir ) <EOL> except OSError , e : <EOL> if e . errno != errno . EEXIST : <EOL> raise <EOL> def addFile ( self , fileObj , hash , precompressed = False , <EOL> integrityCheck = True ) : <EOL> path = self . hashToPath ( hash ) <EOL> self . makeDir ( path ) <EOL> if os . path . exists ( path ) : return <EOL> tmpFd , tmpName = tempfile . mkstemp ( suffix = "<STR_LIT>" , <EOL> dir = os . path . dirname ( path ) ) <EOL> realHash = self . _writeFile ( fileObj , [ tmpFd ] , precompressed , <EOL> computeSha1 = integrityCheck ) <EOL> if integrityCheck and realHash != hash : <EOL> os . unlink ( tmpName ) <EOL> raise errors . IntegrityError <EOL> os . rename ( tmpName , path ) <EOL> def openFile ( self , hash , mode = "<STR_LIT:r>" ) : <EOL> path = self . hashToPath ( hash ) <EOL> f = open ( path , "<STR_LIT:r>" ) <EOL> gzfile = gzip . GzipFile ( path , mode ) <EOL> return gzfile <EOL> def openRawFile ( self , hash ) : <EOL> path = self . hashToPath ( hash ) <EOL> f = open ( path , "<STR_LIT:r>" ) <EOL> return f <EOL> def removeFile ( self , hash ) : <EOL> path = self . hashToPath ( hash ) <EOL> os . unlink ( path ) <EOL> def __init__ ( self , topPath ) : <EOL> self . top = topPath <EOL> if ( not os . path . isdir ( self . top ) ) : <EOL> raise IOError , ( "<STR_LIT>" % topPath ) <EOL> class ShallowDataStore ( DataStore ) : <EOL> def hashToPath ( self , hash ) : <EOL> if len ( hash ) < <NUM_LIT> : <EOL> hash = sha1helper . sha1ToString ( hash ) <EOL> if ( len ( hash ) < <NUM_LIT:5> ) : <EOL> raise KeyError , ( "<STR_LIT>" % hash ) <EOL> return os . sep . join ( ( self . top , hash [ <NUM_LIT:0> : <NUM_LIT:2> ] , hash [ <NUM_LIT:2> : ] ) ) <EOL> def makeDir ( self , path ) : <EOL> d = os . path . dirname ( path ) <EOL> try : <EOL> os . mkdir ( d ) <EOL> except OSError , e : <EOL> if e . args [ <NUM_LIT:0> ] != errno . EEXIST : <EOL> raise <EOL> class FlatDataStore ( DataStore ) : <EOL> def hashToPath ( self , hash ) : <EOL> assert len ( hash ) == <NUM_LIT:20> <EOL> hash = base64 . urlsafe_b64encode ( hash ) <EOL> return os . sep . join ( ( self . top , hash [ : <NUM_LIT> ] ) ) <EOL> def makeDir ( self , path ) : <EOL> pass <EOL> class OverlayDataStoreSet : <EOL> """<STR_LIT>""" <EOL> def hashToPath ( self , hash ) : <EOL> for store in self . stores : <EOL> if store . hasFile ( hash ) : <EOL> return store . hashToPath ( hash ) <EOL> return False <EOL> def hasFile ( self , hash ) : <EOL> for store in self . stores : <EOL> if store . hasFile ( hash ) : <EOL> return True <EOL> return False <EOL> def addFile ( self , f , hash , precompressed = False ) : <EOL> self . stores [ <NUM_LIT:0> ] . addFile ( f , hash , precompressed = precompressed ) <EOL> def addFileReference ( self , hash ) : <EOL> self . stores [ <NUM_LIT:0> ] . addFileReference ( hash ) <EOL> def openFile ( self , hash , mode = "<STR_LIT:r>" ) : <EOL> for store in self . stores : <EOL> if store . hasFile ( hash ) : <EOL> return store . openFile ( hash , mode = mode ) <EOL> assert ( <NUM_LIT:0> ) <EOL> def openRawFile ( self , hash ) : <EOL> for store in self . stores : <EOL> if store . hasFile ( hash ) : <EOL> return store . openRawFile ( hash ) <EOL> assert ( <NUM_LIT:0> ) <EOL> def removeFile ( self , hash ) : <EOL> assert ( <NUM_LIT:0> ) <EOL> def __init__ ( self , * storeList ) : <EOL> self . stores = storeList <EOL> self . storeIter = itertools . cycle ( self . stores ) <EOL> class DataStoreSet ( AbstractDataStore ) : <EOL> """<STR_LIT>""" <EOL> def hashToPath ( self , hash ) : <EOL> store = self . storeIter . next ( ) <EOL> return store . hashToPath ( hash ) <EOL> def hasFile ( self , hash ) : <EOL> store = self . storeIter . next ( ) <EOL> return store . hasFile ( hash ) <EOL> def addFile ( self , f , hash , precompressed = False ) : <EOL> tmpFileList = [ ] <EOL> for store in self . stores : <EOL> path = self . hashToPath ( hash ) <EOL> store . makeDir ( path ) <EOL> if os . path . exists ( path ) : return <EOL> tmpFd , tmpPath = tempfile . mkstemp ( suffix = "<STR_LIT>" , <EOL> dir = os . path . dirname ( path ) ) <EOL> tmpFileList . append ( ( tmpFd , tmpPath , path ) ) <EOL> realHash = self . _writeFile ( f , [ x [ <NUM_LIT:0> ] for x in tmpFileList ] , <EOL> precompressed , computeSha1 = True ) <EOL> if realHash != hash : <EOL> for fd , tmpPath , path in tmpFileList : <EOL> os . unlink ( tmpPath ) <EOL> raise errors . IntegrityError <EOL> for fd , tmpPath , path in tmpFileList : <EOL> os . rename ( tmpPath , path ) <EOL> def addFileReference ( self , hash ) : <EOL> for store in self . stores : <EOL> store . addFileReference ( hash ) <EOL> def openFile ( self , hash , mode = "<STR_LIT:r>" ) : <EOL> store = self . storeIter . next ( ) <EOL> return store . openFile ( hash , mode = mode ) <EOL> def openRawFile ( self , hash ) : <EOL> store = self . storeIter . next ( ) <EOL> return store . openRawFile ( hash ) <EOL> def removeFile ( self , hash ) : <EOL> for store in self . stores : <EOL> store . removeFile ( hash ) <EOL> def __init__ ( self , * storeList ) : <EOL> self . stores = storeList <EOL> self . storeIter = itertools . cycle ( self . stores ) <EOL> class DataStoreRepository : <EOL> """<STR_LIT>""" <EOL> def _storeFileFromContents ( self , contents , sha1 , restoreContents , <EOL> precompressed = False ) : <EOL> if restoreContents : <EOL> self . contentsStore . addFile ( contents . get ( ) , sha1 , <EOL> precompressed = precompressed ) <EOL> else : <EOL> self . contentsStore . addFileReference ( sha1 ) <EOL> return <NUM_LIT:1> <EOL> def _removeFileContents ( self , sha1 ) : <EOL> self . contentsStore . removeFile ( sha1 ) <EOL> def _getFileObject ( self , sha1 ) : <EOL> return self . contentsStore . openFile ( sha1 ) <EOL> def _hasFileContents ( self , sha1 ) : <EOL> return self . contentsStore . hasFile ( sha1 ) <EOL> def getFileContents ( self , fileList ) : <EOL> contentList = [ ] <EOL> for item in fileList : <EOL> ( fileId , fileVersion ) = item [ <NUM_LIT:0> : <NUM_LIT:2> ] <EOL> if len ( item ) == <NUM_LIT:3> : <EOL> fileObj = item [ <NUM_LIT:2> ] <EOL> else : <EOL> fileObj = self . findFileVersion ( fileId ) <EOL> if fileObj and self . _hasFileContents ( fileObj . contents . sha1 ( ) ) : <EOL> cont = filecontents . FromDataStore ( self . contentsStore , <EOL> fileObj . contents . sha1 ( ) ) <EOL> else : <EOL> cont = None <EOL> contentList . append ( cont ) <EOL> return contentList <EOL> def __init__ ( self , dataStore = None ) : <EOL> self . contentsStore = dataStore </s>
