<s> """<STR_LIT>""" <EOL> import os <EOL> import re <EOL> import logging <EOL> import shutil <EOL> import tempfile <EOL> import errno <EOL> import time <EOL> import stat <EOL> import grp <EOL> import pwd <EOL> from ganeti import errors <EOL> from ganeti import constants <EOL> from ganeti import pathutils <EOL> from ganeti . utils import filelock <EOL> _LOST_AND_FOUND = "<STR_LIT>" <EOL> KP_NEVER = <NUM_LIT:0> <EOL> KP_ALWAYS = <NUM_LIT:1> <EOL> KP_IF_EXISTS = <NUM_LIT:2> <EOL> KEEP_PERMS_VALUES = [ <EOL> KP_NEVER , <EOL> KP_ALWAYS , <EOL> KP_IF_EXISTS , <EOL> ] <EOL> def ErrnoOrStr ( err ) : <EOL> """<STR_LIT>""" <EOL> if hasattr ( err , "<STR_LIT>" ) : <EOL> detail = errno . errorcode [ err . errno ] <EOL> else : <EOL> detail = str ( err ) <EOL> return detail <EOL> class FileStatHelper ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self ) : <EOL> """<STR_LIT>""" <EOL> self . st = None <EOL> def __call__ ( self , fh ) : <EOL> """<STR_LIT>""" <EOL> self . st = os . fstat ( fh . fileno ( ) ) <EOL> def ReadFile ( file_name , size = - <NUM_LIT:1> , preread = None ) : <EOL> """<STR_LIT>""" <EOL> f = open ( file_name , "<STR_LIT:r>" ) <EOL> try : <EOL> if preread : <EOL> preread ( f ) <EOL> return f . read ( size ) <EOL> finally : <EOL> f . close ( ) <EOL> def WriteFile ( file_name , fn = None , data = None , <EOL> mode = None , uid = - <NUM_LIT:1> , gid = - <NUM_LIT:1> , <EOL> atime = None , mtime = None , close = True , <EOL> dry_run = False , backup = False , <EOL> prewrite = None , postwrite = None , keep_perms = KP_NEVER ) : <EOL> """<STR_LIT>""" <EOL> if not os . path . isabs ( file_name ) : <EOL> raise errors . ProgrammerError ( "<STR_LIT>" <EOL> "<STR_LIT>" % file_name ) <EOL> if [ fn , data ] . count ( None ) != <NUM_LIT:1> : <EOL> raise errors . ProgrammerError ( "<STR_LIT>" ) <EOL> if [ atime , mtime ] . count ( None ) == <NUM_LIT:1> : <EOL> raise errors . ProgrammerError ( "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> if not keep_perms in KEEP_PERMS_VALUES : <EOL> raise errors . ProgrammerError ( "<STR_LIT>" % <EOL> keep_perms ) <EOL> if keep_perms == KP_ALWAYS and ( uid != - <NUM_LIT:1> or gid != - <NUM_LIT:1> or mode is not None ) : <EOL> raise errors . ProgrammerError ( "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> if backup and not dry_run and os . path . isfile ( file_name ) : <EOL> CreateBackup ( file_name ) <EOL> if keep_perms == KP_ALWAYS or keep_perms == KP_IF_EXISTS : <EOL> try : <EOL> file_stat = os . stat ( file_name ) <EOL> mode = stat . S_IMODE ( file_stat . st_mode ) <EOL> uid = file_stat . st_uid <EOL> gid = file_stat . st_gid <EOL> except OSError : <EOL> if keep_perms == KP_ALWAYS : <EOL> raise <EOL> do_remove = True <EOL> result = None <EOL> ( dir_name , base_name ) = os . path . split ( file_name ) <EOL> ( fd , new_name ) = tempfile . mkstemp ( suffix = "<STR_LIT>" , prefix = base_name , <EOL> dir = dir_name ) <EOL> try : <EOL> try : <EOL> if uid != - <NUM_LIT:1> or gid != - <NUM_LIT:1> : <EOL> os . chown ( new_name , uid , gid ) <EOL> if mode : <EOL> os . chmod ( new_name , mode ) <EOL> if callable ( prewrite ) : <EOL> prewrite ( fd ) <EOL> if data is not None : <EOL> if isinstance ( data , unicode ) : <EOL> data = data . encode ( ) <EOL> assert isinstance ( data , str ) <EOL> to_write = len ( data ) <EOL> offset = <NUM_LIT:0> <EOL> while offset < to_write : <EOL> written = os . write ( fd , buffer ( data , offset ) ) <EOL> assert written >= <NUM_LIT:0> <EOL> assert written <= to_write - offset <EOL> offset += written <EOL> assert offset == to_write <EOL> else : <EOL> fn ( fd ) <EOL> if callable ( postwrite ) : <EOL> postwrite ( fd ) <EOL> os . fsync ( fd ) <EOL> if atime is not None and mtime is not None : <EOL> os . utime ( new_name , ( atime , mtime ) ) <EOL> finally : <EOL> if close : <EOL> os . close ( fd ) <EOL> else : <EOL> result = fd <EOL> if not dry_run : <EOL> os . rename ( new_name , file_name ) <EOL> do_remove = False <EOL> finally : <EOL> if do_remove : <EOL> RemoveFile ( new_name ) <EOL> return result <EOL> def GetFileID ( path = None , fd = None ) : <EOL> """<STR_LIT>""" <EOL> if [ path , fd ] . count ( None ) != <NUM_LIT:1> : <EOL> raise errors . ProgrammerError ( "<STR_LIT>" ) <EOL> if fd is None : <EOL> st = os . stat ( path ) <EOL> else : <EOL> st = os . fstat ( fd ) <EOL> return ( st . st_dev , st . st_ino , st . st_mtime ) <EOL> def VerifyFileID ( fi_disk , fi_ours ) : <EOL> """<STR_LIT>""" <EOL> ( d1 , i1 , m1 ) = fi_disk <EOL> ( d2 , i2 , m2 ) = fi_ours <EOL> return ( d1 , i1 ) == ( d2 , i2 ) and m1 <= m2 <EOL> def SafeWriteFile ( file_name , file_id , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> fd = os . open ( file_name , os . O_RDONLY | os . O_CREAT ) <EOL> try : <EOL> filelock . LockFile ( fd ) <EOL> if file_id is not None : <EOL> disk_id = GetFileID ( fd = fd ) <EOL> if not VerifyFileID ( disk_id , file_id ) : <EOL> raise errors . LockError ( "<STR_LIT>" <EOL> "<STR_LIT>" % file_name ) <EOL> return WriteFile ( file_name , ** kwargs ) <EOL> finally : <EOL> os . close ( fd ) <EOL> def ReadOneLineFile ( file_name , strict = False ) : <EOL> """<STR_LIT>""" <EOL> file_lines = ReadFile ( file_name ) . splitlines ( ) <EOL> full_lines = filter ( bool , file_lines ) <EOL> if not file_lines or not full_lines : <EOL> raise errors . GenericError ( "<STR_LIT>" % file_name ) <EOL> elif strict and len ( full_lines ) > <NUM_LIT:1> : <EOL> raise errors . GenericError ( "<STR_LIT>" % <EOL> file_name ) <EOL> return full_lines [ <NUM_LIT:0> ] <EOL> def RemoveFile ( filename ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> os . unlink ( filename ) <EOL> except OSError , err : <EOL> if err . errno not in ( errno . ENOENT , errno . EISDIR ) : <EOL> raise <EOL> def RemoveDir ( dirname ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> os . rmdir ( dirname ) <EOL> except OSError , err : <EOL> if err . errno != errno . ENOENT : <EOL> raise <EOL> def RenameFile ( old , new , mkdir = False , mkdir_mode = <NUM_LIT:0> <NUM_LIT> , dir_uid = None , <EOL> dir_gid = None ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> return os . rename ( old , new ) <EOL> except OSError , err : <EOL> if mkdir and err . errno == errno . ENOENT : <EOL> dir_path = os . path . dirname ( new ) <EOL> MakeDirWithPerm ( dir_path , mkdir_mode , dir_uid , dir_gid ) <EOL> return os . rename ( old , new ) <EOL> raise <EOL> def EnforcePermission ( path , mode , uid = None , gid = None , must_exist = True , <EOL> _chmod_fn = os . chmod , _chown_fn = os . chown , _stat_fn = os . stat ) : <EOL> """<STR_LIT>""" <EOL> logging . debug ( "<STR_LIT>" , path ) <EOL> if uid is None : <EOL> uid = - <NUM_LIT:1> <EOL> if gid is None : <EOL> gid = - <NUM_LIT:1> <EOL> try : <EOL> st = _stat_fn ( path ) <EOL> fmode = stat . S_IMODE ( st [ stat . ST_MODE ] ) <EOL> if fmode != mode : <EOL> logging . debug ( "<STR_LIT>" , path , fmode , mode ) <EOL> _chmod_fn ( path , mode ) <EOL> if max ( uid , gid ) > - <NUM_LIT:1> : <EOL> fuid = st [ stat . ST_UID ] <EOL> fgid = st [ stat . ST_GID ] <EOL> if fuid != uid or fgid != gid : <EOL> logging . debug ( "<STR_LIT>" <EOL> "<STR_LIT>" , path , fuid , fgid , uid , gid ) <EOL> _chown_fn ( path , uid , gid ) <EOL> except EnvironmentError , err : <EOL> if err . errno == errno . ENOENT : <EOL> if must_exist : <EOL> raise errors . GenericError ( "<STR_LIT>" % path ) <EOL> else : <EOL> raise errors . GenericError ( "<STR_LIT>" % <EOL> ( path , err ) ) <EOL> def MakeDirWithPerm ( path , mode , uid , gid , _lstat_fn = os . lstat , <EOL> _mkdir_fn = os . mkdir , _perm_fn = EnforcePermission ) : <EOL> """<STR_LIT>""" <EOL> logging . debug ( "<STR_LIT>" , path ) <EOL> try : <EOL> st = _lstat_fn ( path ) <EOL> except EnvironmentError , err : <EOL> if err . errno != errno . ENOENT : <EOL> raise errors . GenericError ( "<STR_LIT>" % ( path , err ) ) <EOL> _mkdir_fn ( path ) <EOL> else : <EOL> if not stat . S_ISDIR ( st [ stat . ST_MODE ] ) : <EOL> raise errors . GenericError ( ( "<STR_LIT>" <EOL> "<STR_LIT>" ) % path ) <EOL> _perm_fn ( path , mode , uid = uid , gid = gid ) <EOL> def Makedirs ( path , mode = <NUM_LIT:0> <NUM_LIT> ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> os . makedirs ( path , mode ) <EOL> except OSError , err : <EOL> if err . errno != errno . EEXIST or not os . path . exists ( path ) : <EOL> raise <EOL> def TimestampForFilename ( ) : <EOL> """<STR_LIT>""" <EOL> return time . strftime ( "<STR_LIT>" ) <EOL> def CreateBackup ( file_name ) : <EOL> """<STR_LIT>""" <EOL> if not os . path . isfile ( file_name ) : <EOL> raise errors . ProgrammerError ( "<STR_LIT>" % <EOL> file_name ) <EOL> prefix = ( "<STR_LIT>" % <EOL> ( os . path . basename ( file_name ) , TimestampForFilename ( ) ) ) <EOL> dir_name = os . path . dirname ( file_name ) <EOL> fsrc = open ( file_name , "<STR_LIT:rb>" ) <EOL> try : <EOL> ( fd , backup_name ) = tempfile . mkstemp ( prefix = prefix , dir = dir_name ) <EOL> fdst = os . fdopen ( fd , "<STR_LIT:wb>" ) <EOL> try : <EOL> logging . debug ( "<STR_LIT>" , file_name , backup_name ) <EOL> shutil . copyfileobj ( fsrc , fdst ) <EOL> finally : <EOL> fdst . close ( ) <EOL> finally : <EOL> fsrc . close ( ) <EOL> return backup_name <EOL> def ListVisibleFiles ( path , _is_mountpoint = os . path . ismount ) : <EOL> """<STR_LIT>""" <EOL> if not IsNormAbsPath ( path ) : <EOL> raise errors . ProgrammerError ( "<STR_LIT>" <EOL> "<STR_LIT>" % path ) <EOL> mountpoint = _is_mountpoint ( path ) <EOL> def fn ( name ) : <EOL> """<STR_LIT>""" <EOL> return not ( name . startswith ( "<STR_LIT:.>" ) or <EOL> ( mountpoint and name == _LOST_AND_FOUND and <EOL> os . path . isdir ( os . path . join ( path , name ) ) ) ) <EOL> return filter ( fn , os . listdir ( path ) ) <EOL> def EnsureDirs ( dirs ) : <EOL> """<STR_LIT>""" <EOL> for dir_name , dir_mode in dirs : <EOL> try : <EOL> os . mkdir ( dir_name , dir_mode ) <EOL> except EnvironmentError , err : <EOL> if err . errno != errno . EEXIST : <EOL> raise errors . GenericError ( "<STR_LIT>" <EOL> "<STR_LIT>" % ( dir_name , err ) ) <EOL> try : <EOL> os . chmod ( dir_name , dir_mode ) <EOL> except EnvironmentError , err : <EOL> raise errors . GenericError ( "<STR_LIT>" <EOL> "<STR_LIT>" % ( dir_name , err ) ) <EOL> if not os . path . isdir ( dir_name ) : <EOL> raise errors . GenericError ( "<STR_LIT>" % dir_name ) <EOL> def FindFile ( name , search_path , test = os . path . exists ) : <EOL> """<STR_LIT>""" <EOL> if constants . EXT_PLUGIN_MASK . match ( name ) is None : <EOL> logging . critical ( "<STR_LIT>" , <EOL> name ) <EOL> return None <EOL> for dir_name in search_path : <EOL> item_name = os . path . sep . join ( [ dir_name , name ] ) <EOL> if test ( item_name ) and os . path . basename ( item_name ) == name : <EOL> return item_name <EOL> return None <EOL> def IsNormAbsPath ( path ) : <EOL> """<STR_LIT>""" <EOL> return os . path . normpath ( path ) == path and os . path . isabs ( path ) <EOL> def IsBelowDir ( root , other_path ) : <EOL> """<STR_LIT>""" <EOL> if not ( os . path . isabs ( root ) and os . path . isabs ( other_path ) ) : <EOL> raise ValueError ( "<STR_LIT>" % <EOL> ( root , other_path ) ) <EOL> norm_other = os . path . normpath ( other_path ) <EOL> if norm_other == os . sep : <EOL> return False <EOL> norm_root = os . path . normpath ( root ) <EOL> if norm_root == os . sep : <EOL> prepared_root = norm_root <EOL> else : <EOL> prepared_root = "<STR_LIT>" % ( norm_root , os . sep ) <EOL> return os . path . commonprefix ( [ prepared_root , norm_other ] ) == prepared_root <EOL> URL_RE = re . compile ( r'<STR_LIT>' ) <EOL> def IsUrl ( path ) : <EOL> """<STR_LIT>""" <EOL> return URL_RE . match ( path ) <EOL> def PathJoin ( * args ) : <EOL> """<STR_LIT>""" <EOL> if len ( args ) <= <NUM_LIT:1> : <EOL> raise errors . ProgrammerError ( "<STR_LIT>" ) <EOL> root = args [ <NUM_LIT:0> ] <EOL> if not IsNormAbsPath ( root ) : <EOL> raise ValueError ( "<STR_LIT>" % str ( args [ <NUM_LIT:0> ] ) ) <EOL> result = os . path . join ( * args ) <EOL> if not IsNormAbsPath ( result ) : <EOL> raise ValueError ( "<STR_LIT>" % str ( args ) ) <EOL> if not IsBelowDir ( root , result ) : <EOL> raise ValueError ( "<STR_LIT>" <EOL> "<STR_LIT>" % ( result , root ) ) <EOL> return result <EOL> def TailFile ( fname , lines = <NUM_LIT:20> ) : <EOL> """<STR_LIT>""" <EOL> fd = open ( fname , "<STR_LIT:r>" ) <EOL> try : <EOL> fd . seek ( <NUM_LIT:0> , <NUM_LIT:2> ) <EOL> pos = fd . tell ( ) <EOL> pos = max ( <NUM_LIT:0> , pos - <NUM_LIT> ) <EOL> fd . seek ( pos , <NUM_LIT:0> ) <EOL> raw_data = fd . read ( ) <EOL> finally : <EOL> fd . close ( ) <EOL> rows = raw_data . splitlines ( ) <EOL> return rows [ - lines : ] <EOL> def BytesToMebibyte ( value ) : <EOL> """<STR_LIT>""" <EOL> return int ( round ( value / ( <NUM_LIT> * <NUM_LIT> ) , <NUM_LIT:0> ) ) <EOL> def CalculateDirectorySize ( path ) : <EOL> """<STR_LIT>""" <EOL> size = <NUM_LIT:0> <EOL> for ( curpath , _ , files ) in os . walk ( path ) : <EOL> for filename in files : <EOL> st = os . lstat ( PathJoin ( curpath , filename ) ) <EOL> size += st . st_size <EOL> return BytesToMebibyte ( size ) <EOL> def GetFilesystemStats ( path ) : <EOL> """<STR_LIT>""" <EOL> st = os . statvfs ( path ) <EOL> fsize = BytesToMebibyte ( st . f_bavail * st . f_frsize ) <EOL> tsize = BytesToMebibyte ( st . f_blocks * st . f_frsize ) <EOL> return ( tsize , fsize ) <EOL> def ReadPidFile ( pidfile ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> raw_data = ReadOneLineFile ( pidfile ) <EOL> except EnvironmentError , err : <EOL> if err . errno != errno . ENOENT : <EOL> logging . exception ( "<STR_LIT>" ) <EOL> return <NUM_LIT:0> <EOL> return _ParsePidFileContents ( raw_data ) <EOL> def _ParsePidFileContents ( data ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> pid = int ( data ) <EOL> except ( TypeError , ValueError ) : <EOL> logging . info ( "<STR_LIT>" , exc_info = True ) <EOL> return <NUM_LIT:0> <EOL> else : <EOL> return pid <EOL> def ReadLockedPidFile ( path ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> fd = os . open ( path , os . O_RDONLY ) <EOL> except EnvironmentError , err : <EOL> if err . errno == errno . ENOENT : <EOL> return None <EOL> raise <EOL> try : <EOL> try : <EOL> filelock . LockFile ( fd ) <EOL> except errors . LockError : <EOL> return int ( os . read ( fd , <NUM_LIT:100> ) ) <EOL> finally : <EOL> os . close ( fd ) <EOL> return None <EOL> def DaemonPidFileName ( name ) : <EOL> """<STR_LIT>""" <EOL> return PathJoin ( pathutils . RUN_DIR , "<STR_LIT>" % name ) <EOL> def WritePidFile ( pidfile ) : <EOL> """<STR_LIT>""" <EOL> fd_pidfile = os . open ( pidfile , os . O_RDWR | os . O_CREAT , <NUM_LIT:0> <NUM_LIT> ) <EOL> try : <EOL> filelock . LockFile ( fd_pidfile ) <EOL> except errors . LockError : <EOL> msg = [ "<STR_LIT>" % pidfile ] <EOL> pid = _ParsePidFileContents ( os . read ( fd_pidfile , <NUM_LIT:100> ) ) <EOL> if pid > <NUM_LIT:0> : <EOL> msg . append ( "<STR_LIT>" % pid ) <EOL> raise errors . PidFileLockError ( "<STR_LIT>" . join ( msg ) ) <EOL> os . write ( fd_pidfile , "<STR_LIT>" % os . getpid ( ) ) <EOL> return fd_pidfile <EOL> def ReadWatcherPauseFile ( filename , now = None , remove_after = <NUM_LIT> ) : <EOL> """<STR_LIT>""" <EOL> if now is None : <EOL> now = time . time ( ) <EOL> try : <EOL> value = ReadFile ( filename ) <EOL> except IOError , err : <EOL> if err . errno != errno . ENOENT : <EOL> raise <EOL> value = None <EOL> if value is not None : <EOL> try : <EOL> value = int ( value ) <EOL> except ValueError : <EOL> logging . warning ( ( "<STR_LIT>" <EOL> "<STR_LIT>" ) , filename ) <EOL> RemoveFile ( filename ) <EOL> value = None <EOL> if value is not None : <EOL> if now > ( value + remove_after ) : <EOL> RemoveFile ( filename ) <EOL> value = None <EOL> elif now > value : <EOL> value = None <EOL> return value <EOL> def NewUUID ( ) : <EOL> """<STR_LIT>""" <EOL> return ReadFile ( constants . RANDOM_UUID_FILE , size = <NUM_LIT> ) . rstrip ( "<STR_LIT:\n>" ) <EOL> class TemporaryFileManager ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self ) : <EOL> self . _files = [ ] <EOL> def __del__ ( self ) : <EOL> self . Cleanup ( ) <EOL> def Add ( self , filename ) : <EOL> """<STR_LIT>""" <EOL> self . _files . append ( filename ) <EOL> def Remove ( self , filename ) : <EOL> """<STR_LIT>""" <EOL> self . _files . remove ( filename ) <EOL> def Cleanup ( self ) : <EOL> """<STR_LIT>""" <EOL> while self . _files : <EOL> RemoveFile ( self . _files . pop ( ) ) <EOL> def IsUserInGroup ( uid , gid ) : <EOL> """<STR_LIT>""" <EOL> user = pwd . getpwuid ( uid ) <EOL> group = grp . getgrgid ( gid ) <EOL> return user . pw_gid == gid or user . pw_name in group . gr_mem <EOL> def CanRead ( username , filename ) : <EOL> """<STR_LIT>""" <EOL> filestats = os . stat ( filename ) <EOL> user = pwd . getpwnam ( username ) <EOL> uid = user . pw_uid <EOL> user_readable = filestats . st_mode & stat . S_IRUSR != <NUM_LIT:0> <EOL> group_readable = filestats . st_mode & stat . S_IRGRP != <NUM_LIT:0> <EOL> return ( ( filestats . st_uid == uid and user_readable ) <EOL> or ( filestats . st_uid != uid and <EOL> IsUserInGroup ( uid , filestats . st_gid ) and group_readable ) ) </s>
