<s> """<STR_LIT>""" <EOL> import sys <EOL> if sys . version_info [ : <NUM_LIT:2> ] != ( <NUM_LIT:2> , <NUM_LIT:6> ) : <EOL> msg = "<STR_LIT>" <EOL> sys . stderr . write ( msg % ( sys . version_info [ <NUM_LIT:0> ] , sys . version_info [ <NUM_LIT:1> ] ) ) <EOL> try : <EOL> import cStringIO <EOL> import errno <EOL> import hashlib <EOL> import logging <EOL> import math <EOL> import os <EOL> import sqlite3 <EOL> import stat <EOL> import time <EOL> import traceback <EOL> except ImportError , e : <EOL> msg = "<STR_LIT>" <EOL> sys . stderr . write ( msg % str ( e ) ) <EOL> sys . exit ( <NUM_LIT:1> ) <EOL> try : <EOL> import fuse <EOL> except ImportError : <EOL> sys . stderr . write ( "<STR_LIT>" + "<STR_LIT>" ) <EOL> sys . exit ( <NUM_LIT:1> ) <EOL> from my_formats import format_size , format_timespan <EOL> from get_memory_usage import get_memory_usage <EOL> def main ( ) : <EOL> """<STR_LIT>""" <EOL> dfs = DedupFS ( ) <EOL> fuse_opts = dfs . parse ( [ '<STR_LIT>' , '<STR_LIT>' ] + sys . argv [ <NUM_LIT:1> : ] ) <EOL> dfs_opts = dfs . cmdline [ <NUM_LIT:0> ] <EOL> if dfs_opts . print_stats : <EOL> dfs . read_only = True <EOL> dfs . fsinit ( silent = True ) <EOL> dfs . report_disk_usage ( ) <EOL> dfs . fsdestroy ( silent = True ) <EOL> elif dfs . fuse_args . mount_expected ( ) and not fuse_opts . mountpoint : <EOL> dfs . parse ( [ '<STR_LIT>' ] ) <EOL> elif fuse_opts . mountpoint or not dfs . fuse_args . mount_expected ( ) : <EOL> dfs . main ( ) <EOL> class DedupFS ( fuse . Fuse ) : <EOL> def __init__ ( self , * args , ** kw ) : <EOL> try : <EOL> fuse . fuse_python_api = ( <NUM_LIT:0> , <NUM_LIT:2> ) <EOL> fuse . Fuse . __init__ ( self , * args , ** kw ) <EOL> self . flags = <NUM_LIT:0> <EOL> self . multithreaded = <NUM_LIT:0> <EOL> self . block_size = <NUM_LIT> * <NUM_LIT> <EOL> self . buffers = { } <EOL> self . bytes_read = <NUM_LIT:0> <EOL> self . bytes_written = <NUM_LIT:0> <EOL> self . cache_gc_last_run = time . time ( ) <EOL> self . cache_requests = <NUM_LIT:0> <EOL> self . cache_timeout = <NUM_LIT> <EOL> self . cached_nodes = { } <EOL> self . calls_log_filter = [ ] <EOL> self . datastore_file = '<STR_LIT>' <EOL> self . fs_mounted_at = time . time ( ) <EOL> self . gc_enabled = True <EOL> self . gc_hook_last_run = time . time ( ) <EOL> self . gc_interval = <NUM_LIT> <EOL> self . link_mode = stat . S_IFLNK | <NUM_LIT:0> <NUM_LIT> <EOL> self . memory_usage = <NUM_LIT:0> <EOL> self . metastore_file = '<STR_LIT>' <EOL> self . opcount = <NUM_LIT:0> <EOL> self . read_only = False <EOL> self . root_mode = stat . S_IFDIR | <NUM_LIT:0> <NUM_LIT> <EOL> self . time_spent_caching_nodes = <NUM_LIT:0> <EOL> self . time_spent_hashing = <NUM_LIT:0> <EOL> self . time_spent_interning = <NUM_LIT:0> <EOL> self . time_spent_querying_tree = <NUM_LIT:0> <EOL> self . time_spent_reading = <NUM_LIT:0> <EOL> self . time_spent_traversing_tree = <NUM_LIT:0> <EOL> self . time_spent_writing = <NUM_LIT:0> <EOL> self . time_spent_writing_blocks = <NUM_LIT:0> <EOL> self . __NODE_KEY_VALUE = <NUM_LIT:0> <EOL> self . __NODE_KEY_LAST_USED = <NUM_LIT:1> <EOL> self . logger = logging . getLogger ( '<STR_LIT>' ) <EOL> self . logger . setLevel ( logging . INFO ) <EOL> self . logger . addHandler ( logging . StreamHandler ( sys . stderr ) ) <EOL> option_stored_in_db = "<STR_LIT>" <EOL> self . parser . set_conflict_handler ( '<STR_LIT>' ) <EOL> self . parser . add_option ( '<STR_LIT>' , '<STR_LIT>' , action = '<STR_LIT>' , help = "<STR_LIT>" ) <EOL> self . parser . add_option ( '<STR_LIT>' , '<STR_LIT>' , action = '<STR_LIT:count>' , dest = '<STR_LIT>' , default = <NUM_LIT:0> , help = "<STR_LIT>" ) <EOL> self . parser . add_option ( '<STR_LIT>' , dest = '<STR_LIT>' , action = '<STR_LIT:store_true>' , default = False , help = "<STR_LIT>" ) <EOL> self . parser . add_option ( '<STR_LIT>' , dest = '<STR_LIT>' , help = "<STR_LIT>" ) <EOL> self . parser . add_option ( '<STR_LIT>' , dest = '<STR_LIT>' , metavar = '<STR_LIT>' , default = self . metastore_file , help = "<STR_LIT>" ) <EOL> self . parser . add_option ( '<STR_LIT>' , dest = '<STR_LIT>' , metavar = '<STR_LIT>' , default = self . datastore_file , help = "<STR_LIT>" ) <EOL> self . parser . add_option ( '<STR_LIT>' , dest = '<STR_LIT>' , metavar = '<STR_LIT>' , default = self . block_size , type = '<STR_LIT:int>' , help = "<STR_LIT>" + option_stored_in_db ) <EOL> self . parser . add_option ( '<STR_LIT>' , dest = '<STR_LIT>' , action = '<STR_LIT>' , default = True , help = "<STR_LIT>" ) <EOL> self . parser . add_option ( '<STR_LIT>' , dest = '<STR_LIT>' , action = '<STR_LIT>' , default = True , help = "<STR_LIT>" ) <EOL> self . parser . add_option ( '<STR_LIT>' , dest = '<STR_LIT>' , action = '<STR_LIT>' , default = True , help = "<STR_LIT>" ) <EOL> self . parser . add_option ( '<STR_LIT>' , dest = '<STR_LIT>' , action = '<STR_LIT:store_true>' , default = False , help = "<STR_LIT>" ) <EOL> msg = "<STR_LIT>" + option_stored_in_db <EOL> hash_functions = filter ( lambda m : m [ <NUM_LIT:0> ] != '<STR_LIT:_>' and m != '<STR_LIT>' , dir ( hashlib ) ) <EOL> msg %= '<STR_LIT:U+002CU+0020>' . join ( '<STR_LIT>' % fun for fun in hash_functions ) <EOL> self . parser . add_option ( '<STR_LIT>' , dest = '<STR_LIT>' , metavar = '<STR_LIT>' , type = '<STR_LIT>' , choices = hash_functions , default = '<STR_LIT>' , help = msg ) <EOL> def noop ( s ) : return s <EOL> self . compressors = { '<STR_LIT:none>' : ( noop , noop ) } <EOL> compression_methods = [ '<STR_LIT:none>' ] <EOL> for modname in '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' : <EOL> try : <EOL> module = __import__ ( modname ) <EOL> if hasattr ( module , '<STR_LIT>' ) and hasattr ( module , '<STR_LIT>' ) : <EOL> self . compressors [ modname ] = ( module . compress , module . decompress ) <EOL> compression_methods . append ( modname ) <EOL> except ImportError : <EOL> pass <EOL> msg = "<STR_LIT>" + option_stored_in_db <EOL> msg %= '<STR_LIT:U+002CU+0020>' . join ( '<STR_LIT>' % mth for mth in compression_methods [ <NUM_LIT:1> : ] ) <EOL> self . parser . add_option ( '<STR_LIT>' , dest = '<STR_LIT>' , metavar = '<STR_LIT>' , type = '<STR_LIT>' , choices = compression_methods , default = '<STR_LIT:none>' , help = msg ) <EOL> try : <EOL> for p in '<STR_LIT>' , '<STR_LIT>' : __import__ ( p ) <EOL> self . parser . add_option ( '<STR_LIT>' , action = '<STR_LIT:store_true>' , default = False , help = "<STR_LIT>" ) <EOL> except ImportError : <EOL> self . logger . warning ( "<STR_LIT>" ) <EOL> self . logger . warning ( "<STR_LIT>" ) <EOL> except Exception , e : <EOL> self . __except_to_status ( '<STR_LIT>' , e ) <EOL> sys . exit ( <NUM_LIT:1> ) <EOL> def access ( self , path , flags ) : <EOL> try : <EOL> self . __log_call ( '<STR_LIT>' , '<STR_LIT>' , path , flags ) <EOL> inode = self . __path2keys ( path ) [ <NUM_LIT:1> ] <EOL> if flags != os . F_OK and not self . __access ( inode , flags ) : <EOL> return - errno . EACCES <EOL> return <NUM_LIT:0> <EOL> except Exception , e : <EOL> return self . __except_to_status ( '<STR_LIT>' , e , errno . ENOENT ) <EOL> def chmod ( self , path , mode ) : <EOL> try : <EOL> self . __log_call ( '<STR_LIT>' , '<STR_LIT>' , path , mode ) <EOL> if self . read_only : return - errno . EROFS <EOL> inode = self . __path2keys ( path ) [ <NUM_LIT:1> ] <EOL> self . conn . execute ( '<STR_LIT>' , ( mode , inode ) ) <EOL> self . __gc_hook ( ) <EOL> return <NUM_LIT:0> <EOL> except Exception , e : <EOL> return self . __except_to_status ( '<STR_LIT>' , e , errno . EIO ) <EOL> def chown ( self , path , uid , gid ) : <EOL> try : <EOL> self . __log_call ( '<STR_LIT>' , '<STR_LIT>' , path , uid , gid ) <EOL> if self . read_only : return - errno . EROFS <EOL> inode = self . __path2keys ( path ) [ <NUM_LIT:1> ] <EOL> self . conn . execute ( '<STR_LIT>' , ( uid , gid , inode ) ) <EOL> self . __gc_hook ( ) <EOL> return <NUM_LIT:0> <EOL> except Exception , e : <EOL> return self . __except_to_status ( '<STR_LIT>' , e , errno . EIO ) <EOL> def create ( self , path , flags , mode ) : <EOL> try : <EOL> self . __log_call ( '<STR_LIT>' , '<STR_LIT>' , path , flags , mode ) <EOL> if self . read_only : return - errno . EROFS <EOL> try : <EOL> status = self . open ( path , flags , nested = True ) <EOL> except OSError , e : <EOL> if e . errno != errno . ENOENT : raise <EOL> inode , parent_ino = self . __insert ( path , mode , <NUM_LIT:0> ) <EOL> status = self . open ( path , flags , nested = True , inode = inode ) <EOL> self . __commit_changes ( ) <EOL> self . __gc_hook ( ) <EOL> return status <EOL> except Exception , e : <EOL> self . __rollback_changes ( ) <EOL> return self . __except_to_status ( '<STR_LIT>' , e , errno . EIO ) <EOL> def fsdestroy ( self , silent = False ) : <EOL> try : <EOL> self . __log_call ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> self . __collect_garbage ( ) <EOL> if not silent : <EOL> self . __print_stats ( ) <EOL> if not self . read_only : <EOL> self . logger . info ( "<STR_LIT>" , self . metastore_file ) <EOL> self . __dbmcall ( '<STR_LIT>' ) <EOL> self . conn . commit ( ) <EOL> self . conn . close ( ) <EOL> self . __dbmcall ( '<STR_LIT>' ) <EOL> return <NUM_LIT:0> <EOL> except Exception , e : <EOL> return self . __except_to_status ( '<STR_LIT>' , e , errno . EIO ) <EOL> def fsinit ( self , silent = False ) : <EOL> try : <EOL> options = self . cmdline [ <NUM_LIT:0> ] <EOL> self . block_size = options . block_size <EOL> self . compression_method = options . compression_method <EOL> self . datastore_file = self . __check_data_file ( options . datastore , silent ) <EOL> self . gc_enabled = options . gc_enabled <EOL> self . hash_function = options . hash_function <EOL> self . metastore_file = self . __check_data_file ( options . metastore , silent ) <EOL> self . synchronous = options . synchronous <EOL> self . use_transactions = options . use_transactions <EOL> self . verify_writes = options . verify_writes <EOL> self . __init_logging ( options ) <EOL> self . __log_call ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> self . __setup_database_connections ( silent ) <EOL> if not self . read_only : <EOL> self . __init_metastore ( ) <EOL> self . __get_opts_from_db ( options ) <EOL> if not hasattr ( hashlib , self . hash_function ) : <EOL> self . logger . critical ( "<STR_LIT>" , self . hash_function ) <EOL> sys . exit ( <NUM_LIT:1> ) <EOL> self . hash_function_impl = getattr ( hashlib , self . hash_function ) <EOL> if not self . synchronous and not self . read_only : <EOL> self . logger . warning ( "<STR_LIT>" ) <EOL> self . conn . execute ( '<STR_LIT>' ) <EOL> self . __select_compress_method ( options , silent ) <EOL> return <NUM_LIT:0> <EOL> except Exception , e : <EOL> self . __except_to_status ( '<STR_LIT>' , e , errno . EIO ) <EOL> os . _exit ( <NUM_LIT:1> ) <EOL> def getattr ( self , path ) : <EOL> try : <EOL> self . __log_call ( '<STR_LIT>' , '<STR_LIT>' , path ) <EOL> inode = self . __path2keys ( path ) [ <NUM_LIT:1> ] <EOL> query = '<STR_LIT>' <EOL> attrs = self . conn . execute ( query , ( inode , ) ) . fetchone ( ) <EOL> result = Stat ( st_ino = attrs [ <NUM_LIT:0> ] , <EOL> st_nlink = attrs [ <NUM_LIT:1> ] , <EOL> st_mode = attrs [ <NUM_LIT:2> ] , <EOL> st_uid = attrs [ <NUM_LIT:3> ] , <EOL> st_gid = attrs [ <NUM_LIT:4> ] , <EOL> st_rdev = attrs [ <NUM_LIT:5> ] , <EOL> st_size = attrs [ <NUM_LIT:6> ] , <EOL> st_atime = attrs [ <NUM_LIT:7> ] , <EOL> st_mtime = attrs [ <NUM_LIT:8> ] , <EOL> st_ctime = attrs [ <NUM_LIT:9> ] , <EOL> st_blksize = self . block_size , <EOL> st_blocks = attrs [ <NUM_LIT:6> ] / <NUM_LIT> , <EOL> st_dev = <NUM_LIT:0> ) <EOL> self . logger . debug ( "<STR_LIT>" , path , result ) <EOL> return result <EOL> except Exception , e : <EOL> self . logger . debug ( "<STR_LIT>" , path ) <EOL> return self . __except_to_status ( '<STR_LIT>' , e , errno . ENOENT ) <EOL> def link ( self , target_path , link_path , nested = False ) : <EOL> try : <EOL> self . __log_call ( '<STR_LIT>' , '<STR_LIT>' , nested and '<STR_LIT:U+0020>' or '<STR_LIT>' , target_path , link_path ) <EOL> if self . read_only : return - errno . EROFS <EOL> target_ino = self . __path2keys ( target_path ) [ <NUM_LIT:1> ] <EOL> link_parent , link_name = os . path . split ( link_path ) <EOL> link_parent_id , link_parent_ino = self . __path2keys ( link_parent ) <EOL> string_id = self . __intern ( link_name ) <EOL> self . conn . execute ( '<STR_LIT>' , ( link_parent_id , string_id , target_ino ) ) <EOL> node_id = self . __fetchval ( '<STR_LIT>' ) <EOL> self . conn . execute ( '<STR_LIT>' , ( target_ino , ) ) <EOL> if self . __fetchval ( '<STR_LIT>' , target_ino ) & stat . S_IFDIR : <EOL> self . conn . execute ( '<STR_LIT>' , ( link_parent_ino , ) ) <EOL> self . __cache_set ( link_path , ( node_id , target_ino ) ) <EOL> self . __commit_changes ( nested ) <EOL> self . __gc_hook ( nested ) <EOL> return <NUM_LIT:0> <EOL> except Exception , e : <EOL> self . __rollback_changes ( nested ) <EOL> if nested : raise <EOL> return self . __except_to_status ( '<STR_LIT>' , e , errno . EIO ) <EOL> def mkdir ( self , path , mode ) : <EOL> try : <EOL> self . __log_call ( '<STR_LIT>' , '<STR_LIT>' , path , mode ) <EOL> if self . read_only : return - errno . EROFS <EOL> inode , parent_ino = self . __insert ( path , mode | stat . S_IFDIR , <NUM_LIT> * <NUM_LIT:4> ) <EOL> self . conn . execute ( '<STR_LIT>' , ( parent_ino , ) ) <EOL> self . __commit_changes ( ) <EOL> self . __gc_hook ( ) <EOL> return <NUM_LIT:0> <EOL> except Exception , e : <EOL> self . __rollback_changes ( ) <EOL> return self . __except_to_status ( '<STR_LIT>' , e , errno . EIO ) <EOL> def mknod ( self , path , mode , rdev ) : <EOL> try : <EOL> self . __log_call ( '<STR_LIT>' , '<STR_LIT>' , path , mode ) <EOL> if self . read_only : return - errno . EROFS <EOL> self . __insert ( path , mode , <NUM_LIT:0> , rdev ) <EOL> self . __commit_changes ( ) <EOL> self . __gc_hook ( ) <EOL> return <NUM_LIT:0> <EOL> except Exception , e : <EOL> self . __rollback_changes ( ) <EOL> return self . __except_to_status ( '<STR_LIT>' , e , errno . EIO ) <EOL> def open ( self , path , flags , nested = None , inode = None ) : <EOL> try : <EOL> self . __log_call ( '<STR_LIT>' , '<STR_LIT>' , path , flags ) <EOL> inode = inode or self . __path2keys ( path ) [ <NUM_LIT:1> ] <EOL> access_flags = <NUM_LIT:0> <EOL> if flags & ( os . O_RDONLY | os . O_RDWR ) : access_flags |= os . R_OK <EOL> if flags & ( os . O_WRONLY | os . O_RDWR ) : access_flags |= os . W_OK <EOL> if not self . __access ( inode , access_flags ) : <EOL> return - errno . EACCES <EOL> return <NUM_LIT:0> <EOL> except Exception , e : <EOL> if nested : raise <EOL> return self . __except_to_status ( '<STR_LIT>' , e , errno . ENOENT ) <EOL> def read ( self , path , length , offset ) : <EOL> try : <EOL> self . __log_call ( '<STR_LIT>' , '<STR_LIT>' , path , length , offset ) <EOL> start_time = time . time ( ) <EOL> buf = self . __get_file_buffer ( path ) <EOL> buf . seek ( offset ) <EOL> data = buf . read ( length ) <EOL> self . time_spent_reading += time . time ( ) - start_time <EOL> self . bytes_read += len ( data ) <EOL> return data <EOL> except Exception , e : <EOL> return self . __except_to_status ( '<STR_LIT>' , e , code = errno . EIO ) <EOL> def readdir ( self , path , offset ) : <EOL> try : <EOL> self . __log_call ( '<STR_LIT>' , '<STR_LIT>' , path , offset ) <EOL> node_id , inode = self . __path2keys ( path ) <EOL> yield fuse . Direntry ( '<STR_LIT:.>' , ino = inode ) <EOL> yield fuse . Direntry ( '<STR_LIT:..>' ) <EOL> query = "<STR_LIT>" <EOL> for inode , name in self . conn . execute ( query , ( node_id , ) ) . fetchall ( ) : <EOL> yield fuse . Direntry ( str ( name ) , ino = inode ) <EOL> except Exception , e : <EOL> self . __except_to_status ( '<STR_LIT>' , e ) <EOL> def readlink ( self , path ) : <EOL> try : <EOL> self . __log_call ( '<STR_LIT>' , '<STR_LIT>' , path ) <EOL> inode = self . __path2keys ( path ) [ <NUM_LIT:1> ] <EOL> query = '<STR_LIT>' <EOL> return str ( self . __fetchval ( query , inode ) ) <EOL> except Exception , e : <EOL> return self . __except_to_status ( '<STR_LIT>' , e , errno . ENOENT ) <EOL> def release ( self , path , flags ) : <EOL> try : <EOL> self . __log_call ( '<STR_LIT>' , '<STR_LIT>' , path , flags ) <EOL> if path in self . buffers : <EOL> buf = self . buffers [ path ] <EOL> if buf . dirty : <EOL> start_time = time . time ( ) <EOL> inode = self . __path2keys ( path ) [ <NUM_LIT:1> ] <EOL> apparent_size = len ( buf ) <EOL> try : <EOL> self . __write_blocks ( inode , buf , apparent_size ) <EOL> self . __commit_changes ( ) <EOL> except Exception , e : <EOL> self . __rollback_changes ( ) <EOL> raise <EOL> self . bytes_written += apparent_size <EOL> self . time_spent_writing += time . time ( ) - start_time <EOL> self . __gc_hook ( ) <EOL> buf . close ( ) <EOL> del self . buffers [ path ] <EOL> return <NUM_LIT:0> <EOL> except Exception , e : <EOL> return self . __except_to_status ( '<STR_LIT>' , e , errno . EIO ) <EOL> def rename ( self , old_path , new_path ) : <EOL> try : <EOL> self . __log_call ( '<STR_LIT>' , '<STR_LIT>' , old_path , new_path ) <EOL> if self . read_only : return - errno . EROFS <EOL> try : <EOL> self . __remove ( new_path , check_empty = True ) <EOL> except OSError , e : <EOL> if e . errno != errno . ENOENT : raise <EOL> self . link ( old_path , new_path , nested = True ) <EOL> self . unlink ( old_path , nested = True ) <EOL> self . __commit_changes ( ) <EOL> self . __gc_hook ( ) <EOL> return <NUM_LIT:0> <EOL> except Exception , e : <EOL> self . __rollback_changes ( ) <EOL> return self . __except_to_status ( '<STR_LIT>' , e , errno . ENOENT ) <EOL> def rmdir ( self , path ) : <EOL> try : <EOL> self . __log_call ( '<STR_LIT>' , '<STR_LIT>' , path ) <EOL> if self . read_only : return - errno . EROFS <EOL> self . __remove ( path , check_empty = True ) <EOL> self . __commit_changes ( ) <EOL> return <NUM_LIT:0> <EOL> except Exception , e : <EOL> self . __rollback_changes ( ) <EOL> return self . __except_to_status ( '<STR_LIT>' , e , errno . ENOENT ) <EOL> def statfs ( self ) : <EOL> try : <EOL> self . __log_call ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> host_fs = os . statvfs ( self . metastore_file ) <EOL> return StatVFS ( f_bavail = ( host_fs . f_bsize * host_fs . f_bavail ) / self . block_size , <EOL> f_bfree = ( host_fs . f_frsize * host_fs . f_bfree ) / self . block_size , <EOL> f_blocks = ( host_fs . f_frsize * host_fs . f_blocks ) / self . block_size , <EOL> f_bsize = self . block_size , <EOL> f_favail = <NUM_LIT:0> , <EOL> f_ffree = <NUM_LIT:0> , <EOL> f_files = <NUM_LIT:0> , <EOL> f_flag = <NUM_LIT:0> , <EOL> f_frsize = self . block_size , <EOL> f_namemax = <NUM_LIT> ) <EOL> except Exception , e : <EOL> return self . __except_to_status ( '<STR_LIT>' , e , errno . EIO ) <EOL> def symlink ( self , target_path , link_path ) : <EOL> try : <EOL> self . __log_call ( '<STR_LIT>' , '<STR_LIT>' , link_path , target_path ) <EOL> if self . read_only : return - errno . EROFS <EOL> inode , parent_ino = self . __insert ( link_path , self . link_mode , len ( target_path ) ) <EOL> self . conn . execute ( '<STR_LIT>' , ( inode , sqlite3 . Binary ( target_path ) ) ) <EOL> self . __commit_changes ( ) <EOL> self . __gc_hook ( ) <EOL> return <NUM_LIT:0> <EOL> except Exception , e : <EOL> self . __rollback_changes ( ) <EOL> return self . __except_to_status ( '<STR_LIT>' , e , errno . EIO ) <EOL> def truncate ( self , path , size ) : <EOL> try : <EOL> self . __log_call ( '<STR_LIT>' , '<STR_LIT>' , path , size ) <EOL> if self . read_only : return - errno . EROFS <EOL> inode = self . __path2keys ( path ) [ <NUM_LIT:1> ] <EOL> last_block = size / self . block_size <EOL> self . conn . execute ( '<STR_LIT>' , ( inode , last_block ) ) <EOL> self . conn . execute ( '<STR_LIT>' , ( size , inode ) ) <EOL> self . __gc_hook ( ) <EOL> self . __commit_changes ( ) <EOL> return <NUM_LIT:0> <EOL> except Exception , e : <EOL> self . __rollback_changes ( ) <EOL> return self . __except_to_status ( '<STR_LIT>' , e , errno . ENOENT ) <EOL> def unlink ( self , path , nested = False ) : <EOL> try : <EOL> self . __log_call ( '<STR_LIT>' , '<STR_LIT>' , nested and '<STR_LIT:U+0020>' or '<STR_LIT>' , path ) <EOL> if self . read_only : return - errno . EROFS <EOL> self . __remove ( path ) <EOL> self . __commit_changes ( nested ) <EOL> except Exception , e : <EOL> self . __rollback_changes ( nested ) <EOL> if nested : raise <EOL> return self . __except_to_status ( '<STR_LIT>' , e , errno . ENOENT ) <EOL> def utime ( self , path , times ) : <EOL> try : <EOL> self . __log_call ( '<STR_LIT>' , '<STR_LIT>' , path , * times ) <EOL> if self . read_only : return - errno . EROFS <EOL> inode = self . __path2keys ( path ) [ <NUM_LIT:1> ] <EOL> atime , mtime = times <EOL> self . conn . execute ( '<STR_LIT>' , ( atime , mtime , inode ) ) <EOL> self . __gc_hook ( ) <EOL> return <NUM_LIT:0> <EOL> except Exception , e : <EOL> return self . __except_to_status ( '<STR_LIT>' , e , errno . ENOENT ) <EOL> def utimens ( self , path , ts_acc , ts_mod ) : <EOL> try : <EOL> self . __log_call ( '<STR_LIT>' , '<STR_LIT>' , path , ts_acc . tv_sec , ts_acc . tv_nsec , ts_mod . tv_sec , ts_mod . tv_nsec ) <EOL> if self . read_only : return - errno . EROFS <EOL> inode = self . __path2keys ( path ) [ <NUM_LIT:1> ] <EOL> atime = ts_acc . tv_sec + ( ts_acc . tv_nsec / <NUM_LIT> ) <EOL> mtime = ts_mod . tv_sec + ( ts_mod . tv_nsec / <NUM_LIT> ) <EOL> self . conn . execute ( '<STR_LIT>' , ( atime , mtime , inode ) ) <EOL> self . __gc_hook ( ) <EOL> return <NUM_LIT:0> <EOL> except Exception , e : <EOL> return self . __except_to_status ( '<STR_LIT>' , e , errno . ENOENT ) <EOL> def write ( self , path , data , offset ) : <EOL> try : <EOL> length = len ( data ) <EOL> self . __log_call ( '<STR_LIT>' , '<STR_LIT>' , path , offset , length ) <EOL> start_time = time . time ( ) <EOL> buf = self . __get_file_buffer ( path ) <EOL> buf . seek ( offset ) <EOL> buf . write ( data ) <EOL> self . time_spent_writing += time . time ( ) - start_time <EOL> return length <EOL> except Exception , e : <EOL> return self . __except_to_status ( '<STR_LIT>' , e , errno . EIO ) <EOL> def __init_logging ( self , options ) : <EOL> if options . log_file : <EOL> handler = logging . StreamHandler ( open ( options . log_file , '<STR_LIT:w>' ) ) <EOL> self . logger . addHandler ( handler ) <EOL> if options . verbosity > <NUM_LIT:0> : <EOL> if options . verbosity <= <NUM_LIT:1> : <EOL> self . logger . setLevel ( logging . INFO ) <EOL> elif options . verbosity <= <NUM_LIT:2> : <EOL> self . logger . setLevel ( logging . DEBUG ) <EOL> else : <EOL> self . logger . setLevel ( logging . NOTSET ) <EOL> def __init_metastore ( self ) : <EOL> uid , gid = os . getuid ( ) , os . getgid ( ) <EOL> t = self . __newctime ( ) <EOL> self . conn . executescript ( """<STR_LIT>""" % ( self . root_mode , uid , gid , t , t , t , self . synchronous and <NUM_LIT:1> or <NUM_LIT:0> , <EOL> self . block_size , self . compression_method , self . hash_function ) ) <EOL> def __setup_database_connections ( self , silent ) : <EOL> if not silent : <EOL> self . logger . info ( "<STR_LIT>" , self . metastore_file , self . datastore_file ) <EOL> if not os . path . exists ( self . metastore_file ) : <EOL> self . blocks = self . __open_datastore ( True ) <EOL> else : <EOL> from whichdb import whichdb <EOL> created_by_gdbm = whichdb ( self . metastore_file ) == '<STR_LIT>' <EOL> self . blocks = self . __open_datastore ( created_by_gdbm ) <EOL> self . conn = sqlite3 . connect ( self . metastore_file , isolation_level = None ) <EOL> self . conn . row_factory = sqlite3 . Row <EOL> self . conn . text_factory = str <EOL> self . conn . execute ( '<STR_LIT>' ) <EOL> def __open_datastore ( self , use_gdbm ) : <EOL> mode = self . read_only and '<STR_LIT:r>' or '<STR_LIT:c>' <EOL> if use_gdbm : <EOL> try : <EOL> import gdbm <EOL> mode += self . synchronous and '<STR_LIT:s>' or '<STR_LIT:f>' <EOL> return gdbm . open ( self . datastore_file , mode ) <EOL> except ImportError : <EOL> pass <EOL> import anydbm <EOL> return anydbm . open ( self . datastore_file , mode ) <EOL> def __dbmcall ( self , fun ) : <EOL> if hasattr ( self . blocks , fun ) : <EOL> getattr ( self . blocks , fun ) ( ) <EOL> def __check_data_file ( self , pathname , silent ) : <EOL> pathname = os . path . expanduser ( pathname ) <EOL> if os . access ( pathname , os . F_OK ) : <EOL> if not os . access ( pathname , os . R_OK ) : <EOL> self . logger . critical ( "<STR_LIT>" , pathname ) <EOL> os . _exit ( <NUM_LIT:1> ) <EOL> if not os . access ( pathname , os . W_OK ) : <EOL> if not silent : <EOL> self . logger . warning ( "<STR_LIT>" , pathname ) <EOL> self . read_only = True <EOL> return pathname <EOL> def __log_call ( self , fun , msg , * args ) : <EOL> if self . calls_log_filter == [ ] or fun in self . calls_log_filter : <EOL> self . logger . debug ( msg , * args ) <EOL> def __get_opts_from_db ( self , options ) : <EOL> for name , value in self . conn . execute ( '<STR_LIT>' ) : <EOL> if name == '<STR_LIT>' : <EOL> self . synchronous = int ( value ) != <NUM_LIT:0> <EOL> if not options . synchronous : <EOL> self . synchronous = False <EOL> elif name == '<STR_LIT>' and int ( value ) != self . block_size : <EOL> self . logger . warning ( "<STR_LIT>" , self . block_size , int ( value ) ) <EOL> self . block_size = int ( value ) <EOL> elif name == '<STR_LIT>' and value != self . compression_method : <EOL> if self . compression_method != '<STR_LIT:none>' : <EOL> self . logger . warning ( "<STR_LIT>" , self . compression_method , value ) <EOL> self . compression_method = value <EOL> elif name == '<STR_LIT>' and value != self . hash_function : <EOL> self . logger . warning ( "<STR_LIT>" , self . hash_function , value ) <EOL> self . hash_function = value <EOL> def __select_compress_method ( self , options , silent ) : <EOL> valid_formats = self . compressors . keys ( ) <EOL> selected_format = self . compression_method . lower ( ) <EOL> if selected_format not in valid_formats : <EOL> self . logger . warning ( "<STR_LIT>" , selected_format ) <EOL> selected_format = '<STR_LIT:none>' <EOL> if selected_format != '<STR_LIT:none>' : <EOL> if not silent : <EOL> self . logger . debug ( "<STR_LIT>" , selected_format ) <EOL> if selected_format == '<STR_LIT>' : <EOL> module = __import__ ( '<STR_LIT>' ) <EOL> if hasattr ( module , '<STR_LIT>' ) : <EOL> module . set_block_size ( self . block_size ) <EOL> self . compress , self . decompress = self . compressors [ selected_format ] <EOL> def __write_blocks ( self , inode , buf , apparent_size ) : <EOL> start_time = time . time ( ) <EOL> self . conn . execute ( '<STR_LIT>' , ( inode , ) ) <EOL> storage_size = len ( buf ) <EOL> for block_nr in xrange ( int ( math . ceil ( storage_size / float ( self . block_size ) ) ) ) : <EOL> buf . seek ( self . block_size * block_nr , os . SEEK_SET ) <EOL> new_block = buf . read ( self . block_size ) <EOL> digest = self . __hash ( new_block ) <EOL> encoded_digest = sqlite3 . Binary ( digest ) <EOL> row = self . conn . execute ( '<STR_LIT>' , ( encoded_digest , ) ) . fetchone ( ) <EOL> if row : <EOL> hash_id = row [ <NUM_LIT:0> ] <EOL> existing_block = self . decompress ( self . blocks [ digest ] ) <EOL> if new_block != existing_block : <EOL> dumpfile_collision = '<STR_LIT>' % time . time ( ) <EOL> handle = open ( dumpfile_collision , '<STR_LIT:w>' ) <EOL> handle . write ( '<STR_LIT>' % existing_block ) <EOL> handle . write ( '<STR_LIT>' % new_block ) <EOL> handle . close ( ) <EOL> self . logger . critical ( <EOL> "<STR_LIT>" + "<STR_LIT>" + "<STR_LIT>" + "<STR_LIT>" , <EOL> block_nr , inode , len ( existing_block ) , digest , <EOL> len ( new_block ) , digest , dumpfile_collision ) <EOL> os . _exit ( <NUM_LIT:1> ) <EOL> self . conn . execute ( '<STR_LIT>' , ( inode , hash_id , block_nr ) ) <EOL> else : <EOL> self . blocks [ digest ] = self . compress ( new_block ) <EOL> self . conn . execute ( '<STR_LIT>' , ( encoded_digest , ) ) <EOL> self . conn . execute ( '<STR_LIT>' , ( inode , block_nr ) ) <EOL> self . __verify_write ( new_block , digest , block_nr , inode ) <EOL> block_nr += <NUM_LIT:1> <EOL> self . conn . execute ( '<STR_LIT>' , ( apparent_size , self . __newctime ( ) , inode ) ) <EOL> self . time_spent_writing_blocks += time . time ( ) - start_time <EOL> def __insert ( self , path , mode , size , rdev = <NUM_LIT:0> ) : <EOL> parent , name = os . path . split ( path ) <EOL> parent_id , parent_ino = self . __path2keys ( parent ) <EOL> nlinks = mode & stat . S_IFDIR and <NUM_LIT:2> or <NUM_LIT:1> <EOL> t = self . __newctime ( ) <EOL> uid , gid = self . __getctx ( ) <EOL> self . conn . execute ( '<STR_LIT>' , ( nlinks , mode , uid , gid , rdev , size , t , t , t ) ) <EOL> inode = self . __fetchval ( '<STR_LIT>' ) <EOL> string_id = self . __intern ( name ) <EOL> self . conn . execute ( '<STR_LIT>' , ( parent_id , string_id , inode ) ) <EOL> node_id = self . __fetchval ( '<STR_LIT>' ) <EOL> self . __cache_set ( path , ( node_id , inode ) ) <EOL> return inode , parent_ino <EOL> def __intern ( self , string ) : <EOL> start_time = time . time ( ) <EOL> args = ( sqlite3 . Binary ( string ) , ) <EOL> result = self . conn . execute ( '<STR_LIT>' , args ) . fetchone ( ) <EOL> if not result : <EOL> self . conn . execute ( '<STR_LIT>' , args ) <EOL> result = self . conn . execute ( '<STR_LIT>' ) . fetchone ( ) <EOL> self . time_spent_interning += time . time ( ) - start_time <EOL> return int ( result [ <NUM_LIT:0> ] ) <EOL> def __remove ( self , path , check_empty = False ) : <EOL> node_id , inode = self . __path2keys ( path ) <EOL> query = """<STR_LIT>""" <EOL> if check_empty and self . __fetchval ( query , node_id ) > <NUM_LIT:0> : <EOL> raise OSError , ( errno . ENOTEMPTY , os . strerror ( errno . ENOTEMPTY ) , path ) <EOL> self . __cache_set ( path , None ) <EOL> self . conn . execute ( '<STR_LIT>' , ( node_id , ) ) <EOL> self . conn . execute ( '<STR_LIT>' , ( inode , ) ) <EOL> if self . __fetchval ( '<STR_LIT>' , inode ) & stat . S_IFDIR : <EOL> parent_id , parent_ino = self . __path2keys ( os . path . split ( path ) [ <NUM_LIT:0> ] ) <EOL> self . conn . execute ( '<STR_LIT>' , ( parent_ino , ) ) <EOL> def __verify_write ( self , block , digest , block_nr , inode ) : <EOL> if self . verify_writes : <EOL> saved_value = self . decompress ( self . blocks [ digest ] ) <EOL> if saved_value != block : <EOL> dumpfile_corruption = '<STR_LIT>' % time . time ( ) <EOL> handle = open ( dumpfile_corruption , '<STR_LIT:w>' ) <EOL> handle . write ( '<STR_LIT>' % block ) <EOL> handle . write ( '<STR_LIT>' % saved_value ) <EOL> handle . close ( ) <EOL> self . logger . critical ( <EOL> "<STR_LIT>" + "<STR_LIT>" , <EOL> block_nr , inode , dumpfile_corruption ) <EOL> os . _exit ( <NUM_LIT:1> ) <EOL> def __access ( self , inode , flags ) : <EOL> if self . read_only and flags & os . W_OK : <EOL> return False <EOL> query = '<STR_LIT>' <EOL> attrs = self . conn . execute ( query , ( inode , ) ) . fetchone ( ) <EOL> uid , gid = self . __getctx ( ) <EOL> o = uid == attrs [ '<STR_LIT>' ] <EOL> g = gid == attrs [ '<STR_LIT>' ] and not o <EOL> w = not ( o or g ) <EOL> m = attrs [ '<STR_LIT>' ] <EOL> return ( not ( flags & os . R_OK ) or ( ( o and ( m & <NUM_LIT:0> <NUM_LIT> ) ) or ( g and ( m & <NUM_LIT> <NUM_LIT> ) ) or ( w and ( m & <NUM_LIT> <NUM_LIT:4> ) ) ) ) and ( not ( flags & os . W_OK ) or ( ( o and ( m & <NUM_LIT:0> <NUM_LIT:200> ) ) or ( g and ( m & <NUM_LIT> <NUM_LIT:20> ) ) or ( w and ( m & <NUM_LIT> <NUM_LIT:2> ) ) ) ) and ( not ( flags & os . X_OK ) or ( ( o and ( m & <NUM_LIT:0> <NUM_LIT:100> ) ) or ( g and ( m & <NUM_LIT> <NUM_LIT:10> ) ) or ( w and ( m & <NUM_LIT> <NUM_LIT:1> ) ) ) ) <EOL> def __path2keys ( self , path ) : <EOL> node_id , inode = <NUM_LIT:1> , <NUM_LIT:1> <EOL> if path == '<STR_LIT:/>' : <EOL> return node_id , inode <EOL> start_time = time . time ( ) <EOL> node = self . cached_nodes <EOL> parent_id = node_id <EOL> for segment in self . __split_segments ( path ) : <EOL> if segment in node : <EOL> node = node [ segment ] <EOL> node [ self . __NODE_KEY_LAST_USED ] = start_time <EOL> node_id , inode = node [ self . __NODE_KEY_VALUE ] <EOL> else : <EOL> query_start_time = time . time ( ) <EOL> query = '<STR_LIT>' <EOL> result = self . conn . execute ( query , ( parent_id , sqlite3 . Binary ( segment ) ) ) . fetchone ( ) <EOL> self . time_spent_querying_tree += time . time ( ) - query_start_time <EOL> if result == None : <EOL> self . __cache_check_gc ( ) <EOL> self . time_spent_traversing_tree += time . time ( ) - start_time <EOL> raise OSError , ( errno . ENOENT , os . strerror ( errno . ENOENT ) , path ) <EOL> node_id , inode = result <EOL> new_node = { self . __NODE_KEY_VALUE : ( node_id , inode ) , self . __NODE_KEY_LAST_USED : start_time } <EOL> node [ segment ] = new_node <EOL> node = new_node <EOL> parent_id = node_id <EOL> self . __cache_check_gc ( ) <EOL> self . time_spent_traversing_tree += time . time ( ) - start_time <EOL> return node_id , inode <EOL> def __cache_set ( self , key , value ) : <EOL> segments = self . __split_segments ( key ) <EOL> last_segment = segments . pop ( - <NUM_LIT:1> ) <EOL> node = self . cached_nodes <EOL> time_now = time . time ( ) <EOL> for segment in segments : <EOL> if segment not in node : <EOL> self . time_spent_caching_nodes += time . time ( ) - time_now <EOL> return False <EOL> node = node [ segment ] <EOL> node [ self . __NODE_KEY_LAST_USED ] = time_now <EOL> if not value : <EOL> if last_segment in node : <EOL> del node [ last_segment ] <EOL> elif last_segment not in node : <EOL> node [ last_segment ] = { self . __NODE_KEY_VALUE : value , self . __NODE_KEY_LAST_USED : time_now } <EOL> else : <EOL> node = node [ last_segment ] <EOL> node [ self . __NODE_KEY_VALUE ] = value <EOL> node [ self . __NODE_KEY_LAST_USED ] = time_now <EOL> self . __cache_check_gc ( ) <EOL> self . time_spent_caching_nodes += time . time ( ) - time_now <EOL> return True <EOL> def __cache_check_gc ( self ) : <EOL> self . cache_requests += <NUM_LIT:1> <EOL> if self . cache_requests >= <NUM_LIT> : <EOL> time_now = time . time ( ) <EOL> if time_now - self . cache_gc_last_run >= self . cache_timeout : <EOL> self . __cache_do_gc ( self . cached_nodes , time_now ) <EOL> self . cache_gc_last_run = time_now <EOL> self . cache_requests = <NUM_LIT:0> <EOL> def __cache_do_gc ( self , node , time_now ) : <EOL> for key in node . keys ( ) : <EOL> child = node [ key ] <EOL> if isinstance ( child , dict ) : <EOL> last_used = time_now - child [ self . __NODE_KEY_LAST_USED ] <EOL> if last_used > self . cache_timeout : <EOL> del node [ key ] <EOL> else : <EOL> self . __cache_do_gc ( child , time_now ) <EOL> def __split_segments ( self , key ) : <EOL> return filter ( None , key . split ( '<STR_LIT:/>' ) ) <EOL> def __newctime ( self ) : <EOL> return time . time ( ) <EOL> def __getctx ( self ) : <EOL> c = fuse . FuseGetContext ( ) <EOL> return ( c [ '<STR_LIT>' ] , c [ '<STR_LIT>' ] ) <EOL> def __hash ( self , data ) : <EOL> start_time = time . time ( ) <EOL> context = self . hash_function_impl ( ) <EOL> context . update ( data ) <EOL> digest = context . digest ( ) <EOL> self . time_spent_hashing += time . time ( ) - start_time <EOL> return digest <EOL> def __print_stats ( self ) : <EOL> self . logger . info ( '<STR_LIT:->' * <NUM_LIT> ) <EOL> self . __report_memory_usage ( ) <EOL> self . __report_throughput ( ) <EOL> self . __report_timings ( ) <EOL> def __report_timings ( self ) : <EOL> if self . logger . isEnabledFor ( logging . DEBUG ) : <EOL> timings = [ ( self . time_spent_traversing_tree , '<STR_LIT>' ) , <EOL> ( self . time_spent_caching_nodes , '<STR_LIT>' ) , <EOL> ( self . time_spent_interning , '<STR_LIT>' ) , <EOL> ( self . time_spent_writing_blocks , '<STR_LIT>' ) , <EOL> ( self . time_spent_hashing , '<STR_LIT>' ) , <EOL> ( self . time_spent_querying_tree , '<STR_LIT>' ) ] <EOL> maxdescwidth = max ( [ len ( l ) for t , l in timings ] ) + <NUM_LIT:3> <EOL> timings . sort ( reverse = True ) <EOL> uptime = time . time ( ) - self . fs_mounted_at <EOL> printed_heading = False <EOL> for timespan , description in timings : <EOL> percentage = timespan / ( uptime / <NUM_LIT:100> ) <EOL> if percentage >= <NUM_LIT:1> : <EOL> if not printed_heading : <EOL> self . logger . debug ( "<STR_LIT>" ) <EOL> printed_heading = True <EOL> self . logger . debug ( "<STR_LIT>" % ( maxdescwidth , description + '<STR_LIT::>' , format_timespan ( timespan ) , percentage ) ) <EOL> def report_disk_usage ( self ) : <EOL> disk_usage = self . __fetchval ( '<STR_LIT>' ) * self . __fetchval ( '<STR_LIT>' ) <EOL> disk_usage += os . stat ( self . datastore_file ) . st_size <EOL> apparent_size = self . __fetchval ( '<STR_LIT>' ) <EOL> self . logger . info ( "<STR_LIT>" , <EOL> format_size ( apparent_size ) , format_size ( disk_usage ) , float ( disk_usage ) / ( apparent_size / <NUM_LIT:100> ) ) <EOL> def __report_memory_usage ( self ) : <EOL> memory_usage = get_memory_usage ( ) <EOL> msg = "<STR_LIT>" + format_size ( memory_usage ) <EOL> difference = abs ( memory_usage - self . memory_usage ) <EOL> if self . memory_usage != <NUM_LIT:0> and difference : <EOL> direction = self . memory_usage < memory_usage and '<STR_LIT>' or '<STR_LIT>' <EOL> msg += "<STR_LIT>" % ( direction , format_size ( difference ) ) <EOL> self . logger . info ( msg + '<STR_LIT:.>' ) <EOL> self . memory_usage = memory_usage <EOL> def __report_throughput ( self , nbytes = None , nseconds = None , label = None ) : <EOL> if nbytes == None : <EOL> self . bytes_read , self . time_spent_reading = self . __report_throughput ( self . bytes_read , self . time_spent_reading , "<STR_LIT>" ) <EOL> self . bytes_written , self . time_spent_writing = self . __report_throughput ( self . bytes_written , self . time_spent_writing , "<STR_LIT>" ) <EOL> else : <EOL> if nbytes > <NUM_LIT:0> : <EOL> average = format_size ( nbytes / max ( <NUM_LIT:1> , nseconds ) ) <EOL> self . logger . info ( "<STR_LIT>" , label , average ) <EOL> if nseconds > <NUM_LIT> and nbytes > <NUM_LIT> ** <NUM_LIT:2> : <EOL> return nbytes / <NUM_LIT:2> , nseconds / <NUM_LIT:2> <EOL> return nbytes , nseconds <EOL> def __report_top_blocks ( self ) : <EOL> query = """<STR_LIT>""" <EOL> if self . logger . isEnabledFor ( logging . DEBUG ) : <EOL> printed_header = False <EOL> for row in self . conn . execute ( query ) : <EOL> if not printed_header : <EOL> self . logger . debug ( "<STR_LIT>" ) <EOL> printed_header = True <EOL> msg = "<STR_LIT>" <EOL> preview = row [ '<STR_LIT:value>' ] <EOL> max_length = <NUM_LIT> <EOL> if len ( preview ) < max_length : <EOL> preview = str ( preview ) <EOL> else : <EOL> preview = preview [ <NUM_LIT:0> : max_length ] + '<STR_LIT>' <EOL> nbytes = format_size ( len ( row [ '<STR_LIT:value>' ] ) ) <EOL> self . logger . debug ( msg , row [ '<STR_LIT>' ] , nbytes , row [ '<STR_LIT:count>' ] , preview ) <EOL> def __gc_hook ( self , nested = False ) : <EOL> if not nested : <EOL> self . opcount += <NUM_LIT:1> <EOL> if self . opcount % <NUM_LIT> == <NUM_LIT:0> : <EOL> if time . time ( ) - self . gc_hook_last_run >= self . gc_interval : <EOL> self . __collect_garbage ( ) <EOL> self . __print_stats ( ) <EOL> self . gc_hook_last_run = time . time ( ) <EOL> def __collect_garbage ( self ) : <EOL> if self . gc_enabled and not self . read_only : <EOL> start_time = time . time ( ) <EOL> self . logger . info ( "<STR_LIT>" ) <EOL> self . should_vacuum = False <EOL> for method in self . __collect_strings , self . __collect_inodes , self . __collect_indices , self . __collect_blocks , self . __vacuum_metastore : <EOL> sub_start_time = time . time ( ) <EOL> msg = method ( ) <EOL> if msg : <EOL> elapsed_time = time . time ( ) - sub_start_time <EOL> self . logger . info ( msg , format_timespan ( elapsed_time ) ) <EOL> elapsed_time = time . time ( ) - start_time <EOL> self . logger . info ( "<STR_LIT>" , format_timespan ( elapsed_time ) ) <EOL> def __collect_strings ( self ) : <EOL> count = self . conn . execute ( '<STR_LIT>' ) . rowcount <EOL> if count > <NUM_LIT:0> : <EOL> self . should_vacuum = True <EOL> return "<STR_LIT>" % ( count , count != <NUM_LIT:1> and '<STR_LIT:s>' or '<STR_LIT>' ) <EOL> def __collect_inodes ( self ) : <EOL> count = self . conn . execute ( '<STR_LIT>' ) . rowcount <EOL> if count > <NUM_LIT:0> : <EOL> self . should_vacuum = True <EOL> return "<STR_LIT>" % ( count , count != <NUM_LIT:1> and '<STR_LIT:s>' or '<STR_LIT>' ) <EOL> def __collect_indices ( self ) : <EOL> count = self . conn . execute ( '<STR_LIT>' ) . rowcount <EOL> if count > <NUM_LIT:0> : <EOL> self . should_vacuum = True <EOL> return "<STR_LIT>" % ( count , count != <NUM_LIT:1> and '<STR_LIT>' or '<STR_LIT:y>' ) <EOL> def __collect_blocks ( self ) : <EOL> should_reorganize = False <EOL> for row in self . conn . execute ( '<STR_LIT>' ) : <EOL> del self . blocks [ str ( row [ <NUM_LIT:0> ] ) ] <EOL> should_reorganize = True <EOL> if should_reorganize : <EOL> self . __dbmcall ( '<STR_LIT>' ) <EOL> count = self . conn . execute ( '<STR_LIT>' ) . rowcount <EOL> if count > <NUM_LIT:0> : <EOL> self . should_vacuum = True <EOL> return "<STR_LIT>" % ( count , count != <NUM_LIT:1> and '<STR_LIT:s>' or '<STR_LIT>' ) <EOL> def __vacuum_metastore ( self ) : <EOL> if self . should_vacuum : <EOL> self . conn . execute ( '<STR_LIT>' ) <EOL> return "<STR_LIT>" <EOL> def __commit_changes ( self , nested = False ) : <EOL> if self . use_transactions and not nested : <EOL> self . conn . commit ( ) <EOL> def __rollback_changes ( self , nested = False ) : <EOL> if self . use_transactions and not nested : <EOL> self . logger . info ( '<STR_LIT>' ) <EOL> self . conn . rollback ( ) <EOL> def __get_file_buffer ( self , path ) : <EOL> if path in self . buffers : <EOL> return self . buffers [ path ] <EOL> else : <EOL> buf = Buffer ( ) <EOL> inode = self . __path2keys ( path ) [ <NUM_LIT:1> ] <EOL> query = """<STR_LIT>""" <EOL> for row in self . conn . execute ( query , ( inode , ) ) . fetchall ( ) : <EOL> buf . write ( self . decompress ( self . blocks [ str ( row [ <NUM_LIT:0> ] ) ] ) ) <EOL> self . buffers [ path ] = buf <EOL> return buf <EOL> def __fetchval ( self , query , * values ) : <EOL> return self . conn . execute ( query , values ) . fetchone ( ) [ <NUM_LIT:0> ] <EOL> def __except_to_status ( self , method , exception , code = errno . ENOENT ) : <EOL> if method != '<STR_LIT>' or code != errno . ENOENT : <EOL> sys . stderr . write ( '<STR_LIT>' % ( '<STR_LIT:->' * <NUM_LIT:50> ) ) <EOL> sys . stderr . write ( "<STR_LIT>" % ( method , exception ) ) <EOL> traceback . print_exc ( file = sys . stderr ) <EOL> sys . stderr . write ( '<STR_LIT>' % ( '<STR_LIT:->' * <NUM_LIT:50> ) ) <EOL> sys . stderr . write ( "<STR_LIT>" % - code ) <EOL> sys . stderr . flush ( ) <EOL> if isinstance ( exception , OSError ) : <EOL> return - exception . errno <EOL> else : <EOL> return - code <EOL> class Buffer : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self ) : <EOL> self . buf = cStringIO . StringIO ( ) <EOL> self . dirty = False <EOL> def __getattr__ ( self , attr , default = None ) : <EOL> """<STR_LIT>""" <EOL> return getattr ( self . buf , attr , default ) <EOL> def __len__ ( self ) : <EOL> """<STR_LIT>""" <EOL> position = self . buf . tell ( ) <EOL> self . buf . seek ( <NUM_LIT:0> , os . SEEK_END ) <EOL> length = self . buf . tell ( ) <EOL> self . buf . seek ( position , os . SEEK_SET ) <EOL> return length <EOL> def truncate ( self , * args ) : <EOL> """<STR_LIT>""" <EOL> if len ( self ) > self . buf . tell ( ) : <EOL> self . dirty = True <EOL> return self . buf . truncate ( * args ) <EOL> def write ( self , * args ) : <EOL> """<STR_LIT>""" <EOL> self . dirty = True <EOL> return self . buf . write ( * args ) <EOL> try : <EOL> import collections <EOL> Stat = collections . namedtuple ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> StatVFS = collections . namedtuple ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> except ImportError : <EOL> class __Struct : <EOL> def __init__ ( self , ** kw ) : <EOL> for k , v in kw . iteritems ( ) : <EOL> setattr ( self , k , v ) <EOL> class Stat ( __Struct ) : pass <EOL> class StatVFS ( __Struct ) : pass <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> if '<STR_LIT>' in sys . argv : <EOL> sys . stderr . write ( "<STR_LIT>" ) <EOL> import cProfile , pstats <EOL> profile = '<STR_LIT>' % time . time ( ) <EOL> cProfile . run ( '<STR_LIT>' , profile ) <EOL> sys . stderr . write ( "<STR_LIT>" ) <EOL> s = pstats . Stats ( profile ) <EOL> s . sort_stats ( '<STR_LIT:time>' ) <EOL> s . print_stats ( <NUM_LIT:0.1> ) <EOL> os . unlink ( profile ) <EOL> else : <EOL> main ( ) </s>
