<s> from __future__ import division , unicode_literals <EOL> import hashlib <EOL> import logging <EOL> import os <EOL> import shelve <EOL> from fnmatch import fnmatch <EOL> from . utils import is_unsplitable , get_root_of_unsplitable <EOL> logger = logging . getLogger ( __name__ ) <EOL> class Database ( object ) : <EOL> hash_mode_size_varying = <NUM_LIT> <EOL> def __init__ ( self , db_file , paths , ignore_files , normal_mode , unsplitable_mode , exact_mode , <EOL> hash_name_mode , hash_size_mode , hash_slow_mode ) : <EOL> """<STR_LIT>""" <EOL> self . db = shelve . open ( db_file ) <EOL> self . db_file = db_file <EOL> self . paths = paths <EOL> self . ignore_files = [ self . normalize_filename ( x ) for x in ignore_files ] <EOL> self . normal_mode = normal_mode <EOL> self . unsplitable_mode = unsplitable_mode <EOL> self . exact_mode = exact_mode <EOL> self . hash_name_mode = hash_name_mode <EOL> self . hash_size_mode = hash_size_mode <EOL> self . hash_slow_mode = hash_slow_mode <EOL> self . hash_mode = hash_name_mode or hash_size_mode or hash_slow_mode <EOL> self . hash_size_table = None <EOL> def truncate ( self ) : <EOL> """<STR_LIT>""" <EOL> logger . info ( '<STR_LIT>' ) <EOL> self . db . close ( ) <EOL> self . db = shelve . open ( self . db_file , flag = '<STR_LIT:n>' ) <EOL> def insert_into_database ( self , root , f , mode , prefix = None , unsplitable_name = None ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> self . _insert_into_database ( root , f , mode , prefix , unsplitable_name ) <EOL> except UnicodeDecodeError : <EOL> logger . error ( '<STR_LIT>' % ( root , f , mode ) ) <EOL> def _insert_into_database ( self , root , f , mode , prefix = None , unsplitable_name = None ) : <EOL> """<STR_LIT>""" <EOL> path = os . path . abspath ( os . path . join ( root , f ) ) <EOL> if not os . access ( path , os . R_OK ) : <EOL> logger . warning ( '<STR_LIT>' % path ) <EOL> return <EOL> if mode == '<STR_LIT>' : <EOL> key = self . keyify ( prefix , f ) <EOL> if key in self . db : <EOL> self . db [ key ] = self . db [ key ] + [ path ] <EOL> else : <EOL> self . db [ key ] = [ path ] <EOL> else : <EOL> normalized_filename = self . normalize_filename ( f ) <EOL> size = os . path . getsize ( path ) <EOL> if mode . startswith ( '<STR_LIT>' ) : <EOL> if mode == '<STR_LIT>' : <EOL> key = self . keyify ( normalized_filename ) <EOL> elif mode == '<STR_LIT>' : <EOL> key = str ( '<STR_LIT>' % size ) <EOL> self . db [ key ] = self . db . get ( key , [ ] ) + [ path ] <EOL> else : <EOL> if mode == '<STR_LIT>' : <EOL> split_root = root . split ( os . sep ) <EOL> p_index = len ( split_root ) - split_root [ : : - <NUM_LIT:1> ] . index ( unsplitable_name ) - <NUM_LIT:1> <EOL> p = [ self . normalize_filename ( x ) for x in split_root [ p_index : ] ] + [ normalized_filename ] <EOL> key = self . keyify ( size , * p ) <EOL> elif mode == '<STR_LIT>' : <EOL> key = self . keyify ( size , normalized_filename ) <EOL> if key in self . db : <EOL> old_inode = os . stat ( self . db [ key ] ) . st_ino <EOL> new_inode = os . stat ( path ) . st_ino <EOL> if old_inode != new_inode : <EOL> logger . warning ( '<STR_LIT>' % ( path , self . db [ key ] ) ) <EOL> self . db [ key ] = path <EOL> def skip_file ( self , f ) : <EOL> """<STR_LIT>""" <EOL> normalized_filename = self . normalize_filename ( f ) <EOL> for ignore_file in self . ignore_files : <EOL> if fnmatch ( normalized_filename , ignore_file ) : <EOL> return True <EOL> return False <EOL> def rebuild ( self , paths = None ) : <EOL> """<STR_LIT>""" <EOL> if paths : <EOL> logger . info ( '<STR_LIT>' ) <EOL> else : <EOL> logger . info ( '<STR_LIT>' ) <EOL> self . truncate ( ) <EOL> paths = self . paths <EOL> unsplitable_paths = set ( ) <EOL> if self . unsplitable_mode or self . exact_mode : <EOL> logger . info ( '<STR_LIT>' ) <EOL> for root_path in paths : <EOL> logger . info ( '<STR_LIT>' % root_path ) <EOL> for root , dirs , files in os . walk ( root_path ) : <EOL> if is_unsplitable ( files ) : <EOL> sep_root = root . split ( os . sep ) <EOL> name = get_root_of_unsplitable ( root . split ( os . sep ) ) <EOL> while sep_root [ - <NUM_LIT:1> ] != name : <EOL> sep_root . pop ( ) <EOL> path = os . path . join ( * sep_root ) <EOL> logger . debug ( '<STR_LIT>' % path ) <EOL> unsplitable_paths . add ( path ) <EOL> logger . info ( '<STR_LIT>' % root_path ) <EOL> for root_path in paths : <EOL> logger . info ( '<STR_LIT>' % root_path ) <EOL> for root , dirs , files in os . walk ( root_path ) : <EOL> unsplitable = False <EOL> if self . unsplitable_mode or self . exact_mode : <EOL> sep_root = root . split ( os . sep ) <EOL> while sep_root : <EOL> if os . path . join ( * sep_root ) in unsplitable_paths : <EOL> break <EOL> sep_root . pop ( ) <EOL> if sep_root : <EOL> unsplitable = True <EOL> if self . unsplitable_mode : <EOL> unsplitable_name = sep_root [ - <NUM_LIT:1> ] <EOL> logger . info ( '<STR_LIT>' % ( os . sep . join ( sep_root ) ) ) <EOL> for f in files : <EOL> self . insert_into_database ( root , f , '<STR_LIT>' , unsplitable_name = unsplitable_name ) <EOL> continue <EOL> if not unsplitable : <EOL> if self . normal_mode : <EOL> for f in files : <EOL> if self . skip_file ( f ) : <EOL> continue <EOL> self . insert_into_database ( root , f , '<STR_LIT>' ) <EOL> if self . exact_mode : <EOL> for f in files : <EOL> self . insert_into_database ( root , f , '<STR_LIT>' , '<STR_LIT:f>' ) <EOL> for d in dirs : <EOL> self . insert_into_database ( root , d , '<STR_LIT>' , '<STR_LIT:d>' ) <EOL> if self . hash_name_mode or self . hash_size_mode or self . hash_slow_mode : <EOL> for f in files : <EOL> if self . hash_size_mode or self . hash_slow_mode : <EOL> self . insert_into_database ( root , f , '<STR_LIT>' ) <EOL> if self . hash_name_mode : <EOL> self . insert_into_database ( root , f , '<STR_LIT>' ) <EOL> logger . info ( '<STR_LIT>' % root_path ) <EOL> self . db . sync ( ) <EOL> def clear_hash_size_table ( self ) : <EOL> """<STR_LIT>""" <EOL> self . hash_size_table = None <EOL> def build_hash_size_table ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . hash_size_table is not None : <EOL> logger . debug ( '<STR_LIT>' ) <EOL> return <EOL> self . hash_size_table = set ( ) <EOL> for key in self . db . keys ( ) : <EOL> if not key . startswith ( '<STR_LIT>' ) : <EOL> continue <EOL> _ , size = key . split ( '<STR_LIT::>' ) <EOL> self . hash_size_table . add ( int ( size ) ) <EOL> self . hash_size_table = sorted ( self . hash_size_table ) <EOL> def find_hash_varying_size ( self , size ) : <EOL> """<STR_LIT>""" <EOL> size_span = size * self . hash_mode_size_varying / <NUM_LIT:100> <EOL> min_size_span , max_size_span = size - size_span , size + size_span <EOL> found_sizes = [ ] <EOL> for db_size in self . hash_size_table : <EOL> if db_size < min_size_span : <EOL> continue <EOL> if db_size > max_size_span : <EOL> break <EOL> found_sizes . append ( db_size ) <EOL> found_sizes = sorted ( found_sizes , key = lambda x : abs ( x - size ) ) <EOL> result = [ ] <EOL> for found_size in found_sizes : <EOL> key = str ( '<STR_LIT>' % found_size ) <EOL> result += self . db . get ( key , [ ] ) <EOL> return result <EOL> def find_hash_size ( self , size ) : <EOL> """<STR_LIT>""" <EOL> return self . db . get ( str ( '<STR_LIT>' % size ) , [ ] ) <EOL> def find_hash_name ( self , f ) : <EOL> """<STR_LIT>""" <EOL> key = self . keyify ( self . normalize_filename ( f ) ) <EOL> return self . db . get ( key , [ ] ) <EOL> def find_unsplitable_file_path ( self , rls , f , size ) : <EOL> """<STR_LIT>""" <EOL> f = [ self . normalize_filename ( x ) for x in f ] <EOL> key = self . keyify ( size , self . normalize_filename ( rls ) , * f ) <EOL> return self . db . get ( key ) <EOL> def find_exact_file_path ( self , prefix , rls ) : <EOL> """<STR_LIT>""" <EOL> key = self . keyify ( prefix , rls ) <EOL> return self . db . get ( key ) <EOL> def find_file_path ( self , f , size ) : <EOL> """<STR_LIT>""" <EOL> key = self . keyify ( size , self . normalize_filename ( f ) ) <EOL> return self . db . get ( key ) <EOL> def keyify ( self , size , * names ) : <EOL> """<STR_LIT>""" <EOL> key = '<STR_LIT>' % ( size , '<STR_LIT:|>' . join ( names ) ) <EOL> logger . debug ( '<STR_LIT>' % key ) <EOL> return hashlib . sha256 ( key . encode ( '<STR_LIT:utf-8>' ) ) . hexdigest ( ) <EOL> def normalize_filename ( self , filename ) : <EOL> """<STR_LIT>""" <EOL> return filename . replace ( '<STR_LIT:U+0020>' , '<STR_LIT:_>' ) . lower ( ) </s>
