<s> import os <EOL> from onitu . plug import Plug , DriverError , ServiceError <EOL> from onitu . escalator . client import EscalatorClosed <EOL> from onitu . utils import IS_WINDOWS , u , log_traceback <EOL> if IS_WINDOWS : <EOL> import threading <EOL> import win32api <EOL> import win32file <EOL> import win32con <EOL> else : <EOL> import pyinotify <EOL> TMP_EXT = '<STR_LIT>' <EOL> plug = Plug ( ) <EOL> ignore_delete = set ( ) <EOL> ignore_move = set ( ) <EOL> def to_tmp ( filename ) : <EOL> return os . path . join ( <EOL> os . path . dirname ( filename ) , '<STR_LIT:.>' + os . path . basename ( filename ) + TMP_EXT <EOL> ) <EOL> def walkfiles ( root ) : <EOL> return ( <EOL> os . path . join ( dirpath , f ) <EOL> for dirpath , _ , files in os . walk ( root ) for f in files <EOL> ) <EOL> def update ( metadata , mtime = None ) : <EOL> try : <EOL> metadata . size = os . path . getsize ( metadata . path ) <EOL> if not mtime : <EOL> mtime = os . path . getmtime ( metadata . path ) <EOL> metadata . extra [ '<STR_LIT>' ] = mtime <EOL> except ( IOError , OSError ) as e : <EOL> raise ServiceError ( <EOL> u"<STR_LIT>" . format ( metadata . path , e ) <EOL> ) <EOL> else : <EOL> plug . update_file ( metadata ) <EOL> def delete ( metadata ) : <EOL> if metadata . path in ignore_delete : <EOL> ignore_delete . discard ( metadata . path ) <EOL> return <EOL> plug . delete_file ( metadata ) <EOL> def move ( old_metadata , new_filename ) : <EOL> if old_metadata . path in ignore_move : <EOL> ignore_move . discard ( old_metadata . path ) <EOL> return <EOL> new_metadata = plug . move_file ( old_metadata , new_filename ) <EOL> new_metadata . extra [ '<STR_LIT>' ] = os . path . getmtime ( new_filename ) <EOL> new_metadata . size = os . path . getsize ( new_metadata . path ) <EOL> new_metadata . write ( ) <EOL> def check_changes ( folder ) : <EOL> expected_files = set ( ) <EOL> expected_files . update ( plug . list ( folder ) . keys ( ) ) <EOL> for path in walkfiles ( folder . path ) : <EOL> path = u ( path ) <EOL> if os . path . splitext ( path ) [ <NUM_LIT:1> ] == TMP_EXT : <EOL> continue <EOL> filename = folder . relpath ( path ) <EOL> expected_files . discard ( filename ) <EOL> metadata = plug . get_metadata ( filename , folder ) <EOL> revision = metadata . extra . get ( '<STR_LIT>' , <NUM_LIT:0.> ) <EOL> try : <EOL> mtime = os . path . getmtime ( path ) <EOL> except ( IOError , OSError ) as e : <EOL> raise ServiceError ( <EOL> u"<STR_LIT>" . format ( metadata . path , e ) <EOL> ) <EOL> if mtime > revision : <EOL> update ( metadata , mtime ) <EOL> for filename in expected_files : <EOL> metadata = plug . get_metadata ( filename , folder ) <EOL> plug . delete_file ( metadata ) <EOL> @ plug . handler ( ) <EOL> def normalize_path ( p ) : <EOL> normalized = os . path . normpath ( os . path . expanduser ( p ) ) <EOL> if not os . path . isabs ( normalized ) : <EOL> raise DriverError ( u"<STR_LIT>" . format ( p ) ) <EOL> return normalized <EOL> @ plug . handler ( ) <EOL> def get_chunk ( metadata , offset , size ) : <EOL> try : <EOL> with open ( metadata . path , '<STR_LIT:rb>' ) as f : <EOL> f . seek ( offset ) <EOL> return f . read ( size ) <EOL> except ( IOError , OSError ) as e : <EOL> raise ServiceError ( <EOL> u"<STR_LIT>" . format ( metadata . path , e ) <EOL> ) <EOL> @ plug . handler ( ) <EOL> def start_upload ( metadata ) : <EOL> tmp_file = to_tmp ( metadata . path ) <EOL> try : <EOL> try : <EOL> os . makedirs ( os . path . dirname ( tmp_file ) ) <EOL> except OSError : <EOL> pass <EOL> open ( tmp_file , '<STR_LIT:wb>' ) . close ( ) <EOL> if IS_WINDOWS : <EOL> win32api . SetFileAttributes ( <EOL> tmp_file , win32con . FILE_ATTRIBUTE_HIDDEN ) <EOL> except IOError as e : <EOL> raise ServiceError ( <EOL> u"<STR_LIT>" . format ( tmp_file , e ) <EOL> ) <EOL> @ plug . handler ( ) <EOL> def upload_chunk ( metadata , offset , chunk ) : <EOL> tmp_file = to_tmp ( metadata . path ) <EOL> try : <EOL> with open ( tmp_file , '<STR_LIT>' ) as f : <EOL> f . seek ( offset ) <EOL> f . write ( chunk ) <EOL> except ( IOError , OSError ) as e : <EOL> raise ServiceError ( <EOL> u"<STR_LIT>" . format ( tmp_file , e ) <EOL> ) <EOL> @ plug . handler ( ) <EOL> def end_upload ( metadata ) : <EOL> tmp_file = to_tmp ( metadata . path ) <EOL> try : <EOL> if IS_WINDOWS : <EOL> try : <EOL> os . unlink ( metadata . path ) <EOL> except OSError : <EOL> pass <EOL> os . rename ( tmp_file , metadata . path ) <EOL> mtime = os . path . getmtime ( metadata . path ) <EOL> if IS_WINDOWS : <EOL> win32api . SetFileAttributes ( <EOL> metadata . path , win32con . FILE_ATTRIBUTE_NORMAL ) <EOL> except ( IOError , OSError ) as e : <EOL> raise ServiceError ( <EOL> u"<STR_LIT>" . format ( metadata . path , e ) <EOL> ) <EOL> metadata . extra [ '<STR_LIT>' ] = mtime <EOL> metadata . write ( ) <EOL> @ plug . handler ( ) <EOL> def abort_upload ( metadata ) : <EOL> tmp_file = to_tmp ( metadata . path ) <EOL> try : <EOL> os . unlink ( tmp_file ) <EOL> except ( IOError , OSError ) as e : <EOL> raise ServiceError ( <EOL> u"<STR_LIT>" . format ( tmp_file , e ) <EOL> ) <EOL> @ plug . handler ( ) <EOL> def delete_file ( metadata ) : <EOL> try : <EOL> ignore_delete . add ( metadata . path ) <EOL> os . unlink ( metadata . path ) <EOL> except ( IOError , OSError ) as e : <EOL> ignore_delete . discard ( metadata . path ) <EOL> raise ServiceError ( <EOL> u"<STR_LIT>" . format ( metadata . path , e ) <EOL> ) <EOL> @ plug . handler ( ) <EOL> def move_file ( old_metadata , new_metadata ) : <EOL> try : <EOL> ignore_move . add ( old_metadata . path ) <EOL> os . renames ( old_metadata . path , new_metadata . path ) <EOL> except ( IOError , OSError ) as e : <EOL> ignore_move . discard ( old_metadata . path ) <EOL> raise ServiceError ( <EOL> u"<STR_LIT>" . format ( old_metadata . path , e ) <EOL> ) <EOL> if IS_WINDOWS : <EOL> FILE_LIST_DIRECTORY = <NUM_LIT> <EOL> def win32watcherThread ( root , file_lock ) : <EOL> dirHandle = win32file . CreateFile ( <EOL> root , <EOL> FILE_LIST_DIRECTORY , <EOL> win32con . FILE_SHARE_READ | win32con . FILE_SHARE_WRITE , <EOL> None , <EOL> win32con . OPEN_EXISTING , <EOL> win32con . FILE_FLAG_BACKUP_SEMANTICS , <EOL> None <EOL> ) <EOL> actions_names = { <EOL> <NUM_LIT:1> : '<STR_LIT>' , <EOL> <NUM_LIT:2> : '<STR_LIT>' , <EOL> <NUM_LIT:3> : '<STR_LIT>' , <EOL> <NUM_LIT:4> : '<STR_LIT>' , <EOL> <NUM_LIT:5> : '<STR_LIT>' <EOL> } <EOL> while True : <EOL> results = win32file . ReadDirectoryChangesW ( <EOL> dirHandle , <EOL> <NUM_LIT> , <EOL> True , <EOL> win32con . FILE_NOTIFY_CHANGE_FILE_NAME | <EOL> win32con . FILE_NOTIFY_CHANGE_DIR_NAME | <EOL> win32con . FILE_NOTIFY_CHANGE_ATTRIBUTES | <EOL> win32con . FILE_NOTIFY_CHANGE_SIZE | <EOL> win32con . FILE_NOTIFY_CHANGE_LAST_WRITE | <EOL> win32con . FILE_NOTIFY_CHANGE_SECURITY , <EOL> None <EOL> ) <EOL> for action , file_ in results : <EOL> abs_path = root / file_ <EOL> if ( abs_path . isdir ( ) or abs_path . ext == TMP_EXT or <EOL> not ( win32api . GetFileAttributes ( abs_path ) <EOL> & win32con . FILE_ATTRIBUTE_NORMAL ) ) : <EOL> continue <EOL> with file_lock : <EOL> if actions_names . get ( action ) == '<STR_LIT>' : <EOL> filename = root . relpathto ( abs_path ) <EOL> try : <EOL> metadata = plug . get_metadata ( filename ) <EOL> update ( metadata , abs_path ) <EOL> except EscalatorClosed : <EOL> return <EOL> def watch_changes ( folder ) : <EOL> file_lock = threading . Lock ( ) <EOL> notifier = threading . Thread ( target = win32watcherThread , <EOL> args = ( folder . path , file_lock ) ) <EOL> notifier . setDaemon ( True ) <EOL> notifier . start ( ) <EOL> else : <EOL> class Watcher ( pyinotify . ProcessEvent ) : <EOL> def __init__ ( self , folder , * args , ** kwargs ) : <EOL> pyinotify . ProcessEvent . __init__ ( self , * args , ** kwargs ) <EOL> self . folder = folder <EOL> def process_IN_CLOSE_WRITE ( self , event ) : <EOL> self . process_event ( event . pathname , update ) <EOL> def process_IN_DELETE ( self , event ) : <EOL> self . process_event ( event . pathname , delete ) <EOL> def process_IN_MOVED_TO ( self , event ) : <EOL> if event . dir : <EOL> for new in walkfiles ( event . pathname ) : <EOL> if hasattr ( event , '<STR_LIT>' ) : <EOL> old = new . replace ( event . pathname , event . src_pathname ) <EOL> self . process_event ( old , move , u ( new ) ) <EOL> else : <EOL> self . process_event ( new , update ) <EOL> else : <EOL> if hasattr ( event , '<STR_LIT>' ) : <EOL> self . process_event ( <EOL> event . src_pathname , move , u ( event . pathname ) <EOL> ) <EOL> else : <EOL> self . process_event ( event . pathname , update ) <EOL> def process_event ( self , filename , callback , * args ) : <EOL> filename = os . path . relpath ( u ( filename ) , self . folder . path ) <EOL> if os . path . splitext ( filename ) [ <NUM_LIT:1> ] == TMP_EXT : <EOL> return <EOL> try : <EOL> metadata = plug . get_metadata ( filename , self . folder ) <EOL> callback ( metadata , * args ) <EOL> except EscalatorClosed : <EOL> pass <EOL> except OSError as e : <EOL> plug . logger . warning ( "<STR_LIT>" , e ) <EOL> except ( DriverError , ServiceError ) as e : <EOL> plug . logger . warning ( str ( e ) ) <EOL> except Exception : <EOL> log_traceback ( plug . logger ) <EOL> def watch_changes ( folder ) : <EOL> manager = pyinotify . WatchManager ( ) <EOL> notifier = pyinotify . ThreadedNotifier ( manager , Watcher ( folder ) ) <EOL> notifier . daemon = True <EOL> notifier . start ( ) <EOL> mask = ( pyinotify . IN_CREATE | pyinotify . IN_CLOSE_WRITE | <EOL> pyinotify . IN_DELETE | pyinotify . IN_MOVED_TO | <EOL> pyinotify . IN_MOVED_FROM ) <EOL> manager . add_watch ( folder . path , mask , rec = True , auto_add = True ) <EOL> def start ( ) : <EOL> for folder in plug . folders_to_watch : <EOL> try : <EOL> os . makedirs ( folder . path ) <EOL> except OSError : <EOL> pass <EOL> if not os . path . exists ( folder . path ) : <EOL> raise DriverError ( <EOL> u"<STR_LIT>" . format ( folder . path ) <EOL> ) <EOL> watch_changes ( folder ) <EOL> check_changes ( folder ) <EOL> plug . listen ( ) </s>
