<s> """<STR_LIT>""" <EOL> __author__ = '<STR_LIT>' <EOL> import cStringIO <EOL> import gzip <EOL> import logging <EOL> import subprocess <EOL> import sys <EOL> import traceback <EOL> from functools import partial <EOL> from tornado import gen , options <EOL> from viewfinder . backend . base import main <EOL> from viewfinder . backend . db import db_client <EOL> from viewfinder . backend . storage . object_store import ObjectStore <EOL> def ListBuckets ( ) : <EOL> """<STR_LIT>""" <EOL> return ObjectStore . ListInstances ( ) <EOL> def IsBucket ( name ) : <EOL> """<STR_LIT>""" <EOL> return ObjectStore . HasInstance ( name ) <EOL> def ParseFullPath ( full_path ) : <EOL> """<STR_LIT>""" <EOL> bucket , _ , path = full_path . partition ( '<STR_LIT:/>' ) <EOL> if not bucket or not IsBucket ( bucket ) : <EOL> return None <EOL> else : <EOL> return ( bucket , path ) <EOL> def PrefixFromPattern ( pattern ) : <EOL> """<STR_LIT>""" <EOL> if pattern . endswith ( '<STR_LIT:*>' ) : <EOL> return pattern [ : - <NUM_LIT:1> ] <EOL> elif pattern . endswith ( '<STR_LIT:/>' ) : <EOL> return '<STR_LIT>' if pattern == '<STR_LIT:/>' else pattern <EOL> elif pattern == '<STR_LIT>' : <EOL> return pattern <EOL> else : <EOL> return pattern + '<STR_LIT:/>' <EOL> @ gen . engine <EOL> def FileExists ( store , filename , callback ) : <EOL> """<STR_LIT>""" <EOL> ret = yield gen . Task ( store . ListKeys , prefix = filename , maxkeys = <NUM_LIT:1> ) <EOL> logging . info ( '<STR_LIT>' % ( filename , ret ) ) <EOL> callback ( len ( ret ) > <NUM_LIT:0> and ret [ <NUM_LIT:0> ] == filename ) <EOL> @ gen . engine <EOL> def ListAllKeys ( store , callback , prefix = None , marker = None , batch_size = <NUM_LIT:1000> ) : <EOL> """<STR_LIT>""" <EOL> batch_size = min ( <NUM_LIT:1000> , batch_size ) <EOL> keys = [ ] <EOL> done = False <EOL> while not done : <EOL> new_keys = yield gen . Task ( store . ListKeys , prefix = prefix , marker = marker , maxkeys = batch_size ) <EOL> keys . extend ( new_keys ) <EOL> if len ( new_keys ) < batch_size : <EOL> break <EOL> marker = new_keys [ - <NUM_LIT:1> ] <EOL> callback ( keys ) <EOL> @ gen . engine <EOL> def ListAllCommonPrefixes ( store , delimiter , callback , prefix = None , marker = None ) : <EOL> """<STR_LIT>""" <EOL> prefixes = set ( ) <EOL> keys = [ ] <EOL> while True : <EOL> new_prefixes , new_keys = yield gen . Task ( store . ListCommonPrefixes , delimiter , prefix = prefix , <EOL> marker = marker , maxkeys = <NUM_LIT:1000> ) <EOL> prefixes = prefixes . union ( set ( new_prefixes ) ) <EOL> keys . extend ( new_keys ) <EOL> if ( len ( new_prefixes ) + len ( new_keys ) ) < <NUM_LIT:1000> : <EOL> break <EOL> marker = '<STR_LIT>' <EOL> if len ( new_prefixes ) > <NUM_LIT:0> : <EOL> marker = max ( marker , new_prefixes [ - <NUM_LIT:1> ] ) <EOL> if len ( new_keys ) > <NUM_LIT:0> : <EOL> marker = max ( marker , new_keys [ - <NUM_LIT:1> ] ) <EOL> callback ( ( sorted ( list ( prefixes ) ) , keys ) ) <EOL> @ gen . engine <EOL> def ListFilesAndDirs ( store , pattern , callback ) : <EOL> """<STR_LIT>""" <EOL> result = yield gen . Task ( ListAllCommonPrefixes , store , '<STR_LIT:/>' , prefix = PrefixFromPattern ( pattern ) ) <EOL> callback ( result ) <EOL> @ gen . engine <EOL> def ListRecursively ( store , pattern , callback ) : <EOL> """<STR_LIT>""" <EOL> results = yield gen . Task ( ListAllKeys , store , prefix = PrefixFromPattern ( pattern ) ) <EOL> callback ( results ) <EOL> @ gen . engine <EOL> def GetFileContents ( store , path , callback , auto_gunzip = True ) : <EOL> """<STR_LIT>""" <EOL> buf = yield gen . Task ( store . Get , path ) <EOL> if auto_gunzip and path . endswith ( '<STR_LIT>' ) : <EOL> iobuffer = cStringIO . StringIO ( buf ) <EOL> gzipIO = gzip . GzipFile ( '<STR_LIT:rb>' , fileobj = iobuffer ) <EOL> contents = gzipIO . read ( ) <EOL> iobuffer . close ( ) <EOL> else : <EOL> contents = buf <EOL> callback ( contents ) </s>
