<s> from os import path , walk , unlink <EOL> from os import name as osname <EOL> from tempfile import NamedTemporaryFile <EOL> from shutil import copyfileobj <EOL> from httplib import HTTPResponse <EOL> from StringIO import StringIO <EOL> from urlparse import urlparse <EOL> import mimetypes <EOL> import requests <EOL> from archivekit . util import clean_headers , checksum , fullpath <EOL> from archivekit . util import make_secure_filename , slugify <EOL> def directory_files ( fpath ) : <EOL> for ( dir , _ , files ) in walk ( fpath ) : <EOL> for file_name in files : <EOL> yield path . join ( dir , file_name ) <EOL> class Ingestor ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , file_name = None , file_obj = None , meta = None ) : <EOL> self . _file_name = file_name <EOL> self . _file_obj = file_obj <EOL> self . _file_cache = None <EOL> self . _hash = None <EOL> self . is_local = file_name is not None <EOL> self . meta = meta or { } <EOL> def local ( self ) : <EOL> if self . is_local : <EOL> return self . _file_name <EOL> if self . _file_cache is None : <EOL> tempfile = NamedTemporaryFile ( delete = False ) <EOL> copyfileobj ( self . _file_obj , tempfile ) <EOL> self . _file_cache = tempfile . name <EOL> tempfile . close ( ) <EOL> return self . _file_cache <EOL> def has_local ( self ) : <EOL> cached = self . _file_cache and path . exists ( self . _file_cache ) <EOL> return self . is_local or cached <EOL> def hash ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . _hash is None : <EOL> self . _hash = checksum ( self . local ( ) ) <EOL> return self . _hash <EOL> def generate_meta ( self , meta ) : <EOL> """<STR_LIT>""" <EOL> meta = meta or { } <EOL> for key , value in self . meta . items ( ) : <EOL> if key not in meta : <EOL> meta [ key ] = value <EOL> if '<STR_LIT>' not in meta and self . _file_name : <EOL> meta [ '<STR_LIT>' ] = self . _file_name <EOL> if not meta . get ( '<STR_LIT:name>' ) : <EOL> if meta . get ( '<STR_LIT>' ) and len ( meta . get ( '<STR_LIT>' ) ) : <EOL> path = meta [ '<STR_LIT>' ] <EOL> try : <EOL> path = urlparse ( path ) . path <EOL> except : <EOL> pass <EOL> meta [ '<STR_LIT:name>' ] = path <EOL> elif meta . get ( '<STR_LIT>' ) and len ( meta . get ( '<STR_LIT>' ) ) : <EOL> meta [ '<STR_LIT:name>' ] = meta [ '<STR_LIT>' ] <EOL> name , slug , ext = make_secure_filename ( meta . get ( '<STR_LIT:name>' ) ) <EOL> meta [ '<STR_LIT:name>' ] = name <EOL> if not meta . get ( '<STR_LIT>' ) : <EOL> meta [ '<STR_LIT>' ] = slug <EOL> if not meta . get ( '<STR_LIT>' ) : <EOL> meta [ '<STR_LIT>' ] = ext <EOL> if not meta . get ( '<STR_LIT>' ) and '<STR_LIT>' in meta : <EOL> mime_type = meta . get ( '<STR_LIT>' ) . get ( '<STR_LIT>' ) <EOL> if mime_type not in [ '<STR_LIT>' , '<STR_LIT>' ] : <EOL> meta [ '<STR_LIT>' ] = mime_type <EOL> ext = mimetypes . guess_extension ( mime_type ) <EOL> if ext is not None : <EOL> meta [ '<STR_LIT>' ] = ext . strip ( '<STR_LIT:.>' ) <EOL> elif not meta . get ( '<STR_LIT>' ) and meta . get ( '<STR_LIT:name>' ) : <EOL> mime_type , encoding = mimetypes . guess_type ( meta . get ( '<STR_LIT:name>' ) ) <EOL> meta [ '<STR_LIT>' ] = mime_type <EOL> if meta . get ( '<STR_LIT>' ) : <EOL> meta [ '<STR_LIT>' ] = slugify ( meta . get ( '<STR_LIT>' ) ) <EOL> return meta <EOL> def store ( self , source ) : <EOL> if not self . has_local ( ) : <EOL> source . save_fileobj ( self . _file_obj ) <EOL> elif self . is_local : <EOL> source . save_file ( self . _file_name ) <EOL> elif self . _file_cache : <EOL> source . save_file ( self . _file_cache , destructive = True ) <EOL> def dispose ( self ) : <EOL> if self . _file_cache is not None and path . exists ( self . _file_cache ) : <EOL> unlink ( self . _file_cache ) <EOL> @ classmethod <EOL> def analyze ( cls , something ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( something , cls ) : <EOL> return ( something , ) <EOL> if isinstance ( something , basestring ) : <EOL> url = urlparse ( something ) <EOL> if url . scheme . lower ( ) in [ '<STR_LIT:http>' , '<STR_LIT>' ] : <EOL> something = requests . get ( something ) <EOL> elif url . scheme . lower ( ) in [ '<STR_LIT:file>' , '<STR_LIT>' ] : <EOL> finalpath = url . path <EOL> if osname == '<STR_LIT>' : <EOL> finalpath = finalpath [ <NUM_LIT:1> : ] <EOL> upath = fullpath ( finalpath ) <EOL> if path . isdir ( upath ) : <EOL> return ( cls ( file_name = f ) for f in directory_files ( upath ) ) <EOL> return ( cls ( file_name = upath ) , ) <EOL> if isinstance ( something , requests . Response ) : <EOL> fd = StringIO ( something . content ) <EOL> return ( cls ( file_obj = fd , meta = { <EOL> '<STR_LIT>' : something . status_code , <EOL> '<STR_LIT>' : clean_headers ( something . headers ) , <EOL> '<STR_LIT>' : something . url <EOL> } ) , ) <EOL> if isinstance ( something , HTTPResponse ) : <EOL> return ( cls ( file_obj = something , meta = { <EOL> '<STR_LIT>' : something . status , <EOL> '<STR_LIT>' : clean_headers ( something . getheaders ( ) ) , <EOL> '<STR_LIT>' : something . url <EOL> } ) , ) <EOL> elif hasattr ( something , '<STR_LIT>' ) and hasattr ( something , '<STR_LIT:info>' ) : <EOL> return ( cls ( file_obj = something , meta = { <EOL> '<STR_LIT>' : something . getcode ( ) , <EOL> '<STR_LIT>' : clean_headers ( something . headers ) , <EOL> '<STR_LIT>' : something . url <EOL> } ) , ) <EOL> elif hasattr ( something , '<STR_LIT>' ) : <EOL> return ( cls ( file_obj = something ) , ) <EOL> return [ ] </s>
