<s> """<STR_LIT>""" <EOL> import hashlib <EOL> import six <EOL> try : <EOL> from cStringIO import StringIO as BytesIO <EOL> except ImportError : <EOL> from io import BytesIO <EOL> from PIL import Image <EOL> from scrapy . utils . misc import md5sum <EOL> from scrapy . utils . python import to_bytes <EOL> from scrapy . http import Request <EOL> from scrapy . settings import Settings <EOL> from scrapy . exceptions import DropItem <EOL> from scrapy . pipelines . files import FileException , FilesPipeline <EOL> class NoimagesDrop ( DropItem ) : <EOL> """<STR_LIT>""" <EOL> class ImageException ( FileException ) : <EOL> """<STR_LIT>""" <EOL> class ImagesPipeline ( FilesPipeline ) : <EOL> """<STR_LIT>""" <EOL> MEDIA_NAME = '<STR_LIT:image>' <EOL> def __init__ ( self , store_uri , download_func = None , settings = None ) : <EOL> super ( ImagesPipeline , self ) . __init__ ( store_uri , settings = settings , download_func = download_func ) <EOL> if isinstance ( settings , dict ) or settings is None : <EOL> settings = Settings ( settings ) <EOL> self . expires = settings . getint ( '<STR_LIT>' ) <EOL> self . images_urls_field = settings . get ( '<STR_LIT>' ) <EOL> self . images_result_field = settings . get ( '<STR_LIT>' ) <EOL> self . min_width = settings . getint ( '<STR_LIT>' ) <EOL> self . min_height = settings . getint ( '<STR_LIT>' ) <EOL> self . thumbs = settings . get ( '<STR_LIT>' ) <EOL> @ classmethod <EOL> def from_settings ( cls , settings ) : <EOL> s3store = cls . STORE_SCHEMES [ '<STR_LIT>' ] <EOL> s3store . AWS_ACCESS_KEY_ID = settings [ '<STR_LIT>' ] <EOL> s3store . AWS_SECRET_ACCESS_KEY = settings [ '<STR_LIT>' ] <EOL> store_uri = settings [ '<STR_LIT>' ] <EOL> return cls ( store_uri , settings = settings ) <EOL> def file_downloaded ( self , response , request , info ) : <EOL> return self . image_downloaded ( response , request , info ) <EOL> def image_downloaded ( self , response , request , info ) : <EOL> checksum = None <EOL> for path , image , buf in self . get_images ( response , request , info ) : <EOL> if checksum is None : <EOL> buf . seek ( <NUM_LIT:0> ) <EOL> checksum = md5sum ( buf ) <EOL> width , height = image . size <EOL> self . store . persist_file ( <EOL> path , buf , info , <EOL> meta = { '<STR_LIT:width>' : width , '<STR_LIT>' : height } , <EOL> headers = { '<STR_LIT:Content-Type>' : '<STR_LIT>' } ) <EOL> return checksum <EOL> def get_images ( self , response , request , info ) : <EOL> path = self . file_path ( request , response = response , info = info ) <EOL> orig_image = Image . open ( BytesIO ( response . body ) ) <EOL> width , height = orig_image . size <EOL> if width < self . min_width or height < self . min_height : <EOL> raise ImageException ( "<STR_LIT>" % <EOL> ( width , height , self . min_width , self . min_height ) ) <EOL> image , buf = self . convert_image ( orig_image ) <EOL> yield path , image , buf <EOL> for thumb_id , size in six . iteritems ( self . thumbs ) : <EOL> thumb_path = self . thumb_path ( request , thumb_id , response = response , info = info ) <EOL> thumb_image , thumb_buf = self . convert_image ( image , size ) <EOL> yield thumb_path , thumb_image , thumb_buf <EOL> def convert_image ( self , image , size = None ) : <EOL> if image . format == '<STR_LIT>' and image . mode == '<STR_LIT>' : <EOL> background = Image . new ( '<STR_LIT>' , image . size , ( <NUM_LIT:255> , <NUM_LIT:255> , <NUM_LIT:255> ) ) <EOL> background . paste ( image , image ) <EOL> image = background . convert ( '<STR_LIT>' ) <EOL> elif image . mode != '<STR_LIT>' : <EOL> image = image . convert ( '<STR_LIT>' ) <EOL> if size : <EOL> image = image . copy ( ) <EOL> image . thumbnail ( size , Image . ANTIALIAS ) <EOL> buf = BytesIO ( ) <EOL> image . save ( buf , '<STR_LIT>' ) <EOL> return image , buf <EOL> def get_media_requests ( self , item , info ) : <EOL> return [ Request ( x ) for x in item . get ( self . images_urls_field , [ ] ) ] <EOL> def item_completed ( self , results , item , info ) : <EOL> if isinstance ( item , dict ) or self . images_result_field in item . fields : <EOL> item [ self . images_result_field ] = [ x for ok , x in results if ok ] <EOL> return item <EOL> def file_path ( self , request , response = None , info = None ) : <EOL> def _warn ( ) : <EOL> from scrapy . exceptions import ScrapyDeprecationWarning <EOL> import warnings <EOL> warnings . warn ( '<STR_LIT>' <EOL> '<STR_LIT>' , <EOL> category = ScrapyDeprecationWarning , stacklevel = <NUM_LIT:1> ) <EOL> if not isinstance ( request , Request ) : <EOL> _warn ( ) <EOL> url = request <EOL> else : <EOL> url = request . url <EOL> if not hasattr ( self . file_key , '<STR_LIT>' ) : <EOL> _warn ( ) <EOL> return self . file_key ( url ) <EOL> elif not hasattr ( self . image_key , '<STR_LIT>' ) : <EOL> _warn ( ) <EOL> return self . image_key ( url ) <EOL> image_guid = hashlib . sha1 ( to_bytes ( url ) ) . hexdigest ( ) <EOL> return '<STR_LIT>' % ( image_guid ) <EOL> def thumb_path ( self , request , thumb_id , response = None , info = None ) : <EOL> def _warn ( ) : <EOL> from scrapy . exceptions import ScrapyDeprecationWarning <EOL> import warnings <EOL> warnings . warn ( '<STR_LIT>' <EOL> '<STR_LIT>' , <EOL> category = ScrapyDeprecationWarning , stacklevel = <NUM_LIT:1> ) <EOL> if not isinstance ( request , Request ) : <EOL> _warn ( ) <EOL> url = request <EOL> else : <EOL> url = request . url <EOL> if not hasattr ( self . thumb_key , '<STR_LIT>' ) : <EOL> _warn ( ) <EOL> return self . thumb_key ( url , thumb_id ) <EOL> thumb_guid = hashlib . sha1 ( to_bytes ( url ) ) . hexdigest ( ) <EOL> return '<STR_LIT>' % ( thumb_id , thumb_guid ) <EOL> def file_key ( self , url ) : <EOL> return self . image_key ( url ) <EOL> file_key . _base = True <EOL> def image_key ( self , url ) : <EOL> return self . file_path ( url ) <EOL> image_key . _base = True <EOL> def thumb_key ( self , url , thumb_id ) : <EOL> return self . thumb_path ( url , thumb_id ) <EOL> thumb_key . _base = True </s>
