<s> from abc import abstractmethod <EOL> import datetime <EOL> try : <EOL> import urlparse <EOL> except ImportError : <EOL> import urllib . parse as urlparse <EOL> import warnings <EOL> import lxml . etree <EOL> import lxml . html <EOL> import requests <EOL> import six <EOL> SHARED_SESSION = requests . Session ( ) <EOL> SHARED_SESSION . headers [ '<STR_LIT>' ] = "<STR_LIT>" <EOL> class ScrapedAttribute ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , extract = None , cleanup = None , attribute = None , <EOL> multiple = False ) : <EOL> if extract and attribute : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> self . _cleanup = cleanup <EOL> self . _extract = extract <EOL> self . attribute = attribute <EOL> self . multiple = multiple <EOL> self . _cleanup_method = None <EOL> @ abstractmethod <EOL> def get ( self , doc , scraped_page ) : <EOL> raise NotImplementedError ( ) <EOL> def extract ( self , element , scraped_page ) : <EOL> if self . _extract : <EOL> value = self . _extract ( element ) <EOL> elif self . attribute is None : <EOL> value = element . text_content ( ) <EOL> else : <EOL> value = element . get ( self . attribute ) <EOL> if value is None : <EOL> return <EOL> if six . PY2 and isinstance ( value , str ) : <EOL> value = unicode ( value ) <EOL> return self . perform_cleanups ( value , element , scraped_page ) <EOL> def perform_cleanups ( self , value , element , scraped_page = None ) : <EOL> if self . _cleanup : <EOL> value = self . _cleanup ( value ) <EOL> if self . _cleanup_method : <EOL> value = self . _cleanup_method ( scraped_page , value , element ) <EOL> return self . cleanup ( value , element , scraped_page ) <EOL> def cleanup ( self , value , elements , scraped_page = None ) : <EOL> return value <EOL> def __call__ ( self , func ) : <EOL> self . _cleanup_method = func <EOL> return self <EOL> _SCRAPER_CLASSES = { } <EOL> class _ScrapedMeta ( type ) : <EOL> """<STR_LIT>""" <EOL> def __new__ ( cls , name , bases , namespace ) : <EOL> keys = [ ] <EOL> for key , value in namespace . items ( ) : <EOL> if isinstance ( value , ScrapedAttribute ) : <EOL> def mk_attribute ( selector ) : <EOL> def method ( scraped ) : <EOL> return scraped . _get_value ( selector ) <EOL> return property ( method ) <EOL> namespace [ key ] = mk_attribute ( value ) <EOL> keys . append ( key ) <EOL> result = super ( _ScrapedMeta , cls ) . __new__ ( cls , name , bases , namespace ) <EOL> result . scrape_keys = keys <EOL> _SCRAPER_CLASSES [ name ] = result <EOL> return result <EOL> @ six . add_metaclass ( _ScrapedMeta ) <EOL> class ScrapedPage ( object ) : <EOL> _scrape_doc = None <EOL> scrape_url = None <EOL> scrape_args = [ ] <EOL> scrape_arg_defaults = { } <EOL> scrape_headers = { } <EOL> def __init__ ( self , * pargs , ** kwargs ) : <EOL> scrape_url = kwargs . pop ( "<STR_LIT>" , None ) <EOL> scrape_referer = kwargs . pop ( "<STR_LIT>" , None ) <EOL> arguments = dict ( self . scrape_arg_defaults ) <EOL> arguments . update ( kwargs ) <EOL> arguments . update ( zip ( self . scrape_args , pargs ) ) <EOL> self . scrape_args = arguments <EOL> if scrape_url : <EOL> self . scrape_url = scrape_url <EOL> elif not self . scrape_url : <EOL> raise ValueError ( "<STR_LIT>" % <EOL> type ( self ) . __name__ ) <EOL> else : <EOL> self . scrape_url = self . scrape_url % arguments <EOL> self . scrape_headers = dict ( self . scrape_headers ) <EOL> if scrape_referer : <EOL> self . scrape_headers [ '<STR_LIT>' ] = scrape_referer <EOL> @ property <EOL> def scrape_session ( self ) : <EOL> return SHARED_SESSION <EOL> def scrape_fetch ( self , url ) : <EOL> return self . scrape_session . get ( url , <EOL> headers = self . scrape_headers ) . text <EOL> def scrape_create_document ( self , page ) : <EOL> return lxml . html . fromstring ( page ) <EOL> def _get_value ( self , property_scraper ) : <EOL> if self . _scrape_doc is None : <EOL> page = self . scrape_fetch ( self . scrape_url ) <EOL> self . _scrape_doc = self . scrape_create_document ( page ) <EOL> return property_scraper . get ( self . _scrape_doc , scraped_page = self ) <EOL> @ property <EOL> def _dict ( self ) : <EOL> return dict ( ( key , getattr ( self , key ) ) for key in self . scrape_keys ) <EOL> def __repr__ ( self ) : <EOL> return "<STR_LIT>" % ( type ( self ) . __name__ , self . scrape_url ) <EOL> class Css ( ScrapedAttribute ) : <EOL> def __init__ ( self , selector , ** kwargs ) : <EOL> self . selector = selector <EOL> super ( Css , self ) . __init__ ( ** kwargs ) <EOL> def get ( self , doc , scraped_page ) : <EOL> assert doc is not None <EOL> elements = doc . cssselect ( self . selector ) <EOL> if self . multiple : <EOL> values = [ self . extract ( element , scraped_page ) <EOL> for element in elements ] <EOL> return [ v for v in values if v is not None ] <EOL> elif len ( elements ) : <EOL> return self . extract ( elements [ <NUM_LIT:0> ] , scraped_page ) <EOL> class CssFloat ( Css ) : <EOL> def cleanup ( self , value , elements , scraped_page = None ) : <EOL> try : <EOL> return float ( value ) <EOL> except ValueError : <EOL> return None <EOL> class CssInt ( Css ) : <EOL> def cleanup ( self , value , elements , scraped_page = None ) : <EOL> try : <EOL> return int ( value ) <EOL> except ValueError : <EOL> return None <EOL> class CssDate ( Css ) : <EOL> def __init__ ( self , selector , date_format , ** kwargs ) : <EOL> self . date_format = date_format <EOL> super ( CssDate , self ) . __init__ ( selector , ** kwargs ) <EOL> def cleanup ( self , value , elements , scraped_page = None ) : <EOL> try : <EOL> return datetime . datetime . strptime ( value , self . date_format ) <EOL> except ValueError : <EOL> return None <EOL> class CssBoolean ( Css ) : <EOL> def cleanup ( self , value , elements , scraped_page = None ) : <EOL> return True <EOL> class CssRaw ( Css ) : <EOL> def extract ( self , element , scraped_page ) : <EOL> value = '<STR_LIT>' . join ( lxml . html . tostring ( child , encoding = "<STR_LIT>" ) <EOL> for child in element ) <EOL> return self . perform_cleanups ( value , element , scraped_page ) <EOL> class CssMulti ( Css ) : <EOL> def __init__ ( self , selector , cleanup = None , ** subselectors ) : <EOL> super ( CssMulti , self ) . __init__ ( selector , cleanup = cleanup , <EOL> multiple = True ) <EOL> self . subselectors = subselectors <EOL> warnings . warn ( <EOL> "<STR_LIT>" , <EOL> DeprecationWarning ) <EOL> def extract ( self , element , scraped_page = None ) : <EOL> value = { } <EOL> for key , selector in self . subselectors . items ( ) : <EOL> value [ key ] = selector . get ( element , <EOL> scraped_page = scraped_page ) <EOL> return self . perform_cleanups ( value , element , scraped_page ) <EOL> class CssGroup ( Css ) : <EOL> class _CompoundAttribute ( object ) : <EOL> def __init__ ( self , parent , element , scraped_page ) : <EOL> self . _subselectors = parent . _subselectors <EOL> self . _element = element <EOL> self . _scaped_page = scraped_page <EOL> def __getattr__ ( self , attribute ) : <EOL> try : <EOL> selector = self . _subselectors [ attribute ] <EOL> except KeyError : <EOL> return getattr ( super ( CssGroup . _CompoundAttribute , self ) , <EOL> attribute ) <EOL> return selector . get ( self . _element , self . _scaped_page ) <EOL> def __getitem__ ( self , attribute ) : <EOL> selector = self . _subselectors [ attribute ] <EOL> return selector . get ( self . _element , self . _scaped_page ) <EOL> def __dir__ ( self ) : <EOL> attrs = dir ( super ( CssGroup . _CompoundAttribute , self ) ) <EOL> attrs += self . _subselectors . keys ( ) <EOL> return attrs <EOL> def _dict ( self ) : <EOL> return dict ( <EOL> ( key , selector . get ( self . _element , self . _scaped_page ) ) <EOL> for ( key , selector ) in self . _subselectors . items ( ) ) <EOL> def __init__ ( self , * pargs , ** kwargs ) : <EOL> super ( CssGroup , self ) . __init__ ( * pargs , ** kwargs ) <EOL> self . _subselectors = { } <EOL> def extract ( self , element , scraped_page = None ) : <EOL> value = CssGroup . _CompoundAttribute ( self , element , scraped_page ) <EOL> return self . perform_cleanups ( value , element , scraped_page ) <EOL> def __setattr__ ( self , key , value ) : <EOL> if isinstance ( value , ScrapedAttribute ) : <EOL> self . _subselectors [ key ] = value <EOL> else : <EOL> super ( CssGroup , self ) . __setattr__ ( key , value ) <EOL> class CssLink ( Css ) : <EOL> def __init__ ( self , selector , page_factory , referer = True , ** kwargs ) : <EOL> super ( CssLink , self ) . __init__ ( selector , attribute = "<STR_LIT>" , ** kwargs ) <EOL> self . page_factory = page_factory <EOL> self . referer = referer <EOL> def cleanup ( self , value , elements , scraped_page = None ) : <EOL> url = urlparse . urljoin ( scraped_page . scrape_url , value ) <EOL> factory = ( _SCRAPER_CLASSES [ self . page_factory ] <EOL> if isinstance ( self . page_factory , six . string_types ) <EOL> else self . page_factory ) <EOL> if self . referer is True : <EOL> referer = scraped_page . scrape_url <EOL> elif not self . referer : <EOL> referer = None <EOL> else : <EOL> referer = self . referer <EOL> return factory ( scrape_url = url , scrape_referer = referer ) </s>
