<s> """<STR_LIT>""" <EOL> from collections import defaultdict <EOL> import six <EOL> from scrapy . item import Item <EOL> from scrapy . selector import Selector <EOL> from scrapy . utils . decorators import deprecated <EOL> from scrapy . utils . deprecate import create_deprecated_class <EOL> from scrapy . utils . misc import arg_to_iter , extract_regex <EOL> from scrapy . utils . python import flatten <EOL> from . common import wrap_loader_context <EOL> from . processors import Identity <EOL> class ItemLoader ( object ) : <EOL> default_item_class = Item <EOL> default_input_processor = Identity ( ) <EOL> default_output_processor = Identity ( ) <EOL> default_selector_class = Selector <EOL> def __init__ ( self , item = None , selector = None , response = None , parent = None , ** context ) : <EOL> if selector is None and response is not None : <EOL> selector = self . default_selector_class ( response ) <EOL> self . selector = selector <EOL> context . update ( selector = selector , response = response ) <EOL> if item is None : <EOL> item = self . default_item_class ( ) <EOL> self . context = context <EOL> self . parent = parent <EOL> self . _local_item = context [ '<STR_LIT>' ] = item <EOL> self . _local_values = defaultdict ( list ) <EOL> @ property <EOL> def _values ( self ) : <EOL> if self . parent is not None : <EOL> return self . parent . _values <EOL> else : <EOL> return self . _local_values <EOL> @ property <EOL> def item ( self ) : <EOL> if self . parent is not None : <EOL> return self . parent . item <EOL> else : <EOL> return self . _local_item <EOL> def nested_xpath ( self , xpath , ** context ) : <EOL> selector = self . selector . xpath ( xpath ) <EOL> context . update ( selector = selector ) <EOL> subloader = self . __class__ ( <EOL> item = self . item , parent = self , ** context <EOL> ) <EOL> return subloader <EOL> def nested_css ( self , css , ** context ) : <EOL> selector = self . selector . css ( css ) <EOL> context . update ( selector = selector ) <EOL> subloader = self . __class__ ( <EOL> item = self . item , parent = self , ** context <EOL> ) <EOL> return subloader <EOL> def add_value ( self , field_name , value , * processors , ** kw ) : <EOL> value = self . get_value ( value , * processors , ** kw ) <EOL> if value is None : <EOL> return <EOL> if not field_name : <EOL> for k , v in six . iteritems ( value ) : <EOL> self . _add_value ( k , v ) <EOL> else : <EOL> self . _add_value ( field_name , value ) <EOL> def replace_value ( self , field_name , value , * processors , ** kw ) : <EOL> value = self . get_value ( value , * processors , ** kw ) <EOL> if value is None : <EOL> return <EOL> if not field_name : <EOL> for k , v in six . iteritems ( value ) : <EOL> self . _replace_value ( k , v ) <EOL> else : <EOL> self . _replace_value ( field_name , value ) <EOL> def _add_value ( self , field_name , value ) : <EOL> value = arg_to_iter ( value ) <EOL> processed_value = self . _process_input_value ( field_name , value ) <EOL> if processed_value : <EOL> self . _values [ field_name ] += arg_to_iter ( processed_value ) <EOL> def _replace_value ( self , field_name , value ) : <EOL> self . _values . pop ( field_name , None ) <EOL> self . _add_value ( field_name , value ) <EOL> def get_value ( self , value , * processors , ** kw ) : <EOL> regex = kw . get ( '<STR_LIT>' , None ) <EOL> if regex : <EOL> value = arg_to_iter ( value ) <EOL> value = flatten ( extract_regex ( regex , x ) for x in value ) <EOL> for proc in processors : <EOL> if value is None : <EOL> break <EOL> proc = wrap_loader_context ( proc , self . context ) <EOL> value = proc ( value ) <EOL> return value <EOL> def load_item ( self ) : <EOL> item = self . item <EOL> for field_name in tuple ( self . _values ) : <EOL> value = self . get_output_value ( field_name ) <EOL> if value is not None : <EOL> item [ field_name ] = value <EOL> return item <EOL> def get_output_value ( self , field_name ) : <EOL> proc = self . get_output_processor ( field_name ) <EOL> proc = wrap_loader_context ( proc , self . context ) <EOL> try : <EOL> return proc ( self . _values [ field_name ] ) <EOL> except Exception as e : <EOL> raise ValueError ( "<STR_LIT>" % ( field_name , self . _values [ field_name ] , type ( e ) . __name__ , str ( e ) ) ) <EOL> def get_collected_values ( self , field_name ) : <EOL> return self . _values [ field_name ] <EOL> def get_input_processor ( self , field_name ) : <EOL> proc = getattr ( self , '<STR_LIT>' % field_name , None ) <EOL> if not proc : <EOL> proc = self . _get_item_field_attr ( field_name , '<STR_LIT>' , self . default_input_processor ) <EOL> return proc <EOL> def get_output_processor ( self , field_name ) : <EOL> proc = getattr ( self , '<STR_LIT>' % field_name , None ) <EOL> if not proc : <EOL> proc = self . _get_item_field_attr ( field_name , '<STR_LIT>' , self . default_output_processor ) <EOL> return proc <EOL> def _process_input_value ( self , field_name , value ) : <EOL> proc = self . get_input_processor ( field_name ) <EOL> proc = wrap_loader_context ( proc , self . context ) <EOL> return proc ( value ) <EOL> def _get_item_field_attr ( self , field_name , key , default = None ) : <EOL> if isinstance ( self . item , Item ) : <EOL> value = self . item . fields [ field_name ] . get ( key , default ) <EOL> else : <EOL> value = default <EOL> return value <EOL> def _check_selector_method ( self ) : <EOL> if self . selector is None : <EOL> raise RuntimeError ( "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" % self . __class__ . __name__ ) <EOL> def add_xpath ( self , field_name , xpath , * processors , ** kw ) : <EOL> values = self . _get_xpathvalues ( xpath , ** kw ) <EOL> self . add_value ( field_name , values , * processors , ** kw ) <EOL> def replace_xpath ( self , field_name , xpath , * processors , ** kw ) : <EOL> values = self . _get_xpathvalues ( xpath , ** kw ) <EOL> self . replace_value ( field_name , values , * processors , ** kw ) <EOL> def get_xpath ( self , xpath , * processors , ** kw ) : <EOL> values = self . _get_xpathvalues ( xpath , ** kw ) <EOL> return self . get_value ( values , * processors , ** kw ) <EOL> @ deprecated ( use_instead = '<STR_LIT>' ) <EOL> def _get_values ( self , xpaths , ** kw ) : <EOL> return self . _get_xpathvalues ( xpaths , ** kw ) <EOL> def _get_xpathvalues ( self , xpaths , ** kw ) : <EOL> self . _check_selector_method ( ) <EOL> xpaths = arg_to_iter ( xpaths ) <EOL> return flatten ( self . selector . xpath ( xpath ) . extract ( ) for xpath in xpaths ) <EOL> def add_css ( self , field_name , css , * processors , ** kw ) : <EOL> values = self . _get_cssvalues ( css , ** kw ) <EOL> self . add_value ( field_name , values , * processors , ** kw ) <EOL> def replace_css ( self , field_name , css , * processors , ** kw ) : <EOL> values = self . _get_cssvalues ( css , ** kw ) <EOL> self . replace_value ( field_name , values , * processors , ** kw ) <EOL> def get_css ( self , css , * processors , ** kw ) : <EOL> values = self . _get_cssvalues ( css , ** kw ) <EOL> return self . get_value ( values , * processors , ** kw ) <EOL> def _get_cssvalues ( self , csss , ** kw ) : <EOL> self . _check_selector_method ( ) <EOL> csss = arg_to_iter ( csss ) <EOL> return flatten ( self . selector . css ( css ) . extract ( ) for css in csss ) <EOL> XPathItemLoader = create_deprecated_class ( '<STR_LIT>' , ItemLoader ) </s>
