<s> """<STR_LIT>""" <EOL> import re <EOL> import hashlib <EOL> from importlib import import_module <EOL> from pkgutil import iter_modules <EOL> from w3lib . html import replace_entities <EOL> from ants . utils . python import flatten <EOL> from ants . item import BaseItem <EOL> def arg_to_iter ( arg ) : <EOL> """<STR_LIT>""" <EOL> if arg is None : <EOL> return [ ] <EOL> elif not isinstance ( arg , ( dict , BaseItem ) ) and hasattr ( arg , '<STR_LIT>' ) : <EOL> return arg <EOL> else : <EOL> return [ arg ] <EOL> def load_object ( path ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> dot = path . rindex ( '<STR_LIT:.>' ) <EOL> except ValueError : <EOL> raise ValueError ( "<STR_LIT>" % path ) <EOL> module , name = path [ : dot ] , path [ dot + <NUM_LIT:1> : ] <EOL> try : <EOL> mod = import_module ( module ) <EOL> except ImportError as e : <EOL> raise ImportError ( "<STR_LIT>" % ( path , e ) ) <EOL> try : <EOL> obj = getattr ( mod , name ) <EOL> except AttributeError : <EOL> raise NameError ( "<STR_LIT>" % ( module , name ) ) <EOL> return obj <EOL> def walk_modules ( path , load = False ) : <EOL> """<STR_LIT>""" <EOL> mods = [ ] <EOL> mod = import_module ( path ) <EOL> mods . append ( mod ) <EOL> if hasattr ( mod , '<STR_LIT>' ) : <EOL> for _ , subpath , ispkg in iter_modules ( mod . __path__ ) : <EOL> fullpath = path + '<STR_LIT:.>' + subpath <EOL> if ispkg : <EOL> mods += walk_modules ( fullpath ) <EOL> else : <EOL> submod = import_module ( fullpath ) <EOL> mods . append ( submod ) <EOL> return mods <EOL> def extract_regex ( regex , text , encoding = '<STR_LIT:utf-8>' ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( regex , basestring ) : <EOL> regex = re . compile ( regex , re . UNICODE ) <EOL> try : <EOL> strings = [ regex . search ( text ) . group ( '<STR_LIT>' ) ] <EOL> except : <EOL> strings = regex . findall ( text ) <EOL> strings = flatten ( strings ) <EOL> if isinstance ( text , unicode ) : <EOL> return [ replace_entities ( s , keep = [ '<STR_LIT>' , '<STR_LIT>' ] ) for s in strings ] <EOL> else : <EOL> return [ replace_entities ( unicode ( s , encoding ) , keep = [ '<STR_LIT>' , '<STR_LIT>' ] ) for s in strings ] <EOL> def md5sum ( file ) : <EOL> """<STR_LIT>""" <EOL> m = hashlib . md5 ( ) <EOL> while <NUM_LIT:1> : <EOL> d = file . read ( <NUM_LIT> ) <EOL> if not d : <EOL> break <EOL> m . update ( d ) <EOL> return m . hexdigest ( ) </s>
