<s> import base64 <EOL> from urllib import getproxies , unquote , proxy_bypass <EOL> from urllib2 import _parse_proxy <EOL> from urlparse import urlunparse <EOL> from ants . utils . httpobj import urlparse_cached <EOL> from ants . utils . exceptions import NotConfigured <EOL> class HttpProxyMiddleware ( object ) : <EOL> def __init__ ( self ) : <EOL> self . proxies = { } <EOL> for type , url in getproxies ( ) . items ( ) : <EOL> self . proxies [ type ] = self . _get_proxy ( url , type ) <EOL> if not self . proxies : <EOL> raise NotConfigured <EOL> def _get_proxy ( self , url , orig_type ) : <EOL> proxy_type , user , password , hostport = _parse_proxy ( url ) <EOL> proxy_url = urlunparse ( ( proxy_type or orig_type , hostport , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> if user and password : <EOL> user_pass = '<STR_LIT>' % ( unquote ( user ) , unquote ( password ) ) <EOL> creds = base64 . b64encode ( user_pass ) . strip ( ) <EOL> else : <EOL> creds = None <EOL> return creds , proxy_url <EOL> def process_request ( self , request , spider ) : <EOL> if '<STR_LIT>' in request . meta : <EOL> return <EOL> parsed = urlparse_cached ( request ) <EOL> scheme = parsed . scheme <EOL> if scheme in ( '<STR_LIT:http>' , '<STR_LIT>' ) and proxy_bypass ( parsed . hostname ) : <EOL> return <EOL> if scheme in self . proxies : <EOL> self . _set_proxy ( request , scheme ) <EOL> def _set_proxy ( self , request , scheme ) : <EOL> creds , proxy = self . proxies [ scheme ] <EOL> request . meta [ '<STR_LIT>' ] = proxy <EOL> if creds : <EOL> request . headers [ '<STR_LIT>' ] = '<STR_LIT>' + creds </s>
