<s> '''<STR_LIT>''' <EOL> import urllib2 <EOL> import cookielib <EOL> import gzip <EOL> import socket <EOL> from cola . core . errors import DependencyNotInstalledError <EOL> class Opener ( object ) : <EOL> def open ( self , url ) : <EOL> raise NotImplementedError <EOL> def read ( self ) : <EOL> raise NotImplementedError <EOL> def ungzip ( self , fileobj ) : <EOL> gz = gzip . GzipFile ( fileobj = fileobj , mode = '<STR_LIT:rb>' ) <EOL> try : <EOL> return gz . read ( ) <EOL> finally : <EOL> gz . close ( ) <EOL> class BuiltinOpener ( Opener ) : <EOL> def __init__ ( self , cookie_filename = None , timeout = None , ** kwargs ) : <EOL> self . cj = cookielib . LWPCookieJar ( ) <EOL> if cookie_filename is not None : <EOL> self . cj . load ( cookie_filename ) <EOL> self . cookie_processor = urllib2 . HTTPCookieProcessor ( self . cj ) <EOL> self . _build_opener ( ) <EOL> urllib2 . install_opener ( self . opener ) <EOL> if timeout is None : <EOL> self . _default_timeout = socket . _GLOBAL_DEFAULT_TIMEOUT <EOL> else : <EOL> self . _default_timeout = timeout <EOL> def _build_opener ( self ) : <EOL> self . opener = urllib2 . build_opener ( self . cookie_processor , urllib2 . HTTPHandler ) <EOL> def open ( self , url , data = None , timeout = None ) : <EOL> if timeout is None : <EOL> timeout = self . _default_timeout <EOL> resp = urllib2 . urlopen ( url , data = data , timeout = timeout ) <EOL> is_gzip = resp . headers . dict . get ( '<STR_LIT>' ) == '<STR_LIT>' <EOL> if is_gzip : <EOL> return self . ungzip ( resp ) <EOL> self . content = resp . read ( ) <EOL> return self . content <EOL> def read ( self ) : <EOL> return self . content if hasattr ( self , '<STR_LIT:content>' ) else None <EOL> def add_proxy ( self , addr , proxy_type = '<STR_LIT:all>' , <EOL> user = None , password = None ) : <EOL> if proxy_type == '<STR_LIT:all>' : <EOL> self . proxies = { '<STR_LIT:http>' : addr , '<STR_LIT>' : addr , '<STR_LIT>' : addr } <EOL> else : <EOL> self . proxies [ proxy_type ] = addr <EOL> proxy_handler = urllib2 . ProxyHandler ( self . proxies ) <EOL> self . _build_opener ( ) <EOL> self . opener . add_handler ( proxy_handler ) <EOL> if user and password : <EOL> pwd_manager = urllib2 . HTTPPasswordMgrWithDefaultRealm ( ) <EOL> pwd_manager . add_password ( None , addr , user , password ) <EOL> proxy_auth_handler = urllib2 . ProxyBasicAuthHandler ( pwd_manager ) <EOL> self . opener . add_handler ( proxy_auth_handler ) <EOL> urllib2 . install_opener ( self . opener ) <EOL> def remove_proxy ( self ) : <EOL> self . _build_opener ( ) <EOL> urllib2 . install_opener ( self . opener ) <EOL> class MechanizeOpener ( Opener ) : <EOL> def __init__ ( self , cookie_filename = None , user_agent = None , timeout = None , ** kwargs ) : <EOL> try : <EOL> import mechanize <EOL> except ImportError : <EOL> raise DependencyNotInstalledError ( '<STR_LIT>' ) <EOL> if user_agent is None : <EOL> user_agent = '<STR_LIT>' <EOL> self . browser = mechanize . Browser ( ) <EOL> self . cj = cookielib . LWPCookieJar ( ) <EOL> if cookie_filename is not None : <EOL> self . cj . load ( cookie_filename ) <EOL> self . browser . set_cookiejar ( self . cj ) <EOL> self . browser . set_handle_equiv ( True ) <EOL> self . browser . set_handle_gzip ( True ) <EOL> self . browser . set_handle_redirect ( True ) <EOL> self . browser . set_handle_referer ( True ) <EOL> self . browser . set_handle_robots ( False ) <EOL> self . browser . addheaders = [ <EOL> ( '<STR_LIT>' , user_agent ) ] <EOL> if timeout is None : <EOL> self . _default_timout = mechanize . _sockettimeout . _GLOBAL_DEFAULT_TIMEOUT <EOL> else : <EOL> self . _default_timout = timeout <EOL> def set_default_timeout ( self , timeout ) : <EOL> self . _default_timout = timeout <EOL> def open ( self , url , data = None , timeout = None ) : <EOL> self . _clear_content ( ) <EOL> if timeout is None : <EOL> timeout = self . _default_timout <EOL> self . content = self . browser . open ( url , data = data , timeout = timeout ) . read ( ) <EOL> return self . content <EOL> def add_proxy ( self , addr , proxy_type = '<STR_LIT:all>' , <EOL> user = None , password = None ) : <EOL> if proxy_type == '<STR_LIT:all>' : <EOL> self . proxies = { '<STR_LIT:http>' : addr , '<STR_LIT>' : addr , '<STR_LIT>' : addr } <EOL> else : <EOL> self . proxies [ proxy_type ] = addr <EOL> self . browser . set_proxies ( proxies = self . proxies ) <EOL> if user and password : <EOL> self . browser . add_proxy_password ( user , password ) <EOL> def remove_proxy ( self ) : <EOL> self . browser . set_proxies ( { } ) <EOL> self . proxies = { } <EOL> def browse_open ( self , url , data = None , timeout = None ) : <EOL> if timeout is None : <EOL> timeout = self . _default_timout <EOL> self . _clear_content ( ) <EOL> self . browser . open ( url , data = data , timeout = timeout ) <EOL> return self . browser <EOL> def read ( self ) : <EOL> if hasattr ( self , '<STR_LIT:content>' ) : <EOL> return self . content <EOL> elif self . browser . response ( ) is not None : <EOL> self . content = self . browser . response ( ) . read ( ) <EOL> return self . content <EOL> def _clear_content ( self ) : <EOL> if hasattr ( self , '<STR_LIT:content>' ) : <EOL> del self . content <EOL> def close ( self ) : <EOL> self . _clear_content ( ) <EOL> resp = self . browser . response ( ) <EOL> if resp is not None : <EOL> resp . close ( ) <EOL> self . browser . clear_history ( ) <EOL> class SpynnerOpener ( Opener ) : <EOL> def __init__ ( self , user_agent = None , ** kwargs ) : <EOL> try : <EOL> import spynner <EOL> except ImportError : <EOL> raise DependencyNotInstalledError ( '<STR_LIT>' ) <EOL> if user_agent is None : <EOL> user_agent = '<STR_LIT>' <EOL> self . br = spynner . Browser ( user_agent = user_agent , ** kwargs ) <EOL> def spynner_open ( self , url , data = None , headers = None , method = '<STR_LIT:GET>' , <EOL> wait_for_text = None , wait_for_selector = None , tries = None ) : <EOL> try : <EOL> from PyQt4 . QtNetwork import QNetworkAccessManager <EOL> except ImportError : <EOL> raise DependencyNotInstalledError ( '<STR_LIT>' ) <EOL> if wait_for_text is not None : <EOL> def wait_callback ( br ) : <EOL> return wait_for_text in br . html <EOL> elif wait_for_selector is not None : <EOL> def wait_callback ( br ) : <EOL> return not br . webframe . findFirstElement ( wait_for_selector ) . isNull ( ) <EOL> else : <EOL> wait_callback = None <EOL> operation = QNetworkAccessManager . GetOperation <EOL> if method == '<STR_LIT:POST>' : <EOL> operation = QNetworkAccessManager . PostOperation <EOL> self . br . load ( url , wait_callback = wait_callback , tries = tries , <EOL> operation = operation , body = data , headers = headers ) <EOL> return self . br <EOL> def open ( self , url , data = None , headers = None , method = '<STR_LIT:GET>' , <EOL> wait_for_text = None , wait_for_selector = None , tries = None ) : <EOL> br = self . spynner_open ( url , data = data , headers = headers , method = method , <EOL> wait_for_text = wait_for_text , tries = tries ) <EOL> self . content = br . contents <EOL> return self . content <EOL> def read ( self ) : <EOL> return self . content if hasattr ( self , '<STR_LIT:content>' ) else self . br . contents <EOL> def wait_for_selector ( self , selector , ** kwargs ) : <EOL> self . br . wait_for_content ( <EOL> lambda br : not br . webframe . findFirstElement ( selector ) . isNull ( ) , <EOL> ** kwargs ) </s>
