<s> import os <EOL> import re <EOL> import urllib <EOL> import urlparse <EOL> import requests <EOL> import cookielib <EOL> import socket <EOL> from HTMLParser import HTMLParser <EOL> from fileUtils import fileExists , setFileContent , getFileContent <EOL> socket . setdefaulttimeout ( <NUM_LIT:30> ) <EOL> origGetAddrInfo = socket . getaddrinfo <EOL> def getAddrInfoWrapper ( host , port , family = <NUM_LIT:0> , socktype = <NUM_LIT:0> , proto = <NUM_LIT:0> , flags = <NUM_LIT:0> ) : <EOL> return origGetAddrInfo ( host , port , socket . AF_INET , socktype , proto , flags ) <EOL> socket . getaddrinfo = getAddrInfoWrapper <EOL> '''<STR_LIT>''' <EOL> class BaseRequest ( object ) : <EOL> def __init__ ( self , cookie_file = None ) : <EOL> self . cookie_file = cookie_file <EOL> self . s = requests . Session ( ) <EOL> if fileExists ( self . cookie_file ) : <EOL> self . s . cookies = self . load_cookies_from_lwp ( self . cookie_file ) <EOL> self . s . headers . update ( { '<STR_LIT>' : '<STR_LIT>' } ) <EOL> self . s . headers . update ( { '<STR_LIT>' : '<STR_LIT>' } ) <EOL> self . s . keep_alive = False <EOL> self . url = '<STR_LIT>' <EOL> def save_cookies_lwp ( self , cookiejar , filename ) : <EOL> lwp_cookiejar = cookielib . LWPCookieJar ( ) <EOL> for c in cookiejar : <EOL> args = dict ( vars ( c ) . items ( ) ) <EOL> args [ '<STR_LIT>' ] = args [ '<STR_LIT>' ] <EOL> del args [ '<STR_LIT>' ] <EOL> c = cookielib . Cookie ( ** args ) <EOL> lwp_cookiejar . set_cookie ( c ) <EOL> lwp_cookiejar . save ( filename , ignore_discard = True ) <EOL> def load_cookies_from_lwp ( self , filename ) : <EOL> lwp_cookiejar = cookielib . LWPCookieJar ( ) <EOL> lwp_cookiejar . load ( filename , ignore_discard = True ) <EOL> return lwp_cookiejar <EOL> def fixurl ( self , url ) : <EOL> try : <EOL> url = url . encode ( '<STR_LIT:ascii>' ) <EOL> except : <EOL> parsed_link = urlparse . urlsplit ( url ) <EOL> parsed_link = parsed_link . _replace ( netloc = parsed_link . netloc . encode ( '<STR_LIT>' ) , path = urllib . quote ( parsed_link . path . encode ( '<STR_LIT:utf-8>' ) ) ) <EOL> url = parsed_link . geturl ( ) . encode ( '<STR_LIT:ascii>' ) <EOL> return url <EOL> def getSource ( self , url , form_data , referer , xml = False , mobile = False ) : <EOL> url = self . fixurl ( url ) <EOL> if not referer : <EOL> referer = url <EOL> else : <EOL> referer = self . fixurl ( referer ) <EOL> headers = { '<STR_LIT>' : referer } <EOL> if mobile : <EOL> self . s . headers . update ( { '<STR_LIT>' : '<STR_LIT>' } ) <EOL> if xml : <EOL> headers [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> if '<STR_LIT>' in urlparse . urlsplit ( url ) . netloc : <EOL> headers [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> if '<STR_LIT>' in urlparse . urlsplit ( url ) . netloc : <EOL> headers [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> if '<STR_LIT>' in urlparse . urlsplit ( url ) . netloc : <EOL> headers [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> if '<STR_LIT>' in urlparse . urlsplit ( url ) . netloc : <EOL> self . s . headers . update ( { '<STR_LIT>' : '<STR_LIT>' } ) <EOL> if form_data : <EOL> if '<STR_LIT>' in form_data [ <NUM_LIT:0> ] : <EOL> form_data [ <NUM_LIT:0> ] = ( '<STR_LIT>' , urllib . quote ( self . s . headers [ '<STR_LIT>' ] ) ) <EOL> if '<STR_LIT>' in form_data [ <NUM_LIT:0> ] : <EOL> import random <EOL> cotok = str ( random . randrange ( <NUM_LIT> , <NUM_LIT> ) ) <EOL> form_data [ <NUM_LIT:0> ] = ( '<STR_LIT>' , cotok ) <EOL> r = self . s . post ( url , headers = headers , data = form_data , timeout = <NUM_LIT:20> , cookies = { '<STR_LIT>' : cotok } ) <EOL> else : <EOL> r = self . s . post ( url , headers = headers , data = form_data , timeout = <NUM_LIT:20> ) <EOL> response = r . text <EOL> else : <EOL> try : <EOL> r = self . s . get ( url , headers = headers , timeout = <NUM_LIT:20> ) <EOL> response = r . text <EOL> except ( requests . exceptions . MissingSchema ) : <EOL> response = '<STR_LIT>' <EOL> print ( "<STR_LIT>" , len ( response ) ) <EOL> if self . cookie_file : <EOL> self . save_cookies_lwp ( self . s . cookies , self . cookie_file ) <EOL> return HTMLParser ( ) . unescape ( response ) <EOL> class DemystifiedWebRequest ( BaseRequest ) : <EOL> def __init__ ( self , cookiePath ) : <EOL> super ( DemystifiedWebRequest , self ) . __init__ ( cookiePath ) <EOL> def getSource ( self , url , form_data , referer = '<STR_LIT>' , xml = False , mobile = False , demystify = False ) : <EOL> data = super ( DemystifiedWebRequest , self ) . getSource ( url , form_data , referer , xml , mobile ) <EOL> if not data : <EOL> return None <EOL> if not demystify : <EOL> r = re . compile ( '<STR_LIT>' , re . IGNORECASE + re . DOTALL + re . MULTILINE ) <EOL> m = r . findall ( data ) <EOL> if m : <EOL> for comment in m : <EOL> data = data . replace ( comment , '<STR_LIT>' ) <EOL> else : <EOL> import decryptionUtils as crypt <EOL> data = crypt . doDemystify ( data ) <EOL> return data <EOL> class CachedWebRequest ( DemystifiedWebRequest ) : <EOL> def __init__ ( self , cookiePath , cachePath ) : <EOL> super ( CachedWebRequest , self ) . __init__ ( cookiePath ) <EOL> self . cachePath = cachePath <EOL> self . cachedSourcePath = os . path . join ( self . cachePath , '<STR_LIT>' ) <EOL> self . currentUrlPath = os . path . join ( self . cachePath , '<STR_LIT>' ) <EOL> self . lastUrlPath = os . path . join ( self . cachePath , '<STR_LIT>' ) <EOL> def __setLastUrl ( self , url ) : <EOL> setFileContent ( self . lastUrlPath , url ) <EOL> def __getCachedSource ( self ) : <EOL> try : <EOL> data = getFileContent ( self . cachedSourcePath ) <EOL> except : <EOL> pass <EOL> return data <EOL> def getLastUrl ( self ) : <EOL> return getFileContent ( self . lastUrlPath ) <EOL> def getSource ( self , url , form_data , referer = '<STR_LIT>' , xml = False , mobile = False , ignoreCache = False , demystify = False ) : <EOL> if '<STR_LIT>' in url : <EOL> self . cachedSourcePath = url <EOL> data = self . __getCachedSource ( ) <EOL> return data <EOL> if url == self . getLastUrl ( ) and not ignoreCache : <EOL> data = self . __getCachedSource ( ) <EOL> else : <EOL> data = super ( CachedWebRequest , self ) . getSource ( url , form_data , referer , xml , mobile , demystify ) <EOL> if data : <EOL> self . __setLastUrl ( url ) <EOL> setFileContent ( self . cachedSourcePath , data ) <EOL> return data </s>
