<s> import re <EOL> from functools import partial <EOL> from urlparse import urlparse <EOL> from urllib2 import unquote <EOL> import tornado . httpclient <EOL> from . import LoaderResult <EOL> from thumbor . utils import logger <EOL> from tornado . concurrent import return_future <EOL> def quote_url ( url ) : <EOL> return unquote ( url ) . decode ( '<STR_LIT:utf-8>' ) <EOL> def _normalize_url ( url ) : <EOL> url = quote_url ( url ) <EOL> return url if url . startswith ( '<STR_LIT:http>' ) else '<STR_LIT>' % url <EOL> def validate ( context , url , normalize_url_func = _normalize_url ) : <EOL> url = normalize_url_func ( url ) <EOL> res = urlparse ( url ) <EOL> if not res . hostname : <EOL> return False <EOL> if not context . config . ALLOWED_SOURCES : <EOL> return True <EOL> for pattern in context . config . ALLOWED_SOURCES : <EOL> if isinstance ( pattern , re . _pattern_type ) : <EOL> match = url <EOL> else : <EOL> pattern = '<STR_LIT>' % pattern <EOL> match = res . hostname <EOL> if re . match ( pattern , match ) : <EOL> return True <EOL> return False <EOL> def return_contents ( response , url , callback , context ) : <EOL> result = LoaderResult ( ) <EOL> context . metrics . incr ( '<STR_LIT>' + str ( response . code ) ) <EOL> if response . error : <EOL> result . successful = False <EOL> if response . code == <NUM_LIT> : <EOL> result . error = LoaderResult . ERROR_TIMEOUT <EOL> else : <EOL> result . error = LoaderResult . ERROR_NOT_FOUND <EOL> logger . warn ( u"<STR_LIT>" . format ( url , str ( response . error ) ) ) <EOL> elif response . body is None or len ( response . body ) == <NUM_LIT:0> : <EOL> result . successful = False <EOL> result . error = LoaderResult . ERROR_UPSTREAM <EOL> logger . warn ( u"<STR_LIT>" . format ( url ) ) <EOL> else : <EOL> if response . time_info : <EOL> for x in response . time_info : <EOL> context . metrics . timing ( '<STR_LIT>' + x , response . time_info [ x ] * <NUM_LIT:1000> ) <EOL> context . metrics . timing ( '<STR_LIT>' , len ( response . body ) / response . time_info [ '<STR_LIT>' ] ) <EOL> result . buffer = response . body <EOL> callback ( result ) <EOL> @ return_future <EOL> def load ( context , url , callback , normalize_url_func = _normalize_url ) : <EOL> load_sync ( context , url , callback , normalize_url_func ) <EOL> def load_sync ( context , url , callback , normalize_url_func ) : <EOL> using_proxy = context . config . HTTP_LOADER_PROXY_HOST and context . config . HTTP_LOADER_PROXY_PORT <EOL> if using_proxy or context . config . HTTP_LOADER_CURL_ASYNC_HTTP_CLIENT : <EOL> tornado . httpclient . AsyncHTTPClient . configure ( "<STR_LIT>" ) <EOL> client = tornado . httpclient . AsyncHTTPClient ( max_clients = context . config . HTTP_LOADER_MAX_CLIENTS ) <EOL> user_agent = None <EOL> if context . config . HTTP_LOADER_FORWARD_USER_AGENT : <EOL> if '<STR_LIT>' in context . request_handler . request . headers : <EOL> user_agent = context . request_handler . request . headers [ '<STR_LIT>' ] <EOL> if user_agent is None : <EOL> user_agent = context . config . HTTP_LOADER_DEFAULT_USER_AGENT <EOL> url = normalize_url_func ( url ) <EOL> req = tornado . httpclient . HTTPRequest ( <EOL> url = url , <EOL> connect_timeout = context . config . HTTP_LOADER_CONNECT_TIMEOUT , <EOL> request_timeout = context . config . HTTP_LOADER_REQUEST_TIMEOUT , <EOL> follow_redirects = context . config . HTTP_LOADER_FOLLOW_REDIRECTS , <EOL> max_redirects = context . config . HTTP_LOADER_MAX_REDIRECTS , <EOL> user_agent = user_agent , <EOL> proxy_host = encode ( context . config . HTTP_LOADER_PROXY_HOST ) , <EOL> proxy_port = context . config . HTTP_LOADER_PROXY_PORT , <EOL> proxy_username = encode ( context . config . HTTP_LOADER_PROXY_USERNAME ) , <EOL> proxy_password = encode ( context . config . HTTP_LOADER_PROXY_PASSWORD ) , <EOL> ca_certs = encode ( context . config . HTTP_LOADER_CA_CERTS ) , <EOL> client_key = encode ( context . config . HTTP_LOADER_CLIENT_KEY ) , <EOL> client_cert = encode ( context . config . HTTP_LOADER_CLIENT_CERT ) <EOL> ) <EOL> client . fetch ( req , callback = partial ( return_contents , url = url , callback = callback , context = context ) ) <EOL> def encode ( string ) : <EOL> return None if string is None else string . encode ( '<STR_LIT:ascii>' ) </s>
