<s> import datetime <EOL> import json <EOL> from twisted . web import http <EOL> from twisted . internet import defer <EOL> from twisted . web import client <EOL> from zope import interface <EOL> from piped import util , exceptions , processing , yamlutil <EOL> from piped . processors import base <EOL> class HttpRequestProcessor ( base . Processor ) : <EOL> def __init__ ( self , request_path = '<STR_LIT>' , skip_if_request_stopped = True , ** kw ) : <EOL> super ( HttpRequestProcessor , self ) . __init__ ( ** kw ) <EOL> self . request_path = request_path <EOL> self . skip_if_request_stopped = skip_if_request_stopped <EOL> def process ( self , baton ) : <EOL> request = self . get_request_or_fail ( baton ) <EOL> if self . skip_if_request_stopped and not request . channel : <EOL> return baton <EOL> return self . process_request ( request , baton ) <EOL> def get_request_or_fail ( self , baton ) : <EOL> request = util . dict_get_path ( baton , self . request_path ) <EOL> if not request : <EOL> self . _fail_because_request_is_invalid ( ) <EOL> return request <EOL> def _fail_because_request_is_invalid ( self ) : <EOL> raise exceptions . PipedError ( '<STR_LIT>' % self . request_path ) <EOL> def process_request ( self , request , baton ) : <EOL> raise NotImplementedError ( ) <EOL> class ResponseWriter ( HttpRequestProcessor ) : <EOL> """<STR_LIT>""" <EOL> interface . classProvides ( processing . IProcessor ) <EOL> name = '<STR_LIT>' <EOL> def __init__ ( self , content_path = '<STR_LIT:content>' , content_type = None , response_code = None , <EOL> encoding = '<STR_LIT:utf8>' , finish = True , fallback_content = Ellipsis , ** kw ) : <EOL> super ( ResponseWriter , self ) . __init__ ( ** kw ) <EOL> self . content_path = content_path <EOL> self . content_type = content_type <EOL> self . finish = finish <EOL> self . fallback_content = fallback_content <EOL> self . encoding = encoding <EOL> if isinstance ( response_code , basestring ) : <EOL> response_code = getattr ( http , response_code , Ellipsis ) <EOL> self . _fail_if_response_code_is_invalid ( response_code ) <EOL> self . response_code = response_code <EOL> def _fail_if_response_code_is_invalid ( self , response_code ) : <EOL> if response_code is not None and not isinstance ( response_code , int ) : <EOL> e_msg = '<STR_LIT>' <EOL> hint = '<STR_LIT>' <EOL> valid_strings = set ( ) <EOL> for code_name in dir ( http ) : <EOL> code = getattr ( http , code_name , Ellipsis ) <EOL> if isinstance ( code , int ) and code in http . RESPONSES : <EOL> valid_strings . add ( '<STR_LIT>' % code_name ) <EOL> detail = '<STR_LIT>' % ( '<STR_LIT:U+002C>' . join ( valid_strings ) ) <EOL> raise exceptions . ConfigurationError ( e_msg , hint , detail ) <EOL> def process_request ( self , request , baton ) : <EOL> content = util . dict_get_path ( baton , self . content_path , self . fallback_content ) <EOL> if self . response_code : <EOL> request . setResponseCode ( self . response_code ) <EOL> if self . content_type : <EOL> request . setHeader ( '<STR_LIT>' , self . content_type ) <EOL> if isinstance ( content , unicode ) : <EOL> content = content . encode ( self . encoding ) <EOL> if content is not Ellipsis : <EOL> request . write ( content ) <EOL> if self . finish : <EOL> request . finish ( ) <EOL> return baton <EOL> class IPDeterminer ( HttpRequestProcessor ) : <EOL> """<STR_LIT>""" <EOL> interface . classProvides ( processing . IProcessor ) <EOL> name = '<STR_LIT>' <EOL> def __init__ ( self , output_path = '<STR_LIT>' , <EOL> proxied = False , proxy_header = '<STR_LIT>' , fail_if_not_proxied = False , ** kw ) : <EOL> kw . setdefault ( '<STR_LIT>' , False ) <EOL> super ( IPDeterminer , self ) . __init__ ( ** kw ) <EOL> self . output_path = output_path <EOL> self . proxied = proxied <EOL> self . proxy_header = proxy_header <EOL> self . fail_if_not_proxied = fail_if_not_proxied <EOL> def process_request ( self , request , baton ) : <EOL> ip = self . _determine_ip ( request ) <EOL> util . dict_set_path ( baton , self . output_path , ip ) <EOL> return baton <EOL> def _determine_ip ( self , request ) : <EOL> if self . proxied : <EOL> ip = request . getHeader ( self . proxy_header ) <EOL> if ip : <EOL> return ip . split ( '<STR_LIT:U+002C>' ) [ <NUM_LIT:0> ] . strip ( ) <EOL> if self . fail_if_not_proxied : <EOL> e_msg = '<STR_LIT>' <EOL> detail = '<STR_LIT>' % self . proxy_header <EOL> hint = ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> raise exceptions . PipedError ( e_msg , detail , hint ) <EOL> return request . getClientIP ( ) <EOL> class SetHttpHeaders ( HttpRequestProcessor ) : <EOL> """<STR_LIT>""" <EOL> interface . classProvides ( processing . IProcessor ) <EOL> name = '<STR_LIT>' <EOL> def __init__ ( self , headers , ** kw ) : <EOL> super ( SetHttpHeaders , self ) . __init__ ( ** kw ) <EOL> self . headers = headers <EOL> def process_request ( self , request , baton ) : <EOL> for name , value in self . headers . items ( ) : <EOL> request . setHeader ( name , value ) <EOL> return baton <EOL> class SetExpireHeader ( HttpRequestProcessor ) : <EOL> """<STR_LIT>""" <EOL> interface . classProvides ( processing . IProcessor ) <EOL> name = '<STR_LIT>' <EOL> def __init__ ( self , timedelta , ** kw ) : <EOL> super ( SetExpireHeader , self ) . __init__ ( ** kw ) <EOL> self . timedelta_kwargs = timedelta <EOL> def process_request ( self , request , baton ) : <EOL> delta = datetime . timedelta ( ** self . timedelta_kwargs ) <EOL> seconds = <NUM_LIT> * delta . days + delta . seconds <EOL> until = datetime . datetime . now ( ) + delta <EOL> request . setHeader ( '<STR_LIT>' , until . strftime ( '<STR_LIT>' ) . strip ( ) ) <EOL> request . setHeader ( '<STR_LIT>' , '<STR_LIT>' % seconds ) <EOL> return baton <EOL> class ExtractRequestArguments ( base . MappingProcessor ) : <EOL> """<STR_LIT>""" <EOL> interface . classProvides ( processing . IProcessor ) <EOL> name = '<STR_LIT>' <EOL> def __init__ ( self , request_path = '<STR_LIT>' , * a , ** kw ) : <EOL> """<STR_LIT>""" <EOL> super ( ExtractRequestArguments , self ) . __init__ ( * a , ** kw ) <EOL> self . request_path = request_path <EOL> def get_input ( self , baton , input_path , ** kwargs ) : <EOL> request = util . dict_get_path ( baton , self . request_path ) <EOL> return request . args . get ( input_path , Ellipsis ) <EOL> def process_mapping ( self , input , input_path , output_path , baton , only_first = True , load_json = False ) : <EOL> request = util . dict_get_path ( baton , self . request_path ) <EOL> if input_path not in request . args : <EOL> return input <EOL> if load_json : <EOL> for i , value in enumerate ( input ) : <EOL> input [ i ] = json . loads ( value ) <EOL> if only_first : <EOL> return input [ <NUM_LIT:0> ] <EOL> return input <EOL> class ClientGetPage ( base . Processor ) : <EOL> """<STR_LIT>""" <EOL> interface . classProvides ( processing . IProcessor ) <EOL> name = '<STR_LIT>' <EOL> def __init__ ( self , base_url = None , url = yamlutil . BatonPath ( '<STR_LIT:url>' ) , method = '<STR_LIT:GET>' , headers = None , agent = None , timeout = <NUM_LIT:0> , cookies = None , <EOL> follow_redirect = True , redirect_limit = <NUM_LIT:20> , after_found_get = False , output_path = '<STR_LIT>' , postdata = None , * a , ** kw ) : <EOL> """<STR_LIT>""" <EOL> super ( ClientGetPage , self ) . __init__ ( * a , ** kw ) <EOL> self . base_url = base_url <EOL> self . kwargs = dict ( <EOL> url = url , <EOL> method = method , <EOL> headers = headers , <EOL> agent = agent , <EOL> timeout = timeout , <EOL> cookies = cookies , <EOL> followRedirect = follow_redirect , <EOL> redirectLimit = redirect_limit , <EOL> afterFoundGet = after_found_get , <EOL> postdata = postdata <EOL> ) <EOL> self . output_path = output_path <EOL> @ defer . inlineCallbacks <EOL> def process ( self , baton ) : <EOL> kwargs = self . kwargs . copy ( ) <EOL> for key , value in kwargs . items ( ) : <EOL> value = self . get_input ( baton , value ) <EOL> if hasattr ( value , '<STR_LIT>' ) : <EOL> value = yield value ( ) <EOL> kwargs [ key ] = yield value <EOL> postdata = kwargs [ '<STR_LIT>' ] <EOL> if hasattr ( postdata , '<STR_LIT>' ) and hasattr ( postdata . read , '<STR_LIT>' ) : <EOL> kwargs [ '<STR_LIT>' ] = yield postdata . read ( ) <EOL> base_url = self . get_input ( baton , self . base_url ) or '<STR_LIT>' <EOL> kwargs [ '<STR_LIT:url>' ] = base_url + self . _flatten ( kwargs [ '<STR_LIT:url>' ] ) <EOL> response = yield client . getPage ( ** kwargs ) <EOL> baton = self . get_resulting_baton ( baton , self . output_path , response ) <EOL> defer . returnValue ( baton ) <EOL> def _flatten ( self , string_or_list , separator = '<STR_LIT:/>' ) : <EOL> if isinstance ( string_or_list , list ) : <EOL> return separator . join ( string_or_list ) <EOL> return string_or_list </s>
