<s> """<STR_LIT>""" <EOL> from __future__ import absolute_import <EOL> import os <EOL> import gc <EOL> import time <EOL> import json <EOL> import resource <EOL> from twisted . web . server import NOT_DONE_YET <EOL> from twisted . web . resource import Resource <EOL> from twisted . web . static import File <EOL> from twisted . internet import reactor , defer <EOL> from twisted . python import log <EOL> import six <EOL> import splash <EOL> from splash . argument_cache import ArgumentCache <EOL> from splash . qtrender import ( <EOL> HtmlRender , PngRender , JsonRender , HarRender , JpegRender <EOL> ) <EOL> from splash . lua import is_supported as lua_is_supported <EOL> from splash . utils import ( <EOL> get_num_fds , <EOL> get_leaks , <EOL> BinaryCapsule , <EOL> SplashJSONEncoder , <EOL> get_ru_maxrss , <EOL> to_bytes ) <EOL> from splash import sentry <EOL> from splash . render_options import RenderOptions <EOL> from splash . qtutils import clear_caches <EOL> from splash . exceptions import ( <EOL> BadOption , RenderError , InternalError , <EOL> GlobalTimeoutError , UnsupportedContentType , <EOL> ExpiredArguments , <EOL> ) <EOL> if lua_is_supported ( ) : <EOL> from splash . qtrender_lua import LuaRender <EOL> else : <EOL> LuaRender = None <EOL> class _ValidatingResource ( Resource ) : <EOL> def render ( self , request ) : <EOL> try : <EOL> return Resource . render ( self , request ) <EOL> except BadOption as e : <EOL> self . _write_error ( request , <NUM_LIT> , e ) <EOL> self . _log_stats ( request , { } , error = self . _format_error ( <NUM_LIT> , e ) ) <EOL> return b"<STR_LIT:\n>" <EOL> def _write_error_content ( self , request , code , err , content_type = b'<STR_LIT>' ) : <EOL> request . setHeader ( b"<STR_LIT>" , content_type ) <EOL> request . setResponseCode ( code ) <EOL> content = json . dumps ( err ) . encode ( '<STR_LIT:utf8>' ) <EOL> request . write ( content ) <EOL> return err <EOL> def _write_error ( self , request , code , exc ) : <EOL> """<STR_LIT>""" <EOL> err = self . _format_error ( code , exc ) <EOL> return self . _write_error_content ( request , code , err , <EOL> content_type = b"<STR_LIT:application/json>" ) <EOL> def _format_error ( self , code , exc ) : <EOL> err = { <EOL> "<STR_LIT:error>" : code , <EOL> "<STR_LIT:type>" : exc . __class__ . __name__ , <EOL> "<STR_LIT:description>" : ( exc . __doc__ or '<STR_LIT>' ) . strip ( ) , <EOL> "<STR_LIT:info>" : None , <EOL> } <EOL> if len ( exc . args ) == <NUM_LIT:1> : <EOL> err [ '<STR_LIT:info>' ] = exc . args [ <NUM_LIT:0> ] <EOL> elif len ( exc . args ) > <NUM_LIT:1> : <EOL> err [ '<STR_LIT:info>' ] = exc . args <EOL> return err <EOL> class BaseRenderResource ( _ValidatingResource ) : <EOL> isLeaf = True <EOL> content_type = "<STR_LIT>" <EOL> def __init__ ( self , pool , max_timeout , argument_cache ) : <EOL> Resource . __init__ ( self ) <EOL> self . pool = pool <EOL> self . js_profiles_path = self . pool . js_profiles_path <EOL> self . max_timeout = max_timeout <EOL> self . argument_cache = argument_cache <EOL> def render_GET ( self , request ) : <EOL> request . starttime = time . time ( ) <EOL> render_options = RenderOptions . fromrequest ( request , self . max_timeout ) <EOL> original_options = render_options . data . copy ( ) <EOL> expired_args = render_options . get_expired_args ( self . argument_cache ) <EOL> if expired_args : <EOL> error = self . _write_expired_args ( request , expired_args ) <EOL> self . _log_stats ( request , original_options , error ) <EOL> return b"<STR_LIT:\n>" <EOL> saved_args = render_options . save_args_to_cache ( self . argument_cache ) <EOL> if saved_args : <EOL> value = '<STR_LIT:;>' . join ( "<STR_LIT>" . format ( name , value ) <EOL> for name , value in saved_args ) <EOL> request . setHeader ( b'<STR_LIT>' , value . encode ( '<STR_LIT:utf8>' ) ) <EOL> render_options . load_cached_args ( self . argument_cache ) <EOL> render_options . get_filters ( self . pool ) <EOL> timeout = render_options . get_timeout ( ) <EOL> wait_time = render_options . get_wait ( ) <EOL> pool_d = self . _get_render ( request , render_options ) <EOL> timer = reactor . callLater ( timeout + wait_time , pool_d . cancel ) <EOL> request . notifyFinish ( ) . addErrback ( self . _request_failed , pool_d , timer ) <EOL> pool_d . addCallback ( self . _cancel_timer , timer ) <EOL> pool_d . addCallback ( self . _write_output , request ) <EOL> pool_d . addErrback ( self . _on_timeout_error , request , timeout = timeout ) <EOL> pool_d . addErrback ( self . _on_render_error , request ) <EOL> pool_d . addErrback ( self . _on_bad_request , request ) <EOL> pool_d . addErrback ( self . _on_internal_error , request ) <EOL> pool_d . addBoth ( self . _finish_request , request , <EOL> options = original_options ) <EOL> return NOT_DONE_YET <EOL> def render_POST ( self , request ) : <EOL> request_content_type = request . getHeader ( b'<STR_LIT>' ) . decode ( '<STR_LIT>' ) <EOL> supported_types = [ '<STR_LIT>' , '<STR_LIT:application/json>' ] <EOL> if not any ( ct in request_content_type for ct in supported_types ) : <EOL> ex = UnsupportedContentType ( { <EOL> '<STR_LIT>' : supported_types , <EOL> '<STR_LIT>' : request_content_type , <EOL> } ) <EOL> return self . _write_error ( request , <NUM_LIT> , ex ) <EOL> return self . render_GET ( request ) <EOL> def _cancel_timer ( self , _ , timer ) : <EOL> timer . cancel ( ) <EOL> return _ <EOL> def _request_failed ( self , _ , pool_d , timer ) : <EOL> log . msg ( "<STR_LIT>" % _ . value ) <EOL> timer . cancel ( ) <EOL> pool_d . cancel ( ) <EOL> def _write_output ( self , data , request , content_type = None ) : <EOL> if content_type is None : <EOL> content_type = self . content_type <EOL> if isinstance ( data , ( dict , list ) ) : <EOL> data = json . dumps ( data , cls = SplashJSONEncoder ) <EOL> return self . _write_output ( data , request , b"<STR_LIT:application/json>" ) <EOL> if isinstance ( data , tuple ) and len ( data ) == <NUM_LIT:4> : <EOL> data , content_type , headers , status_code = data <EOL> request . setResponseCode ( status_code ) <EOL> for name , value in headers : <EOL> request . setHeader ( name , value ) <EOL> return self . _write_output ( data , request , content_type ) <EOL> if data is None or isinstance ( data , ( bool , six . integer_types , float ) ) : <EOL> return self . _write_output ( str ( data ) , request , content_type ) <EOL> if isinstance ( data , BinaryCapsule ) : <EOL> return self . _write_output ( data . data , request , data . content_type ) <EOL> if not isinstance ( data , bytes ) : <EOL> data = data . encode ( '<STR_LIT:utf8>' ) <EOL> if not isinstance ( content_type , bytes ) : <EOL> content_type = content_type . encode ( '<STR_LIT>' ) <EOL> request . setHeader ( b"<STR_LIT>" , content_type ) <EOL> if not isinstance ( data , bytes ) : <EOL> data = data . encode ( '<STR_LIT:utf-8>' ) <EOL> request . write ( data ) <EOL> def _write_expired_args ( self , request , expired_args ) : <EOL> ex = ExpiredArguments ( { '<STR_LIT>' : expired_args } ) <EOL> return self . _write_error ( request , <NUM_LIT> , ex ) <EOL> def _log_stats ( self , request , options , error = None ) : <EOL> msg = { <EOL> "<STR_LIT:path>" : request . path . decode ( '<STR_LIT:utf-8>' ) , <EOL> "<STR_LIT>" : time . time ( ) - request . starttime , <EOL> "<STR_LIT>" : resource . getrusage ( resource . RUSAGE_SELF ) . ru_maxrss , <EOL> "<STR_LIT>" : os . getloadavg ( ) , <EOL> "<STR_LIT>" : get_num_fds ( ) , <EOL> "<STR_LIT>" : len ( self . pool . active ) , <EOL> "<STR_LIT>" : len ( self . pool . queue . pending ) , <EOL> "<STR_LIT>" : id ( request ) , <EOL> "<STR_LIT>" : request . method . decode ( '<STR_LIT:ascii>' ) , <EOL> "<STR_LIT>" : int ( time . time ( ) ) , <EOL> "<STR_LIT>" : ( request . getHeader ( b"<STR_LIT>" ) . decode ( '<STR_LIT:utf-8>' ) <EOL> if request . getHeader ( b"<STR_LIT>" ) else None ) , <EOL> "<STR_LIT:args>" : options , <EOL> "<STR_LIT>" : request . code , <EOL> "<STR_LIT>" : request . client . host <EOL> } <EOL> if error : <EOL> msg [ "<STR_LIT:error>" ] = error <EOL> msg = json . dumps ( msg ) . encode ( "<STR_LIT:utf8>" ) <EOL> log . msg ( msg , system = "<STR_LIT>" ) <EOL> def _on_timeout_error ( self , failure , request , timeout = None ) : <EOL> failure . trap ( defer . CancelledError ) <EOL> ex = GlobalTimeoutError ( { '<STR_LIT>' : timeout } ) <EOL> return self . _write_error ( request , <NUM_LIT> , ex ) <EOL> def _on_render_error ( self , failure , request ) : <EOL> failure . trap ( RenderError ) <EOL> return self . _write_error ( request , <NUM_LIT> , failure . value ) <EOL> def _on_internal_error ( self , failure , request ) : <EOL> log . err ( ) <EOL> sentry . capture ( failure ) <EOL> ex = InternalError ( str ( failure . value ) ) <EOL> return self . _write_error ( request , <NUM_LIT> , ex ) <EOL> def _on_bad_request ( self , failure , request ) : <EOL> failure . trap ( BadOption ) <EOL> return self . _write_error ( request , <NUM_LIT> , failure . value ) <EOL> def _finish_request ( self , failure , request , options ) : <EOL> self . _log_stats ( request , options , error = failure ) <EOL> if not request . _disconnected : <EOL> request . finish ( ) <EOL> def _get_render ( self , request , options ) : <EOL> raise NotImplementedError ( ) <EOL> class RenderHtmlResource ( BaseRenderResource ) : <EOL> content_type = "<STR_LIT>" <EOL> def _get_render ( self , request , options ) : <EOL> params = options . get_common_params ( self . js_profiles_path ) <EOL> return self . pool . render ( HtmlRender , options , ** params ) <EOL> class ExecuteLuaScriptResource ( BaseRenderResource ) : <EOL> content_type = "<STR_LIT>" <EOL> def __init__ ( self , pool , sandboxed , <EOL> lua_package_path , <EOL> lua_sandbox_allowed_modules , <EOL> max_timeout , <EOL> argument_cache , <EOL> ) : <EOL> BaseRenderResource . __init__ ( self , pool , max_timeout , argument_cache ) <EOL> self . sandboxed = sandboxed <EOL> self . lua_package_path = lua_package_path <EOL> self . lua_sandbox_allowed_modules = lua_sandbox_allowed_modules <EOL> def _get_render ( self , request , options ) : <EOL> params = dict ( <EOL> proxy = options . get_proxy ( ) , <EOL> lua_source = options . get_lua_source ( ) , <EOL> sandboxed = self . sandboxed , <EOL> lua_package_path = self . lua_package_path , <EOL> lua_sandbox_allowed_modules = self . lua_sandbox_allowed_modules , <EOL> ) <EOL> return self . pool . render ( LuaRender , options , ** params ) <EOL> class RenderPngResource ( BaseRenderResource ) : <EOL> content_type = "<STR_LIT>" <EOL> def _get_render ( self , request , options ) : <EOL> params = options . get_common_params ( self . js_profiles_path ) <EOL> params . update ( options . get_png_params ( ) ) <EOL> return self . pool . render ( PngRender , options , ** params ) <EOL> class RenderJpegResource ( BaseRenderResource ) : <EOL> content_type = "<STR_LIT>" <EOL> def _get_render ( self , request , options ) : <EOL> params = options . get_common_params ( self . js_profiles_path ) <EOL> params . update ( options . get_jpeg_params ( ) ) <EOL> return self . pool . render ( JpegRender , options , ** params ) <EOL> class RenderJsonResource ( BaseRenderResource ) : <EOL> content_type = "<STR_LIT:application/json>" <EOL> def _get_render ( self , request , options ) : <EOL> params = options . get_common_params ( self . js_profiles_path ) <EOL> params . update ( options . get_jpeg_params ( ) ) <EOL> params . update ( options . get_include_params ( ) ) <EOL> return self . pool . render ( JsonRender , options , ** params ) <EOL> class RenderHarResource ( BaseRenderResource ) : <EOL> content_type = "<STR_LIT:application/json>" <EOL> def _get_render ( self , request , options ) : <EOL> params = options . get_common_params ( self . js_profiles_path ) <EOL> return self . pool . render ( HarRender , options , ** params ) <EOL> class DebugResource ( Resource ) : <EOL> isLeaf = True <EOL> def __init__ ( self , pool , argument_cache , warn = False ) : <EOL> Resource . __init__ ( self ) <EOL> self . argument_cache = argument_cache <EOL> self . pool = pool <EOL> self . warn = warn <EOL> def render_GET ( self , request ) : <EOL> request . setHeader ( b"<STR_LIT>" , b"<STR_LIT:application/json>" ) <EOL> info = { <EOL> "<STR_LIT>" : get_leaks ( ) , <EOL> "<STR_LIT>" : [ self . get_repr ( r ) for r in self . pool . active ] , <EOL> "<STR_LIT>" : len ( self . pool . queue . pending ) , <EOL> "<STR_LIT>" : resource . getrusage ( resource . RUSAGE_SELF ) . ru_maxrss , <EOL> "<STR_LIT>" : get_num_fds ( ) , <EOL> "<STR_LIT>" : len ( self . argument_cache ) <EOL> } <EOL> if self . warn : <EOL> info [ '<STR_LIT>' ] = "<STR_LIT>" "<STR_LIT>" <EOL> return ( json . dumps ( info , sort_keys = True ) ) . encode ( '<STR_LIT:utf-8>' ) <EOL> def get_repr ( self , render ) : <EOL> if hasattr ( render , '<STR_LIT:url>' ) : <EOL> return render . url <EOL> return render . tab . url <EOL> class ClearCachesResource ( Resource ) : <EOL> isLeaf = True <EOL> content_type = "<STR_LIT:application/json>" <EOL> def __init__ ( self , argument_cache ) : <EOL> Resource . __init__ ( self ) <EOL> self . argument_cache = argument_cache <EOL> def render_POST ( self , request ) : <EOL> argcache_size = len ( self . argument_cache ) <EOL> self . argument_cache . clear ( ) <EOL> clear_caches ( ) <EOL> unreachable = gc . collect ( ) <EOL> return json . dumps ( { <EOL> "<STR_LIT:status>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : unreachable , <EOL> "<STR_LIT>" : argcache_size , <EOL> } , sort_keys = True ) . encode ( '<STR_LIT:utf-8>' ) <EOL> class PingResource ( Resource ) : <EOL> isLeaf = True <EOL> def render_GET ( self , request ) : <EOL> request . setHeader ( b"<STR_LIT>" , b"<STR_LIT:application/json>" ) <EOL> return ( json . dumps ( { <EOL> "<STR_LIT:status>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : get_ru_maxrss ( ) , <EOL> } , sort_keys = True ) ) . encode ( '<STR_LIT:utf-8>' ) <EOL> HARVIEWER_PATH = '<STR_LIT>' <EOL> BOOTSTRAP_THEME = '<STR_LIT>' <EOL> CODEMIRROR_RESOURCES = """<STR_LIT>""" <EOL> def safe_json ( obj ) : <EOL> """<STR_LIT>""" <EOL> return json . dumps ( obj ) . replace ( '<STR_LIT:<>' , '<STR_LIT>' ) <EOL> class DemoUI ( _ValidatingResource ) : <EOL> isLeaf = True <EOL> content_type = "<STR_LIT>" <EOL> PATH = b'<STR_LIT:info>' <EOL> def __init__ ( self , pool , lua_enabled , max_timeout ) : <EOL> Resource . __init__ ( self ) <EOL> self . pool = pool <EOL> self . lua_enabled = lua_enabled <EOL> self . max_timeout = max_timeout <EOL> def _validate_params ( self , request ) : <EOL> options = RenderOptions . fromrequest ( request , self . max_timeout ) <EOL> options . get_filters ( self . pool ) <EOL> params = options . get_common_params ( self . pool . js_profiles_path ) <EOL> params . update ( { <EOL> '<STR_LIT>' : options . get_save_args ( ) , <EOL> '<STR_LIT>' : options . get_load_args ( ) , <EOL> '<STR_LIT>' : options . get_timeout ( ) , <EOL> '<STR_LIT>' : <NUM_LIT:1> , <EOL> '<STR_LIT>' : <NUM_LIT:1> , <EOL> '<STR_LIT:html>' : <NUM_LIT:1> , <EOL> } ) <EOL> filters = options . get ( '<STR_LIT>' , default = '<STR_LIT>' ) <EOL> if filters : <EOL> params [ '<STR_LIT>' ] = filters <EOL> if self . lua_enabled : <EOL> params . update ( { <EOL> '<STR_LIT>' : options . get_lua_source ( ) , <EOL> } ) <EOL> return params <EOL> def render_GET ( self , request ) : <EOL> params = self . _validate_params ( request ) <EOL> url = params . get ( '<STR_LIT:url>' , '<STR_LIT>' ) . strip ( ) <EOL> if url and not url . lower ( ) . startswith ( '<STR_LIT:http>' ) : <EOL> url = '<STR_LIT>' + url <EOL> params [ '<STR_LIT:url>' ] = url <EOL> timeout = params [ '<STR_LIT>' ] <EOL> params = { k : v for k , v in params . items ( ) if v is not None } <EOL> request . addCookie ( '<STR_LIT>' , <NUM_LIT> ) <EOL> return ( """<STR_LIT>""" % dict ( <EOL> version = splash . __version__ , <EOL> harviewer_path = HARVIEWER_PATH , <EOL> options = safe_json ( { <EOL> "<STR_LIT>" : params , <EOL> "<STR_LIT>" : "<STR_LIT>" if self . lua_enabled else "<STR_LIT>" , <EOL> "<STR_LIT>" : HARVIEWER_PATH , <EOL> "<STR_LIT>" : self . lua_enabled , <EOL> } ) , <EOL> timeout = timeout , <EOL> url = url , <EOL> theme = BOOTSTRAP_THEME , <EOL> cm_resources = CODEMIRROR_RESOURCES if self . lua_enabled else "<STR_LIT>" , <EOL> ) ) . encode ( '<STR_LIT:utf-8>' ) <EOL> class Root ( Resource ) : <EOL> def __init__ ( self , pool , ui_enabled , lua_enabled , lua_sandbox_enabled , <EOL> lua_package_path , <EOL> lua_sandbox_allowed_modules , <EOL> max_timeout , <EOL> argument_cache_max_entries , <EOL> ) : <EOL> Resource . __init__ ( self ) <EOL> self . argument_cache = ArgumentCache ( argument_cache_max_entries ) <EOL> self . ui_enabled = ui_enabled <EOL> self . lua_enabled = lua_enabled <EOL> _args = pool , max_timeout , self . argument_cache <EOL> self . putChild ( b"<STR_LIT>" , RenderHtmlResource ( * _args ) ) <EOL> self . putChild ( b"<STR_LIT>" , RenderPngResource ( * _args ) ) <EOL> self . putChild ( b"<STR_LIT>" , RenderJpegResource ( * _args ) ) <EOL> self . putChild ( b"<STR_LIT>" , RenderJsonResource ( * _args ) ) <EOL> self . putChild ( b"<STR_LIT>" , RenderHarResource ( * _args ) ) <EOL> self . putChild ( b"<STR_LIT>" , DebugResource ( pool , self . argument_cache ) ) <EOL> self . putChild ( b"<STR_LIT>" , ClearCachesResource ( self . argument_cache ) ) <EOL> self . putChild ( b"<STR_LIT>" , PingResource ( ) ) <EOL> self . putChild ( b"<STR_LIT>" , DebugResource ( pool , self . argument_cache , warn = True ) ) <EOL> if self . lua_enabled and ExecuteLuaScriptResource is not None : <EOL> self . putChild ( b"<STR_LIT>" , ExecuteLuaScriptResource ( <EOL> pool = pool , <EOL> sandboxed = lua_sandbox_enabled , <EOL> lua_package_path = lua_package_path , <EOL> lua_sandbox_allowed_modules = lua_sandbox_allowed_modules , <EOL> max_timeout = max_timeout , <EOL> argument_cache = self . argument_cache , <EOL> ) ) <EOL> if self . ui_enabled : <EOL> root = os . path . dirname ( __file__ ) <EOL> ui = File ( os . path . join ( root , '<STR_LIT>' ) ) <EOL> har_path = os . path . join ( root , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) <EOL> ui . putChild ( to_bytes ( HARVIEWER_PATH ) , File ( har_path ) ) <EOL> inspections_path = os . path . join ( root , '<STR_LIT>' , '<STR_LIT>' ) <EOL> ui . putChild ( b"<STR_LIT>" , File ( inspections_path ) ) <EOL> examples_path = os . path . join ( root , '<STR_LIT>' ) <EOL> ui . putChild ( b"<STR_LIT>" , File ( examples_path ) ) <EOL> self . putChild ( b"<STR_LIT>" , ui ) <EOL> self . putChild ( DemoUI . PATH , DemoUI ( <EOL> pool = pool , <EOL> lua_enabled = self . lua_enabled , <EOL> max_timeout = max_timeout <EOL> ) ) <EOL> self . max_timeout = max_timeout <EOL> def getChild ( self , name , request ) : <EOL> if name == b"<STR_LIT>" and self . ui_enabled : <EOL> return self <EOL> return Resource . getChild ( self , name , request ) <EOL> def get_example_script ( self ) : <EOL> return """<STR_LIT>""" . strip ( ) <EOL> def render_GET ( self , request ) : <EOL> """<STR_LIT>""" <EOL> result = """<STR_LIT>""" % dict ( <EOL> version = splash . __version__ , <EOL> theme = BOOTSTRAP_THEME , <EOL> options = safe_json ( { <EOL> "<STR_LIT>" : "<STR_LIT>" if self . lua_enabled else "<STR_LIT>" , <EOL> "<STR_LIT>" : self . lua_enabled , <EOL> "<STR_LIT>" : self . get_example_script ( ) , <EOL> } ) , <EOL> cm_resources = CODEMIRROR_RESOURCES , <EOL> timeout = self . max_timeout , <EOL> ) <EOL> return result . encode ( '<STR_LIT:utf8>' ) </s>
