<s> """<STR_LIT>""" <EOL> import json <EOL> import os <EOL> import six <EOL> from six . moves . configparser import ConfigParser , NoSectionError , NoOptionError <EOL> from six . moves . urllib . parse import unquote <EOL> from hashlib import md5 <EOL> from swift . common import constraints <EOL> from swift . common . exceptions import ListingIterError , SegmentError <EOL> from swift . common . http import is_success <EOL> from swift . common . swob import Request , Response , HTTPRequestedRangeNotSatisfiable , HTTPBadRequest , HTTPConflict <EOL> from swift . common . utils import get_logger , RateLimitedIterator , read_conf_dir , quote , close_if_possible , closing_if_possible <EOL> from swift . common . request_helpers import SegmentedIterable <EOL> from swift . common . wsgi import WSGIContext , make_subrequest <EOL> class GetContext ( WSGIContext ) : <EOL> def __init__ ( self , dlo , logger ) : <EOL> super ( GetContext , self ) . __init__ ( dlo . app ) <EOL> self . dlo = dlo <EOL> self . logger = logger <EOL> def _get_container_listing ( self , req , version , account , container , <EOL> prefix , marker = '<STR_LIT>' ) : <EOL> con_req = make_subrequest ( <EOL> req . environ , path = '<STR_LIT:/>' . join ( [ '<STR_LIT>' , version , account , container ] ) , <EOL> method = '<STR_LIT:GET>' , <EOL> headers = { '<STR_LIT>' : req . headers . get ( '<STR_LIT>' ) } , <EOL> agent = ( '<STR_LIT>' + '<STR_LIT>' ) , swift_source = '<STR_LIT>' ) <EOL> con_req . query_string = '<STR_LIT>' % quote ( prefix ) <EOL> if marker : <EOL> con_req . query_string += '<STR_LIT>' % quote ( marker ) <EOL> con_resp = con_req . get_response ( self . dlo . app ) <EOL> if not is_success ( con_resp . status_int ) : <EOL> return con_resp , None <EOL> with closing_if_possible ( con_resp . app_iter ) : <EOL> return None , json . loads ( '<STR_LIT>' . join ( con_resp . app_iter ) ) <EOL> def _segment_listing_iterator ( self , req , version , account , container , <EOL> prefix , segments , first_byte = None , <EOL> last_byte = None ) : <EOL> if first_byte is None : <EOL> first_byte = <NUM_LIT:0> <EOL> if last_byte is None : <EOL> last_byte = float ( "<STR_LIT>" ) <EOL> marker = '<STR_LIT>' <EOL> while True : <EOL> for segment in segments : <EOL> seg_length = int ( segment [ '<STR_LIT>' ] ) <EOL> if first_byte >= seg_length : <EOL> first_byte = max ( first_byte - seg_length , - <NUM_LIT:1> ) <EOL> last_byte = max ( last_byte - seg_length , - <NUM_LIT:1> ) <EOL> continue <EOL> elif last_byte < <NUM_LIT:0> : <EOL> break <EOL> seg_name = segment [ '<STR_LIT:name>' ] <EOL> if isinstance ( seg_name , six . text_type ) : <EOL> seg_name = seg_name . encode ( "<STR_LIT:utf-8>" ) <EOL> yield ( "<STR_LIT:/>" + "<STR_LIT:/>" . join ( ( version , account , container , <EOL> seg_name ) ) , <EOL> None , None , <EOL> ( None if first_byte <= <NUM_LIT:0> else first_byte ) , <EOL> ( None if last_byte >= seg_length - <NUM_LIT:1> else last_byte ) ) <EOL> first_byte = max ( first_byte - seg_length , - <NUM_LIT:1> ) <EOL> last_byte = max ( last_byte - seg_length , - <NUM_LIT:1> ) <EOL> if len ( segments ) < constraints . CONTAINER_LISTING_LIMIT : <EOL> break <EOL> elif last_byte < <NUM_LIT:0> : <EOL> break <EOL> marker = segments [ - <NUM_LIT:1> ] [ '<STR_LIT:name>' ] <EOL> error_response , segments = self . _get_container_listing ( <EOL> req , version , account , container , prefix , marker ) <EOL> if error_response : <EOL> close_if_possible ( error_response . app_iter ) <EOL> raise ListingIterError ( <EOL> "<STR_LIT>" % <EOL> ( error_response . status_int , account , container ) ) <EOL> def get_or_head_response ( self , req , x_object_manifest , <EOL> response_headers = None ) : <EOL> if response_headers is None : <EOL> response_headers = self . _response_headers <EOL> container , obj_prefix = x_object_manifest . split ( '<STR_LIT:/>' , <NUM_LIT:1> ) <EOL> container = unquote ( container ) <EOL> obj_prefix = unquote ( obj_prefix ) <EOL> req . acl = None <EOL> version , account , _junk = req . split_path ( <NUM_LIT:2> , <NUM_LIT:3> , True ) <EOL> error_response , segments = self . _get_container_listing ( <EOL> req , version , account , container , obj_prefix ) <EOL> if error_response : <EOL> return error_response <EOL> have_complete_listing = len ( segments ) < constraints . CONTAINER_LISTING_LIMIT <EOL> first_byte = last_byte = None <EOL> actual_content_length = None <EOL> content_length_for_swob_range = None <EOL> if req . range and len ( req . range . ranges ) == <NUM_LIT:1> : <EOL> content_length_for_swob_range = sum ( o [ '<STR_LIT>' ] for o in segments ) <EOL> _junk , range_end = req . range . ranges_for_length ( float ( "<STR_LIT>" ) ) [ <NUM_LIT:0> ] <EOL> if ( have_complete_listing <EOL> or range_end < content_length_for_swob_range ) : <EOL> byteranges = req . range . ranges_for_length ( <EOL> content_length_for_swob_range ) <EOL> if not byteranges : <EOL> return HTTPRequestedRangeNotSatisfiable ( request = req ) <EOL> first_byte , last_byte = byteranges [ <NUM_LIT:0> ] <EOL> last_byte -= <NUM_LIT:1> <EOL> actual_content_length = last_byte - first_byte + <NUM_LIT:1> <EOL> else : <EOL> actual_content_length = None <EOL> content_length_for_swob_range = None <EOL> req . range = None <EOL> response_headers = [ <EOL> ( h , v ) for h , v in response_headers <EOL> if h . lower ( ) not in ( "<STR_LIT>" , "<STR_LIT>" ) ] <EOL> if content_length_for_swob_range is not None : <EOL> response_headers . append ( ( '<STR_LIT>' , <EOL> str ( content_length_for_swob_range ) ) ) <EOL> elif have_complete_listing : <EOL> actual_content_length = sum ( o [ '<STR_LIT>' ] for o in segments ) <EOL> response_headers . append ( ( '<STR_LIT>' , <EOL> str ( actual_content_length ) ) ) <EOL> if have_complete_listing : <EOL> response_headers = [ ( h , v ) for h , v in response_headers <EOL> if h . lower ( ) != "<STR_LIT>" ] <EOL> etag = md5 ( ) <EOL> for seg_dict in segments : <EOL> etag . update ( seg_dict [ '<STR_LIT>' ] . strip ( '<STR_LIT:">' ) ) <EOL> response_headers . append ( ( '<STR_LIT>' , '<STR_LIT>' % etag . hexdigest ( ) ) ) <EOL> app_iter = None <EOL> if req . method == '<STR_LIT:GET>' : <EOL> listing_iter = RateLimitedIterator ( <EOL> self . _segment_listing_iterator ( <EOL> req , version , account , container , obj_prefix , segments , <EOL> first_byte = first_byte , last_byte = last_byte ) , <EOL> self . dlo . rate_limit_segments_per_sec , <EOL> limit_after = self . dlo . rate_limit_after_segment ) <EOL> app_iter = SegmentedIterable ( <EOL> req , self . dlo . app , listing_iter , ua_suffix = "<STR_LIT>" , <EOL> swift_source = "<STR_LIT>" , name = req . path , logger = self . logger , <EOL> max_get_time = self . dlo . max_get_time , <EOL> response_body_length = actual_content_length ) <EOL> try : <EOL> app_iter . validate_first_segment ( ) <EOL> except ( SegmentError , ListingIterError ) : <EOL> return HTTPConflict ( request = req ) <EOL> resp = Response ( request = req , headers = response_headers , <EOL> conditional_response = True , <EOL> app_iter = app_iter ) <EOL> return resp <EOL> def handle_request ( self , req , start_response ) : <EOL> """<STR_LIT>""" <EOL> resp_iter = self . _app_call ( req . environ ) <EOL> for header , value in self . _response_headers : <EOL> if ( header . lower ( ) == '<STR_LIT>' ) : <EOL> close_if_possible ( resp_iter ) <EOL> response = self . get_or_head_response ( req , value ) <EOL> return response ( req . environ , start_response ) <EOL> start_response ( self . _response_status , <EOL> self . _response_headers , <EOL> self . _response_exc_info ) <EOL> return resp_iter <EOL> class DynamicLargeObject ( object ) : <EOL> def __init__ ( self , app , conf ) : <EOL> self . app = app <EOL> self . logger = get_logger ( conf , log_route = '<STR_LIT>' ) <EOL> self . _populate_config_from_old_location ( conf ) <EOL> self . max_get_time = int ( conf . get ( '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> self . rate_limit_after_segment = int ( conf . get ( <EOL> '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> self . rate_limit_segments_per_sec = int ( conf . get ( <EOL> '<STR_LIT>' , '<STR_LIT:1>' ) ) <EOL> def _populate_config_from_old_location ( self , conf ) : <EOL> if ( '<STR_LIT>' in conf or <EOL> '<STR_LIT>' in conf or <EOL> '<STR_LIT>' in conf or <EOL> '<STR_LIT>' not in conf ) : <EOL> return <EOL> cp = ConfigParser ( ) <EOL> if os . path . isdir ( conf [ '<STR_LIT>' ] ) : <EOL> read_conf_dir ( cp , conf [ '<STR_LIT>' ] ) <EOL> else : <EOL> cp . read ( conf [ '<STR_LIT>' ] ) <EOL> try : <EOL> pipe = cp . get ( "<STR_LIT>" , "<STR_LIT>" ) <EOL> except ( NoSectionError , NoOptionError ) : <EOL> return <EOL> proxy_name = pipe . rsplit ( None , <NUM_LIT:1> ) [ - <NUM_LIT:1> ] <EOL> proxy_section = "<STR_LIT>" + proxy_name <EOL> for setting in ( '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' ) : <EOL> try : <EOL> conf [ setting ] = cp . get ( proxy_section , setting ) <EOL> except ( NoSectionError , NoOptionError ) : <EOL> pass <EOL> def __call__ ( self , env , start_response ) : <EOL> """<STR_LIT>""" <EOL> req = Request ( env ) <EOL> try : <EOL> vrs , account , container , obj = req . split_path ( <NUM_LIT:4> , <NUM_LIT:4> , True ) <EOL> except ValueError : <EOL> return self . app ( env , start_response ) <EOL> env [ '<STR_LIT>' ] = self . copy_hook ( <EOL> env . get ( '<STR_LIT>' , <EOL> lambda src_req , src_resp , sink_req : src_resp ) ) <EOL> if ( ( req . method == '<STR_LIT:GET>' or req . method == '<STR_LIT>' ) and <EOL> req . params . get ( '<STR_LIT>' ) != '<STR_LIT>' ) : <EOL> return GetContext ( self , self . logger ) . handle_request ( req , start_response ) <EOL> elif req . method == '<STR_LIT>' : <EOL> error_response = self . _validate_x_object_manifest_header ( req ) <EOL> if error_response : <EOL> return error_response ( env , start_response ) <EOL> return self . app ( env , start_response ) <EOL> def _validate_x_object_manifest_header ( self , req ) : <EOL> """<STR_LIT>""" <EOL> if '<STR_LIT>' in req . headers : <EOL> value = req . headers [ '<STR_LIT>' ] <EOL> container = prefix = None <EOL> try : <EOL> container , prefix = value . split ( '<STR_LIT:/>' , <NUM_LIT:1> ) <EOL> except ValueError : <EOL> pass <EOL> if not container or not prefix or '<STR_LIT:?>' in value or '<STR_LIT:&>' in value or prefix . startswith ( '<STR_LIT:/>' ) : <EOL> return HTTPBadRequest ( <EOL> request = req , <EOL> body = ( '<STR_LIT>' <EOL> '<STR_LIT>' ) ) <EOL> def copy_hook ( self , inner_hook ) : <EOL> def dlo_copy_hook ( source_req , source_resp , sink_req ) : <EOL> x_o_m = source_resp . headers . get ( '<STR_LIT>' ) <EOL> if x_o_m : <EOL> if source_req . params . get ( '<STR_LIT>' ) == '<STR_LIT>' : <EOL> sink_req . headers [ '<STR_LIT>' ] = x_o_m <EOL> else : <EOL> ctx = GetContext ( self , self . logger ) <EOL> source_resp = ctx . get_or_head_response ( <EOL> source_req , x_o_m , source_resp . headers . items ( ) ) <EOL> return inner_hook ( source_req , source_resp , sink_req ) <EOL> return dlo_copy_hook <EOL> def filter_factory ( global_conf , ** local_conf ) : <EOL> conf = global_conf . copy ( ) <EOL> conf . update ( local_conf ) <EOL> def dlo_filter ( app ) : <EOL> return DynamicLargeObject ( app , conf ) <EOL> return dlo_filter </s>
