<s> from time import time , gmtime , strftime <EOL> from urllib import unquote , quote <EOL> from urlparse import urlparse <EOL> from hashlib import md5 , sha1 <EOL> from xml . sax import saxutils <EOL> import hmac <EOL> import re <EOL> from swift . common import utils <EOL> from swift . common . utils import get_logger , TRUE_VALUES , readconf , list_from_csv , register_swift_info <EOL> from swift . common . constraints import check_utf8 <EOL> from swift . common . wsgi import make_pre_authed_request <EOL> from swift . common . swob import Response , Request , HTTPBadRequest , HTTPForbidden , HTTPNotFound , HTTPNoContent , HTTPOk , HTTPAccepted , HTTPCreated , HTTPMethodNotAllowed , HTTPRequestedRangeNotSatisfiable , HTTPInternalServerError , HTTPPreconditionFailed , HTTPNotModified , HTTPMovedPermanently , HTTPUnauthorized <EOL> try : <EOL> import simplejson as json <EOL> except ImportError : <EOL> import json <EOL> CACHE_BAD_URL = <NUM_LIT> <EOL> CACHE_404 = <NUM_LIT:30> <EOL> MEMCACHE_TIMEOUT = <NUM_LIT> <EOL> MIN_TTL = <NUM_LIT:15> * <NUM_LIT> <EOL> MAX_TTL = <NUM_LIT> * <NUM_LIT> * <NUM_LIT> * <NUM_LIT> <EOL> DEFAULT_TTL = <NUM_LIT> <EOL> MAX_CDN_FILE_SIZE = <NUM_LIT:10> * <NUM_LIT> ** <NUM_LIT:3> <EOL> class InvalidContentType ( Exception ) : <EOL> pass <EOL> class InvalidUtf8 ( Exception ) : <EOL> pass <EOL> class OriginDbFailure ( Exception ) : <EOL> pass <EOL> class OriginDbNotFound ( Exception ) : <EOL> pass <EOL> class InvalidConfiguration ( Exception ) : <EOL> pass <EOL> class OriginRequestNotAllowed ( Exception ) : <EOL> pass <EOL> ACCEPTABLE_FORMATS = [ '<STR_LIT>' , <EOL> '<STR_LIT:application/json>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> class SosResponse ( Response ) : <EOL> def absolute_location ( self ) : <EOL> """<STR_LIT>""" <EOL> return self . location <EOL> def split_path ( path , minsegs = <NUM_LIT:1> , maxsegs = None , rest_with_last = False ) : <EOL> """<STR_LIT>""" <EOL> path = unquote ( path ) <EOL> if not check_utf8 ( path ) : <EOL> raise InvalidUtf8 ( '<STR_LIT>' ) <EOL> if not maxsegs : <EOL> maxsegs = minsegs <EOL> if minsegs > maxsegs : <EOL> raise ValueError ( '<STR_LIT>' % ( minsegs , maxsegs ) ) <EOL> if rest_with_last : <EOL> segs = path . split ( '<STR_LIT:/>' , maxsegs ) <EOL> minsegs += <NUM_LIT:1> <EOL> maxsegs += <NUM_LIT:1> <EOL> count = len ( segs ) <EOL> if segs [ <NUM_LIT:0> ] or count < minsegs or count > maxsegs or '<STR_LIT>' in segs [ <NUM_LIT:1> : minsegs ] : <EOL> raise ValueError ( '<STR_LIT>' % quote ( path ) ) <EOL> else : <EOL> minsegs += <NUM_LIT:1> <EOL> maxsegs += <NUM_LIT:1> <EOL> segs = path . split ( '<STR_LIT:/>' , maxsegs ) <EOL> count = len ( segs ) <EOL> if segs [ <NUM_LIT:0> ] or count < minsegs or count > maxsegs + <NUM_LIT:1> or '<STR_LIT>' in segs [ <NUM_LIT:1> : minsegs ] or ( count == maxsegs + <NUM_LIT:1> and <EOL> segs [ maxsegs ] ) : <EOL> raise ValueError ( '<STR_LIT>' % quote ( path ) ) <EOL> segs = segs [ <NUM_LIT:1> : maxsegs ] <EOL> segs . extend ( [ None ] * ( maxsegs - <NUM_LIT:1> - len ( segs ) ) ) <EOL> return segs <EOL> class HashData ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , account , container , ttl , cdn_enabled , logs_enabled , <EOL> deleted = False ) : <EOL> try : <EOL> if not isinstance ( account , unicode ) : <EOL> account = unicode ( account , '<STR_LIT:utf-8>' ) <EOL> if not isinstance ( container , unicode ) : <EOL> container = unicode ( container , '<STR_LIT:utf-8>' ) <EOL> except UnicodeDecodeError : <EOL> raise InvalidUtf8 ( ) <EOL> self . account = account <EOL> self . container = container <EOL> self . ttl = int ( ttl ) <EOL> self . logs_enabled = bool ( logs_enabled ) <EOL> self . cdn_enabled = bool ( cdn_enabled ) <EOL> self . deleted = bool ( deleted ) <EOL> def get_json_str ( self ) : <EOL> return json . dumps ( { <EOL> '<STR_LIT>' : self . account , '<STR_LIT>' : self . container , <EOL> '<STR_LIT>' : self . ttl , '<STR_LIT>' : self . logs_enabled , <EOL> '<STR_LIT>' : self . cdn_enabled , '<STR_LIT>' : self . deleted } ) <EOL> def __str__ ( self ) : <EOL> return self . get_json_str ( ) <EOL> def __eq__ ( self , other ) : <EOL> return ( self . account == other . account and <EOL> self . container == other . container and <EOL> self . ttl == other . ttl and <EOL> self . logs_enabled == other . logs_enabled and <EOL> self . cdn_enabled == other . cdn_enabled and <EOL> self . deleted == other . deleted ) <EOL> def __ne__ ( self , other ) : <EOL> return not self == other <EOL> @ classmethod <EOL> def create_from_json ( cls , json_str ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> data = json . loads ( json_str ) <EOL> return HashData ( data [ '<STR_LIT>' ] , data [ '<STR_LIT>' ] , data [ '<STR_LIT>' ] , <EOL> data [ '<STR_LIT>' ] , data [ '<STR_LIT>' ] , <EOL> deleted = data . get ( '<STR_LIT>' , False ) ) <EOL> except ( KeyError , ValueError , TypeError ) , e : <EOL> raise ValueError ( "<STR_LIT>" % ( e , json_str ) ) <EOL> def gen_listing_content_type ( self ) : <EOL> return '<STR_LIT>' % { <EOL> '<STR_LIT>' : self . cdn_enabled , '<STR_LIT>' : self . ttl , <EOL> '<STR_LIT>' : self . logs_enabled } <EOL> class OriginBase ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , app , conf , logger ) : <EOL> self . app = app <EOL> self . conf = conf <EOL> self . logger = logger <EOL> self . hash_suffix = conf . get ( '<STR_LIT>' ) <EOL> self . origin_account = conf . get ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> self . num_hash_cont = int ( conf . get ( '<STR_LIT>' , <NUM_LIT:100> ) ) <EOL> self . hmac_signed_url_secret = self . conf . get ( '<STR_LIT>' ) <EOL> self . token_length = int ( self . conf . get ( '<STR_LIT>' , <NUM_LIT:30> ) ) <EOL> self . min_ttl = int ( conf . get ( '<STR_LIT>' , MIN_TTL ) ) <EOL> self . max_ttl = int ( conf . get ( '<STR_LIT>' , MAX_TTL ) ) <EOL> self . log_access_requests = conf . get ( '<STR_LIT>' , '<STR_LIT:t>' ) in TRUE_VALUES <EOL> self . number_dns_shards = int ( conf . get ( '<STR_LIT>' , <NUM_LIT:100> ) ) <EOL> self . forward_origin = conf . get ( '<STR_LIT>' , '<STR_LIT:t>' ) in TRUE_VALUES <EOL> if not self . hash_suffix : <EOL> raise InvalidConfiguration ( '<STR_LIT>' ) <EOL> def hash_path ( self , account , container ) : <EOL> """<STR_LIT>""" <EOL> return md5 ( '<STR_LIT>' % ( account , container , <EOL> self . hash_suffix ) ) . hexdigest ( ) <EOL> def get_hsh_obj_path ( self , hsh ) : <EOL> """<STR_LIT>""" <EOL> hsh_num = int ( hsh , <NUM_LIT:16> ) % self . num_hash_cont <EOL> return '<STR_LIT>' % ( self . origin_account , hsh_num , hsh ) <EOL> def cdn_data_memcache_key ( self , cdn_obj_path ) : <EOL> return '<STR_LIT>' % ( self . origin_account , cdn_obj_path ) <EOL> def get_cdn_data ( self , env , cdn_obj_path ) : <EOL> """<STR_LIT>""" <EOL> memcache_client = utils . cache_from_env ( env ) <EOL> memcache_key = self . cdn_data_memcache_key ( cdn_obj_path ) <EOL> if memcache_client : <EOL> cached_cdn_data = memcache_client . get ( memcache_key ) <EOL> if cached_cdn_data == '<STR_LIT>' : <EOL> return None <EOL> if cached_cdn_data : <EOL> try : <EOL> hash_data = HashData . create_from_json ( cached_cdn_data ) <EOL> if hash_data . deleted : <EOL> return None <EOL> else : <EOL> return hash_data <EOL> except ValueError : <EOL> pass <EOL> resp = make_pre_authed_request ( <EOL> env , '<STR_LIT:GET>' , cdn_obj_path , <EOL> agent = '<STR_LIT>' , swift_source = '<STR_LIT>' ) . get_response ( self . app ) <EOL> if resp . status_int // <NUM_LIT:100> == <NUM_LIT:2> : <EOL> try : <EOL> if memcache_client : <EOL> memcache_client . set ( memcache_key , resp . body , <EOL> serialize = False , <EOL> time = MEMCACHE_TIMEOUT ) <EOL> hash_data = HashData . create_from_json ( resp . body ) <EOL> if not hash_data . deleted : <EOL> return hash_data <EOL> except ValueError : <EOL> self . logger . warn ( '<STR_LIT>' % cdn_obj_path ) <EOL> if resp . status_int == <NUM_LIT> : <EOL> if memcache_client : <EOL> memcache_client . set ( memcache_key , '<STR_LIT>' , <EOL> serialize = False , time = CACHE_404 ) <EOL> return None <EOL> def get_cdn_urls ( self , hsh , request_type , request_format_tag = '<STR_LIT>' ) : <EOL> """<STR_LIT>""" <EOL> format_section = None <EOL> section_names = [ '<STR_LIT>' % ( request_type . lower ( ) , <EOL> request_format_tag ) , <EOL> '<STR_LIT>' % request_type . lower ( ) , <EOL> '<STR_LIT>' ] <EOL> for section_name in section_names : <EOL> format_section = self . conf . get ( section_name ) <EOL> if format_section : <EOL> break <EOL> else : <EOL> raise InvalidConfiguration ( '<STR_LIT>' <EOL> % ( request_type , request_format_tag ) ) <EOL> url_vars = { '<STR_LIT>' : hsh , <EOL> '<STR_LIT>' : int ( hsh , <NUM_LIT:16> ) % self . number_dns_shards } <EOL> cdn_urls = { } <EOL> for key , url in format_section . items ( ) : <EOL> cdn_urls [ key ] = ( url % url_vars ) . rstrip ( '<STR_LIT:/>' ) <EOL> if self . hmac_signed_url_secret : <EOL> for key , url in cdn_urls . iteritems ( ) : <EOL> parsed = urlparse ( url ) <EOL> token = hmac . new ( key = self . hmac_signed_url_secret , <EOL> msg = parsed . hostname , <EOL> digestmod = sha1 ) . hexdigest ( ) <EOL> cdn_urls [ key ] = '<STR_LIT>' % ( parsed . scheme , <EOL> token [ : self . token_length ] , <EOL> parsed . hostname ) <EOL> return cdn_urls <EOL> class AdminHandler ( OriginBase ) : <EOL> def __init__ ( self , app , conf , logger ) : <EOL> OriginBase . __init__ ( self , app , conf , logger ) <EOL> self . admin_key = conf . get ( '<STR_LIT>' ) <EOL> def is_origin_admin ( self , req ) : <EOL> """<STR_LIT>""" <EOL> return ( self . admin_key and <EOL> req . headers . get ( '<STR_LIT>' ) == '<STR_LIT>' and <EOL> req . headers . get ( '<STR_LIT>' ) == self . admin_key ) <EOL> def handle_request ( self , env , req ) : <EOL> """<STR_LIT>""" <EOL> if not self . is_origin_admin ( req ) : <EOL> return HTTPForbidden ( request = req ) <EOL> try : <EOL> vsn , account = split_path ( req . path , <NUM_LIT:2> , <NUM_LIT:2> ) <EOL> except ValueError : <EOL> return HTTPBadRequest ( request = req ) <EOL> if account == '<STR_LIT>' : <EOL> path = '<STR_LIT>' % self . origin_account <EOL> resp = make_pre_authed_request ( <EOL> req . environ , '<STR_LIT>' , path , <EOL> agent = '<STR_LIT>' , swift_source = '<STR_LIT>' ) . get_response ( self . app ) <EOL> if resp . status_int // <NUM_LIT:100> != <NUM_LIT:2> : <EOL> raise Exception ( <EOL> '<STR_LIT>' % <EOL> ( path , resp . status ) ) <EOL> for i in xrange ( self . num_hash_cont ) : <EOL> cont_name = '<STR_LIT>' % i <EOL> path = '<STR_LIT>' % ( self . origin_account , cont_name ) <EOL> resp = make_pre_authed_request ( <EOL> req . environ , '<STR_LIT>' , path , <EOL> agent = '<STR_LIT>' , <EOL> swift_source = '<STR_LIT>' ) . get_response ( self . app ) <EOL> if resp . status_int // <NUM_LIT:100> != <NUM_LIT:2> : <EOL> raise Exception ( '<STR_LIT>' % <EOL> ( cont_name , path , resp . status ) ) <EOL> return HTTPNoContent ( request = req ) <EOL> return HTTPNotFound ( request = req ) <EOL> class CdnHandler ( OriginBase ) : <EOL> def __init__ ( self , app , conf , logger ) : <EOL> OriginBase . __init__ ( self , app , conf , logger ) <EOL> self . logger = logger <EOL> self . max_cdn_file_size = int ( conf . get ( '<STR_LIT>' , <EOL> MAX_CDN_FILE_SIZE ) ) <EOL> self . allowed_origin_remote_ips = [ ] <EOL> remote_ips = conf . get ( '<STR_LIT>' ) <EOL> if remote_ips : <EOL> self . allowed_origin_remote_ips = [ ip . strip ( ) for ip in remote_ips . split ( '<STR_LIT:U+002C>' ) if ip . strip ( ) ] <EOL> if not bool ( conf . get ( '<STR_LIT>' ) ) : <EOL> raise InvalidConfiguration ( '<STR_LIT>' ) <EOL> self . cdn_regexes = [ ] <EOL> for key , val in conf [ '<STR_LIT>' ] . items ( ) : <EOL> regex = re . compile ( val ) <EOL> self . cdn_regexes . append ( regex ) <EOL> def _getCacheHeaders ( self , ttl ) : <EOL> return { '<STR_LIT>' : strftime ( "<STR_LIT>" , <EOL> gmtime ( time ( ) + ttl ) ) , <EOL> '<STR_LIT>' : '<STR_LIT>' % ttl } <EOL> def _getCdnHeaders ( self , req ) : <EOL> headers = { '<STR_LIT>' : '<STR_LIT:True>' , '<STR_LIT>' : '<STR_LIT>' } <EOL> for header in [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] : <EOL> if header in req . headers : <EOL> headers [ header ] = req . headers [ header ] <EOL> return headers <EOL> def handle_request ( self , env , req ) : <EOL> if req . method not in ( '<STR_LIT:GET>' , '<STR_LIT>' , '<STR_LIT>' ) : <EOL> headers = self . _getCacheHeaders ( CACHE_BAD_URL ) <EOL> return HTTPMethodNotAllowed ( request = req , headers = headers ) <EOL> if req . method == '<STR_LIT>' and req . headers . get ( '<STR_LIT>' ) not in ( '<STR_LIT:GET>' , '<STR_LIT>' , '<STR_LIT>' ) : <EOL> headers = self . _getCacheHeaders ( CACHE_BAD_URL ) <EOL> return HTTPUnauthorized ( request = req , <EOL> headers = { '<STR_LIT>' : '<STR_LIT>' } ) <EOL> if self . allowed_origin_remote_ips and req . remote_addr not in self . allowed_origin_remote_ips : <EOL> raise OriginRequestNotAllowed ( <EOL> '<STR_LIT>' % req . remote_addr ) <EOL> hsh = env . get ( '<STR_LIT>' ) <EOL> object_name = env . get ( '<STR_LIT>' ) <EOL> if hsh is None or object_name is None : <EOL> for regex in self . cdn_regexes : <EOL> match_obj = regex . match ( req . url ) <EOL> if match_obj : <EOL> match_dict = match_obj . groupdict ( ) <EOL> if not hsh : <EOL> hsh = match_dict . get ( '<STR_LIT>' ) <EOL> if not object_name : <EOL> object_name = match_dict . get ( '<STR_LIT:object_name>' ) <EOL> break <EOL> if not hsh : <EOL> self . logger . debug ( '<STR_LIT>' % ( hsh , req . url ) ) <EOL> headers = self . _getCacheHeaders ( CACHE_BAD_URL ) <EOL> return HTTPNotFound ( request = req , headers = headers ) <EOL> if hsh . find ( '<STR_LIT:->' ) >= <NUM_LIT:0> : <EOL> hsh = hsh . split ( '<STR_LIT:->' , <NUM_LIT:1> ) [ <NUM_LIT:1> ] <EOL> try : <EOL> cdn_obj_path = self . get_hsh_obj_path ( hsh ) <EOL> except ValueError , e : <EOL> self . logger . debug ( '<STR_LIT>' % e ) <EOL> headers = self . _getCacheHeaders ( CACHE_BAD_URL ) <EOL> return HTTPBadRequest ( request = req , headers = headers ) <EOL> hash_data = self . get_cdn_data ( env , cdn_obj_path ) <EOL> if hash_data and hash_data . cdn_enabled : <EOL> if env . get ( '<STR_LIT>' ) : <EOL> auth_resp , ttl = env [ '<STR_LIT>' ] ( <EOL> env , hash_data . account . encode ( '<STR_LIT:utf-8>' ) ) <EOL> if auth_resp : <EOL> return auth_resp ( <EOL> request = req , headers = self . _getCacheHeaders ( ttl ) ) <EOL> swift_path = quote ( '<STR_LIT>' % ( <EOL> hash_data . account . encode ( '<STR_LIT:utf-8>' ) , <EOL> hash_data . container . encode ( '<STR_LIT:utf-8>' ) ) ) <EOL> if object_name : <EOL> swift_path += object_name <EOL> headers = self . _getCdnHeaders ( req ) <EOL> env [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> env [ '<STR_LIT>' ] = True <EOL> if not self . forward_origin : <EOL> env . pop ( '<STR_LIT>' , None ) <EOL> resp = make_pre_authed_request ( <EOL> env , req . method , swift_path , headers = headers , <EOL> agent = '<STR_LIT>' , swift_source = '<STR_LIT>' ) . get_response ( self . app ) <EOL> if resp . status_int == <NUM_LIT> and '<STR_LIT>' in resp . headers : <EOL> loc_parsed = urlparse ( resp . headers [ '<STR_LIT>' ] ) <EOL> acc_cont_path = '<STR_LIT>' % ( <EOL> hash_data . account . encode ( '<STR_LIT:utf-8>' ) , <EOL> hash_data . container . encode ( '<STR_LIT:utf-8>' ) ) <EOL> if loc_parsed . path . startswith ( acc_cont_path ) : <EOL> sos_loc = loc_parsed . path [ len ( acc_cont_path ) : ] <EOL> resp = SosResponse ( <EOL> headers = self . _getCacheHeaders ( hash_data . ttl ) ) <EOL> resp . headers [ '<STR_LIT>' ] = sos_loc <EOL> resp . status = <NUM_LIT> <EOL> return resp <EOL> else : <EOL> self . logger . exception ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' % ( loc_parsed . geturl ( ) , acc_cont_path ) ) <EOL> return HTTPInternalServerError ( '<STR_LIT>' ) <EOL> if resp . status_int == <NUM_LIT> : <EOL> return HTTPNotModified ( <EOL> request = req , headers = self . _getCacheHeaders ( hash_data . ttl ) ) <EOL> if resp . status_int == <NUM_LIT> : <EOL> return HTTPRequestedRangeNotSatisfiable ( <EOL> request = req , headers = self . _getCacheHeaders ( CACHE_404 ) ) <EOL> if resp . status_int // <NUM_LIT:100> == <NUM_LIT:2> or resp . status_int == <NUM_LIT> or ( <EOL> resp . status_int == <NUM_LIT> and req . method == '<STR_LIT>' ) : <EOL> if resp . content_length > self . max_cdn_file_size : <EOL> return HTTPBadRequest ( <EOL> request = req , headers = self . _getCacheHeaders ( CACHE_404 ) ) <EOL> cdn_resp = Response ( request = req , app_iter = resp . app_iter ) <EOL> cdn_resp . status = resp . status_int <EOL> cdn_resp . headers . update ( resp . headers ) <EOL> for allow_header in [ '<STR_LIT>' , '<STR_LIT>' ] : <EOL> if allow_header in cdn_resp . headers : <EOL> cdn_resp . headers [ allow_header ] = '<STR_LIT>' <EOL> if resp . status_int in ( <NUM_LIT> , <NUM_LIT> ) : <EOL> cdn_resp . headers . update ( self . _getCacheHeaders ( CACHE_404 ) ) <EOL> else : <EOL> cdn_resp . headers . update ( <EOL> self . _getCacheHeaders ( hash_data . ttl ) ) <EOL> return cdn_resp <EOL> self . logger . error ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' % ( resp . status , swift_path , <EOL> resp . body [ : <NUM_LIT> ] ) ) <EOL> return HTTPNotFound ( request = req , <EOL> headers = self . _getCacheHeaders ( CACHE_404 ) ) <EOL> class OriginDbHandler ( OriginBase ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , app , conf , logger ) : <EOL> OriginBase . __init__ ( self , app , conf , logger ) <EOL> self . conf = conf <EOL> self . logger = logger <EOL> self . default_ttl = int ( self . conf . get ( '<STR_LIT>' , DEFAULT_TTL ) ) <EOL> self . extra_header_for_deletes = self . conf . get ( <EOL> '<STR_LIT>' , '<STR_LIT>' ) <EOL> def _parse_container_listing ( self , account , listing_dict , output_format , <EOL> only_cdn_enabled = None ) : <EOL> """<STR_LIT>""" <EOL> container = listing_dict [ '<STR_LIT:name>' ] <EOL> if isinstance ( container , unicode ) : <EOL> container = container . encode ( '<STR_LIT:utf-8>' ) <EOL> cdn_data = listing_dict [ '<STR_LIT>' ] <EOL> hsh = self . hash_path ( account , container ) <EOL> if not cdn_data . startswith ( '<STR_LIT>' ) : <EOL> raise InvalidContentType ( '<STR_LIT>' % <EOL> ( account , container , cdn_data ) ) <EOL> try : <EOL> cdn_enabled , ttl , log_ret = cdn_data [ len ( '<STR_LIT>' ) : ] . split ( '<STR_LIT:->' ) <EOL> cdn_enabled = cdn_enabled . lower ( ) in TRUE_VALUES <EOL> log_ret = log_ret . lower ( ) in TRUE_VALUES <EOL> ttl = int ( ttl ) <EOL> if ( only_cdn_enabled is not None and <EOL> only_cdn_enabled != cdn_enabled ) : <EOL> return None <EOL> except ValueError : <EOL> raise InvalidContentType ( <EOL> '<STR_LIT>' % ( account , container , <EOL> cdn_data ) ) <EOL> if output_format not in ( '<STR_LIT>' , '<STR_LIT>' ) : <EOL> return container <EOL> cdn_url_dict = self . get_cdn_urls ( <EOL> hsh , '<STR_LIT:GET>' , request_format_tag = output_format ) <EOL> output_dict = { '<STR_LIT:name>' : container . decode ( '<STR_LIT:utf-8>' ) , <EOL> '<STR_LIT>' : cdn_enabled , <EOL> '<STR_LIT>' : ttl , '<STR_LIT>' : log_ret } <EOL> output_dict . update ( cdn_url_dict ) <EOL> def sos_escape ( val ) : <EOL> if isinstance ( val , basestring ) : <EOL> return saxutils . escape ( val ) <EOL> return val <EOL> if output_format == '<STR_LIT>' : <EOL> xml_data = '<STR_LIT:\n>' . join ( <EOL> [ '<STR_LIT>' % ( tag , sos_escape ( val ) , tag ) <EOL> for tag , val in output_dict . items ( ) ] ) <EOL> return """<STR_LIT>""" % xml_data <EOL> return output_dict <EOL> def origin_db_get ( self , env , req ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> vsn , account , junk = split_path ( req . path , <NUM_LIT:2> , <NUM_LIT:3> , <EOL> rest_with_last = True ) <EOL> except ValueError : <EOL> return HTTPBadRequest ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> if not account : <EOL> return HTTPBadRequest ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> list_format = req . params . get ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> if list_format : <EOL> list_format = list_format . lower ( ) <EOL> else : <EOL> list_format = req . accept . best_match ( ACCEPTABLE_FORMATS ) <EOL> if list_format : <EOL> list_format = list_format . split ( '<STR_LIT:/>' ) [ <NUM_LIT:1> ] <EOL> enabled_only = None <EOL> if req . params . get ( '<STR_LIT>' ) : <EOL> enabled_only = req . params [ '<STR_LIT>' ] . lower ( ) in TRUE_VALUES <EOL> limit = req . params . get ( '<STR_LIT>' ) <EOL> if limit : <EOL> try : <EOL> limit = int ( limit ) <EOL> except ValueError : <EOL> return HTTPBadRequest ( '<STR_LIT>' ) <EOL> def get_listings ( marker ) : <EOL> listing_path = quote ( '<STR_LIT>' % ( self . origin_account , account ) ) <EOL> listing_path += '<STR_LIT>' + quote ( marker ) <EOL> resp = make_pre_authed_request ( <EOL> env , '<STR_LIT:GET>' , listing_path , <EOL> agent = '<STR_LIT>' , swift_source = '<STR_LIT>' ) . get_response ( self . app ) <EOL> resp_headers = { } <EOL> listing_formatted = [ ] <EOL> if resp . status_int // <NUM_LIT:100> == <NUM_LIT:2> : <EOL> cont_listing = json . loads ( resp . body ) <EOL> for listing_dict in cont_listing : <EOL> if limit is None or len ( listing_formatted ) < limit : <EOL> try : <EOL> formatted_data = self . _parse_container_listing ( <EOL> account , listing_dict , list_format , <EOL> only_cdn_enabled = enabled_only ) <EOL> if formatted_data : <EOL> listing_formatted . append ( formatted_data ) <EOL> except InvalidContentType , e : <EOL> self . logger . exception ( e ) <EOL> continue <EOL> else : <EOL> break <EOL> if cont_listing and not listing_formatted : <EOL> new_marker = cont_listing [ - <NUM_LIT:1> ] [ '<STR_LIT:name>' ] <EOL> if isinstance ( new_marker , unicode ) : <EOL> new_marker = new_marker . encode ( '<STR_LIT:utf-8>' ) <EOL> return get_listings ( new_marker ) <EOL> elif resp . status_int == <NUM_LIT> : <EOL> raise OriginDbNotFound ( ) <EOL> else : <EOL> raise OriginDbFailure ( '<STR_LIT>' ) <EOL> return resp_headers , listing_formatted <EOL> try : <EOL> resp_headers , listing_formatted = get_listings ( <EOL> req . params . get ( '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> if list_format == '<STR_LIT>' : <EOL> resp_headers [ '<STR_LIT:Content-Type>' ] = '<STR_LIT>' <EOL> response_body = ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) % ( <EOL> account , '<STR_LIT:\n>' . join ( listing_formatted ) ) <EOL> elif list_format == '<STR_LIT>' : <EOL> resp_headers [ '<STR_LIT:Content-Type>' ] = '<STR_LIT:application/json>' <EOL> response_body = json . dumps ( listing_formatted ) <EOL> else : <EOL> resp_headers [ '<STR_LIT:Content-Type>' ] = '<STR_LIT>' <EOL> response_body = '<STR_LIT:\n>' . join ( listing_formatted ) + '<STR_LIT:\n>' <EOL> return Response ( body = response_body , headers = resp_headers ) <EOL> except OriginDbNotFound : <EOL> if list_format == '<STR_LIT>' : <EOL> empty_xml = '<STR_LIT:\n>' . join ( <EOL> [ '<STR_LIT>' , <EOL> '<STR_LIT>' % account , <EOL> '<STR_LIT>' ] ) <EOL> return HTTPOk ( request = req , <EOL> headers = { '<STR_LIT:Content-Type>' : '<STR_LIT>' } , <EOL> body = empty_xml ) <EOL> elif list_format == '<STR_LIT>' : <EOL> return HTTPOk ( request = req , <EOL> headers = { '<STR_LIT:Content-Type>' : '<STR_LIT:application/json>' } , <EOL> body = json . dumps ( [ ] ) ) <EOL> return HTTPNoContent ( request = req ) <EOL> def origin_db_delete ( self , env , req ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> vsn , account , container = split_path ( req . path , <NUM_LIT:3> , <NUM_LIT:3> ) <EOL> except ValueError : <EOL> return HTTPBadRequest ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> if self . extra_header_for_deletes and not req . headers . get ( <EOL> self . extra_header_for_deletes , '<STR_LIT:f>' ) . lower ( ) in TRUE_VALUES : <EOL> return HTTPMethodNotAllowed ( request = req ) <EOL> hsh = self . hash_path ( account , container ) <EOL> cdn_obj_path = self . get_hsh_obj_path ( hsh ) <EOL> memcache_client = utils . cache_from_env ( env ) <EOL> if memcache_client : <EOL> memcache_key = self . cdn_data_memcache_key ( cdn_obj_path ) <EOL> memcache_client . delete ( memcache_key ) <EOL> ref_hash_data = HashData ( <EOL> account , container , self . default_ttl , False , False , deleted = True ) <EOL> self . _set_hash_data ( env , cdn_obj_path , ref_hash_data , <EOL> update_listings = False ) <EOL> cdn_list_path = quote ( '<STR_LIT>' % ( self . origin_account , <EOL> account , container ) ) <EOL> list_resp = make_pre_authed_request ( <EOL> env , '<STR_LIT>' , cdn_list_path , <EOL> agent = '<STR_LIT>' , swift_source = '<STR_LIT>' ) . get_response ( self . app ) <EOL> if list_resp . status_int // <NUM_LIT:100> != <NUM_LIT:2> and list_resp . status_int != <NUM_LIT> : <EOL> raise OriginDbFailure ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' % ( cdn_list_path , list_resp . status_int ) ) <EOL> if list_resp . status_int == <NUM_LIT> : <EOL> return HTTPNotFound ( request = req ) <EOL> return HTTPNoContent ( request = req ) <EOL> def origin_db_head ( self , env , req ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> vsn , account , container = split_path ( req . path , <NUM_LIT:3> , <NUM_LIT:3> ) <EOL> except ValueError : <EOL> return HTTPBadRequest ( ) <EOL> hsh = self . hash_path ( account , container ) <EOL> cdn_obj_path = self . get_hsh_obj_path ( hsh ) <EOL> hash_data = self . get_cdn_data ( env , cdn_obj_path ) <EOL> if hash_data : <EOL> headers = self . get_cdn_urls ( hsh , '<STR_LIT>' ) <EOL> headers . update ( <EOL> { '<STR_LIT>' : hash_data . ttl , <EOL> '<STR_LIT>' : hash_data . logs_enabled and <EOL> '<STR_LIT:True>' or '<STR_LIT:False>' , <EOL> '<STR_LIT>' : hash_data . cdn_enabled and '<STR_LIT:True>' or '<STR_LIT:False>' } ) <EOL> return HTTPNoContent ( headers = headers ) <EOL> return HTTPNotFound ( request = req ) <EOL> def _set_hash_data ( self , env , cdn_obj_path , new_hash_data , <EOL> update_listings = True ) : <EOL> """<STR_LIT>""" <EOL> cdn_obj_data = new_hash_data . get_json_str ( ) <EOL> cdn_obj_etag = md5 ( cdn_obj_data ) . hexdigest ( ) <EOL> cdn_obj_resp = make_pre_authed_request ( <EOL> env , '<STR_LIT>' , cdn_obj_path , body = cdn_obj_data , <EOL> headers = { '<STR_LIT>' : cdn_obj_etag } , <EOL> agent = '<STR_LIT>' , swift_source = '<STR_LIT>' ) . get_response ( self . app ) <EOL> if cdn_obj_resp . status_int // <NUM_LIT:100> != <NUM_LIT:2> : <EOL> raise OriginDbFailure ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' % ( cdn_obj_path , cdn_obj_resp . status_int ) ) <EOL> memcache_client = utils . cache_from_env ( env ) <EOL> if memcache_client : <EOL> memcache_key = self . cdn_data_memcache_key ( cdn_obj_path ) <EOL> memcache_client . delete ( memcache_key ) <EOL> if not update_listings : <EOL> return <EOL> listing_cont_path = quote ( '<STR_LIT>' % ( self . origin_account , <EOL> new_hash_data . account ) ) <EOL> resp = make_pre_authed_request ( <EOL> env , '<STR_LIT>' , listing_cont_path , <EOL> agent = '<STR_LIT>' , swift_source = '<STR_LIT>' ) . get_response ( self . app ) <EOL> if resp . status_int == <NUM_LIT> : <EOL> resp = make_pre_authed_request ( <EOL> env , '<STR_LIT>' , listing_cont_path , <EOL> agent = '<STR_LIT>' , swift_source = '<STR_LIT>' ) . get_response ( self . app ) <EOL> if resp . status_int // <NUM_LIT:100> != <NUM_LIT:2> : <EOL> raise OriginDbFailure ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' % ( listing_cont_path , resp . status ) ) <EOL> cdn_list_path = quote ( '<STR_LIT>' % ( <EOL> self . origin_account , new_hash_data . account . encode ( '<STR_LIT:utf-8>' ) , <EOL> new_hash_data . container . encode ( '<STR_LIT:utf-8>' ) ) ) <EOL> listing_content_type = new_hash_data . gen_listing_content_type ( ) <EOL> cdn_list_resp = make_pre_authed_request ( <EOL> env , '<STR_LIT>' , cdn_list_path , <EOL> headers = { '<STR_LIT:Content-Type>' : listing_content_type , <EOL> '<STR_LIT>' : <NUM_LIT:0> } , <EOL> agent = '<STR_LIT>' , swift_source = '<STR_LIT>' ) . get_response ( self . app ) <EOL> if cdn_list_resp . status_int // <NUM_LIT:100> != <NUM_LIT:2> : <EOL> raise OriginDbFailure ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' % ( cdn_list_path , cdn_list_resp . status_int ) ) <EOL> def origin_db_puts_posts ( self , env , req ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> vsn , account , container = split_path ( req . path , <NUM_LIT:3> , <NUM_LIT:3> ) <EOL> except ValueError , e : <EOL> return HTTPBadRequest ( ) <EOL> hsh = self . hash_path ( account , container ) <EOL> cdn_obj_path = self . get_hsh_obj_path ( hsh ) <EOL> ttl , cdn_enabled , logs_enabled = self . default_ttl , True , False <EOL> hash_data = self . get_cdn_data ( env , cdn_obj_path ) <EOL> if hash_data : <EOL> ttl = hash_data . ttl <EOL> cdn_enabled = hash_data . cdn_enabled <EOL> logs_enabled = hash_data . logs_enabled <EOL> else : <EOL> if req . method == '<STR_LIT:POST>' : <EOL> return HTTPNotFound ( request = req ) <EOL> try : <EOL> ttl = int ( req . headers . get ( '<STR_LIT>' , ttl ) ) <EOL> except ValueError : <EOL> return HTTPBadRequest ( _ ( '<STR_LIT>' ) ) <EOL> ttl = min ( max ( ttl , self . min_ttl ) , self . max_ttl ) <EOL> if '<STR_LIT>' in req . headers : <EOL> logs_enabled = req . headers . get ( '<STR_LIT>' ) . lower ( ) in TRUE_VALUES <EOL> if '<STR_LIT>' in req . headers : <EOL> cdn_enabled = req . headers . get ( '<STR_LIT>' ) . lower ( ) in TRUE_VALUES <EOL> new_hash_data = HashData ( account , container , ttl , cdn_enabled , <EOL> logs_enabled ) <EOL> if hash_data and hash_data == new_hash_data : <EOL> cdn_url_headers = self . get_cdn_urls ( hsh , '<STR_LIT>' ) <EOL> return HTTPAccepted ( request = req , <EOL> headers = cdn_url_headers ) <EOL> self . _set_hash_data ( env , cdn_obj_path , new_hash_data ) <EOL> cdn_url_headers = self . get_cdn_urls ( hsh , '<STR_LIT>' ) <EOL> if req . method == '<STR_LIT:POST>' : <EOL> resp = HTTPAccepted ( request = req , <EOL> headers = cdn_url_headers ) <EOL> else : <EOL> resp = HTTPCreated ( request = req , <EOL> headers = cdn_url_headers ) <EOL> listing_content_type = new_hash_data . gen_listing_content_type ( ) <EOL> resp . extra_log_data = listing_content_type <EOL> return resp <EOL> def handle_request ( self , env , req ) : <EOL> """<STR_LIT>""" <EOL> if '<STR_LIT>' in req . environ : <EOL> aresp = req . environ [ '<STR_LIT>' ] ( req ) <EOL> if aresp : <EOL> return aresp <EOL> try : <EOL> if req . method in ( '<STR_LIT>' , '<STR_LIT:POST>' ) : <EOL> return self . origin_db_puts_posts ( env , req ) <EOL> if req . method == '<STR_LIT:GET>' : <EOL> return self . origin_db_get ( env , req ) <EOL> if req . method == '<STR_LIT>' : <EOL> return self . origin_db_head ( env , req ) <EOL> if req . method == '<STR_LIT>' : <EOL> return self . origin_db_delete ( env , req ) <EOL> except OriginDbFailure , e : <EOL> self . logger . exception ( e ) <EOL> return HTTPInternalServerError ( '<STR_LIT>' ) <EOL> return HTTPNotFound ( ) <EOL> class OriginServer ( object ) : <EOL> @ classmethod <EOL> def _translate_conf ( cls , conf ) : <EOL> origin_conf = conf [ '<STR_LIT>' ] <EOL> conf = readconf ( origin_conf , raw = True ) <EOL> xconf = conf [ '<STR_LIT>' ] <EOL> for format_section in [ <EOL> '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' ] : <EOL> if conf . get ( format_section , None ) : <EOL> xconf [ format_section ] = conf [ format_section ] <EOL> return xconf <EOL> def __init__ ( self , app , conf ) : <EOL> self . app = app <EOL> self . logger = get_logger ( conf , log_route = '<STR_LIT>' ) <EOL> self . conf = OriginServer . _translate_conf ( conf ) <EOL> self . origin_prefix = self . conf . get ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> self . origin_db_hosts = [ <EOL> host . strip ( ) for host in <EOL> self . conf . get ( '<STR_LIT>' , '<STR_LIT>' ) . split ( '<STR_LIT:U+002C>' ) if host . strip ( ) ] <EOL> self . origin_cdn_host_suffixes = [ <EOL> host . strip ( ) for host in <EOL> self . conf . get ( '<STR_LIT>' , '<STR_LIT>' ) . split ( '<STR_LIT:U+002C>' ) <EOL> if host . strip ( ) ] <EOL> if not self . origin_cdn_host_suffixes : <EOL> raise InvalidConfiguration ( '<STR_LIT>' ) <EOL> self . ignore_hosts = [ <EOL> host . strip ( ) for host in <EOL> self . conf . get ( '<STR_LIT>' , '<STR_LIT>' ) . split ( '<STR_LIT:U+002C>' ) <EOL> if host . strip ( ) ] <EOL> self . log_access_requests = self . conf . get ( '<STR_LIT>' , '<STR_LIT:t>' ) in TRUE_VALUES <EOL> def __call__ ( self , env , start_response ) : <EOL> """<STR_LIT>""" <EOL> env [ '<STR_LIT>' ] = time ( ) <EOL> host = env . get ( '<STR_LIT>' , '<STR_LIT>' ) . split ( '<STR_LIT::>' ) [ <NUM_LIT:0> ] <EOL> try : <EOL> handler = None <EOL> request_type = '<STR_LIT>' <EOL> if host in self . origin_db_hosts : <EOL> handler = OriginDbHandler ( self . app , self . conf , self . logger ) <EOL> request_type = '<STR_LIT>' <EOL> for cdn_host_suffix in self . origin_cdn_host_suffixes : <EOL> if host . endswith ( cdn_host_suffix ) and host not in self . ignore_hosts : <EOL> handler = CdnHandler ( self . app , self . conf , self . logger ) <EOL> request_type = '<STR_LIT>' <EOL> break <EOL> if env [ '<STR_LIT>' ] . startswith ( self . origin_prefix ) : <EOL> handler = AdminHandler ( self . app , self . conf , self . logger ) <EOL> request_type = '<STR_LIT>' <EOL> if handler : <EOL> req = Request ( env ) <EOL> resp = handler . handle_request ( env , req ) <EOL> if request_type != '<STR_LIT>' : <EOL> self . _log_request ( env , resp , request_type ) <EOL> return resp ( env , start_response ) <EOL> except InvalidConfiguration , e : <EOL> self . logger . exception ( e ) <EOL> return HTTPInternalServerError ( body = str ( e ) ) ( env , start_response ) <EOL> except InvalidUtf8 : <EOL> return HTTPPreconditionFailed ( <EOL> body = '<STR_LIT>' ) ( env , start_response ) <EOL> except OriginRequestNotAllowed , e : <EOL> self . logger . debug ( e ) <EOL> return self . app ( env , start_response ) <EOL> def _log_request ( self , env , response , request_type ) : <EOL> """<STR_LIT>""" <EOL> if not self . log_access_requests : <EOL> return <EOL> trans_time = '<STR_LIT>' % ( time ( ) - <EOL> env . get ( '<STR_LIT>' , time ( ) ) ) <EOL> the_request = quote ( unquote ( env [ '<STR_LIT>' ] ) ) <EOL> if env . get ( '<STR_LIT>' ) : <EOL> the_request = the_request + '<STR_LIT:?>' + env [ '<STR_LIT>' ] <EOL> client = env . get ( '<STR_LIT>' ) <EOL> if not client and '<STR_LIT>' in env : <EOL> client = env [ '<STR_LIT>' ] . split ( '<STR_LIT:U+002C>' ) [ <NUM_LIT:0> ] . strip ( ) <EOL> extra_data = '<STR_LIT:->' <EOL> if hasattr ( response , '<STR_LIT>' ) : <EOL> extra_data = response . extra_log_data <EOL> self . logger . info ( '<STR_LIT:U+0020>' . join ( quote ( str ( x ) ) for x in ( <EOL> client or '<STR_LIT:->' , <EOL> env . get ( '<STR_LIT>' , '<STR_LIT:->' ) , <EOL> strftime ( '<STR_LIT>' , gmtime ( ) ) , <EOL> request_type , <EOL> env [ '<STR_LIT>' ] , <EOL> env . get ( '<STR_LIT>' , '<STR_LIT:->' ) , <EOL> the_request , <EOL> env [ '<STR_LIT>' ] , <EOL> response . status_int , <EOL> env . get ( '<STR_LIT>' , '<STR_LIT:->' ) , <EOL> env . get ( '<STR_LIT>' , '<STR_LIT:->' ) , <EOL> env . get ( '<STR_LIT>' , '<STR_LIT:->' ) , <EOL> env . get ( '<STR_LIT>' , '<STR_LIT:->' ) , <EOL> env . get ( '<STR_LIT>' , '<STR_LIT:->' ) , <EOL> trans_time , <EOL> extra_data ) ) ) <EOL> def filter_factory ( global_conf , ** local_conf ) : <EOL> """<STR_LIT>""" <EOL> conf = global_conf . copy ( ) <EOL> conf . update ( local_conf ) <EOL> if conf . get ( '<STR_LIT>' , True ) : <EOL> xconf = OriginServer . _translate_conf ( conf ) <EOL> min_ttl = int ( xconf . get ( '<STR_LIT>' , MIN_TTL ) ) <EOL> max_ttl = int ( xconf . get ( '<STR_LIT>' , MAX_TTL ) ) <EOL> default_ttl = int ( xconf . get ( '<STR_LIT>' , DEFAULT_TTL ) ) <EOL> max_cdn_file_size = int ( xconf . get ( '<STR_LIT>' , <EOL> MAX_CDN_FILE_SIZE ) ) <EOL> register_swift_info ( '<STR_LIT>' , <EOL> min_ttl = min_ttl , <EOL> max_ttl = max_ttl , <EOL> default_ttl = default_ttl , <EOL> max_cdn_file_size = max_cdn_file_size ) <EOL> def origin ( app ) : <EOL> return OriginServer ( app , conf ) <EOL> return origin </s>
