<s> """<STR_LIT>""" <EOL> from __future__ import print_function <EOL> import errno <EOL> import fcntl <EOL> import grp <EOL> import hmac <EOL> import json <EOL> import math <EOL> import operator <EOL> import os <EOL> import pwd <EOL> import re <EOL> import sys <EOL> import time <EOL> import uuid <EOL> import functools <EOL> import email . parser <EOL> from hashlib import md5 , sha1 <EOL> from random import random , shuffle <EOL> from contextlib import contextmanager , closing <EOL> import ctypes <EOL> import ctypes . util <EOL> from optparse import OptionParser <EOL> from tempfile import mkstemp , NamedTemporaryFile <EOL> import glob <EOL> import itertools <EOL> import stat <EOL> import datetime <EOL> import eventlet <EOL> import eventlet . semaphore <EOL> from eventlet import GreenPool , sleep , Timeout , tpool , greenthread , greenio , event <EOL> from eventlet . green import socket , threading <EOL> import eventlet . queue <EOL> import netifaces <EOL> import codecs <EOL> utf8_decoder = codecs . getdecoder ( '<STR_LIT:utf-8>' ) <EOL> utf8_encoder = codecs . getencoder ( '<STR_LIT:utf-8>' ) <EOL> import six <EOL> from six . moves import cPickle as pickle <EOL> from six . moves . configparser import ( ConfigParser , NoSectionError , <EOL> NoOptionError , RawConfigParser ) <EOL> from six . moves import range <EOL> from six . moves . urllib . parse import ParseResult <EOL> from six . moves . urllib . parse import quote as _quote <EOL> from six . moves . urllib . parse import urlparse as stdlib_urlparse <EOL> from swift import gettext_ as _ <EOL> import swift . common . exceptions <EOL> from swift . common . http import is_success , is_redirection , HTTP_NOT_FOUND , HTTP_PRECONDITION_FAILED , HTTP_REQUESTED_RANGE_NOT_SATISFIABLE <EOL> from swift . common . header_key_dict import HeaderKeyDict <EOL> if six . PY3 : <EOL> stdlib_queue = eventlet . patcher . original ( '<STR_LIT>' ) <EOL> else : <EOL> stdlib_queue = eventlet . patcher . original ( '<STR_LIT>' ) <EOL> stdlib_threading = eventlet . patcher . original ( '<STR_LIT>' ) <EOL> from logging . handlers import SysLogHandler <EOL> import logging <EOL> logging . thread = eventlet . green . thread <EOL> logging . threading = eventlet . green . threading <EOL> logging . _lock = logging . threading . RLock ( ) <EOL> NOTICE = <NUM_LIT> <EOL> logging . addLevelName ( NOTICE , '<STR_LIT>' ) <EOL> SysLogHandler . priority_map [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> _sys_fallocate = None <EOL> _posix_fadvise = None <EOL> _libc_socket = None <EOL> _libc_bind = None <EOL> _libc_accept = None <EOL> FALLOCATE_RESERVE = <NUM_LIT:0> <EOL> HASH_PATH_SUFFIX = '<STR_LIT>' <EOL> HASH_PATH_PREFIX = '<STR_LIT>' <EOL> SWIFT_CONF_FILE = '<STR_LIT>' <EOL> AF_ALG = getattr ( socket , '<STR_LIT>' , <NUM_LIT> ) <EOL> F_SETPIPE_SZ = getattr ( fcntl , '<STR_LIT>' , <NUM_LIT> ) <EOL> IPV6_RE = re . compile ( "<STR_LIT>" ) <EOL> class InvalidHashPathConfigError ( ValueError ) : <EOL> def __str__ ( self ) : <EOL> return "<STR_LIT>" "<STR_LIT>" % SWIFT_CONF_FILE <EOL> def validate_hash_conf ( ) : <EOL> global HASH_PATH_SUFFIX <EOL> global HASH_PATH_PREFIX <EOL> if not HASH_PATH_SUFFIX and not HASH_PATH_PREFIX : <EOL> hash_conf = ConfigParser ( ) <EOL> if hash_conf . read ( SWIFT_CONF_FILE ) : <EOL> try : <EOL> HASH_PATH_SUFFIX = hash_conf . get ( '<STR_LIT>' , <EOL> '<STR_LIT>' ) <EOL> except ( NoSectionError , NoOptionError ) : <EOL> pass <EOL> try : <EOL> HASH_PATH_PREFIX = hash_conf . get ( '<STR_LIT>' , <EOL> '<STR_LIT>' ) <EOL> except ( NoSectionError , NoOptionError ) : <EOL> pass <EOL> if not HASH_PATH_SUFFIX and not HASH_PATH_PREFIX : <EOL> raise InvalidHashPathConfigError ( ) <EOL> try : <EOL> validate_hash_conf ( ) <EOL> except InvalidHashPathConfigError : <EOL> pass <EOL> def get_hmac ( request_method , path , expires , key ) : <EOL> """<STR_LIT>""" <EOL> return hmac . new ( <EOL> key , '<STR_LIT>' % ( request_method , expires , path ) , sha1 ) . hexdigest ( ) <EOL> _swift_info = { } <EOL> _swift_admin_info = { } <EOL> def get_swift_info ( admin = False , disallowed_sections = None ) : <EOL> """<STR_LIT>""" <EOL> disallowed_sections = disallowed_sections or [ ] <EOL> info = dict ( _swift_info ) <EOL> for section in disallowed_sections : <EOL> key_to_pop = None <EOL> sub_section_dict = info <EOL> for sub_section in section . split ( '<STR_LIT:.>' ) : <EOL> if key_to_pop : <EOL> sub_section_dict = sub_section_dict . get ( key_to_pop , { } ) <EOL> if not isinstance ( sub_section_dict , dict ) : <EOL> sub_section_dict = { } <EOL> break <EOL> key_to_pop = sub_section <EOL> sub_section_dict . pop ( key_to_pop , None ) <EOL> if admin : <EOL> info [ '<STR_LIT>' ] = dict ( _swift_admin_info ) <EOL> info [ '<STR_LIT>' ] [ '<STR_LIT>' ] = list ( disallowed_sections ) <EOL> return info <EOL> def register_swift_info ( name = '<STR_LIT>' , admin = False , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> if name == '<STR_LIT>' or name == '<STR_LIT>' : <EOL> raise ValueError ( '<STR_LIT>' . format ( name ) ) <EOL> if admin : <EOL> dict_to_use = _swift_admin_info <EOL> else : <EOL> dict_to_use = _swift_info <EOL> if name not in dict_to_use : <EOL> if "<STR_LIT:.>" in name : <EOL> raise ValueError ( '<STR_LIT>' % name ) <EOL> dict_to_use [ name ] = { } <EOL> for key , val in kwargs . items ( ) : <EOL> if "<STR_LIT:.>" in key : <EOL> raise ValueError ( '<STR_LIT>' % key ) <EOL> dict_to_use [ name ] [ key ] = val <EOL> def backward ( f , blocksize = <NUM_LIT> ) : <EOL> """<STR_LIT>""" <EOL> f . seek ( <NUM_LIT:0> , os . SEEK_END ) <EOL> if f . tell ( ) == <NUM_LIT:0> : <EOL> return <EOL> last_row = b'<STR_LIT>' <EOL> while f . tell ( ) != <NUM_LIT:0> : <EOL> try : <EOL> f . seek ( - blocksize , os . SEEK_CUR ) <EOL> except IOError : <EOL> blocksize = f . tell ( ) <EOL> f . seek ( - blocksize , os . SEEK_CUR ) <EOL> block = f . read ( blocksize ) <EOL> f . seek ( - blocksize , os . SEEK_CUR ) <EOL> rows = block . split ( b'<STR_LIT:\n>' ) <EOL> rows [ - <NUM_LIT:1> ] = rows [ - <NUM_LIT:1> ] + last_row <EOL> while rows : <EOL> last_row = rows . pop ( - <NUM_LIT:1> ) <EOL> if rows and last_row : <EOL> yield last_row <EOL> yield last_row <EOL> TRUE_VALUES = set ( ( '<STR_LIT:true>' , '<STR_LIT:1>' , '<STR_LIT:yes>' , '<STR_LIT>' , '<STR_LIT:t>' , '<STR_LIT:y>' ) ) <EOL> def config_true_value ( value ) : <EOL> """<STR_LIT>""" <EOL> return value is True or ( isinstance ( value , six . string_types ) and value . lower ( ) in TRUE_VALUES ) <EOL> def config_auto_int_value ( value , default ) : <EOL> """<STR_LIT>""" <EOL> if value is None or ( isinstance ( value , six . string_types ) and value . lower ( ) == '<STR_LIT>' ) : <EOL> return default <EOL> try : <EOL> value = int ( value ) <EOL> except ( TypeError , ValueError ) : <EOL> raise ValueError ( '<STR_LIT>' <EOL> '<STR_LIT>' % value ) <EOL> return value <EOL> def append_underscore ( prefix ) : <EOL> if prefix and not prefix . endswith ( '<STR_LIT:_>' ) : <EOL> prefix += '<STR_LIT:_>' <EOL> return prefix <EOL> def config_read_reseller_options ( conf , defaults ) : <EOL> """<STR_LIT>""" <EOL> reseller_prefix_opt = conf . get ( '<STR_LIT>' , '<STR_LIT>' ) . split ( '<STR_LIT:U+002C>' ) <EOL> reseller_prefixes = [ ] <EOL> for prefix in [ pre . strip ( ) for pre in reseller_prefix_opt if pre . strip ( ) ] : <EOL> if prefix == "<STR_LIT>" : <EOL> prefix = '<STR_LIT>' <EOL> prefix = append_underscore ( prefix ) <EOL> if prefix not in reseller_prefixes : <EOL> reseller_prefixes . append ( prefix ) <EOL> if len ( reseller_prefixes ) == <NUM_LIT:0> : <EOL> reseller_prefixes . append ( '<STR_LIT>' ) <EOL> associated_options = { } <EOL> for prefix in reseller_prefixes : <EOL> associated_options [ prefix ] = dict ( defaults ) <EOL> associated_options [ prefix ] . update ( <EOL> config_read_prefixed_options ( conf , '<STR_LIT>' , defaults ) ) <EOL> prefix_name = prefix if prefix != '<STR_LIT>' else "<STR_LIT>" <EOL> associated_options [ prefix ] . update ( <EOL> config_read_prefixed_options ( conf , prefix_name , defaults ) ) <EOL> return reseller_prefixes , associated_options <EOL> def config_read_prefixed_options ( conf , prefix_name , defaults ) : <EOL> """<STR_LIT>""" <EOL> params = { } <EOL> for option_name in defaults . keys ( ) : <EOL> value = conf . get ( '<STR_LIT>' % ( prefix_name , option_name ) ) <EOL> if value : <EOL> if isinstance ( defaults . get ( option_name ) , list ) : <EOL> params [ option_name ] = [ ] <EOL> for role in value . lower ( ) . split ( '<STR_LIT:U+002C>' ) : <EOL> params [ option_name ] . append ( role . strip ( ) ) <EOL> else : <EOL> params [ option_name ] = value . strip ( ) <EOL> return params <EOL> def noop_libc_function ( * args ) : <EOL> return <NUM_LIT:0> <EOL> def validate_configuration ( ) : <EOL> try : <EOL> validate_hash_conf ( ) <EOL> except InvalidHashPathConfigError as e : <EOL> sys . exit ( "<STR_LIT>" % e ) <EOL> def load_libc_function ( func_name , log_error = True , <EOL> fail_if_missing = False ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> libc = ctypes . CDLL ( ctypes . util . find_library ( '<STR_LIT:c>' ) , use_errno = True ) <EOL> return getattr ( libc , func_name ) <EOL> except AttributeError : <EOL> if fail_if_missing : <EOL> raise <EOL> if log_error : <EOL> logging . warning ( _ ( "<STR_LIT>" <EOL> "<STR_LIT>" ) , func_name ) <EOL> return noop_libc_function <EOL> def generate_trans_id ( trans_id_suffix ) : <EOL> return '<STR_LIT>' % ( <EOL> uuid . uuid4 ( ) . hex [ : <NUM_LIT> ] , time . time ( ) , quote ( trans_id_suffix ) ) <EOL> def get_policy_index ( req_headers , res_headers ) : <EOL> """<STR_LIT>""" <EOL> header = '<STR_LIT>' <EOL> policy_index = res_headers . get ( header , req_headers . get ( header ) ) <EOL> return str ( policy_index ) if policy_index is not None else None <EOL> def get_log_line ( req , res , trans_time , additional_info ) : <EOL> """<STR_LIT>""" <EOL> policy_index = get_policy_index ( req . headers , res . headers ) <EOL> return '<STR_LIT>' % ( <EOL> req . remote_addr , <EOL> time . strftime ( '<STR_LIT>' , time . gmtime ( ) ) , <EOL> req . method , req . path , res . status . split ( ) [ <NUM_LIT:0> ] , <EOL> res . content_length or '<STR_LIT:->' , req . referer or '<STR_LIT:->' , <EOL> req . headers . get ( '<STR_LIT>' , '<STR_LIT:->' ) , <EOL> req . user_agent or '<STR_LIT:->' , trans_time , additional_info or '<STR_LIT:->' , <EOL> os . getpid ( ) , policy_index or '<STR_LIT:->' ) <EOL> def get_trans_id_time ( trans_id ) : <EOL> if len ( trans_id ) >= <NUM_LIT> and trans_id . startswith ( '<STR_LIT>' ) and trans_id [ <NUM_LIT> ] == '<STR_LIT:->' : <EOL> try : <EOL> return int ( trans_id [ <NUM_LIT> : <NUM_LIT> ] , <NUM_LIT:16> ) <EOL> except ValueError : <EOL> pass <EOL> return None <EOL> class FileLikeIter ( object ) : <EOL> def __init__ ( self , iterable ) : <EOL> """<STR_LIT>""" <EOL> self . iterator = iter ( iterable ) <EOL> self . buf = None <EOL> self . closed = False <EOL> def __iter__ ( self ) : <EOL> return self <EOL> def next ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . closed : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> if self . buf : <EOL> rv = self . buf <EOL> self . buf = None <EOL> return rv <EOL> else : <EOL> return next ( self . iterator ) <EOL> __next__ = next <EOL> def read ( self , size = - <NUM_LIT:1> ) : <EOL> """<STR_LIT>""" <EOL> if self . closed : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> if size < <NUM_LIT:0> : <EOL> return b'<STR_LIT>' . join ( self ) <EOL> elif not size : <EOL> chunk = b'<STR_LIT>' <EOL> elif self . buf : <EOL> chunk = self . buf <EOL> self . buf = None <EOL> else : <EOL> try : <EOL> chunk = next ( self . iterator ) <EOL> except StopIteration : <EOL> return b'<STR_LIT>' <EOL> if len ( chunk ) > size : <EOL> self . buf = chunk [ size : ] <EOL> chunk = chunk [ : size ] <EOL> return chunk <EOL> def readline ( self , size = - <NUM_LIT:1> ) : <EOL> """<STR_LIT>""" <EOL> if self . closed : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> data = b'<STR_LIT>' <EOL> while b'<STR_LIT:\n>' not in data and ( size < <NUM_LIT:0> or len ( data ) < size ) : <EOL> if size < <NUM_LIT:0> : <EOL> chunk = self . read ( <NUM_LIT> ) <EOL> else : <EOL> chunk = self . read ( size - len ( data ) ) <EOL> if not chunk : <EOL> break <EOL> data += chunk <EOL> if b'<STR_LIT:\n>' in data : <EOL> data , sep , rest = data . partition ( b'<STR_LIT:\n>' ) <EOL> data += sep <EOL> if self . buf : <EOL> self . buf = rest + self . buf <EOL> else : <EOL> self . buf = rest <EOL> return data <EOL> def readlines ( self , sizehint = - <NUM_LIT:1> ) : <EOL> """<STR_LIT>""" <EOL> if self . closed : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> lines = [ ] <EOL> while True : <EOL> line = self . readline ( sizehint ) <EOL> if not line : <EOL> break <EOL> lines . append ( line ) <EOL> if sizehint >= <NUM_LIT:0> : <EOL> sizehint -= len ( line ) <EOL> if sizehint <= <NUM_LIT:0> : <EOL> break <EOL> return lines <EOL> def close ( self ) : <EOL> """<STR_LIT>""" <EOL> self . iterator = None <EOL> self . closed = True <EOL> class FallocateWrapper ( object ) : <EOL> def __init__ ( self , noop = False ) : <EOL> if noop : <EOL> self . func_name = '<STR_LIT>' <EOL> self . fallocate = noop_libc_function <EOL> return <EOL> for func in ( '<STR_LIT>' , '<STR_LIT>' ) : <EOL> self . func_name = func <EOL> self . fallocate = load_libc_function ( func , log_error = False ) <EOL> if self . fallocate is not noop_libc_function : <EOL> break <EOL> if self . fallocate is noop_libc_function : <EOL> logging . warning ( _ ( "<STR_LIT>" <EOL> "<STR_LIT>" ) ) <EOL> def __call__ ( self , fd , mode , offset , length ) : <EOL> """<STR_LIT>""" <EOL> if FALLOCATE_RESERVE > <NUM_LIT:0> : <EOL> st = os . fstatvfs ( fd ) <EOL> free = st . f_frsize * st . f_bavail - length . value <EOL> if free <= FALLOCATE_RESERVE : <EOL> raise OSError ( <EOL> errno . ENOSPC , <EOL> '<STR_LIT>' % ( free , <EOL> FALLOCATE_RESERVE ) ) <EOL> args = { <EOL> '<STR_LIT>' : ( fd , mode , offset , length ) , <EOL> '<STR_LIT>' : ( fd , offset , length ) <EOL> } <EOL> return self . fallocate ( * args [ self . func_name ] ) <EOL> def disable_fallocate ( ) : <EOL> global _sys_fallocate <EOL> _sys_fallocate = FallocateWrapper ( noop = True ) <EOL> def fallocate ( fd , size ) : <EOL> """<STR_LIT>""" <EOL> global _sys_fallocate <EOL> if _sys_fallocate is None : <EOL> _sys_fallocate = FallocateWrapper ( ) <EOL> if size < <NUM_LIT:0> : <EOL> size = <NUM_LIT:0> <EOL> ret = _sys_fallocate ( fd , <NUM_LIT:1> , <NUM_LIT:0> , ctypes . c_uint64 ( size ) ) <EOL> err = ctypes . get_errno ( ) <EOL> if ret and err not in ( <NUM_LIT:0> , errno . ENOSYS , errno . EOPNOTSUPP , <EOL> errno . EINVAL ) : <EOL> raise OSError ( err , '<STR_LIT>' % size ) <EOL> def fsync ( fd ) : <EOL> """<STR_LIT>""" <EOL> if hasattr ( fcntl , '<STR_LIT>' ) : <EOL> try : <EOL> fcntl . fcntl ( fd , fcntl . F_FULLSYNC ) <EOL> except IOError as e : <EOL> raise OSError ( e . errno , '<STR_LIT>' % fd ) <EOL> else : <EOL> os . fsync ( fd ) <EOL> def fdatasync ( fd ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> os . fdatasync ( fd ) <EOL> except AttributeError : <EOL> fsync ( fd ) <EOL> def fsync_dir ( dirpath ) : <EOL> """<STR_LIT>""" <EOL> dirfd = None <EOL> try : <EOL> dirfd = os . open ( dirpath , os . O_DIRECTORY | os . O_RDONLY ) <EOL> fsync ( dirfd ) <EOL> except OSError as err : <EOL> if err . errno == errno . ENOTDIR : <EOL> raise <EOL> logging . warning ( _ ( '<STR_LIT>' <EOL> '<STR_LIT>' ) , <EOL> { '<STR_LIT>' : dirpath , '<STR_LIT>' : os . strerror ( err . errno ) } ) <EOL> finally : <EOL> if dirfd : <EOL> os . close ( dirfd ) <EOL> def drop_buffer_cache ( fd , offset , length ) : <EOL> """<STR_LIT>""" <EOL> global _posix_fadvise <EOL> if _posix_fadvise is None : <EOL> _posix_fadvise = load_libc_function ( '<STR_LIT>' ) <EOL> ret = _posix_fadvise ( fd , ctypes . c_uint64 ( offset ) , <EOL> ctypes . c_uint64 ( length ) , <NUM_LIT:4> ) <EOL> if ret != <NUM_LIT:0> : <EOL> logging . warning ( "<STR_LIT>" <EOL> "<STR_LIT>" , { '<STR_LIT>' : fd , '<STR_LIT>' : offset , <EOL> '<STR_LIT>' : length , '<STR_LIT>' : ret } ) <EOL> NORMAL_FORMAT = "<STR_LIT>" <EOL> INTERNAL_FORMAT = NORMAL_FORMAT + '<STR_LIT>' <EOL> SHORT_FORMAT = NORMAL_FORMAT + '<STR_LIT>' <EOL> MAX_OFFSET = ( <NUM_LIT:16> ** <NUM_LIT:16> ) - <NUM_LIT:1> <EOL> PRECISION = <NUM_LIT> <EOL> FORCE_INTERNAL = False <EOL> @ functools . total_ordering <EOL> class Timestamp ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , timestamp , offset = <NUM_LIT:0> , delta = <NUM_LIT:0> ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( timestamp , six . string_types ) : <EOL> parts = timestamp . split ( '<STR_LIT:_>' , <NUM_LIT:1> ) <EOL> self . timestamp = float ( parts . pop ( <NUM_LIT:0> ) ) <EOL> if parts : <EOL> self . offset = int ( parts [ <NUM_LIT:0> ] , <NUM_LIT:16> ) <EOL> else : <EOL> self . offset = <NUM_LIT:0> <EOL> else : <EOL> self . timestamp = float ( timestamp ) <EOL> self . offset = getattr ( timestamp , '<STR_LIT>' , <NUM_LIT:0> ) <EOL> if offset >= <NUM_LIT:0> : <EOL> self . offset += offset <EOL> else : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> if self . offset > MAX_OFFSET : <EOL> raise ValueError ( '<STR_LIT>' % MAX_OFFSET ) <EOL> self . raw = int ( round ( self . timestamp / PRECISION ) ) <EOL> if delta : <EOL> self . raw = self . raw + delta <EOL> if self . raw <= <NUM_LIT:0> : <EOL> raise ValueError ( <EOL> '<STR_LIT>' % ( - <NUM_LIT:1> * self . raw ) ) <EOL> self . timestamp = float ( self . raw * PRECISION ) <EOL> if self . timestamp < <NUM_LIT:0> : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> if self . timestamp >= <NUM_LIT> : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> def __repr__ ( self ) : <EOL> return INTERNAL_FORMAT % ( self . timestamp , self . offset ) <EOL> def __str__ ( self ) : <EOL> raise TypeError ( '<STR_LIT>' ) <EOL> def __float__ ( self ) : <EOL> return self . timestamp <EOL> def __int__ ( self ) : <EOL> return int ( self . timestamp ) <EOL> def __nonzero__ ( self ) : <EOL> return bool ( self . timestamp or self . offset ) <EOL> def __bool__ ( self ) : <EOL> return self . __nonzero__ ( ) <EOL> @ property <EOL> def normal ( self ) : <EOL> return NORMAL_FORMAT % self . timestamp <EOL> @ property <EOL> def internal ( self ) : <EOL> if self . offset or FORCE_INTERNAL : <EOL> return INTERNAL_FORMAT % ( self . timestamp , self . offset ) <EOL> else : <EOL> return self . normal <EOL> @ property <EOL> def short ( self ) : <EOL> if self . offset or FORCE_INTERNAL : <EOL> return SHORT_FORMAT % ( self . timestamp , self . offset ) <EOL> else : <EOL> return self . normal <EOL> @ property <EOL> def isoformat ( self ) : <EOL> t = float ( self . normal ) <EOL> if six . PY3 : <EOL> frac , t = math . modf ( t ) <EOL> us = round ( frac * <NUM_LIT> ) <EOL> if us >= <NUM_LIT> : <EOL> t += <NUM_LIT:1> <EOL> us -= <NUM_LIT> <EOL> elif us < <NUM_LIT:0> : <EOL> t -= <NUM_LIT:1> <EOL> us += <NUM_LIT> <EOL> dt = datetime . datetime . utcfromtimestamp ( t ) <EOL> dt = dt . replace ( microsecond = us ) <EOL> else : <EOL> dt = datetime . datetime . utcfromtimestamp ( t ) <EOL> isoformat = dt . isoformat ( ) <EOL> if len ( isoformat ) < len ( "<STR_LIT>" ) : <EOL> isoformat += "<STR_LIT>" <EOL> return isoformat <EOL> def __eq__ ( self , other ) : <EOL> if other is None : <EOL> return False <EOL> if not isinstance ( other , Timestamp ) : <EOL> other = Timestamp ( other ) <EOL> return self . internal == other . internal <EOL> def __ne__ ( self , other ) : <EOL> if other is None : <EOL> return True <EOL> if not isinstance ( other , Timestamp ) : <EOL> other = Timestamp ( other ) <EOL> return self . internal != other . internal <EOL> def __lt__ ( self , other ) : <EOL> if other is None : <EOL> return False <EOL> if not isinstance ( other , Timestamp ) : <EOL> other = Timestamp ( other ) <EOL> return self . internal < other . internal <EOL> def __hash__ ( self ) : <EOL> return hash ( self . internal ) <EOL> def encode_timestamps ( t1 , t2 = None , t3 = None , explicit = False ) : <EOL> """<STR_LIT>""" <EOL> form = '<STR_LIT>' <EOL> values = [ t1 . short ] <EOL> if t2 is not None : <EOL> t2_t1_delta = t2 . raw - t1 . raw <EOL> explicit = explicit or ( t2_t1_delta != <NUM_LIT:0> ) <EOL> values . append ( t2_t1_delta ) <EOL> if t3 is not None : <EOL> t3_t2_delta = t3 . raw - t2 . raw <EOL> explicit = explicit or ( t3_t2_delta != <NUM_LIT:0> ) <EOL> values . append ( t3_t2_delta ) <EOL> if explicit : <EOL> form += '<STR_LIT>' <EOL> if t3 is not None : <EOL> form += '<STR_LIT>' <EOL> return form . format ( * values ) <EOL> def decode_timestamps ( encoded , explicit = False ) : <EOL> """<STR_LIT>""" <EOL> if not isinstance ( encoded , basestring ) : <EOL> ts = Timestamp ( encoded ) <EOL> return ts , ts , ts <EOL> parts = [ ] <EOL> signs = [ ] <EOL> pos_parts = encoded . split ( '<STR_LIT:+>' ) <EOL> for part in pos_parts : <EOL> neg_parts = part . split ( '<STR_LIT:->' ) <EOL> parts = parts + neg_parts <EOL> signs = signs + [ <NUM_LIT:1> ] + [ - <NUM_LIT:1> ] * ( len ( neg_parts ) - <NUM_LIT:1> ) <EOL> t1 = Timestamp ( parts [ <NUM_LIT:0> ] ) <EOL> t2 = t3 = None <EOL> if len ( parts ) > <NUM_LIT:1> : <EOL> t2 = t1 <EOL> delta = signs [ <NUM_LIT:1> ] * int ( parts [ <NUM_LIT:1> ] , <NUM_LIT:16> ) <EOL> if delta : <EOL> t2 = Timestamp ( ( t1 . raw + delta ) * PRECISION ) <EOL> elif not explicit : <EOL> t2 = t1 <EOL> if len ( parts ) > <NUM_LIT:2> : <EOL> t3 = t2 <EOL> delta = signs [ <NUM_LIT:2> ] * int ( parts [ <NUM_LIT:2> ] , <NUM_LIT:16> ) <EOL> if delta : <EOL> t3 = Timestamp ( ( t2 . raw + delta ) * PRECISION ) <EOL> elif not explicit : <EOL> t3 = t2 <EOL> return t1 , t2 , t3 <EOL> def normalize_timestamp ( timestamp ) : <EOL> """<STR_LIT>""" <EOL> return Timestamp ( timestamp ) . normal <EOL> EPOCH = datetime . datetime ( <NUM_LIT> , <NUM_LIT:1> , <NUM_LIT:1> ) <EOL> def last_modified_date_to_timestamp ( last_modified_date_str ) : <EOL> """<STR_LIT>""" <EOL> start = datetime . datetime . strptime ( last_modified_date_str , <EOL> '<STR_LIT>' ) <EOL> delta = start - EPOCH <EOL> return Timestamp ( delta . total_seconds ( ) ) <EOL> def normalize_delete_at_timestamp ( timestamp ) : <EOL> """<STR_LIT>""" <EOL> return '<STR_LIT>' % min ( max ( <NUM_LIT:0> , float ( timestamp ) ) , <NUM_LIT> ) <EOL> def mkdirs ( path ) : <EOL> """<STR_LIT>""" <EOL> if not os . path . isdir ( path ) : <EOL> try : <EOL> os . makedirs ( path ) <EOL> except OSError as err : <EOL> if err . errno != errno . EEXIST or not os . path . isdir ( path ) : <EOL> raise <EOL> def makedirs_count ( path , count = <NUM_LIT:0> ) : <EOL> """<STR_LIT>""" <EOL> head , tail = os . path . split ( path ) <EOL> if not tail : <EOL> head , tail = os . path . split ( head ) <EOL> if head and tail and not os . path . exists ( head ) : <EOL> count = makedirs_count ( head , count ) <EOL> if tail == os . path . curdir : <EOL> return <EOL> try : <EOL> os . mkdir ( path ) <EOL> except OSError as e : <EOL> if e . errno != errno . EEXIST or not os . path . isdir ( path ) : <EOL> raise <EOL> else : <EOL> count += <NUM_LIT:1> <EOL> return count <EOL> def renamer ( old , new , fsync = True ) : <EOL> """<STR_LIT>""" <EOL> dirpath = os . path . dirname ( new ) <EOL> try : <EOL> count = makedirs_count ( dirpath ) <EOL> os . rename ( old , new ) <EOL> except OSError : <EOL> count = makedirs_count ( dirpath ) <EOL> os . rename ( old , new ) <EOL> if fsync : <EOL> for i in range ( <NUM_LIT:0> , count + <NUM_LIT:1> ) : <EOL> fsync_dir ( dirpath ) <EOL> dirpath = os . path . dirname ( dirpath ) <EOL> def split_path ( path , minsegs = <NUM_LIT:1> , maxsegs = None , rest_with_last = False ) : <EOL> """<STR_LIT>""" <EOL> if not maxsegs : <EOL> maxsegs = minsegs <EOL> if minsegs > maxsegs : <EOL> raise ValueError ( '<STR_LIT>' % ( minsegs , maxsegs ) ) <EOL> if rest_with_last : <EOL> segs = path . split ( '<STR_LIT:/>' , maxsegs ) <EOL> minsegs += <NUM_LIT:1> <EOL> maxsegs += <NUM_LIT:1> <EOL> count = len ( segs ) <EOL> if ( segs [ <NUM_LIT:0> ] or count < minsegs or count > maxsegs or <EOL> '<STR_LIT>' in segs [ <NUM_LIT:1> : minsegs ] ) : <EOL> raise ValueError ( '<STR_LIT>' % quote ( path ) ) <EOL> else : <EOL> minsegs += <NUM_LIT:1> <EOL> maxsegs += <NUM_LIT:1> <EOL> segs = path . split ( '<STR_LIT:/>' , maxsegs ) <EOL> count = len ( segs ) <EOL> if ( segs [ <NUM_LIT:0> ] or count < minsegs or count > maxsegs + <NUM_LIT:1> or <EOL> '<STR_LIT>' in segs [ <NUM_LIT:1> : minsegs ] or <EOL> ( count == maxsegs + <NUM_LIT:1> and segs [ maxsegs ] ) ) : <EOL> raise ValueError ( '<STR_LIT>' % quote ( path ) ) <EOL> segs = segs [ <NUM_LIT:1> : maxsegs ] <EOL> segs . extend ( [ None ] * ( maxsegs - <NUM_LIT:1> - len ( segs ) ) ) <EOL> return segs <EOL> def validate_device_partition ( device , partition ) : <EOL> """<STR_LIT>""" <EOL> if not device or '<STR_LIT:/>' in device or device in [ '<STR_LIT:.>' , '<STR_LIT:..>' ] : <EOL> raise ValueError ( '<STR_LIT>' % quote ( device or '<STR_LIT>' ) ) <EOL> if not partition or '<STR_LIT:/>' in partition or partition in [ '<STR_LIT:.>' , '<STR_LIT:..>' ] : <EOL> raise ValueError ( '<STR_LIT>' % quote ( partition or '<STR_LIT>' ) ) <EOL> class RateLimitedIterator ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , iterable , elements_per_second , limit_after = <NUM_LIT:0> , <EOL> ratelimit_if = lambda _junk : True ) : <EOL> self . iterator = iter ( iterable ) <EOL> self . elements_per_second = elements_per_second <EOL> self . limit_after = limit_after <EOL> self . running_time = <NUM_LIT:0> <EOL> self . ratelimit_if = ratelimit_if <EOL> def __iter__ ( self ) : <EOL> return self <EOL> def next ( self ) : <EOL> next_value = next ( self . iterator ) <EOL> if self . ratelimit_if ( next_value ) : <EOL> if self . limit_after > <NUM_LIT:0> : <EOL> self . limit_after -= <NUM_LIT:1> <EOL> else : <EOL> self . running_time = ratelimit_sleep ( self . running_time , <EOL> self . elements_per_second ) <EOL> return next_value <EOL> __next__ = next <EOL> class GreenthreadSafeIterator ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , unsafe_iterable ) : <EOL> self . unsafe_iter = iter ( unsafe_iterable ) <EOL> self . semaphore = eventlet . semaphore . Semaphore ( value = <NUM_LIT:1> ) <EOL> def __iter__ ( self ) : <EOL> return self <EOL> def next ( self ) : <EOL> with self . semaphore : <EOL> return next ( self . unsafe_iter ) <EOL> __next__ = next <EOL> class NullLogger ( object ) : <EOL> """<STR_LIT>""" <EOL> def write ( self , * args ) : <EOL> pass <EOL> class LoggerFileObject ( object ) : <EOL> _cls_thread_local = threading . local ( ) <EOL> def __init__ ( self , logger , log_type = '<STR_LIT>' ) : <EOL> self . logger = logger <EOL> self . log_type = log_type <EOL> def write ( self , value ) : <EOL> if getattr ( self . _cls_thread_local , '<STR_LIT>' , False ) : <EOL> return <EOL> self . _cls_thread_local . already_called_write = True <EOL> try : <EOL> value = value . strip ( ) <EOL> if value : <EOL> if '<STR_LIT>' in value : <EOL> self . logger . error ( <EOL> _ ( '<STR_LIT>' ) , self . log_type ) <EOL> else : <EOL> self . logger . error ( _ ( '<STR_LIT>' ) , <EOL> { '<STR_LIT:type>' : self . log_type , '<STR_LIT:value>' : value } ) <EOL> finally : <EOL> self . _cls_thread_local . already_called_write = False <EOL> def writelines ( self , values ) : <EOL> if getattr ( self . _cls_thread_local , '<STR_LIT>' , False ) : <EOL> return <EOL> self . _cls_thread_local . already_called_writelines = True <EOL> try : <EOL> self . logger . error ( _ ( '<STR_LIT>' ) , <EOL> { '<STR_LIT:type>' : self . log_type , <EOL> '<STR_LIT:value>' : '<STR_LIT>' . join ( values ) } ) <EOL> finally : <EOL> self . _cls_thread_local . already_called_writelines = False <EOL> def close ( self ) : <EOL> pass <EOL> def flush ( self ) : <EOL> pass <EOL> def __iter__ ( self ) : <EOL> return self <EOL> def next ( self ) : <EOL> raise IOError ( errno . EBADF , '<STR_LIT>' ) <EOL> __next__ = next <EOL> def read ( self , size = - <NUM_LIT:1> ) : <EOL> raise IOError ( errno . EBADF , '<STR_LIT>' ) <EOL> def readline ( self , size = - <NUM_LIT:1> ) : <EOL> raise IOError ( errno . EBADF , '<STR_LIT>' ) <EOL> def tell ( self ) : <EOL> return <NUM_LIT:0> <EOL> def xreadlines ( self ) : <EOL> return self <EOL> class StatsdClient ( object ) : <EOL> def __init__ ( self , host , port , base_prefix = '<STR_LIT>' , tail_prefix = '<STR_LIT>' , <EOL> default_sample_rate = <NUM_LIT:1> , sample_rate_factor = <NUM_LIT:1> , logger = None ) : <EOL> self . _host = host <EOL> self . _port = port <EOL> self . _base_prefix = base_prefix <EOL> self . set_prefix ( tail_prefix ) <EOL> self . _default_sample_rate = default_sample_rate <EOL> self . _sample_rate_factor = sample_rate_factor <EOL> self . random = random <EOL> self . logger = logger <EOL> addr_info = None <EOL> try : <EOL> addr_info = socket . getaddrinfo ( host , port , socket . AF_INET ) <EOL> self . _sock_family = socket . AF_INET <EOL> except socket . gaierror : <EOL> try : <EOL> addr_info = socket . getaddrinfo ( host , port , socket . AF_INET6 ) <EOL> self . _sock_family = socket . AF_INET6 <EOL> except socket . gaierror : <EOL> self . _sock_family = socket . AF_INET <EOL> if addr_info is not None : <EOL> sockaddr = addr_info [ <NUM_LIT:0> ] [ - <NUM_LIT:1> ] <EOL> self . _target = ( host , ) + ( sockaddr [ <NUM_LIT:1> : ] ) <EOL> else : <EOL> self . _target = ( host , port ) <EOL> def set_prefix ( self , new_prefix ) : <EOL> if new_prefix and self . _base_prefix : <EOL> self . _prefix = '<STR_LIT:.>' . join ( [ self . _base_prefix , new_prefix , '<STR_LIT>' ] ) <EOL> elif new_prefix : <EOL> self . _prefix = new_prefix + '<STR_LIT:.>' <EOL> elif self . _base_prefix : <EOL> self . _prefix = self . _base_prefix + '<STR_LIT:.>' <EOL> else : <EOL> self . _prefix = '<STR_LIT>' <EOL> def _send ( self , m_name , m_value , m_type , sample_rate ) : <EOL> if sample_rate is None : <EOL> sample_rate = self . _default_sample_rate <EOL> sample_rate = sample_rate * self . _sample_rate_factor <EOL> parts = [ '<STR_LIT>' % ( self . _prefix , m_name , m_value ) , m_type ] <EOL> if sample_rate < <NUM_LIT:1> : <EOL> if self . random ( ) < sample_rate : <EOL> parts . append ( '<STR_LIT>' % ( sample_rate , ) ) <EOL> else : <EOL> return <EOL> if six . PY3 : <EOL> parts = [ part . encode ( '<STR_LIT:utf-8>' ) for part in parts ] <EOL> with closing ( self . _open_socket ( ) ) as sock : <EOL> try : <EOL> return sock . sendto ( b'<STR_LIT:|>' . join ( parts ) , self . _target ) <EOL> except IOError as err : <EOL> if self . logger : <EOL> self . logger . warning ( <EOL> '<STR_LIT>' , <EOL> self . _target , err ) <EOL> def _open_socket ( self ) : <EOL> return socket . socket ( self . _sock_family , socket . SOCK_DGRAM ) <EOL> def update_stats ( self , m_name , m_value , sample_rate = None ) : <EOL> return self . _send ( m_name , m_value , '<STR_LIT:c>' , sample_rate ) <EOL> def increment ( self , metric , sample_rate = None ) : <EOL> return self . update_stats ( metric , <NUM_LIT:1> , sample_rate ) <EOL> def decrement ( self , metric , sample_rate = None ) : <EOL> return self . update_stats ( metric , - <NUM_LIT:1> , sample_rate ) <EOL> def timing ( self , metric , timing_ms , sample_rate = None ) : <EOL> return self . _send ( metric , timing_ms , '<STR_LIT>' , sample_rate ) <EOL> def timing_since ( self , metric , orig_time , sample_rate = None ) : <EOL> return self . timing ( metric , ( time . time ( ) - orig_time ) * <NUM_LIT:1000> , <EOL> sample_rate ) <EOL> def transfer_rate ( self , metric , elapsed_time , byte_xfer , sample_rate = None ) : <EOL> if byte_xfer : <EOL> return self . timing ( metric , <EOL> elapsed_time * <NUM_LIT:1000> / byte_xfer * <NUM_LIT:1000> , <EOL> sample_rate ) <EOL> def server_handled_successfully ( status_int ) : <EOL> """<STR_LIT>""" <EOL> return ( is_success ( status_int ) or <EOL> is_redirection ( status_int ) or <EOL> status_int == HTTP_NOT_FOUND or <EOL> status_int == HTTP_PRECONDITION_FAILED or <EOL> status_int == HTTP_REQUESTED_RANGE_NOT_SATISFIABLE ) <EOL> def timing_stats ( ** dec_kwargs ) : <EOL> """<STR_LIT>""" <EOL> def decorating_func ( func ) : <EOL> method = func . __name__ <EOL> @ functools . wraps ( func ) <EOL> def _timing_stats ( ctrl , * args , ** kwargs ) : <EOL> start_time = time . time ( ) <EOL> resp = func ( ctrl , * args , ** kwargs ) <EOL> if server_handled_successfully ( resp . status_int ) : <EOL> ctrl . logger . timing_since ( method + '<STR_LIT>' , <EOL> start_time , ** dec_kwargs ) <EOL> else : <EOL> ctrl . logger . timing_since ( method + '<STR_LIT>' , <EOL> start_time , ** dec_kwargs ) <EOL> return resp <EOL> return _timing_stats <EOL> return decorating_func <EOL> class LogAdapter ( logging . LoggerAdapter , object ) : <EOL> """<STR_LIT>""" <EOL> _cls_thread_local = threading . local ( ) <EOL> def __init__ ( self , logger , server ) : <EOL> logging . LoggerAdapter . __init__ ( self , logger , { } ) <EOL> self . server = server <EOL> self . warn = self . warning <EOL> @ property <EOL> def txn_id ( self ) : <EOL> if hasattr ( self . _cls_thread_local , '<STR_LIT>' ) : <EOL> return self . _cls_thread_local . txn_id <EOL> @ txn_id . setter <EOL> def txn_id ( self , value ) : <EOL> self . _cls_thread_local . txn_id = value <EOL> @ property <EOL> def client_ip ( self ) : <EOL> if hasattr ( self . _cls_thread_local , '<STR_LIT>' ) : <EOL> return self . _cls_thread_local . client_ip <EOL> @ client_ip . setter <EOL> def client_ip ( self , value ) : <EOL> self . _cls_thread_local . client_ip = value <EOL> @ property <EOL> def thread_locals ( self ) : <EOL> return ( self . txn_id , self . client_ip ) <EOL> @ thread_locals . setter <EOL> def thread_locals ( self , value ) : <EOL> self . txn_id , self . client_ip = value <EOL> def getEffectiveLevel ( self ) : <EOL> return self . logger . getEffectiveLevel ( ) <EOL> def process ( self , msg , kwargs ) : <EOL> """<STR_LIT>""" <EOL> kwargs [ '<STR_LIT>' ] = { '<STR_LIT>' : self . server , '<STR_LIT>' : self . txn_id , <EOL> '<STR_LIT>' : self . client_ip } <EOL> return msg , kwargs <EOL> def notice ( self , msg , * args , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> self . log ( NOTICE , msg , * args , ** kwargs ) <EOL> def _exception ( self , msg , * args , ** kwargs ) : <EOL> logging . LoggerAdapter . exception ( self , msg , * args , ** kwargs ) <EOL> def exception ( self , msg , * args , ** kwargs ) : <EOL> _junk , exc , _junk = sys . exc_info ( ) <EOL> call = self . error <EOL> emsg = '<STR_LIT>' <EOL> if isinstance ( exc , ( OSError , socket . error ) ) : <EOL> if exc . errno in ( errno . EIO , errno . ENOSPC ) : <EOL> emsg = str ( exc ) <EOL> elif exc . errno == errno . ECONNREFUSED : <EOL> emsg = _ ( '<STR_LIT>' ) <EOL> elif exc . errno == errno . EHOSTUNREACH : <EOL> emsg = _ ( '<STR_LIT>' ) <EOL> elif exc . errno == errno . ETIMEDOUT : <EOL> emsg = _ ( '<STR_LIT>' ) <EOL> else : <EOL> call = self . _exception <EOL> elif isinstance ( exc , eventlet . Timeout ) : <EOL> emsg = exc . __class__ . __name__ <EOL> if hasattr ( exc , '<STR_LIT>' ) : <EOL> emsg += '<STR_LIT>' % exc . seconds <EOL> if isinstance ( exc , swift . common . exceptions . MessageTimeout ) : <EOL> if exc . msg : <EOL> emsg += '<STR_LIT>' % exc . msg <EOL> else : <EOL> call = self . _exception <EOL> call ( '<STR_LIT>' % ( msg , emsg ) , * args , ** kwargs ) <EOL> def set_statsd_prefix ( self , prefix ) : <EOL> """<STR_LIT>""" <EOL> if self . logger . statsd_client : <EOL> self . logger . statsd_client . set_prefix ( prefix ) <EOL> def statsd_delegate ( statsd_func_name ) : <EOL> """<STR_LIT>""" <EOL> func = getattr ( StatsdClient , statsd_func_name ) <EOL> @ functools . wraps ( func ) <EOL> def wrapped ( self , * a , ** kw ) : <EOL> if getattr ( self . logger , '<STR_LIT>' ) : <EOL> return func ( self . logger . statsd_client , * a , ** kw ) <EOL> return wrapped <EOL> update_stats = statsd_delegate ( '<STR_LIT>' ) <EOL> increment = statsd_delegate ( '<STR_LIT>' ) <EOL> decrement = statsd_delegate ( '<STR_LIT>' ) <EOL> timing = statsd_delegate ( '<STR_LIT>' ) <EOL> timing_since = statsd_delegate ( '<STR_LIT>' ) <EOL> transfer_rate = statsd_delegate ( '<STR_LIT>' ) <EOL> class SwiftLogFormatter ( logging . Formatter ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , fmt = None , datefmt = None , max_line_length = <NUM_LIT:0> ) : <EOL> logging . Formatter . __init__ ( self , fmt = fmt , datefmt = datefmt ) <EOL> self . max_line_length = max_line_length <EOL> def format ( self , record ) : <EOL> if not hasattr ( record , '<STR_LIT>' ) : <EOL> record . server = record . name <EOL> record . message = record . getMessage ( ) <EOL> if self . _fmt . find ( '<STR_LIT>' ) >= <NUM_LIT:0> : <EOL> record . asctime = self . formatTime ( record , self . datefmt ) <EOL> msg = ( self . _fmt % record . __dict__ ) . replace ( '<STR_LIT:\n>' , '<STR_LIT>' ) <EOL> if record . exc_info : <EOL> if not record . exc_text : <EOL> record . exc_text = self . formatException ( <EOL> record . exc_info ) . replace ( '<STR_LIT:\n>' , '<STR_LIT>' ) <EOL> if record . exc_text : <EOL> if not msg . endswith ( '<STR_LIT>' ) : <EOL> msg = msg + '<STR_LIT>' <EOL> msg = msg + record . exc_text <EOL> if ( hasattr ( record , '<STR_LIT>' ) and record . txn_id and <EOL> record . levelno != logging . INFO and <EOL> record . txn_id not in msg ) : <EOL> msg = "<STR_LIT>" % ( msg , record . txn_id ) <EOL> if ( hasattr ( record , '<STR_LIT>' ) and record . client_ip and <EOL> record . levelno != logging . INFO and <EOL> record . client_ip not in msg ) : <EOL> msg = "<STR_LIT>" % ( msg , record . client_ip ) <EOL> if self . max_line_length > <NUM_LIT:0> and len ( msg ) > self . max_line_length : <EOL> if self . max_line_length < <NUM_LIT:7> : <EOL> msg = msg [ : self . max_line_length ] <EOL> else : <EOL> approxhalf = ( self . max_line_length - <NUM_LIT:5> ) // <NUM_LIT:2> <EOL> msg = msg [ : approxhalf ] + "<STR_LIT>" + msg [ - approxhalf : ] <EOL> return msg <EOL> def get_logger ( conf , name = None , log_to_console = False , log_route = None , <EOL> fmt = "<STR_LIT>" ) : <EOL> """<STR_LIT>""" <EOL> if not conf : <EOL> conf = { } <EOL> if name is None : <EOL> name = conf . get ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> if not log_route : <EOL> log_route = name <EOL> logger = logging . getLogger ( log_route ) <EOL> logger . propagate = False <EOL> formatter = SwiftLogFormatter ( <EOL> fmt = fmt , max_line_length = int ( conf . get ( '<STR_LIT>' , <NUM_LIT:0> ) ) ) <EOL> if not hasattr ( get_logger , '<STR_LIT>' ) : <EOL> get_logger . handler4logger = { } <EOL> if logger in get_logger . handler4logger : <EOL> logger . removeHandler ( get_logger . handler4logger [ logger ] ) <EOL> facility = getattr ( SysLogHandler , conf . get ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> SysLogHandler . LOG_LOCAL0 ) <EOL> udp_host = conf . get ( '<STR_LIT>' ) <EOL> if udp_host : <EOL> udp_port = int ( conf . get ( '<STR_LIT>' , <EOL> logging . handlers . SYSLOG_UDP_PORT ) ) <EOL> handler = SysLogHandler ( address = ( udp_host , udp_port ) , <EOL> facility = facility ) <EOL> else : <EOL> log_address = conf . get ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> try : <EOL> handler = SysLogHandler ( address = log_address , facility = facility ) <EOL> except socket . error as e : <EOL> if e . errno not in [ errno . ENOTSOCK , errno . ENOENT ] : <EOL> raise e <EOL> handler = SysLogHandler ( facility = facility ) <EOL> handler . setFormatter ( formatter ) <EOL> logger . addHandler ( handler ) <EOL> get_logger . handler4logger [ logger ] = handler <EOL> if log_to_console or hasattr ( get_logger , '<STR_LIT>' ) : <EOL> if not hasattr ( get_logger , '<STR_LIT>' ) : <EOL> get_logger . console_handler4logger = { } <EOL> if logger in get_logger . console_handler4logger : <EOL> logger . removeHandler ( get_logger . console_handler4logger [ logger ] ) <EOL> console_handler = logging . StreamHandler ( sys . __stderr__ ) <EOL> console_handler . setFormatter ( formatter ) <EOL> logger . addHandler ( console_handler ) <EOL> get_logger . console_handler4logger [ logger ] = console_handler <EOL> logger . setLevel ( <EOL> getattr ( logging , conf . get ( '<STR_LIT>' , '<STR_LIT>' ) . upper ( ) , logging . INFO ) ) <EOL> statsd_host = conf . get ( '<STR_LIT>' ) <EOL> if statsd_host : <EOL> statsd_port = int ( conf . get ( '<STR_LIT>' , <NUM_LIT> ) ) <EOL> base_prefix = conf . get ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> default_sample_rate = float ( conf . get ( <EOL> '<STR_LIT>' , <NUM_LIT:1> ) ) <EOL> sample_rate_factor = float ( conf . get ( <EOL> '<STR_LIT>' , <NUM_LIT:1> ) ) <EOL> statsd_client = StatsdClient ( statsd_host , statsd_port , base_prefix , <EOL> name , default_sample_rate , <EOL> sample_rate_factor , logger = logger ) <EOL> logger . statsd_client = statsd_client <EOL> else : <EOL> logger . statsd_client = None <EOL> adapted_logger = LogAdapter ( logger , name ) <EOL> other_handlers = conf . get ( '<STR_LIT>' , None ) <EOL> if other_handlers : <EOL> log_custom_handlers = [ s . strip ( ) for s in other_handlers . split ( '<STR_LIT:U+002C>' ) <EOL> if s . strip ( ) ] <EOL> for hook in log_custom_handlers : <EOL> try : <EOL> mod , fnc = hook . rsplit ( '<STR_LIT:.>' , <NUM_LIT:1> ) <EOL> logger_hook = getattr ( __import__ ( mod , fromlist = [ fnc ] ) , fnc ) <EOL> logger_hook ( conf , name , log_to_console , log_route , fmt , <EOL> logger , adapted_logger ) <EOL> except ( AttributeError , ImportError ) : <EOL> print ( '<STR_LIT>' % hook , <EOL> file = sys . stderr ) <EOL> except ValueError : <EOL> print ( '<STR_LIT>' % hook , <EOL> file = sys . stderr ) <EOL> return adapted_logger <EOL> def get_hub ( ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> import select <EOL> if hasattr ( select , "<STR_LIT>" ) : <EOL> return "<STR_LIT>" <EOL> return "<STR_LIT>" <EOL> except ImportError : <EOL> return None <EOL> def drop_privileges ( user , call_setsid = True ) : <EOL> """<STR_LIT>""" <EOL> if os . geteuid ( ) == <NUM_LIT:0> : <EOL> groups = [ g . gr_gid for g in grp . getgrall ( ) if user in g . gr_mem ] <EOL> os . setgroups ( groups ) <EOL> user = pwd . getpwnam ( user ) <EOL> os . setgid ( user [ <NUM_LIT:3> ] ) <EOL> os . setuid ( user [ <NUM_LIT:2> ] ) <EOL> os . environ [ '<STR_LIT>' ] = user [ <NUM_LIT:5> ] <EOL> if call_setsid : <EOL> try : <EOL> os . setsid ( ) <EOL> except OSError : <EOL> pass <EOL> os . chdir ( '<STR_LIT:/>' ) <EOL> os . umask ( <NUM_LIT> ) <EOL> def capture_stdio ( logger , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> sys . excepthook = lambda * exc_info : logger . critical ( _ ( '<STR_LIT>' ) , exc_info = exc_info ) <EOL> stdio_files = [ sys . stdin , sys . stdout , sys . stderr ] <EOL> console_fds = [ h . stream . fileno ( ) for _junk , h in getattr ( <EOL> get_logger , '<STR_LIT>' , { } ) . items ( ) ] <EOL> stdio_files = [ f for f in stdio_files if f . fileno ( ) not in console_fds ] <EOL> with open ( os . devnull , '<STR_LIT>' ) as nullfile : <EOL> for f in stdio_files : <EOL> try : <EOL> f . flush ( ) <EOL> except IOError : <EOL> pass <EOL> try : <EOL> os . dup2 ( nullfile . fileno ( ) , f . fileno ( ) ) <EOL> except OSError : <EOL> pass <EOL> if kwargs . pop ( '<STR_LIT>' , True ) : <EOL> sys . stdout = LoggerFileObject ( logger ) <EOL> if kwargs . pop ( '<STR_LIT>' , True ) : <EOL> sys . stderr = LoggerFileObject ( logger , '<STR_LIT>' ) <EOL> def parse_options ( parser = None , once = False , test_args = None ) : <EOL> """<STR_LIT>""" <EOL> if not parser : <EOL> parser = OptionParser ( usage = "<STR_LIT>" ) <EOL> parser . add_option ( "<STR_LIT>" , "<STR_LIT>" , default = False , action = "<STR_LIT:store_true>" , <EOL> help = "<STR_LIT>" ) <EOL> if once : <EOL> parser . add_option ( "<STR_LIT>" , "<STR_LIT>" , default = False , action = "<STR_LIT:store_true>" , <EOL> help = "<STR_LIT>" ) <EOL> options , args = parser . parse_args ( args = test_args ) <EOL> if not args : <EOL> parser . print_usage ( ) <EOL> print ( _ ( "<STR_LIT>" ) ) <EOL> sys . exit ( <NUM_LIT:1> ) <EOL> config = os . path . abspath ( args . pop ( <NUM_LIT:0> ) ) <EOL> if not os . path . exists ( config ) : <EOL> parser . print_usage ( ) <EOL> print ( _ ( "<STR_LIT>" ) % config ) <EOL> sys . exit ( <NUM_LIT:1> ) <EOL> extra_args = [ ] <EOL> for arg in args : <EOL> if arg in options . __dict__ : <EOL> setattr ( options , arg , True ) <EOL> else : <EOL> extra_args . append ( arg ) <EOL> options = vars ( options ) <EOL> if extra_args : <EOL> options [ '<STR_LIT>' ] = extra_args <EOL> return config , options <EOL> def expand_ipv6 ( address ) : <EOL> """<STR_LIT>""" <EOL> packed_ip = socket . inet_pton ( socket . AF_INET6 , address ) <EOL> return socket . inet_ntop ( socket . AF_INET6 , packed_ip ) <EOL> def whataremyips ( bind_ip = None ) : <EOL> """<STR_LIT>""" <EOL> if bind_ip : <EOL> try : <EOL> _ , _ , _ , _ , sockaddr = socket . getaddrinfo ( <EOL> bind_ip , None , <NUM_LIT:0> , socket . SOCK_STREAM , <NUM_LIT:0> , <EOL> socket . AI_NUMERICHOST ) [ <NUM_LIT:0> ] <EOL> if sockaddr [ <NUM_LIT:0> ] not in ( '<STR_LIT>' , '<STR_LIT>' ) : <EOL> return [ bind_ip ] <EOL> except socket . gaierror : <EOL> pass <EOL> addresses = [ ] <EOL> for interface in netifaces . interfaces ( ) : <EOL> try : <EOL> iface_data = netifaces . ifaddresses ( interface ) <EOL> for family in iface_data : <EOL> if family not in ( netifaces . AF_INET , netifaces . AF_INET6 ) : <EOL> continue <EOL> for address in iface_data [ family ] : <EOL> addr = address [ '<STR_LIT>' ] <EOL> if family == netifaces . AF_INET6 : <EOL> addr = expand_ipv6 ( addr . split ( '<STR_LIT:%>' ) [ <NUM_LIT:0> ] ) <EOL> addresses . append ( addr ) <EOL> except ValueError : <EOL> pass <EOL> return addresses <EOL> def parse_socket_string ( socket_string , default_port ) : <EOL> """<STR_LIT>""" <EOL> port = default_port <EOL> if socket_string . startswith ( '<STR_LIT:[>' ) : <EOL> match = IPV6_RE . match ( socket_string ) <EOL> if not match : <EOL> raise ValueError ( "<STR_LIT>" % socket_string ) <EOL> host = match . group ( '<STR_LIT:address>' ) <EOL> port = match . group ( '<STR_LIT:port>' ) or port <EOL> else : <EOL> if '<STR_LIT::>' in socket_string : <EOL> tokens = socket_string . split ( '<STR_LIT::>' ) <EOL> if len ( tokens ) > <NUM_LIT:2> : <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> host , port = tokens <EOL> else : <EOL> host = socket_string <EOL> return ( host , port ) <EOL> def storage_directory ( datadir , partition , name_hash ) : <EOL> """<STR_LIT>""" <EOL> return os . path . join ( datadir , str ( partition ) , name_hash [ - <NUM_LIT:3> : ] , name_hash ) <EOL> def hash_path ( account , container = None , object = None , raw_digest = False ) : <EOL> """<STR_LIT>""" <EOL> if object and not container : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> paths = [ account ] <EOL> if container : <EOL> paths . append ( container ) <EOL> if object : <EOL> paths . append ( object ) <EOL> if raw_digest : <EOL> return md5 ( HASH_PATH_PREFIX + '<STR_LIT:/>' + '<STR_LIT:/>' . join ( paths ) <EOL> + HASH_PATH_SUFFIX ) . digest ( ) <EOL> else : <EOL> return md5 ( HASH_PATH_PREFIX + '<STR_LIT:/>' + '<STR_LIT:/>' . join ( paths ) <EOL> + HASH_PATH_SUFFIX ) . hexdigest ( ) <EOL> @ contextmanager <EOL> def lock_path ( directory , timeout = <NUM_LIT:10> , timeout_class = None ) : <EOL> """<STR_LIT>""" <EOL> if timeout_class is None : <EOL> timeout_class = swift . common . exceptions . LockTimeout <EOL> mkdirs ( directory ) <EOL> lockpath = '<STR_LIT>' % directory <EOL> fd = os . open ( lockpath , os . O_WRONLY | os . O_CREAT ) <EOL> sleep_time = <NUM_LIT> <EOL> slower_sleep_time = max ( timeout * <NUM_LIT> , sleep_time ) <EOL> slowdown_at = timeout * <NUM_LIT> <EOL> time_slept = <NUM_LIT:0> <EOL> try : <EOL> with timeout_class ( timeout , lockpath ) : <EOL> while True : <EOL> try : <EOL> fcntl . flock ( fd , fcntl . LOCK_EX | fcntl . LOCK_NB ) <EOL> break <EOL> except IOError as err : <EOL> if err . errno != errno . EAGAIN : <EOL> raise <EOL> if time_slept > slowdown_at : <EOL> sleep_time = slower_sleep_time <EOL> sleep ( sleep_time ) <EOL> time_slept += sleep_time <EOL> yield True <EOL> finally : <EOL> os . close ( fd ) <EOL> @ contextmanager <EOL> def lock_file ( filename , timeout = <NUM_LIT:10> , append = False , unlink = True ) : <EOL> """<STR_LIT>""" <EOL> flags = os . O_CREAT | os . O_RDWR <EOL> if append : <EOL> flags |= os . O_APPEND <EOL> mode = '<STR_LIT>' <EOL> else : <EOL> mode = '<STR_LIT>' <EOL> while True : <EOL> fd = os . open ( filename , flags ) <EOL> file_obj = os . fdopen ( fd , mode ) <EOL> try : <EOL> with swift . common . exceptions . LockTimeout ( timeout , filename ) : <EOL> while True : <EOL> try : <EOL> fcntl . flock ( fd , fcntl . LOCK_EX | fcntl . LOCK_NB ) <EOL> break <EOL> except IOError as err : <EOL> if err . errno != errno . EAGAIN : <EOL> raise <EOL> sleep ( <NUM_LIT> ) <EOL> try : <EOL> if os . stat ( filename ) . st_ino != os . fstat ( fd ) . st_ino : <EOL> continue <EOL> except OSError as err : <EOL> if err . errno == errno . ENOENT : <EOL> continue <EOL> raise <EOL> yield file_obj <EOL> if unlink : <EOL> os . unlink ( filename ) <EOL> break <EOL> finally : <EOL> file_obj . close ( ) <EOL> def lock_parent_directory ( filename , timeout = <NUM_LIT:10> ) : <EOL> """<STR_LIT>""" <EOL> return lock_path ( os . path . dirname ( filename ) , timeout = timeout ) <EOL> def get_time_units ( time_amount ) : <EOL> """<STR_LIT>""" <EOL> time_unit = '<STR_LIT:s>' <EOL> if time_amount > <NUM_LIT> : <EOL> time_amount /= <NUM_LIT> <EOL> time_unit = '<STR_LIT:m>' <EOL> if time_amount > <NUM_LIT> : <EOL> time_amount /= <NUM_LIT> <EOL> time_unit = '<STR_LIT:h>' <EOL> return time_amount , time_unit <EOL> def compute_eta ( start_time , current_value , final_value ) : <EOL> """<STR_LIT>""" <EOL> elapsed = time . time ( ) - start_time <EOL> completion = ( float ( current_value ) / final_value ) or <NUM_LIT> <EOL> return get_time_units ( <NUM_LIT:1.0> / completion * elapsed - elapsed ) <EOL> def unlink_older_than ( path , mtime ) : <EOL> """<STR_LIT>""" <EOL> filepaths = map ( functools . partial ( os . path . join , path ) , listdir ( path ) ) <EOL> return unlink_paths_older_than ( filepaths , mtime ) <EOL> def unlink_paths_older_than ( filepaths , mtime ) : <EOL> """<STR_LIT>""" <EOL> for fpath in filepaths : <EOL> try : <EOL> if os . path . getmtime ( fpath ) < mtime : <EOL> os . unlink ( fpath ) <EOL> except OSError : <EOL> pass <EOL> def item_from_env ( env , item_name , allow_none = False ) : <EOL> """<STR_LIT>""" <EOL> item = env . get ( item_name , None ) <EOL> if item is None and not allow_none : <EOL> logging . error ( "<STR_LIT>" , item_name ) <EOL> return item <EOL> def cache_from_env ( env , allow_none = False ) : <EOL> """<STR_LIT>""" <EOL> return item_from_env ( env , '<STR_LIT>' , allow_none ) <EOL> def read_conf_dir ( parser , conf_dir ) : <EOL> conf_files = [ ] <EOL> for f in os . listdir ( conf_dir ) : <EOL> if f . endswith ( '<STR_LIT>' ) and not f . startswith ( '<STR_LIT:.>' ) : <EOL> conf_files . append ( os . path . join ( conf_dir , f ) ) <EOL> return parser . read ( sorted ( conf_files ) ) <EOL> def readconf ( conf_path , section_name = None , log_name = None , defaults = None , <EOL> raw = False ) : <EOL> """<STR_LIT>""" <EOL> if defaults is None : <EOL> defaults = { } <EOL> if raw : <EOL> c = RawConfigParser ( defaults ) <EOL> else : <EOL> c = ConfigParser ( defaults ) <EOL> if hasattr ( conf_path , '<STR_LIT>' ) : <EOL> c . readfp ( conf_path ) <EOL> else : <EOL> if os . path . isdir ( conf_path ) : <EOL> success = read_conf_dir ( c , conf_path ) <EOL> else : <EOL> success = c . read ( conf_path ) <EOL> if not success : <EOL> print ( _ ( "<STR_LIT>" ) % conf_path ) <EOL> sys . exit ( <NUM_LIT:1> ) <EOL> if section_name : <EOL> if c . has_section ( section_name ) : <EOL> conf = dict ( c . items ( section_name ) ) <EOL> else : <EOL> print ( _ ( "<STR_LIT>" ) % <EOL> { '<STR_LIT>' : section_name , '<STR_LIT>' : conf_path } ) <EOL> sys . exit ( <NUM_LIT:1> ) <EOL> if "<STR_LIT>" not in conf : <EOL> if log_name is not None : <EOL> conf [ '<STR_LIT>' ] = log_name <EOL> else : <EOL> conf [ '<STR_LIT>' ] = section_name <EOL> else : <EOL> conf = { } <EOL> for s in c . sections ( ) : <EOL> conf . update ( { s : dict ( c . items ( s ) ) } ) <EOL> if '<STR_LIT>' not in conf : <EOL> conf [ '<STR_LIT>' ] = log_name <EOL> conf [ '<STR_LIT>' ] = conf_path <EOL> return conf <EOL> def write_pickle ( obj , dest , tmp = None , pickle_protocol = <NUM_LIT:0> ) : <EOL> """<STR_LIT>""" <EOL> if tmp is None : <EOL> tmp = os . path . dirname ( dest ) <EOL> fd , tmppath = mkstemp ( dir = tmp , suffix = '<STR_LIT>' ) <EOL> with os . fdopen ( fd , '<STR_LIT:wb>' ) as fo : <EOL> pickle . dump ( obj , fo , pickle_protocol ) <EOL> fo . flush ( ) <EOL> os . fsync ( fd ) <EOL> renamer ( tmppath , dest ) <EOL> def search_tree ( root , glob_match , ext = '<STR_LIT>' , exts = None , dir_ext = None ) : <EOL> """<STR_LIT>""" <EOL> exts = exts or [ ext ] <EOL> found_files = [ ] <EOL> for path in glob . glob ( os . path . join ( root , glob_match ) ) : <EOL> if os . path . isdir ( path ) : <EOL> for root , dirs , files in os . walk ( path ) : <EOL> if dir_ext and root . endswith ( dir_ext ) : <EOL> found_files . append ( root ) <EOL> break <EOL> for file_ in files : <EOL> if any ( exts ) and not any ( file_ . endswith ( e ) for e in exts ) : <EOL> continue <EOL> found_files . append ( os . path . join ( root , file_ ) ) <EOL> found_dir = False <EOL> for dir_ in dirs : <EOL> if dir_ext and dir_ . endswith ( dir_ext ) : <EOL> found_dir = True <EOL> found_files . append ( os . path . join ( root , dir_ ) ) <EOL> if found_dir : <EOL> break <EOL> else : <EOL> if ext and not path . endswith ( ext ) : <EOL> continue <EOL> found_files . append ( path ) <EOL> return sorted ( found_files ) <EOL> def write_file ( path , contents ) : <EOL> """<STR_LIT>""" <EOL> dirname , name = os . path . split ( path ) <EOL> if not os . path . exists ( dirname ) : <EOL> try : <EOL> os . makedirs ( dirname ) <EOL> except OSError as err : <EOL> if err . errno == errno . EACCES : <EOL> sys . exit ( '<STR_LIT>' <EOL> '<STR_LIT>' % dirname ) <EOL> with open ( path , '<STR_LIT:w>' ) as f : <EOL> f . write ( '<STR_LIT:%s>' % contents ) <EOL> def remove_file ( path ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> os . unlink ( path ) <EOL> except OSError : <EOL> pass <EOL> def audit_location_generator ( devices , datadir , suffix = '<STR_LIT>' , <EOL> mount_check = True , logger = None ) : <EOL> '''<STR_LIT>''' <EOL> device_dir = listdir ( devices ) <EOL> shuffle ( device_dir ) <EOL> for device in device_dir : <EOL> if mount_check and not ismount ( os . path . join ( devices , device ) ) : <EOL> if logger : <EOL> logger . warning ( <EOL> _ ( '<STR_LIT>' ) , device ) <EOL> continue <EOL> datadir_path = os . path . join ( devices , device , datadir ) <EOL> try : <EOL> partitions = listdir ( datadir_path ) <EOL> except OSError as e : <EOL> if logger : <EOL> logger . warning ( '<STR_LIT>' , datadir_path , e ) <EOL> continue <EOL> for partition in partitions : <EOL> part_path = os . path . join ( datadir_path , partition ) <EOL> try : <EOL> suffixes = listdir ( part_path ) <EOL> except OSError as e : <EOL> if e . errno != errno . ENOTDIR : <EOL> raise <EOL> continue <EOL> for asuffix in suffixes : <EOL> suff_path = os . path . join ( part_path , asuffix ) <EOL> try : <EOL> hashes = listdir ( suff_path ) <EOL> except OSError as e : <EOL> if e . errno != errno . ENOTDIR : <EOL> raise <EOL> continue <EOL> for hsh in hashes : <EOL> hash_path = os . path . join ( suff_path , hsh ) <EOL> try : <EOL> files = sorted ( listdir ( hash_path ) , reverse = True ) <EOL> except OSError as e : <EOL> if e . errno != errno . ENOTDIR : <EOL> raise <EOL> continue <EOL> for fname in files : <EOL> if suffix and not fname . endswith ( suffix ) : <EOL> continue <EOL> path = os . path . join ( hash_path , fname ) <EOL> yield path , device , partition <EOL> def ratelimit_sleep ( running_time , max_rate , incr_by = <NUM_LIT:1> , rate_buffer = <NUM_LIT:5> ) : <EOL> '''<STR_LIT>''' <EOL> if max_rate <= <NUM_LIT:0> or incr_by <= <NUM_LIT:0> : <EOL> return running_time <EOL> clock_accuracy = <NUM_LIT> <EOL> now = time . time ( ) * clock_accuracy <EOL> time_per_request = clock_accuracy * ( float ( incr_by ) / max_rate ) <EOL> if now - running_time > rate_buffer * clock_accuracy : <EOL> running_time = now <EOL> elif running_time - now > time_per_request : <EOL> eventlet . sleep ( ( running_time - now ) / clock_accuracy ) <EOL> return running_time + time_per_request <EOL> class ContextPool ( GreenPool ) : <EOL> "<STR_LIT>" <EOL> def __enter__ ( self ) : <EOL> return self <EOL> def __exit__ ( self , type , value , traceback ) : <EOL> for coro in list ( self . coroutines_running ) : <EOL> coro . kill ( ) <EOL> class GreenAsyncPileWaitallTimeout ( Timeout ) : <EOL> pass <EOL> class GreenAsyncPile ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , size_or_pool ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( size_or_pool , GreenPool ) : <EOL> self . _pool = size_or_pool <EOL> size = self . _pool . size <EOL> else : <EOL> self . _pool = GreenPool ( size_or_pool ) <EOL> size = size_or_pool <EOL> self . _responses = eventlet . queue . LightQueue ( size ) <EOL> self . _inflight = <NUM_LIT:0> <EOL> self . _pending = <NUM_LIT:0> <EOL> def _run_func ( self , func , args , kwargs ) : <EOL> try : <EOL> self . _responses . put ( func ( * args , ** kwargs ) ) <EOL> finally : <EOL> self . _inflight -= <NUM_LIT:1> <EOL> @ property <EOL> def inflight ( self ) : <EOL> return self . _inflight <EOL> def spawn ( self , func , * args , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> self . _pending += <NUM_LIT:1> <EOL> self . _inflight += <NUM_LIT:1> <EOL> self . _pool . spawn ( self . _run_func , func , args , kwargs ) <EOL> def waitfirst ( self , timeout ) : <EOL> """<STR_LIT>""" <EOL> for result in self . _wait ( timeout , first_n = <NUM_LIT:1> ) : <EOL> return result <EOL> def waitall ( self , timeout ) : <EOL> """<STR_LIT>""" <EOL> return self . _wait ( timeout ) <EOL> def _wait ( self , timeout , first_n = None ) : <EOL> results = [ ] <EOL> try : <EOL> with GreenAsyncPileWaitallTimeout ( timeout ) : <EOL> while True : <EOL> results . append ( next ( self ) ) <EOL> if first_n and len ( results ) >= first_n : <EOL> break <EOL> except ( GreenAsyncPileWaitallTimeout , StopIteration ) : <EOL> pass <EOL> return results <EOL> def __iter__ ( self ) : <EOL> return self <EOL> def next ( self ) : <EOL> try : <EOL> rv = self . _responses . get_nowait ( ) <EOL> except eventlet . queue . Empty : <EOL> if self . _inflight == <NUM_LIT:0> : <EOL> raise StopIteration ( ) <EOL> rv = self . _responses . get ( ) <EOL> self . _pending -= <NUM_LIT:1> <EOL> return rv <EOL> __next__ = next <EOL> class ModifiedParseResult ( ParseResult ) : <EOL> "<STR_LIT>" <EOL> @ property <EOL> def hostname ( self ) : <EOL> netloc = self . netloc . split ( '<STR_LIT:@>' , <NUM_LIT:1> ) [ - <NUM_LIT:1> ] <EOL> if netloc . startswith ( '<STR_LIT:[>' ) : <EOL> return netloc [ <NUM_LIT:1> : ] . split ( '<STR_LIT:]>' ) [ <NUM_LIT:0> ] <EOL> elif '<STR_LIT::>' in netloc : <EOL> return netloc . rsplit ( '<STR_LIT::>' ) [ <NUM_LIT:0> ] <EOL> return netloc <EOL> @ property <EOL> def port ( self ) : <EOL> netloc = self . netloc . split ( '<STR_LIT:@>' , <NUM_LIT:1> ) [ - <NUM_LIT:1> ] <EOL> if netloc . startswith ( '<STR_LIT:[>' ) : <EOL> netloc = netloc . rsplit ( '<STR_LIT:]>' ) [ <NUM_LIT:1> ] <EOL> if '<STR_LIT::>' in netloc : <EOL> return int ( netloc . rsplit ( '<STR_LIT::>' ) [ <NUM_LIT:1> ] ) <EOL> return None <EOL> def urlparse ( url ) : <EOL> """<STR_LIT>""" <EOL> return ModifiedParseResult ( * stdlib_urlparse ( url ) ) <EOL> def validate_sync_to ( value , allowed_sync_hosts , realms_conf ) : <EOL> """<STR_LIT>""" <EOL> orig_value = value <EOL> value = value . rstrip ( '<STR_LIT:/>' ) <EOL> if not value : <EOL> return ( None , None , None , None ) <EOL> if value . startswith ( '<STR_LIT>' ) : <EOL> if not realms_conf : <EOL> return ( None , None , None , None ) <EOL> data = value [ <NUM_LIT:2> : ] . split ( '<STR_LIT:/>' ) <EOL> if len ( data ) != <NUM_LIT:4> : <EOL> return ( <EOL> _ ( '<STR_LIT>' ) % orig_value , <EOL> None , None , None ) <EOL> realm , cluster , account , container = data <EOL> realm_key = realms_conf . key ( realm ) <EOL> if not realm_key : <EOL> return ( _ ( '<STR_LIT>' ) % realm , None , None , None ) <EOL> endpoint = realms_conf . endpoint ( realm , cluster ) <EOL> if not endpoint : <EOL> return ( <EOL> _ ( '<STR_LIT>' ) % ( realm , cluster ) , <EOL> None , None , None ) <EOL> return ( <EOL> None , <EOL> '<STR_LIT>' % ( endpoint . rstrip ( '<STR_LIT:/>' ) , account , container ) , <EOL> realm . upper ( ) , realm_key ) <EOL> p = urlparse ( value ) <EOL> if p . scheme not in ( '<STR_LIT:http>' , '<STR_LIT>' ) : <EOL> return ( <EOL> _ ( '<STR_LIT>' <EOL> '<STR_LIT>' ) % p . scheme , <EOL> None , None , None ) <EOL> if not p . path : <EOL> return ( _ ( '<STR_LIT>' ) , None , None , None ) <EOL> if p . params or p . query or p . fragment : <EOL> return ( <EOL> _ ( '<STR_LIT>' <EOL> '<STR_LIT>' ) , <EOL> None , None , None ) <EOL> if p . hostname not in allowed_sync_hosts : <EOL> return ( <EOL> _ ( '<STR_LIT>' ) % p . hostname , <EOL> None , None , None ) <EOL> return ( None , value , None , None ) <EOL> def affinity_key_function ( affinity_str ) : <EOL> """<STR_LIT>""" <EOL> affinity_str = affinity_str . strip ( ) <EOL> if not affinity_str : <EOL> return lambda x : <NUM_LIT:0> <EOL> priority_matchers = [ ] <EOL> pieces = [ s . strip ( ) for s in affinity_str . split ( '<STR_LIT:U+002C>' ) ] <EOL> for piece in pieces : <EOL> match = re . match ( "<STR_LIT>" , piece ) <EOL> if match : <EOL> region , zone , priority = match . groups ( ) <EOL> region = int ( region ) <EOL> priority = int ( priority ) <EOL> zone = int ( zone ) if zone else None <EOL> matcher = { '<STR_LIT>' : region , '<STR_LIT>' : priority } <EOL> if zone is not None : <EOL> matcher [ '<STR_LIT>' ] = zone <EOL> priority_matchers . append ( matcher ) <EOL> else : <EOL> raise ValueError ( "<STR_LIT>" % affinity_str ) <EOL> priority_matchers . sort ( key = operator . itemgetter ( '<STR_LIT>' ) ) <EOL> def keyfn ( ring_node ) : <EOL> for matcher in priority_matchers : <EOL> if ( matcher [ '<STR_LIT>' ] == ring_node [ '<STR_LIT>' ] <EOL> and ( '<STR_LIT>' not in matcher <EOL> or matcher [ '<STR_LIT>' ] == ring_node [ '<STR_LIT>' ] ) ) : <EOL> return matcher [ '<STR_LIT>' ] <EOL> return <NUM_LIT> <EOL> return keyfn <EOL> def affinity_locality_predicate ( write_affinity_str ) : <EOL> """<STR_LIT>""" <EOL> affinity_str = write_affinity_str . strip ( ) <EOL> if not affinity_str : <EOL> return None <EOL> matchers = [ ] <EOL> pieces = [ s . strip ( ) for s in affinity_str . split ( '<STR_LIT:U+002C>' ) ] <EOL> for piece in pieces : <EOL> match = re . match ( "<STR_LIT>" , piece ) <EOL> if match : <EOL> region , zone = match . groups ( ) <EOL> region = int ( region ) <EOL> zone = int ( zone ) if zone else None <EOL> matcher = { '<STR_LIT>' : region } <EOL> if zone is not None : <EOL> matcher [ '<STR_LIT>' ] = zone <EOL> matchers . append ( matcher ) <EOL> else : <EOL> raise ValueError ( "<STR_LIT>" % affinity_str ) <EOL> def is_local ( ring_node ) : <EOL> for matcher in matchers : <EOL> if ( matcher [ '<STR_LIT>' ] == ring_node [ '<STR_LIT>' ] <EOL> and ( '<STR_LIT>' not in matcher <EOL> or matcher [ '<STR_LIT>' ] == ring_node [ '<STR_LIT>' ] ) ) : <EOL> return True <EOL> return False <EOL> return is_local <EOL> def get_remote_client ( req ) : <EOL> client = req . headers . get ( '<STR_LIT>' ) <EOL> if not client and '<STR_LIT>' in req . headers : <EOL> client = req . headers [ '<STR_LIT>' ] . split ( '<STR_LIT:U+002C>' ) [ <NUM_LIT:0> ] . strip ( ) <EOL> if not client : <EOL> client = req . remote_addr <EOL> return client <EOL> def human_readable ( value ) : <EOL> """<STR_LIT>""" <EOL> value = float ( value ) <EOL> index = - <NUM_LIT:1> <EOL> suffixes = '<STR_LIT>' <EOL> while value >= <NUM_LIT> and index + <NUM_LIT:1> < len ( suffixes ) : <EOL> index += <NUM_LIT:1> <EOL> value = round ( value / <NUM_LIT> ) <EOL> if index == - <NUM_LIT:1> : <EOL> return '<STR_LIT>' % value <EOL> return '<STR_LIT>' % ( round ( value ) , suffixes [ index ] ) <EOL> def put_recon_cache_entry ( cache_entry , key , item ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( item , dict ) : <EOL> if key not in cache_entry or key in cache_entry and not isinstance ( cache_entry [ key ] , dict ) : <EOL> cache_entry [ key ] = { } <EOL> elif key in cache_entry and item == { } : <EOL> cache_entry . pop ( key , None ) <EOL> return <EOL> for k , v in item . items ( ) : <EOL> if v == { } : <EOL> cache_entry [ key ] . pop ( k , None ) <EOL> else : <EOL> cache_entry [ key ] [ k ] = v <EOL> else : <EOL> cache_entry [ key ] = item <EOL> def dump_recon_cache ( cache_dict , cache_file , logger , lock_timeout = <NUM_LIT:2> ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> with lock_file ( cache_file , lock_timeout , unlink = False ) as cf : <EOL> cache_entry = { } <EOL> try : <EOL> existing_entry = cf . readline ( ) <EOL> if existing_entry : <EOL> cache_entry = json . loads ( existing_entry ) <EOL> except ValueError : <EOL> pass <EOL> for cache_key , cache_value in cache_dict . items ( ) : <EOL> put_recon_cache_entry ( cache_entry , cache_key , cache_value ) <EOL> tf = None <EOL> try : <EOL> with NamedTemporaryFile ( dir = os . path . dirname ( cache_file ) , <EOL> delete = False ) as tf : <EOL> tf . write ( json . dumps ( cache_entry ) + '<STR_LIT:\n>' ) <EOL> renamer ( tf . name , cache_file , fsync = False ) <EOL> finally : <EOL> if tf is not None : <EOL> try : <EOL> os . unlink ( tf . name ) <EOL> except OSError as err : <EOL> if err . errno != errno . ENOENT : <EOL> raise <EOL> except ( Exception , Timeout ) : <EOL> logger . exception ( _ ( '<STR_LIT>' ) ) <EOL> def listdir ( path ) : <EOL> try : <EOL> return os . listdir ( path ) <EOL> except OSError as err : <EOL> if err . errno != errno . ENOENT : <EOL> raise <EOL> return [ ] <EOL> def streq_const_time ( s1 , s2 ) : <EOL> """<STR_LIT>""" <EOL> if len ( s1 ) != len ( s2 ) : <EOL> return False <EOL> result = <NUM_LIT:0> <EOL> for ( a , b ) in zip ( s1 , s2 ) : <EOL> result |= ord ( a ) ^ ord ( b ) <EOL> return result == <NUM_LIT:0> <EOL> def pairs ( item_list ) : <EOL> """<STR_LIT>""" <EOL> for i , item1 in enumerate ( item_list ) : <EOL> for item2 in item_list [ ( i + <NUM_LIT:1> ) : ] : <EOL> yield ( item1 , item2 ) <EOL> def replication ( func ) : <EOL> """<STR_LIT>""" <EOL> func . replication = True <EOL> return func <EOL> def public ( func ) : <EOL> """<STR_LIT>""" <EOL> func . publicly_accessible = True <EOL> return func <EOL> def quorum_size ( n ) : <EOL> """<STR_LIT>""" <EOL> return ( n // <NUM_LIT:2> ) + <NUM_LIT:1> <EOL> def rsync_ip ( ip ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> socket . inet_pton ( socket . AF_INET6 , ip ) <EOL> except socket . error : <EOL> return ip <EOL> else : <EOL> return '<STR_LIT>' % ip <EOL> def rsync_module_interpolation ( template , device ) : <EOL> """<STR_LIT>""" <EOL> replacements = { <EOL> '<STR_LIT>' : rsync_ip ( device . get ( '<STR_LIT>' , '<STR_LIT>' ) ) , <EOL> '<STR_LIT:port>' : device . get ( '<STR_LIT:port>' , '<STR_LIT>' ) , <EOL> '<STR_LIT>' : rsync_ip ( device . get ( '<STR_LIT>' , '<STR_LIT>' ) ) , <EOL> '<STR_LIT>' : device . get ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> '<STR_LIT>' : device . get ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> '<STR_LIT>' : device . get ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> '<STR_LIT>' : device . get ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> '<STR_LIT>' : device . get ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> } <EOL> try : <EOL> module = template . format ( ** replacements ) <EOL> except KeyError as e : <EOL> raise ValueError ( '<STR_LIT>' <EOL> '<STR_LIT:%s>' % e ) <EOL> return module <EOL> def get_valid_utf8_str ( str_or_unicode ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( str_or_unicode , six . text_type ) : <EOL> ( str_or_unicode , _len ) = utf8_encoder ( str_or_unicode , '<STR_LIT:replace>' ) <EOL> ( valid_utf8_str , _len ) = utf8_decoder ( str_or_unicode , '<STR_LIT:replace>' ) <EOL> return valid_utf8_str . encode ( '<STR_LIT:utf-8>' ) <EOL> def list_from_csv ( comma_separated_str ) : <EOL> """<STR_LIT>""" <EOL> if comma_separated_str : <EOL> return [ v . strip ( ) for v in comma_separated_str . split ( '<STR_LIT:U+002C>' ) if v . strip ( ) ] <EOL> return [ ] <EOL> def csv_append ( csv_string , item ) : <EOL> """<STR_LIT>""" <EOL> if csv_string : <EOL> return "<STR_LIT:U+002C>" . join ( ( csv_string , item ) ) <EOL> else : <EOL> return item <EOL> class CloseableChain ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , * iterables ) : <EOL> self . iterables = iterables <EOL> def __iter__ ( self ) : <EOL> return iter ( itertools . chain ( * ( self . iterables ) ) ) <EOL> def close ( self ) : <EOL> for it in self . iterables : <EOL> close_method = getattr ( it , '<STR_LIT>' , None ) <EOL> if close_method : <EOL> close_method ( ) <EOL> def reiterate ( iterable ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( iterable , ( list , tuple ) ) : <EOL> return iterable <EOL> else : <EOL> iterator = iter ( iterable ) <EOL> try : <EOL> chunk = '<STR_LIT>' <EOL> while not chunk : <EOL> chunk = next ( iterator ) <EOL> return CloseableChain ( [ chunk ] , iterator ) <EOL> except StopIteration : <EOL> return [ ] <EOL> class InputProxy ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , wsgi_input ) : <EOL> """<STR_LIT>""" <EOL> self . wsgi_input = wsgi_input <EOL> self . bytes_received = <NUM_LIT:0> <EOL> self . client_disconnect = False <EOL> def read ( self , * args , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> chunk = self . wsgi_input . read ( * args , ** kwargs ) <EOL> except Exception : <EOL> self . client_disconnect = True <EOL> raise <EOL> self . bytes_received += len ( chunk ) <EOL> return chunk <EOL> def readline ( self , * args , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> line = self . wsgi_input . readline ( * args , ** kwargs ) <EOL> except Exception : <EOL> self . client_disconnect = True <EOL> raise <EOL> self . bytes_received += len ( line ) <EOL> return line <EOL> class LRUCache ( object ) : <EOL> """<STR_LIT>""" <EOL> PREV , NEXT , KEY , CACHED_AT , VALUE = <NUM_LIT:0> , <NUM_LIT:1> , <NUM_LIT:2> , <NUM_LIT:3> , <NUM_LIT:4> <EOL> def __init__ ( self , maxsize = <NUM_LIT:1000> , maxtime = <NUM_LIT> ) : <EOL> self . maxsize = maxsize <EOL> self . maxtime = maxtime <EOL> self . reset ( ) <EOL> def reset ( self ) : <EOL> self . mapping = { } <EOL> self . head = [ None , None , None , None , None ] <EOL> self . tail = [ self . head , None , None , None , None ] <EOL> self . head [ self . NEXT ] = self . tail <EOL> def set_cache ( self , value , * key ) : <EOL> while len ( self . mapping ) >= self . maxsize : <EOL> old_next , old_key = self . head [ self . NEXT ] [ self . NEXT : self . NEXT + <NUM_LIT:2> ] <EOL> self . head [ self . NEXT ] , old_next [ self . PREV ] = old_next , self . head <EOL> del self . mapping [ old_key ] <EOL> last = self . tail [ self . PREV ] <EOL> link = [ last , self . tail , key , time . time ( ) , value ] <EOL> self . mapping [ key ] = last [ self . NEXT ] = self . tail [ self . PREV ] = link <EOL> return value <EOL> def get_cached ( self , link , * key ) : <EOL> link_prev , link_next , key , cached_at , value = link <EOL> if cached_at + self . maxtime < time . time ( ) : <EOL> raise KeyError ( '<STR_LIT>' % ( key , ) ) <EOL> link_prev [ self . NEXT ] = link_next <EOL> link_next [ self . PREV ] = link_prev <EOL> last = self . tail [ self . PREV ] <EOL> last [ self . NEXT ] = self . tail [ self . PREV ] = link <EOL> link [ self . PREV ] = last <EOL> link [ self . NEXT ] = self . tail <EOL> return value <EOL> def __call__ ( self , f ) : <EOL> class LRUCacheWrapped ( object ) : <EOL> @ functools . wraps ( f ) <EOL> def __call__ ( im_self , * key ) : <EOL> link = self . mapping . get ( key , self . head ) <EOL> if link is not self . head : <EOL> try : <EOL> return self . get_cached ( link , * key ) <EOL> except KeyError : <EOL> pass <EOL> value = f ( * key ) <EOL> self . set_cache ( value , * key ) <EOL> return value <EOL> def size ( im_self ) : <EOL> """<STR_LIT>""" <EOL> return len ( self . mapping ) <EOL> def reset ( im_self ) : <EOL> return self . reset ( ) <EOL> def get_maxsize ( im_self ) : <EOL> return self . maxsize <EOL> def set_maxsize ( im_self , i ) : <EOL> self . maxsize = i <EOL> def get_maxtime ( im_self ) : <EOL> return self . maxtime <EOL> def set_maxtime ( im_self , i ) : <EOL> self . maxtime = i <EOL> maxsize = property ( get_maxsize , set_maxsize ) <EOL> maxtime = property ( get_maxtime , set_maxtime ) <EOL> def __repr__ ( im_self ) : <EOL> return '<STR_LIT>' % ( im_self . __class__ . __name__ , f ) <EOL> return LRUCacheWrapped ( ) <EOL> def tpool_reraise ( func , * args , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> def inner ( ) : <EOL> try : <EOL> return func ( * args , ** kwargs ) <EOL> except BaseException as err : <EOL> return err <EOL> resp = tpool . execute ( inner ) <EOL> if isinstance ( resp , BaseException ) : <EOL> raise resp <EOL> return resp <EOL> class ThreadPool ( object ) : <EOL> """<STR_LIT>""" <EOL> BYTE = '<STR_LIT:a>' . encode ( '<STR_LIT:utf-8>' ) <EOL> def __init__ ( self , nthreads = <NUM_LIT:2> ) : <EOL> self . nthreads = nthreads <EOL> self . _run_queue = stdlib_queue . Queue ( ) <EOL> self . _result_queue = stdlib_queue . Queue ( ) <EOL> self . _threads = [ ] <EOL> self . _alive = True <EOL> if nthreads <= <NUM_LIT:0> : <EOL> return <EOL> _raw_rpipe , self . wpipe = os . pipe ( ) <EOL> self . rpipe = greenio . GreenPipe ( _raw_rpipe , '<STR_LIT:rb>' ) <EOL> for _junk in range ( nthreads ) : <EOL> thr = stdlib_threading . Thread ( <EOL> target = self . _worker , <EOL> args = ( self . _run_queue , self . _result_queue ) ) <EOL> thr . daemon = True <EOL> thr . start ( ) <EOL> self . _threads . append ( thr ) <EOL> self . _consumer_coro = greenthread . spawn_n ( self . _consume_results , <EOL> self . _result_queue ) <EOL> def _worker ( self , work_queue , result_queue ) : <EOL> """<STR_LIT>""" <EOL> while True : <EOL> item = work_queue . get ( ) <EOL> if item is None : <EOL> break <EOL> ev , func , args , kwargs = item <EOL> try : <EOL> result = func ( * args , ** kwargs ) <EOL> result_queue . put ( ( ev , True , result ) ) <EOL> except BaseException : <EOL> result_queue . put ( ( ev , False , sys . exc_info ( ) ) ) <EOL> finally : <EOL> work_queue . task_done ( ) <EOL> os . write ( self . wpipe , self . BYTE ) <EOL> def _consume_results ( self , queue ) : <EOL> """<STR_LIT>""" <EOL> while True : <EOL> try : <EOL> self . rpipe . read ( <NUM_LIT:1> ) <EOL> except ValueError : <EOL> break <EOL> while True : <EOL> try : <EOL> ev , success , result = queue . get ( block = False ) <EOL> except stdlib_queue . Empty : <EOL> break <EOL> try : <EOL> if success : <EOL> ev . send ( result ) <EOL> else : <EOL> ev . send_exception ( * result ) <EOL> finally : <EOL> queue . task_done ( ) <EOL> def run_in_thread ( self , func , * args , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> if not self . _alive : <EOL> raise swift . common . exceptions . ThreadPoolDead ( ) <EOL> if self . nthreads <= <NUM_LIT:0> : <EOL> result = func ( * args , ** kwargs ) <EOL> sleep ( ) <EOL> return result <EOL> ev = event . Event ( ) <EOL> self . _run_queue . put ( ( ev , func , args , kwargs ) , block = False ) <EOL> result = ev . wait ( ) <EOL> return result <EOL> def _run_in_eventlet_tpool ( self , func , * args , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> def inner ( ) : <EOL> try : <EOL> return ( True , func ( * args , ** kwargs ) ) <EOL> except ( Timeout , BaseException ) as err : <EOL> return ( False , err ) <EOL> success , result = tpool . execute ( inner ) <EOL> if success : <EOL> return result <EOL> else : <EOL> raise result <EOL> def force_run_in_thread ( self , func , * args , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> if not self . _alive : <EOL> raise swift . common . exceptions . ThreadPoolDead ( ) <EOL> if self . nthreads <= <NUM_LIT:0> : <EOL> return self . _run_in_eventlet_tpool ( func , * args , ** kwargs ) <EOL> else : <EOL> return self . run_in_thread ( func , * args , ** kwargs ) <EOL> def terminate ( self ) : <EOL> """<STR_LIT>""" <EOL> self . _alive = False <EOL> if self . nthreads <= <NUM_LIT:0> : <EOL> return <EOL> for _junk in range ( self . nthreads ) : <EOL> self . _run_queue . put ( None ) <EOL> for thr in self . _threads : <EOL> thr . join ( ) <EOL> self . _threads = [ ] <EOL> self . nthreads = <NUM_LIT:0> <EOL> greenthread . kill ( self . _consumer_coro ) <EOL> self . rpipe . close ( ) <EOL> os . close ( self . wpipe ) <EOL> def ismount ( path ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> return ismount_raw ( path ) <EOL> except OSError : <EOL> return False <EOL> def ismount_raw ( path ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> s1 = os . lstat ( path ) <EOL> except os . error as err : <EOL> if err . errno == errno . ENOENT : <EOL> return False <EOL> raise <EOL> if stat . S_ISLNK ( s1 . st_mode ) : <EOL> return False <EOL> s2 = os . lstat ( os . path . join ( path , '<STR_LIT:..>' ) ) <EOL> dev1 = s1 . st_dev <EOL> dev2 = s2 . st_dev <EOL> if dev1 != dev2 : <EOL> return True <EOL> ino1 = s1 . st_ino <EOL> ino2 = s2 . st_ino <EOL> if ino1 == ino2 : <EOL> return True <EOL> return False <EOL> def close_if_possible ( maybe_closable ) : <EOL> close_method = getattr ( maybe_closable , '<STR_LIT>' , None ) <EOL> if callable ( close_method ) : <EOL> return close_method ( ) <EOL> @ contextmanager <EOL> def closing_if_possible ( maybe_closable ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> yield maybe_closable <EOL> finally : <EOL> close_if_possible ( maybe_closable ) <EOL> _rfc_token = r'<STR_LIT>' <EOL> _rfc_extension_pattern = re . compile ( <EOL> r'<STR_LIT>' + _rfc_token + r"<STR_LIT>" + _rfc_token + <EOL> r'<STR_LIT>' ) <EOL> _content_range_pattern = re . compile ( r'<STR_LIT>' ) <EOL> def parse_content_range ( content_range ) : <EOL> """<STR_LIT>""" <EOL> found = re . search ( _content_range_pattern , content_range ) <EOL> if not found : <EOL> raise ValueError ( "<STR_LIT>" % ( content_range , ) ) <EOL> return tuple ( int ( x ) for x in found . groups ( ) ) <EOL> def parse_content_type ( content_type ) : <EOL> """<STR_LIT>""" <EOL> parm_list = [ ] <EOL> if '<STR_LIT:;>' in content_type : <EOL> content_type , parms = content_type . split ( '<STR_LIT:;>' , <NUM_LIT:1> ) <EOL> parms = '<STR_LIT:;>' + parms <EOL> for m in _rfc_extension_pattern . findall ( parms ) : <EOL> key = m [ <NUM_LIT:0> ] . strip ( ) <EOL> value = m [ <NUM_LIT:1> ] . strip ( ) <EOL> parm_list . append ( ( key , value ) ) <EOL> return content_type , parm_list <EOL> def extract_swift_bytes ( content_type ) : <EOL> """<STR_LIT>""" <EOL> content_type , params = parse_content_type ( content_type ) <EOL> swift_bytes = None <EOL> for k , v in params : <EOL> if k == '<STR_LIT>' : <EOL> swift_bytes = v <EOL> else : <EOL> content_type += '<STR_LIT>' % ( k , v ) <EOL> return content_type , swift_bytes <EOL> def override_bytes_from_content_type ( listing_dict , logger = None ) : <EOL> """<STR_LIT>""" <EOL> content_type , params = parse_content_type ( listing_dict [ '<STR_LIT>' ] ) <EOL> for key , value in params : <EOL> if key == '<STR_LIT>' : <EOL> try : <EOL> listing_dict [ '<STR_LIT>' ] = int ( value ) <EOL> except ValueError : <EOL> if logger : <EOL> logger . exception ( "<STR_LIT>" ) <EOL> else : <EOL> content_type += '<STR_LIT>' % ( key , value ) <EOL> listing_dict [ '<STR_LIT>' ] = content_type <EOL> def clean_content_type ( value ) : <EOL> if '<STR_LIT:;>' in value : <EOL> left , right = value . rsplit ( '<STR_LIT:;>' , <NUM_LIT:1> ) <EOL> if right . lstrip ( ) . startswith ( '<STR_LIT>' ) : <EOL> return left <EOL> return value <EOL> def quote ( value , safe = '<STR_LIT:/>' ) : <EOL> """<STR_LIT>""" <EOL> return _quote ( get_valid_utf8_str ( value ) , safe ) <EOL> def get_expirer_container ( x_delete_at , expirer_divisor , acc , cont , obj ) : <EOL> """<STR_LIT>""" <EOL> shard_int = int ( hash_path ( acc , cont , obj ) , <NUM_LIT:16> ) % <NUM_LIT:100> <EOL> return normalize_delete_at_timestamp ( <EOL> int ( x_delete_at ) / expirer_divisor * expirer_divisor - shard_int ) <EOL> class _MultipartMimeFileLikeObject ( object ) : <EOL> def __init__ ( self , wsgi_input , boundary , input_buffer , read_chunk_size ) : <EOL> self . no_more_data_for_this_file = False <EOL> self . no_more_files = False <EOL> self . wsgi_input = wsgi_input <EOL> self . boundary = boundary <EOL> self . input_buffer = input_buffer <EOL> self . read_chunk_size = read_chunk_size <EOL> def read ( self , length = None ) : <EOL> if not length : <EOL> length = self . read_chunk_size <EOL> if self . no_more_data_for_this_file : <EOL> return b'<STR_LIT>' <EOL> if len ( self . input_buffer ) < length + len ( self . boundary ) + <NUM_LIT:2> : <EOL> to_read = length + len ( self . boundary ) + <NUM_LIT:2> <EOL> while to_read > <NUM_LIT:0> : <EOL> try : <EOL> chunk = self . wsgi_input . read ( to_read ) <EOL> except ( IOError , ValueError ) as e : <EOL> raise swift . common . exceptions . ChunkReadError ( str ( e ) ) <EOL> to_read -= len ( chunk ) <EOL> self . input_buffer += chunk <EOL> if not chunk : <EOL> self . no_more_files = True <EOL> break <EOL> boundary_pos = self . input_buffer . find ( self . boundary ) <EOL> if boundary_pos == - <NUM_LIT:1> or boundary_pos > length : <EOL> ret = self . input_buffer [ : length ] <EOL> self . input_buffer = self . input_buffer [ length : ] <EOL> else : <EOL> ret , self . input_buffer = self . input_buffer . split ( self . boundary , <NUM_LIT:1> ) <EOL> self . no_more_files = self . input_buffer . startswith ( b'<STR_LIT>' ) <EOL> self . no_more_data_for_this_file = True <EOL> self . input_buffer = self . input_buffer [ <NUM_LIT:2> : ] <EOL> return ret <EOL> def readline ( self ) : <EOL> if self . no_more_data_for_this_file : <EOL> return b'<STR_LIT>' <EOL> boundary_pos = newline_pos = - <NUM_LIT:1> <EOL> while newline_pos < <NUM_LIT:0> and boundary_pos < <NUM_LIT:0> : <EOL> try : <EOL> chunk = self . wsgi_input . read ( self . read_chunk_size ) <EOL> except ( IOError , ValueError ) as e : <EOL> raise swift . common . exceptions . ChunkReadError ( str ( e ) ) <EOL> self . input_buffer += chunk <EOL> newline_pos = self . input_buffer . find ( b'<STR_LIT:\r\n>' ) <EOL> boundary_pos = self . input_buffer . find ( self . boundary ) <EOL> if not chunk : <EOL> self . no_more_files = True <EOL> break <EOL> if newline_pos >= <NUM_LIT:0> and ( boundary_pos < <NUM_LIT:0> or newline_pos < boundary_pos ) : <EOL> ret = b'<STR_LIT>' <EOL> to_read = newline_pos + <NUM_LIT:2> <EOL> while to_read > <NUM_LIT:0> : <EOL> chunk = self . read ( to_read ) <EOL> if not chunk : <EOL> break <EOL> to_read -= len ( chunk ) <EOL> ret += chunk <EOL> return ret <EOL> else : <EOL> return self . read ( len ( self . input_buffer ) ) <EOL> def iter_multipart_mime_documents ( wsgi_input , boundary , read_chunk_size = <NUM_LIT> ) : <EOL> """<STR_LIT>""" <EOL> boundary = '<STR_LIT>' + boundary <EOL> blen = len ( boundary ) + <NUM_LIT:2> <EOL> try : <EOL> got = wsgi_input . readline ( blen ) <EOL> while got == '<STR_LIT:\r\n>' : <EOL> got = wsgi_input . readline ( blen ) <EOL> except ( IOError , ValueError ) as e : <EOL> raise swift . common . exceptions . ChunkReadError ( str ( e ) ) <EOL> if got . strip ( ) != boundary : <EOL> raise swift . common . exceptions . MimeInvalid ( <EOL> '<STR_LIT>' , ( boundary , got ) ) <EOL> boundary = '<STR_LIT:\r\n>' + boundary <EOL> input_buffer = '<STR_LIT>' <EOL> done = False <EOL> while not done : <EOL> it = _MultipartMimeFileLikeObject ( wsgi_input , boundary , input_buffer , <EOL> read_chunk_size ) <EOL> yield it <EOL> done = it . no_more_files <EOL> input_buffer = it . input_buffer <EOL> def parse_mime_headers ( doc_file ) : <EOL> """<STR_LIT>""" <EOL> headers = [ ] <EOL> while True : <EOL> line = doc_file . readline ( ) <EOL> done = line in ( b'<STR_LIT:\r\n>' , b'<STR_LIT:\n>' , b'<STR_LIT>' ) <EOL> if six . PY3 : <EOL> try : <EOL> line = line . decode ( '<STR_LIT:utf-8>' ) <EOL> except UnicodeDecodeError : <EOL> line = line . decode ( '<STR_LIT>' ) <EOL> headers . append ( line ) <EOL> if done : <EOL> break <EOL> if six . PY3 : <EOL> header_string = '<STR_LIT>' . join ( headers ) <EOL> else : <EOL> header_string = b'<STR_LIT>' . join ( headers ) <EOL> headers = email . parser . Parser ( ) . parsestr ( header_string ) <EOL> return HeaderKeyDict ( headers ) <EOL> def mime_to_document_iters ( input_file , boundary , read_chunk_size = <NUM_LIT> ) : <EOL> """<STR_LIT>""" <EOL> doc_files = iter_multipart_mime_documents ( input_file , boundary , <EOL> read_chunk_size ) <EOL> for i , doc_file in enumerate ( doc_files ) : <EOL> headers = parse_mime_headers ( doc_file ) <EOL> yield ( headers , doc_file ) <EOL> def maybe_multipart_byteranges_to_document_iters ( app_iter , content_type ) : <EOL> """<STR_LIT>""" <EOL> content_type , params_list = parse_content_type ( content_type ) <EOL> if content_type != '<STR_LIT>' : <EOL> yield app_iter <EOL> return <EOL> body_file = FileLikeIter ( app_iter ) <EOL> boundary = dict ( params_list ) [ '<STR_LIT>' ] <EOL> for _headers , body in mime_to_document_iters ( body_file , boundary ) : <EOL> yield ( chunk for chunk in iter ( lambda : body . read ( <NUM_LIT> ) , '<STR_LIT>' ) ) <EOL> def document_iters_to_multipart_byteranges ( ranges_iter , boundary ) : <EOL> """<STR_LIT>""" <EOL> divider = "<STR_LIT>" + boundary + "<STR_LIT:\r\n>" <EOL> terminator = "<STR_LIT>" + boundary + "<STR_LIT>" <EOL> for range_spec in ranges_iter : <EOL> start_byte = range_spec [ "<STR_LIT>" ] <EOL> end_byte = range_spec [ "<STR_LIT>" ] <EOL> entity_length = range_spec . get ( "<STR_LIT>" , "<STR_LIT:*>" ) <EOL> content_type = range_spec [ "<STR_LIT>" ] <EOL> part_iter = range_spec [ "<STR_LIT>" ] <EOL> part_header = '<STR_LIT>' . join ( ( <EOL> divider , <EOL> "<STR_LIT>" , str ( content_type ) , "<STR_LIT:\r\n>" , <EOL> "<STR_LIT>" , "<STR_LIT>" % ( <EOL> start_byte , end_byte , entity_length ) , <EOL> "<STR_LIT:\r\n>" <EOL> ) ) <EOL> yield part_header <EOL> for chunk in part_iter : <EOL> yield chunk <EOL> yield "<STR_LIT:\r\n>" <EOL> yield terminator <EOL> def document_iters_to_http_response_body ( ranges_iter , boundary , multipart , <EOL> logger ) : <EOL> """<STR_LIT>""" <EOL> if multipart : <EOL> return document_iters_to_multipart_byteranges ( ranges_iter , boundary ) <EOL> else : <EOL> try : <EOL> response_body_iter = next ( ranges_iter ) [ '<STR_LIT>' ] <EOL> except StopIteration : <EOL> return '<STR_LIT>' <EOL> def string_along ( useful_iter , useless_iter_iter , logger ) : <EOL> for x in useful_iter : <EOL> yield x <EOL> try : <EOL> next ( useless_iter_iter ) <EOL> except StopIteration : <EOL> pass <EOL> else : <EOL> logger . warning ( <EOL> "<STR_LIT>" ) <EOL> return string_along ( response_body_iter , ranges_iter , logger ) <EOL> def multipart_byteranges_to_document_iters ( input_file , boundary , <EOL> read_chunk_size = <NUM_LIT> ) : <EOL> """<STR_LIT>""" <EOL> for headers , body in mime_to_document_iters ( input_file , boundary , <EOL> read_chunk_size ) : <EOL> first_byte , last_byte , length = parse_content_range ( <EOL> headers . get ( '<STR_LIT>' ) ) <EOL> yield ( first_byte , last_byte , length , headers . items ( ) , body ) <EOL> ATTRIBUTES_RE = re . compile ( r'<STR_LIT>' ) <EOL> def parse_content_disposition ( header ) : <EOL> """<STR_LIT>""" <EOL> attributes = { } <EOL> attrs = '<STR_LIT>' <EOL> if '<STR_LIT:;>' in header : <EOL> header , attrs = [ x . strip ( ) for x in header . split ( '<STR_LIT:;>' , <NUM_LIT:1> ) ] <EOL> m = True <EOL> while m : <EOL> m = ATTRIBUTES_RE . match ( attrs ) <EOL> if m : <EOL> attrs = attrs [ len ( m . group ( <NUM_LIT:0> ) ) : ] <EOL> attributes [ m . group ( <NUM_LIT:1> ) ] = m . group ( <NUM_LIT:2> ) . strip ( '<STR_LIT:">' ) <EOL> return header , attributes <EOL> class sockaddr_alg ( ctypes . Structure ) : <EOL> _fields_ = [ ( "<STR_LIT>" , ctypes . c_ushort ) , <EOL> ( "<STR_LIT>" , ctypes . c_ubyte * <NUM_LIT> ) , <EOL> ( "<STR_LIT>" , ctypes . c_uint ) , <EOL> ( "<STR_LIT>" , ctypes . c_uint ) , <EOL> ( "<STR_LIT>" , ctypes . c_ubyte * <NUM_LIT:64> ) ] <EOL> _bound_md5_sockfd = None <EOL> def get_md5_socket ( ) : <EOL> """<STR_LIT>""" <EOL> global _bound_md5_sockfd <EOL> global _libc_socket <EOL> global _libc_bind <EOL> global _libc_accept <EOL> if _libc_accept is None : <EOL> _libc_accept = load_libc_function ( '<STR_LIT>' , fail_if_missing = True ) <EOL> if _libc_socket is None : <EOL> _libc_socket = load_libc_function ( '<STR_LIT>' , fail_if_missing = True ) <EOL> if _libc_bind is None : <EOL> _libc_bind = load_libc_function ( '<STR_LIT>' , fail_if_missing = True ) <EOL> if _bound_md5_sockfd is None : <EOL> sockaddr_setup = sockaddr_alg ( <EOL> AF_ALG , <EOL> ( ord ( '<STR_LIT:h>' ) , ord ( '<STR_LIT:a>' ) , ord ( '<STR_LIT:s>' ) , ord ( '<STR_LIT:h>' ) , <NUM_LIT:0> ) , <EOL> <NUM_LIT:0> , <NUM_LIT:0> , <EOL> ( ord ( '<STR_LIT:m>' ) , ord ( '<STR_LIT:d>' ) , ord ( '<STR_LIT:5>' ) , <NUM_LIT:0> ) ) <EOL> hash_sockfd = _libc_socket ( ctypes . c_int ( AF_ALG ) , <EOL> ctypes . c_int ( socket . SOCK_SEQPACKET ) , <EOL> ctypes . c_int ( <NUM_LIT:0> ) ) <EOL> if hash_sockfd < <NUM_LIT:0> : <EOL> raise IOError ( ctypes . get_errno ( ) , <EOL> "<STR_LIT>" ) <EOL> bind_result = _libc_bind ( ctypes . c_int ( hash_sockfd ) , <EOL> ctypes . pointer ( sockaddr_setup ) , <EOL> ctypes . c_int ( ctypes . sizeof ( sockaddr_alg ) ) ) <EOL> if bind_result < <NUM_LIT:0> : <EOL> os . close ( hash_sockfd ) <EOL> raise IOError ( ctypes . get_errno ( ) , "<STR_LIT>" ) <EOL> _bound_md5_sockfd = hash_sockfd <EOL> md5_sockfd = _libc_accept ( ctypes . c_int ( _bound_md5_sockfd ) , None , <NUM_LIT:0> ) <EOL> if md5_sockfd < <NUM_LIT:0> : <EOL> raise IOError ( ctypes . get_errno ( ) , "<STR_LIT>" ) <EOL> return md5_sockfd </s>
