<s> """<STR_LIT>""" <EOL> import os <EOL> import re <EOL> import sys <EOL> import logging <EOL> from functools import partial <EOL> from datetime import datetime <EOL> from abc import ABCMeta , abstractmethod <EOL> import json <EOL> from _config import AttrDict <EOL> __all__ = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> def multikey_getter_gen ( parser , keys , is_indices = False , delimiter = "<STR_LIT:\t>" ) : <EOL> """<STR_LIT>""" <EOL> if is_indices : <EOL> keys = map ( int , keys ) <EOL> def multikey_getter ( line , parser , keyset ) : <EOL> data = parser ( line . strip ( ) ) <EOL> return delimiter . join ( ( unicode ( data [ k ] ) for k in keyset ) ) <EOL> def multiindex_getter ( line , parser , keyset ) : <EOL> data = parser ( line . strip ( ) ) <EOL> return delimiter . join ( ( unicode ( data . by_index ( idx - <NUM_LIT:1> , raw = True ) ) for idx in keys ) ) <EOL> if is_indices is True : <EOL> return partial ( multiindex_getter , parser = parser , keyset = keys ) <EOL> else : <EOL> return partial ( multikey_getter , parser = parser , keyset = keys ) <EOL> def unescape_json ( s ) : <EOL> """<STR_LIT>""" <EOL> return s . replace ( "<STR_LIT>" , '<STR_LIT:/>' ) . replace ( '<STR_LIT>' , '<STR_LIT:">' ) . decode ( '<STR_LIT>' ) <EOL> class LogParser ( object ) : <EOL> """<STR_LIT>""" <EOL> __metaclass__ = ABCMeta <EOL> def __call__ ( self , line ) : <EOL> """<STR_LIT>""" <EOL> return self . parse ( line ) <EOL> @ abstractmethod <EOL> def parse ( self , line ) : <EOL> """<STR_LIT>""" <EOL> def set_format ( self , format ) : <EOL> """<STR_LIT>""" <EOL> class LogLine ( dict ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , fieldnames = None ) : <EOL> """<STR_LIT>""" <EOL> self . _fieldnames = None <EOL> if fieldnames : <EOL> self . fieldnames = fieldnames <EOL> @ property <EOL> def fieldnames ( self ) : <EOL> """<STR_LIT>""" <EOL> return self . _fieldnames <EOL> @ fieldnames . setter <EOL> def fieldnames ( self , fieldnames ) : <EOL> """<STR_LIT>""" <EOL> self . _fieldnames = dict ( enumerate ( fieldnames ) ) <EOL> def by_index ( self , i , raw = False ) : <EOL> return self . by_key ( self . _fieldnames [ i ] , raw = raw ) <EOL> def by_key ( self , key , raw = False ) : <EOL> """<STR_LIT>""" <EOL> val = None <EOL> if raw is True : <EOL> return self [ key ] <EOL> if key == '<STR_LIT>' : <EOL> val = datetime . strptime ( self [ key ] [ <NUM_LIT:1> : - <NUM_LIT:7> ] , '<STR_LIT>' ) <EOL> else : <EOL> val = self [ key ] <EOL> return val <EOL> class JSONParser ( LogParser ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self ) : <EOL> LogParser . __init__ ( self ) <EOL> self . _logline_wrapper = LogLine ( ) <EOL> def parse ( self , line ) : <EOL> """<STR_LIT>""" <EOL> parsed_row = json . loads ( line ) <EOL> data = self . _logline_wrapper <EOL> self . _logline_wrapper . fieldnames = parsed_row . keys ( ) <EOL> data . clear ( ) <EOL> for k , v in parsed_row . iteritems ( ) : <EOL> data [ k ] = v <EOL> return data <EOL> class AccessLog ( LogParser ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , format = None ) : <EOL> LogParser . __init__ ( self ) <EOL> self . fieldnames = None <EOL> self . fieldselector = None <EOL> self . _logline_wrapper = None <EOL> if format : <EOL> self . fieldselector = self . _parse_log_format ( format ) <EOL> self . _logline_wrapper = LogLine ( self . fieldnames ) <EOL> def set_format ( self , format ) : <EOL> """<STR_LIT>""" <EOL> self . fieldselector = self . _parse_log_format ( format ) <EOL> self . _logline_wrapper = LogLine ( self . fieldnames ) <EOL> def parse ( self , logline ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> match = self . fieldselector . match ( logline ) <EOL> except AttributeError , exc : <EOL> raise AttributeError ( "<STR_LIT>" % self . __class__ . __name__ ) <EOL> if match : <EOL> data = self . _logline_wrapper <EOL> data . clear ( ) <EOL> for k , v in zip ( self . fieldnames , match . groups ( ) ) : <EOL> data [ k ] = v <EOL> return data <EOL> else : <EOL> raise ValueError ( "<STR_LIT>" % logline ) <EOL> def _parse_log_format ( self , format ) : <EOL> """<STR_LIT>""" <EOL> format = format . strip ( ) <EOL> format = re . sub ( '<STR_LIT>' , '<STR_LIT:U+0020>' , format ) <EOL> subpatterns = [ ] <EOL> findquotes = re . compile ( r'<STR_LIT>' ) <EOL> findreferreragent = re . compile ( '<STR_LIT>' ) <EOL> findpercent = re . compile ( r'<STR_LIT>' ) <EOL> lstripquotes = re . compile ( r'<STR_LIT>' ) <EOL> rstripquotes = re . compile ( r'<STR_LIT>' ) <EOL> self . fieldnames = [ ] <EOL> for element in format . split ( '<STR_LIT:U+0020>' ) : <EOL> hasquotes = <NUM_LIT:0> <EOL> if findquotes . match ( element ) : <EOL> hasquotes = <NUM_LIT:1> <EOL> if hasquotes : <EOL> element = lstripquotes . sub ( '<STR_LIT>' , element ) <EOL> element = rstripquotes . sub ( '<STR_LIT>' , element ) <EOL> self . fieldnames . append ( element ) <EOL> subpattern = '<STR_LIT>' <EOL> if hasquotes : <EOL> if element == '<STR_LIT>' or findreferreragent . search ( element ) : <EOL> subpattern = r'<STR_LIT>' <EOL> else : <EOL> subpattern = r'<STR_LIT>' <EOL> elif findpercent . search ( element ) : <EOL> subpattern = r'<STR_LIT>' <EOL> elif element == '<STR_LIT>' : <EOL> subpattern = '<STR_LIT>' <EOL> subpatterns . append ( subpattern ) <EOL> _pattern = '<STR_LIT>' + '<STR_LIT:U+0020>' . join ( subpatterns ) + '<STR_LIT:$>' <EOL> _regex = re . compile ( _pattern ) <EOL> return _regex <EOL> class CommonLogFormat ( AccessLog ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self ) : <EOL> AccessLog . __init__ ( self , format = '<STR_LIT>' ) <EOL> class uWSGIParser ( LogParser ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self ) : <EOL> LogParser . __init__ ( self ) <EOL> self . _re = re . compile ( r'<STR_LIT>' ) <EOL> self . fieldnames = ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:path>' , '<STR_LIT>' , '<STR_LIT>' ) <EOL> self . _logline_wrapper = LogLine ( self . fieldnames ) <EOL> def parse ( self , logline ) : <EOL> """<STR_LIT>""" <EOL> match = self . _re . match ( logline ) <EOL> if match : <EOL> data = self . _logline_wrapper <EOL> data . clear ( ) <EOL> for k , v in zip ( self . fieldnames , match . groups ( ) ) : <EOL> data [ k ] = v <EOL> return data <EOL> else : <EOL> raise ValueError ( "<STR_LIT>" % logline ) </s>
