<s> """<STR_LIT>""" <EOL> import os <EOL> import signal <EOL> import argparse <EOL> import datetime <EOL> from configobj import ConfigObj <EOL> from uwsgi_sloth . analyzer import format_data , RealtimeLogAnalyzer , URLClassifier <EOL> from uwsgi_sloth . tailer import Tailer , no_new_line <EOL> from uwsgi_sloth . template import render_template <EOL> from uwsgi_sloth . utils import makedir_if_none_exists , total_seconds , parse_url_rules , smart_str <EOL> from uwsgi_sloth . models import merge_requests_data_to , RequestsData , SavePoint <EOL> from uwsgi_sloth . settings import REALTIME_UPDATE_INTERVAL , DEFAULT_MIN_MSECS <EOL> import logging <EOL> logger = logging . getLogger ( '<STR_LIT>' ) <EOL> class HTMLRender ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , html_dir , domain = None ) : <EOL> self . html_dir = html_dir <EOL> self . domain = domain <EOL> def render_requests_data_to_html ( self , data , file_name , context = { } ) : <EOL> """<STR_LIT>""" <EOL> file_path = os . path . join ( self . html_dir , file_name ) <EOL> logger . info ( '<STR_LIT>' % file_path ) <EOL> data = format_data ( data ) <EOL> data . update ( context ) <EOL> data . update ( domain = self . domain ) <EOL> with open ( file_path , '<STR_LIT:w>' ) as fp : <EOL> fp . write ( smart_str ( render_template ( '<STR_LIT>' , data ) ) ) <EOL> def update_html_symlink ( html_dir ) : <EOL> """<STR_LIT>""" <EOL> today = datetime . date . today ( ) <EOL> yesterday = datetime . date . today ( ) - datetime . timedelta ( days = <NUM_LIT:1> ) <EOL> for from_date , alias_name in ( <EOL> ( today , '<STR_LIT>' ) , ( yesterday , '<STR_LIT>' ) ) : <EOL> from_date_file_path = os . path . join ( html_dir , '<STR_LIT>' % from_date ) <EOL> symlink_path = os . path . join ( html_dir , alias_name ) <EOL> try : <EOL> os . unlink ( symlink_path ) <EOL> except OSError : <EOL> pass <EOL> os . symlink ( from_date_file_path , symlink_path ) <EOL> def start ( args ) : <EOL> config = ConfigObj ( infile = args . config . name ) <EOL> data_dir = config [ '<STR_LIT>' ] <EOL> uwsgi_log_path = config [ '<STR_LIT>' ] <EOL> min_msecs = int ( config . get ( '<STR_LIT>' , DEFAULT_MIN_MSECS ) ) <EOL> url_file = config . get ( '<STR_LIT>' ) <EOL> url_rules = [ ] <EOL> if url_file : <EOL> with open ( url_file , '<STR_LIT:r>' ) as fp : <EOL> url_rules = parse_url_rules ( fp ) <EOL> html_dir = os . path . join ( data_dir , '<STR_LIT:html>' ) <EOL> db_dir = os . path . join ( data_dir , '<STR_LIT:data>' ) <EOL> makedir_if_none_exists ( html_dir ) <EOL> makedir_if_none_exists ( db_dir ) <EOL> save_point = SavePoint ( db_dir ) <EOL> last_log_datetime = save_point . get_last_datetime ( ) or ( datetime . datetime . now ( ) - datetime . timedelta ( seconds = REALTIME_UPDATE_INTERVAL ) ) <EOL> logger . info ( '<STR_LIT>' % last_log_datetime ) <EOL> last_update_datetime = None <EOL> url_classifier = URLClassifier ( user_defined_rules = url_rules ) <EOL> analyzer = RealtimeLogAnalyzer ( url_classifier = url_classifier , min_msecs = min_msecs , <EOL> start_from_datetime = last_log_datetime ) <EOL> file_tailer = Tailer ( uwsgi_log_path ) <EOL> html_render = HTMLRender ( html_dir , domain = config . get ( '<STR_LIT>' ) ) <EOL> def gracefully_exit ( * args ) : <EOL> logger . info ( '<STR_LIT>' ) <EOL> file_tailer . stop_follow ( ) <EOL> signal . signal ( signal . SIGINT , gracefully_exit ) <EOL> for line in file_tailer : <EOL> if line != no_new_line : <EOL> analyzer . analyze_line ( line ) <EOL> now = datetime . datetime . now ( ) <EOL> if not file_tailer . trailing : <EOL> continue <EOL> if last_update_datetime and total_seconds ( now - last_update_datetime ) < REALTIME_UPDATE_INTERVAL : <EOL> continue <EOL> html_render . render_requests_data_to_html ( analyzer . get_data ( '<STR_LIT>' ) , <EOL> '<STR_LIT>' , context = { '<STR_LIT>' : '<STR_LIT>' } ) <EOL> analyzer . clean_data_by_key ( '<STR_LIT>' ) <EOL> for date in analyzer . data . keys ( ) : <EOL> day_requests_data = RequestsData ( date , db_dir ) <EOL> merge_requests_data_to ( day_requests_data . data , analyzer . get_data ( date ) ) <EOL> html_render . render_requests_data_to_html ( day_requests_data . data , <EOL> '<STR_LIT>' % date , context = { '<STR_LIT>' : date } ) <EOL> day_requests_data . save ( ) <EOL> analyzer . clean_data_by_key ( date ) <EOL> update_html_symlink ( html_dir ) <EOL> last_update_datetime = now <EOL> if analyzer . last_analyzed_datetime : <EOL> save_point . set_last_datetime ( analyzer . last_analyzed_datetime ) <EOL> save_point . save ( ) <EOL> def load_subcommand ( subparsers ) : <EOL> """<STR_LIT>""" <EOL> parser_start = subparsers . add_parser ( '<STR_LIT:start>' , help = '<STR_LIT>' ) <EOL> parser_start . add_argument ( '<STR_LIT:-c>' , '<STR_LIT>' , type = argparse . FileType ( '<STR_LIT:r>' ) , dest = '<STR_LIT>' , <EOL> help = '<STR_LIT>' , required = True ) <EOL> parser_start . set_defaults ( func = start ) </s>
