<s> import sys <EOL> import os <EOL> import time <EOL> import signal <EOL> import errno <EOL> import multiprocessing <EOL> sys . path . insert ( <NUM_LIT:0> , os . path . abspath ( os . path . join ( os . path . dirname ( sys . argv [ <NUM_LIT:0> ] ) , "<STR_LIT:..>" ) ) ) <EOL> import configuration <EOL> import background . utils <EOL> import mailutils <EOL> def getRSS ( pid ) : <EOL> for line in open ( "<STR_LIT>" % pid ) : <EOL> words = line . split ( ) <EOL> if words [ <NUM_LIT:0> ] == "<STR_LIT>" : <EOL> if words [ <NUM_LIT:2> ] . lower ( ) == "<STR_LIT>" : unit = <NUM_LIT> <EOL> elif words [ <NUM_LIT:2> ] . lower ( ) == "<STR_LIT>" : unit = <NUM_LIT> ** <NUM_LIT:2> <EOL> elif words [ <NUM_LIT:2> ] . lower ( ) == "<STR_LIT>" : unit = <NUM_LIT> ** <NUM_LIT:3> <EOL> else : raise Exception ( "<STR_LIT>" % words [ <NUM_LIT:2> ] ) <EOL> return int ( words [ <NUM_LIT:1> ] ) * unit <EOL> else : raise Exception ( "<STR_LIT>" ) <EOL> class Watchdog ( background . utils . BackgroundProcess ) : <EOL> def __init__ ( self ) : <EOL> service = configuration . services . WATCHDOG <EOL> super ( Watchdog , self ) . __init__ ( service = service ) <EOL> cpu_count = multiprocessing . cpu_count ( ) <EOL> self . load1_limit = service . get ( "<STR_LIT>" , <NUM_LIT:0> ) * cpu_count <EOL> self . load5_limit = service . get ( "<STR_LIT>" , <NUM_LIT:0> ) * cpu_count <EOL> self . load15_limit = service . get ( "<STR_LIT>" , <NUM_LIT:0> ) * cpu_count <EOL> def run ( self ) : <EOL> soft_restart_attempted = set ( ) <EOL> previous = { } <EOL> getloadavg_failed = False <EOL> load1_has_warned = <NUM_LIT:0> <EOL> load1_last_time = <NUM_LIT:0> <EOL> load5_has_warned = <NUM_LIT:0> <EOL> load5_last_time = <NUM_LIT:0> <EOL> load15_has_warned = <NUM_LIT:0> <EOL> load15_last_time = <NUM_LIT:0> <EOL> while not self . terminated : <EOL> self . interrupted = False <EOL> def sendLoadAverageWarning ( interval , limit , load ) : <EOL> cpu_count = multiprocessing . cpu_count ( ) <EOL> mailutils . sendAdministratorMessage ( <EOL> "<STR_LIT>" , "<STR_LIT>" % interval , <EOL> ( "<STR_LIT>" <EOL> % ( interval , load ) ) + <EOL> ( "<STR_LIT>" <EOL> % ( limit , limit / cpu_count , cpu_count ) ) + <EOL> "<STR_LIT>" ) <EOL> try : <EOL> load1 , load5 , load15 = os . getloadavg ( ) <EOL> self . debug ( "<STR_LIT>" % ( load1 , load5 , load15 ) ) <EOL> except OSError : <EOL> load1 , load5 , load15 = <NUM_LIT:0> , <NUM_LIT:0> , <NUM_LIT:0> <EOL> if not getloadavg_failed : <EOL> self . exception ( "<STR_LIT>" ) <EOL> getloadavg_failed = True <EOL> now = time . time ( ) <EOL> if self . load1_limit and load1 > self . load1_limit : <EOL> if load1 > load1_has_warned * <NUM_LIT> or now - load1_last_time > <NUM_LIT> : <EOL> sendLoadAverageWarning ( <NUM_LIT:1> , self . load1_limit , load1 ) <EOL> load1_has_warned = load1 <EOL> load1_last_time = now <EOL> else : <EOL> load1_has_warned = <NUM_LIT:0> <EOL> load1_last_time = <NUM_LIT:0> <EOL> if self . load5_limit and load5 > self . load5_limit : <EOL> if load5 > load5_has_warned * <NUM_LIT> or now - load5_last_time > <NUM_LIT:5> * <NUM_LIT> : <EOL> sendLoadAverageWarning ( <NUM_LIT:5> , self . load5_limit , load5 ) <EOL> load5_has_warned = load5 <EOL> load5_last_time = now <EOL> else : <EOL> load5_has_warned = <NUM_LIT:0> <EOL> load5_last_time = <NUM_LIT:0> <EOL> if self . load15_limit and load15 > self . load15_limit : <EOL> if load15 > load15_has_warned * <NUM_LIT> or now - load15_last_time > <NUM_LIT:15> * <NUM_LIT> : <EOL> sendLoadAverageWarning ( <NUM_LIT:15> , self . load15_limit , load15 ) <EOL> load15_has_warned = load15 <EOL> load15_last_time = now <EOL> else : <EOL> load15_has_warned = <NUM_LIT:0> <EOL> load15_last_time = <NUM_LIT:0> <EOL> pidfile_dir = configuration . paths . WSGI_PIDFILE_DIR <EOL> if os . path . isdir ( pidfile_dir ) : <EOL> pids = set ( map ( int , os . listdir ( pidfile_dir ) ) ) <EOL> else : <EOL> pids = set ( ) <EOL> for pid in pids : <EOL> try : rss = getRSS ( pid ) <EOL> except IOError as error : <EOL> if error . errno == errno . ENOENT : <EOL> self . warning ( "<STR_LIT>" % os . path . join ( pidfile_dir , str ( pid ) ) ) <EOL> os . unlink ( os . path . join ( pidfile_dir , str ( pid ) ) ) <EOL> continue <EOL> else : raise <EOL> if previous . get ( pid ) != rss : <EOL> self . debug ( "<STR_LIT>" % ( pid , rss ) ) <EOL> previous [ pid ] = rss <EOL> if rss > configuration . services . WATCHDOG [ "<STR_LIT>" ] : <EOL> mailutils . sendAdministratorMessage ( <EOL> "<STR_LIT>" , "<STR_LIT>" % pid , <EOL> ( "<STR_LIT>" <EOL> % ( rss , signal . SIGKILL ) ) ) <EOL> self . info ( "<STR_LIT>" % ( pid , rss ) ) <EOL> os . kill ( pid , signal . SIGKILL ) <EOL> elif rss > configuration . services . WATCHDOG [ "<STR_LIT>" ] and pid not in soft_restart_attempted : <EOL> mailutils . sendAdministratorMessage ( <EOL> "<STR_LIT>" , "<STR_LIT>" % pid , <EOL> ( "<STR_LIT>" <EOL> % ( rss , signal . SIGINT ) ) ) <EOL> self . info ( "<STR_LIT>" % ( pid , rss ) ) <EOL> os . kill ( pid , signal . SIGINT ) <EOL> soft_restart_attempted . add ( pid ) <EOL> for pid in previous . keys ( ) : <EOL> if pid not in pids : del previous [ pid ] <EOL> soft_restart_attempted = soft_restart_attempted & pids <EOL> self . signal_idle_state ( ) <EOL> time . sleep ( <NUM_LIT:10> ) <EOL> def start_service ( ) : <EOL> watchdog = Watchdog ( ) <EOL> return watchdog . start ( ) <EOL> background . utils . call ( "<STR_LIT>" , start_service ) </s>
