<s> """<STR_LIT>""" <EOL> import hashlib <EOL> import os <EOL> import pdb <EOL> import posixpath <EOL> import Queue <EOL> import sys <EOL> import threading <EOL> import time <EOL> import traceback <EOL> import urllib2 <EOL> from M2Crypto import BIO <EOL> from M2Crypto import EVP <EOL> from M2Crypto import RSA <EOL> from M2Crypto import X509 <EOL> import psutil <EOL> import logging <EOL> from grr . client import actions <EOL> from grr . client import client_stats <EOL> from grr . client import client_utils <EOL> from grr . lib import communicator <EOL> from grr . lib import config_lib <EOL> from grr . lib import flags <EOL> from grr . lib import queues <EOL> from grr . lib import rdfvalue <EOL> from grr . lib import registry <EOL> from grr . lib import stats <EOL> from grr . lib import type_info <EOL> from grr . lib import utils <EOL> from grr . lib . rdfvalues import client as rdf_client <EOL> from grr . lib . rdfvalues import crypto as rdf_crypto <EOL> from grr . lib . rdfvalues import flows as rdf_flows <EOL> from grr . lib . rdfvalues import protodict as rdf_protodict <EOL> class HTTPObject ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , url = "<STR_LIT>" , data = "<STR_LIT>" , proxy = "<STR_LIT>" , code = <NUM_LIT> , duration = <NUM_LIT:0> ) : <EOL> self . url = url <EOL> self . data = data <EOL> self . proxy = proxy <EOL> self . code = code <EOL> self . messages = self . source = self . nonce = None <EOL> self . duration = duration <EOL> def Success ( self ) : <EOL> """<STR_LIT>""" <EOL> return self . code in ( <NUM_LIT:200> , <NUM_LIT> ) <EOL> class HTTPManager ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , heart_beat_cb = None ) : <EOL> self . heart_beat_cb = heart_beat_cb <EOL> self . proxies = self . _GetProxies ( ) <EOL> self . base_urls = self . _GetBaseURLs ( ) <EOL> self . last_proxy_index = <NUM_LIT:0> <EOL> self . last_base_url_index = <NUM_LIT:0> <EOL> self . consecutive_connection_errors = <NUM_LIT:0> <EOL> self . retry_error_limit = config_lib . CONFIG [ "<STR_LIT>" ] <EOL> self . active_base_url = None <EOL> self . error_poll_min = config_lib . CONFIG [ "<STR_LIT>" ] <EOL> def _GetBaseURLs ( self ) : <EOL> """<STR_LIT>""" <EOL> result = config_lib . CONFIG [ "<STR_LIT>" ] <EOL> if not result : <EOL> for control_url in config_lib . CONFIG [ "<STR_LIT>" ] : <EOL> result . append ( posixpath . dirname ( control_url ) + "<STR_LIT:/>" ) <EOL> for url in result : <EOL> if not url . endswith ( "<STR_LIT:/>" ) : <EOL> raise RuntimeError ( "<STR_LIT>" ) <EOL> return result <EOL> def _GetProxies ( self ) : <EOL> """<STR_LIT>""" <EOL> result = client_utils . FindProxies ( ) <EOL> result . append ( "<STR_LIT>" ) <EOL> result . extend ( config_lib . CONFIG [ "<STR_LIT>" ] ) <EOL> return result <EOL> def _ConcatenateURL ( self , base , url ) : <EOL> if not url . startswith ( "<STR_LIT:/>" ) : <EOL> url = "<STR_LIT:/>" + url <EOL> if base . endswith ( "<STR_LIT:/>" ) : <EOL> base = base [ : - <NUM_LIT:1> ] <EOL> return base + url <EOL> def OpenServerEndpoint ( self , path , verify_cb = lambda x : True , data = None , <EOL> request_opts = None ) : <EOL> """<STR_LIT>""" <EOL> tries = <NUM_LIT:0> <EOL> last_error = HTTPObject ( code = <NUM_LIT> ) <EOL> while tries < len ( self . base_urls ) : <EOL> base_url_index = self . last_base_url_index % len ( self . base_urls ) <EOL> active_base_url = self . base_urls [ base_url_index ] <EOL> result = self . OpenURL ( self . _ConcatenateURL ( active_base_url , path ) , <EOL> data = data , verify_cb = verify_cb , <EOL> request_opts = request_opts ) <EOL> if not result . Success ( ) : <EOL> tries += <NUM_LIT:1> <EOL> self . last_base_url_index += <NUM_LIT:1> <EOL> last_error = result <EOL> continue <EOL> self . active_base_url = active_base_url <EOL> return result <EOL> logging . info ( "<STR_LIT>" <EOL> "<STR_LIT>" , self . base_urls , self . proxies ) <EOL> return last_error <EOL> def OpenURL ( self , url , verify_cb = lambda x : True , data = None , <EOL> request_opts = None ) : <EOL> """<STR_LIT>""" <EOL> tries = <NUM_LIT:0> <EOL> last_error = <NUM_LIT> <EOL> while tries < len ( self . proxies ) : <EOL> proxy_index = self . last_proxy_index % len ( self . proxies ) <EOL> proxy = self . proxies [ proxy_index ] <EOL> try : <EOL> proxydict = { } <EOL> if proxy : <EOL> proxydict [ "<STR_LIT:http>" ] = proxy <EOL> proxydict [ "<STR_LIT>" ] = proxy <EOL> proxy_support = urllib2 . ProxyHandler ( proxydict ) <EOL> opener = urllib2 . build_opener ( proxy_support ) <EOL> urllib2 . install_opener ( opener ) <EOL> request_opts = ( request_opts or { } ) . copy ( ) <EOL> request_opts [ "<STR_LIT>" ] = "<STR_LIT>" <EOL> request = urllib2 . Request ( url , data , request_opts ) <EOL> duration , handle = self . _RetryRequest ( request ) <EOL> data = handle . read ( ) <EOL> result = HTTPObject ( url = url , data = data , proxy = proxy , code = <NUM_LIT:200> , <EOL> duration = duration ) <EOL> if not verify_cb ( result ) : <EOL> raise urllib2 . HTTPError ( url = url , code = <NUM_LIT> , msg = "<STR_LIT>" , <EOL> hdrs = [ ] , fp = handle ) <EOL> self . consecutive_connection_errors = <NUM_LIT:0> <EOL> return result <EOL> except urllib2 . HTTPError as e : <EOL> if e . code == <NUM_LIT> : <EOL> self . consecutive_connection_errors = <NUM_LIT:0> <EOL> return HTTPObject ( code = <NUM_LIT> ) <EOL> self . last_proxy_index = proxy_index + <NUM_LIT:1> <EOL> tries += <NUM_LIT:1> <EOL> last_error = e . code <EOL> except urllib2 . URLError as e : <EOL> self . last_proxy_index = proxy_index + <NUM_LIT:1> <EOL> tries += <NUM_LIT:1> <EOL> last_error = <NUM_LIT> <EOL> return HTTPObject ( code = last_error ) <EOL> def _RetryRequest ( self , request ) : <EOL> """<STR_LIT>""" <EOL> while True : <EOL> try : <EOL> now = time . time ( ) <EOL> result = urllib2 . urlopen ( request , timeout = config_lib . CONFIG [ <EOL> "<STR_LIT>" ] ) <EOL> return time . time ( ) - now , result <EOL> except ( urllib2 . HTTPError , urllib2 . URLError ) as e : <EOL> self . consecutive_connection_errors += <NUM_LIT:1> <EOL> if self . active_base_url is not None : <EOL> if getattr ( e , "<STR_LIT:code>" , None ) == <NUM_LIT> : <EOL> raise <EOL> if self . consecutive_connection_errors >= self . retry_error_limit : <EOL> logging . info ( <EOL> "<STR_LIT>" , <EOL> self . active_base_url ) <EOL> self . active_base_url = None <EOL> raise e <EOL> logging . debug ( <EOL> "<STR_LIT>" , <EOL> self . error_poll_min ) <EOL> self . Wait ( self . error_poll_min ) <EOL> else : <EOL> raise e <EOL> def Wait ( self , timeout ) : <EOL> """<STR_LIT>""" <EOL> time . sleep ( timeout - int ( timeout ) ) <EOL> for _ in xrange ( int ( timeout ) ) : <EOL> time . sleep ( <NUM_LIT:1> ) <EOL> if self . heart_beat_cb : <EOL> self . heart_beat_cb ( ) <EOL> class Timer ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , heart_beat_cb = None ) : <EOL> self . heart_beat_cb = heart_beat_cb <EOL> self . poll_min = config_lib . CONFIG [ "<STR_LIT>" ] <EOL> self . sleep_time = self . poll_max = config_lib . CONFIG [ "<STR_LIT>" ] <EOL> self . poll_slew = config_lib . CONFIG [ "<STR_LIT>" ] <EOL> def FastPoll ( self ) : <EOL> """<STR_LIT>""" <EOL> self . sleep_time = self . poll_min <EOL> def SlowPoll ( self ) : <EOL> """<STR_LIT>""" <EOL> self . sleep_time = self . poll_max <EOL> def Wait ( self ) : <EOL> """<STR_LIT>""" <EOL> time . sleep ( self . sleep_time - int ( self . sleep_time ) ) <EOL> for _ in xrange ( int ( self . sleep_time ) ) : <EOL> time . sleep ( <NUM_LIT:1> ) <EOL> if self . heart_beat_cb : <EOL> self . heart_beat_cb ( ) <EOL> self . sleep_time = min ( <EOL> self . poll_max , <EOL> max ( self . poll_min , self . sleep_time ) * self . poll_slew ) <EOL> class CommsInit ( registry . InitHook ) : <EOL> def RunOnce ( self ) : <EOL> stats . STATS . RegisterGaugeMetric ( "<STR_LIT>" , long ) <EOL> stats . STATS . RegisterCounterMetric ( "<STR_LIT>" ) <EOL> stats . STATS . RegisterCounterMetric ( "<STR_LIT>" ) <EOL> stats . STATS . RegisterCounterMetric ( "<STR_LIT>" ) <EOL> stats . STATS . RegisterCounterMetric ( "<STR_LIT>" ) <EOL> stats . STATS . RegisterCounterMetric ( "<STR_LIT>" ) <EOL> class Status ( object ) : <EOL> """<STR_LIT>""" <EOL> received_count = <NUM_LIT:0> <EOL> sent_count = <NUM_LIT:0> <EOL> sent_len = <NUM_LIT:0> <EOL> sent = { } <EOL> require_fastpoll = False <EOL> code = <NUM_LIT:200> <EOL> def __init__ ( self , ** kwargs ) : <EOL> self . __dict__ . update ( kwargs ) <EOL> class GRRClientWorker ( object ) : <EOL> """<STR_LIT>""" <EOL> stats_collector = None <EOL> IDLE_THRESHOLD = <NUM_LIT> <EOL> sent_bytes_per_flow = { } <EOL> STATS_MAX_SEND_INTERVAL = rdfvalue . Duration ( "<STR_LIT>" ) <EOL> STATS_MIN_SEND_INTERVAL = rdfvalue . Duration ( "<STR_LIT>" ) <EOL> def __init__ ( self ) : <EOL> """<STR_LIT>""" <EOL> super ( GRRClientWorker , self ) . __init__ ( ) <EOL> self . _in_queue = [ ] <EOL> self . _out_queue = [ ] <EOL> self . _out_queue_size = <NUM_LIT:0> <EOL> self . _is_active = False <EOL> self . _send_stats_on_check = False <EOL> self . last_stats_sent_time = None <EOL> self . proc = psutil . Process ( os . getpid ( ) ) <EOL> self . suspended_actions = { } <EOL> self . nanny_controller = client_utils . NannyController ( ) <EOL> self . nanny_controller . StartNanny ( ) <EOL> if not GRRClientWorker . stats_collector : <EOL> GRRClientWorker . stats_collector = client_stats . ClientStatsCollector ( self ) <EOL> GRRClientWorker . stats_collector . start ( ) <EOL> self . lock = threading . RLock ( ) <EOL> self . http_manager = HTTPManager ( <EOL> heart_beat_cb = self . nanny_controller . Heartbeat ) <EOL> def Sleep ( self , timeout ) : <EOL> """<STR_LIT>""" <EOL> self . nanny_controller . Heartbeat ( ) <EOL> time . sleep ( timeout - int ( timeout ) ) <EOL> for _ in xrange ( int ( timeout ) ) : <EOL> time . sleep ( <NUM_LIT:1> ) <EOL> self . nanny_controller . Heartbeat ( ) <EOL> def ClientMachineIsIdle ( self ) : <EOL> return psutil . cpu_percent ( <NUM_LIT> ) <= <NUM_LIT:100> * self . IDLE_THRESHOLD <EOL> def __del__ ( self ) : <EOL> self . nanny_controller . StopNanny ( ) <EOL> def Drain ( self , max_size = <NUM_LIT> ) : <EOL> """<STR_LIT>""" <EOL> queue = rdf_flows . MessageList ( ) <EOL> length = <NUM_LIT:0> <EOL> self . _out_queue . sort ( key = lambda msg : msg [ <NUM_LIT:0> ] ) <EOL> self . _out_queue . reverse ( ) <EOL> while self . _out_queue and length < max_size : <EOL> message = self . _out_queue . pop ( ) [ <NUM_LIT:1> ] <EOL> queue . job . Append ( message ) <EOL> stats . STATS . IncrementCounter ( "<STR_LIT>" ) <EOL> message_length = len ( message . Get ( "<STR_LIT:args>" ) ) <EOL> length += message_length <EOL> self . _out_queue_size -= message_length <EOL> self . _out_queue . reverse ( ) <EOL> return queue <EOL> def SendReply ( self , rdf_value = None , request_id = None , response_id = None , <EOL> priority = None , session_id = "<STR_LIT>" , message_type = None , name = None , <EOL> require_fastpoll = None , ttl = None , blocking = True , task_id = None ) : <EOL> """<STR_LIT>""" <EOL> if not isinstance ( rdf_value , rdfvalue . RDFValue ) : <EOL> raise RuntimeError ( "<STR_LIT>" ) <EOL> message = rdf_flows . GrrMessage ( <EOL> session_id = session_id , task_id = task_id , name = name , <EOL> response_id = response_id , request_id = request_id , <EOL> priority = priority , require_fastpoll = require_fastpoll , <EOL> ttl = ttl , type = message_type ) <EOL> if rdf_value : <EOL> message . payload = rdf_value <EOL> serialized_message = message . SerializeToString ( ) <EOL> self . ChargeBytesToSession ( session_id , len ( serialized_message ) ) <EOL> if message . type == rdf_flows . GrrMessage . Type . STATUS : <EOL> rdf_value . network_bytes_sent = self . sent_bytes_per_flow [ session_id ] <EOL> del self . sent_bytes_per_flow [ session_id ] <EOL> message . payload = rdf_value <EOL> try : <EOL> self . QueueResponse ( message , priority = message . priority , blocking = blocking ) <EOL> except Queue . Full : <EOL> if not blocking : <EOL> raise <EOL> logging . info ( "<STR_LIT>" ) <EOL> @ utils . Synchronized <EOL> def ChargeBytesToSession ( self , session_id , length , limit = <NUM_LIT:0> ) : <EOL> self . sent_bytes_per_flow . setdefault ( session_id , <NUM_LIT:0> ) <EOL> self . sent_bytes_per_flow [ session_id ] += length <EOL> if limit and ( self . sent_bytes_per_flow [ session_id ] > limit ) : <EOL> self . SendClientAlert ( "<STR_LIT>" ) <EOL> raise actions . NetworkBytesExceededError ( <EOL> "<STR_LIT>" ) <EOL> def QueueResponse ( self , message , <EOL> priority = rdf_flows . GrrMessage . Priority . MEDIUM_PRIORITY , <EOL> blocking = True ) : <EOL> """<STR_LIT>""" <EOL> _ = blocking <EOL> self . _out_queue . append ( ( - <NUM_LIT:1> * priority , message ) ) <EOL> self . _out_queue_size += len ( message . Get ( "<STR_LIT:args>" ) ) <EOL> def HandleMessage ( self , message ) : <EOL> """<STR_LIT>""" <EOL> self . _is_active = True <EOL> try : <EOL> try : <EOL> suspended_action_id = message . payload . iterator . suspended_action <EOL> action = self . suspended_actions [ suspended_action_id ] <EOL> except ( AttributeError , KeyError ) : <EOL> action_cls = actions . ActionPlugin . classes . get ( message . name ) <EOL> if action_cls is None : <EOL> raise RuntimeError ( "<STR_LIT>" % message . name ) <EOL> action = action_cls ( grr_worker = self ) <EOL> self . nanny_controller . WriteTransactionLog ( message ) <EOL> action . Progress ( ) <EOL> action . Execute ( message ) <EOL> self . nanny_controller . CleanTransactionLog ( ) <EOL> finally : <EOL> self . _is_active = False <EOL> self . _send_stats_on_check = True <EOL> def QueueMessages ( self , messages ) : <EOL> """<STR_LIT>""" <EOL> for message in messages : <EOL> self . _in_queue . append ( message ) <EOL> stats . STATS . IncrementCounter ( "<STR_LIT>" ) <EOL> while self . _in_queue and ( <EOL> self . _out_queue_size < config_lib . CONFIG [ "<STR_LIT>" ] ) : <EOL> message = self . _in_queue . pop ( <NUM_LIT:0> ) <EOL> try : <EOL> self . HandleMessage ( message ) <EOL> except Exception as e : <EOL> self . SendReply ( <EOL> rdf_flows . GrrStatus ( <EOL> status = rdf_flows . GrrStatus . ReturnedStatus . GENERIC_ERROR , <EOL> error_message = utils . SmartUnicode ( e ) ) , <EOL> request_id = message . request_id , <EOL> response_id = message . response_id , <EOL> session_id = message . session_id , <EOL> task_id = message . task_id , <EOL> message_type = rdf_flows . GrrMessage . Type . STATUS ) <EOL> if flags . FLAGS . debug : <EOL> pdb . post_mortem ( ) <EOL> def MemoryExceeded ( self ) : <EOL> """<STR_LIT>""" <EOL> rss_size = self . proc . memory_info ( ) . rss <EOL> return rss_size / <NUM_LIT> / <NUM_LIT> > config_lib . CONFIG [ "<STR_LIT>" ] <EOL> def InQueueSize ( self ) : <EOL> """<STR_LIT>""" <EOL> return len ( self . _in_queue ) <EOL> def OutQueueSize ( self ) : <EOL> """<STR_LIT>""" <EOL> return len ( self . _out_queue ) <EOL> def IsActive ( self ) : <EOL> """<STR_LIT>""" <EOL> return self . _is_active <EOL> def CheckStats ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . last_stats_sent_time is None : <EOL> self . last_stats_sent_time = rdfvalue . RDFDatetime ( ) . Now ( ) <EOL> stats . STATS . SetGaugeValue ( "<STR_LIT>" , <EOL> self . last_stats_sent_time . AsSecondsFromEpoch ( ) ) <EOL> time_since_last_check = ( rdfvalue . RDFDatetime ( ) . Now ( ) - <EOL> self . last_stats_sent_time ) <EOL> if time_since_last_check < self . STATS_MIN_SEND_INTERVAL : <EOL> return <EOL> if ( time_since_last_check > self . STATS_MAX_SEND_INTERVAL or <EOL> self . _is_active or self . _send_stats_on_check ) : <EOL> self . _send_stats_on_check = False <EOL> logging . info ( "<STR_LIT>" ) <EOL> action_cls = actions . ActionPlugin . classes . get ( <EOL> "<STR_LIT>" , actions . ActionPlugin ) <EOL> action = action_cls ( grr_worker = self ) <EOL> action . Run ( rdf_client . GetClientStatsRequest ( <EOL> start_time = self . last_stats_sent_time ) ) <EOL> self . last_stats_sent_time = rdfvalue . RDFDatetime ( ) . Now ( ) <EOL> stats . STATS . SetGaugeValue ( "<STR_LIT>" , <EOL> self . last_stats_sent_time . AsSecondsFromEpoch ( ) ) <EOL> def SendNannyMessage ( self ) : <EOL> msg = self . nanny_controller . GetNannyMessage ( ) <EOL> if msg : <EOL> self . SendReply ( <EOL> rdf_protodict . DataBlob ( string = msg ) , <EOL> session_id = rdfvalue . FlowSessionID ( flow_name = "<STR_LIT>" ) , <EOL> priority = rdf_flows . GrrMessage . Priority . LOW_PRIORITY , <EOL> require_fastpoll = False ) <EOL> self . nanny_controller . ClearNannyMessage ( ) <EOL> def SendClientAlert ( self , msg ) : <EOL> self . SendReply ( <EOL> rdf_protodict . DataBlob ( string = msg ) , <EOL> session_id = rdfvalue . FlowSessionID ( flow_name = "<STR_LIT>" ) , <EOL> priority = rdf_flows . GrrMessage . Priority . LOW_PRIORITY , <EOL> require_fastpoll = False ) <EOL> class SizeQueue ( object ) : <EOL> """<STR_LIT>""" <EOL> total_size = <NUM_LIT:0> <EOL> def __init__ ( self , maxsize = <NUM_LIT> , nanny = None ) : <EOL> self . lock = threading . RLock ( ) <EOL> self . queue = [ ] <EOL> self . _reversed = [ ] <EOL> self . total_size = <NUM_LIT:0> <EOL> self . maxsize = maxsize <EOL> self . nanny = nanny <EOL> def Put ( self , item , priority = rdf_flows . GrrMessage . Priority . MEDIUM_PRIORITY , <EOL> block = True , timeout = <NUM_LIT:1000> ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( item , rdfvalue . RDFValue ) : <EOL> item = item . SerializeToString ( ) <EOL> if priority >= rdf_flows . GrrMessage . Priority . HIGH_PRIORITY : <EOL> pass <EOL> elif not block : <EOL> if self . total_size >= self . maxsize : <EOL> raise Queue . Full <EOL> else : <EOL> count = <NUM_LIT:0> <EOL> while self . total_size >= self . maxsize : <EOL> time . sleep ( <NUM_LIT:1> ) <EOL> self . nanny . Heartbeat ( ) <EOL> count += <NUM_LIT:1> <EOL> if timeout and count > timeout : <EOL> raise Queue . Full <EOL> with self . lock : <EOL> self . queue . append ( ( - <NUM_LIT:1> * priority , item ) ) <EOL> self . total_size += len ( item ) <EOL> def Get ( self ) : <EOL> """<STR_LIT>""" <EOL> with self . lock : <EOL> if self . _reversed : <EOL> self . _reversed . reverse ( ) <EOL> self . queue = self . _reversed + self . queue <EOL> self . queue . sort ( key = lambda msg : msg [ <NUM_LIT:0> ] ) <EOL> self . queue . reverse ( ) <EOL> self . _reversed , self . queue = self . queue , [ ] <EOL> while self . _reversed : <EOL> item = self . _reversed . pop ( ) [ <NUM_LIT:1> ] <EOL> self . total_size -= len ( item ) <EOL> yield item <EOL> def Size ( self ) : <EOL> return self . total_size <EOL> def Full ( self ) : <EOL> return self . total_size >= self . maxsize <EOL> class GRRThreadedWorker ( GRRClientWorker , threading . Thread ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self ) : <EOL> super ( GRRThreadedWorker , self ) . __init__ ( ) <EOL> self . _in_queue = utils . HeartbeatQueue ( <EOL> callback = self . nanny_controller . Heartbeat , maxsize = <NUM_LIT> ) <EOL> self . _out_queue = SizeQueue ( <EOL> maxsize = config_lib . CONFIG [ "<STR_LIT>" ] , <EOL> nanny = self . nanny_controller ) <EOL> self . daemon = True <EOL> self . start ( ) <EOL> def Sleep ( self , timeout ) : <EOL> """<STR_LIT>""" <EOL> self . nanny_controller . Heartbeat ( ) <EOL> time . sleep ( timeout - int ( timeout ) ) <EOL> for _ in xrange ( int ( timeout ) ) : <EOL> time . sleep ( <NUM_LIT:1> ) <EOL> if self . _out_queue . Full ( ) : <EOL> return <EOL> self . nanny_controller . Heartbeat ( ) <EOL> def Drain ( self , max_size = <NUM_LIT> ) : <EOL> """<STR_LIT>""" <EOL> queue = rdf_flows . MessageList ( ) <EOL> length = <NUM_LIT:0> <EOL> for message in self . _out_queue . Get ( ) : <EOL> queue . job . Append ( rdf_flows . GrrMessage ( message ) ) <EOL> stats . STATS . IncrementCounter ( "<STR_LIT>" ) <EOL> length += len ( message ) <EOL> if length > max_size : <EOL> break <EOL> return queue <EOL> def QueueResponse ( self , message , <EOL> priority = rdf_flows . GrrMessage . Priority . MEDIUM_PRIORITY , <EOL> blocking = True ) : <EOL> """<STR_LIT>""" <EOL> self . _out_queue . Put ( message , priority = priority , block = blocking ) <EOL> def QueueMessages ( self , messages ) : <EOL> """<STR_LIT>""" <EOL> for message in messages : <EOL> self . _in_queue . put ( message , block = True ) <EOL> stats . STATS . IncrementCounter ( "<STR_LIT>" ) <EOL> def InQueueSize ( self ) : <EOL> """<STR_LIT>""" <EOL> return self . _in_queue . qsize ( ) <EOL> def OutQueueSize ( self ) : <EOL> """<STR_LIT>""" <EOL> return self . _out_queue . Size ( ) <EOL> def __del__ ( self ) : <EOL> self . _in_queue . put ( None , block = True ) <EOL> self . nanny_controller . StopNanny ( ) <EOL> def OnStartup ( self ) : <EOL> """<STR_LIT>""" <EOL> last_request = self . nanny_controller . GetTransactionLog ( ) <EOL> if last_request : <EOL> status = rdf_flows . GrrStatus ( <EOL> status = rdf_flows . GrrStatus . ReturnedStatus . CLIENT_KILLED , <EOL> error_message = "<STR_LIT>" ) <EOL> nanny_status = self . nanny_controller . GetNannyStatus ( ) <EOL> if nanny_status : <EOL> status . nanny_status = nanny_status <EOL> self . SendReply ( status , <EOL> request_id = last_request . request_id , <EOL> response_id = <NUM_LIT:1> , <EOL> session_id = last_request . session_id , <EOL> message_type = rdf_flows . GrrMessage . Type . STATUS ) <EOL> self . nanny_controller . CleanTransactionLog ( ) <EOL> action_cls = actions . ActionPlugin . classes . get ( <EOL> "<STR_LIT>" , actions . ActionPlugin ) <EOL> action = action_cls ( grr_worker = self ) <EOL> action . Run ( None , ttl = <NUM_LIT:1> ) <EOL> def run ( self ) : <EOL> """<STR_LIT>""" <EOL> self . OnStartup ( ) <EOL> while True : <EOL> message = self . _in_queue . get ( ) <EOL> if message is None : <EOL> break <EOL> try : <EOL> self . HandleMessage ( message ) <EOL> except Exception as e : <EOL> logging . warn ( "<STR_LIT:%s>" , e ) <EOL> self . SendReply ( <EOL> rdf_flows . GrrStatus ( <EOL> status = rdf_flows . GrrStatus . ReturnedStatus . GENERIC_ERROR , <EOL> error_message = utils . SmartUnicode ( e ) ) , <EOL> request_id = message . request_id , <EOL> response_id = <NUM_LIT:1> , <EOL> session_id = message . session_id , <EOL> task_id = message . task_id , <EOL> message_type = rdf_flows . GrrMessage . Type . STATUS ) <EOL> if flags . FLAGS . debug : <EOL> pdb . post_mortem ( ) <EOL> class GRRHTTPClient ( object ) : <EOL> """<STR_LIT>""" <EOL> http_manager_class = HTTPManager <EOL> def __init__ ( self , ca_cert = None , worker = None , private_key = None ) : <EOL> """<STR_LIT>""" <EOL> self . ca_cert = ca_cert <EOL> if private_key is None : <EOL> private_key = config_lib . CONFIG . Get ( "<STR_LIT>" , default = None ) <EOL> self . server_certificate = None <EOL> self . http_manager = self . http_manager_class ( ) <EOL> self . communicator = ClientCommunicator ( private_key = private_key ) <EOL> self . timer = Timer ( ) <EOL> self . last_enrollment_time = <NUM_LIT:0> <EOL> self . last_foreman_check = <NUM_LIT:0> <EOL> if worker : <EOL> self . client_worker = worker ( ) <EOL> else : <EOL> self . client_worker = GRRThreadedWorker ( ) <EOL> def VerifyServerPEM ( self , http_object ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> server_pem = http_object . data <EOL> server_url = http_object . url <EOL> if "<STR_LIT>" in server_pem : <EOL> self . communicator . LoadServerCertificate ( <EOL> server_certificate = server_pem , ca_certificate = self . ca_cert ) <EOL> logging . info ( "<STR_LIT>" ) <EOL> return True <EOL> except Exception as e : <EOL> logging . info ( "<STR_LIT>" , <EOL> server_url , e ) <EOL> return False <EOL> def VerifyServerControlResponse ( self , http_object ) : <EOL> """<STR_LIT>""" <EOL> if http_object . code != <NUM_LIT:200> : <EOL> return False <EOL> try : <EOL> http_object . messages , http_object . source , http_object . nonce = ( <EOL> self . communicator . DecryptMessage ( http_object . data ) ) <EOL> return True <EOL> except communicator . DecodingError as e : <EOL> logging . info ( "<STR_LIT>" , e ) <EOL> return False <EOL> def MakeRequest ( self , data ) : <EOL> """<STR_LIT>""" <EOL> stats . STATS . IncrementCounter ( "<STR_LIT>" , len ( data ) ) <EOL> response = self . http_manager . OpenServerEndpoint ( <EOL> path = "<STR_LIT>" % config_lib . CONFIG [ "<STR_LIT>" ] , <EOL> verify_cb = self . VerifyServerControlResponse , <EOL> data = data , request_opts = { "<STR_LIT:Content-Type>" : "<STR_LIT>" } ) <EOL> if response . code == <NUM_LIT> : <EOL> self . InitiateEnrolment ( ) <EOL> return response <EOL> if response . code == <NUM_LIT:200> : <EOL> stats . STATS . IncrementCounter ( "<STR_LIT>" , <EOL> len ( response . data ) ) <EOL> return response <EOL> return response <EOL> def RunOnce ( self ) : <EOL> """<STR_LIT>""" <EOL> if not self . _FetchServerCertificate ( ) : <EOL> self . timer . Wait ( ) <EOL> return HTTPObject ( code = <NUM_LIT> ) <EOL> if self . http_manager . consecutive_connection_errors == <NUM_LIT:0> : <EOL> message_list = self . client_worker . Drain ( <EOL> max_size = config_lib . CONFIG [ "<STR_LIT>" ] ) <EOL> else : <EOL> message_list = rdf_flows . MessageList ( ) <EOL> for message in message_list . job : <EOL> if message . require_fastpoll : <EOL> self . timer . FastPoll ( ) <EOL> break <EOL> payload = rdf_flows . ClientCommunication ( ) <EOL> if self . client_worker . MemoryExceeded ( ) : <EOL> logging . info ( "<STR_LIT>" ) <EOL> payload . queue_size = <NUM_LIT> <EOL> else : <EOL> payload . queue_size = self . client_worker . InQueueSize ( ) <EOL> nonce = self . communicator . EncodeMessages ( message_list , payload ) <EOL> payload_data = payload . SerializeToString ( ) <EOL> response = self . MakeRequest ( payload_data ) <EOL> if response . code != <NUM_LIT:200> or response . messages is None : <EOL> logging . info ( "<STR_LIT>" , <EOL> self . communicator . common_name , <EOL> self . http_manager . active_base_url , <EOL> response . code ) <EOL> self . server_certificate = None <EOL> messages = list ( message_list . job ) <EOL> for message in messages : <EOL> message . priority = rdf_flows . GrrMessage . Priority . HIGH_PRIORITY <EOL> message . require_fastpoll = False <EOL> message . ttl -= <NUM_LIT:1> <EOL> if message . ttl > <NUM_LIT:0> : <EOL> self . client_worker . QueueResponse ( <EOL> message , rdf_flows . GrrMessage . Priority . HIGH_PRIORITY + <NUM_LIT:1> ) <EOL> else : <EOL> logging . info ( "<STR_LIT>" ) <EOL> return response <EOL> if response . nonce != nonce : <EOL> logging . info ( "<STR_LIT>" ) <EOL> response . code = <NUM_LIT> <EOL> return response <EOL> if response . source != self . communicator . server_name : <EOL> logging . info ( "<STR_LIT>" <EOL> "<STR_LIT>" , response . source , <EOL> self . communicator . server_name ) <EOL> response . code = <NUM_LIT> <EOL> return response <EOL> for message in response . messages : <EOL> if message . require_fastpoll : <EOL> self . timer . FastPoll ( ) <EOL> break <EOL> self . client_worker . QueueMessages ( response . messages ) <EOL> cn = self . communicator . common_name <EOL> logging . info ( "<STR_LIT>" <EOL> "<STR_LIT>" , <EOL> cn , len ( message_list ) , len ( payload_data ) , <EOL> len ( response . messages ) , <EOL> response . duration , self . timer . sleep_time ) <EOL> return response <EOL> def SendForemanRequest ( self ) : <EOL> self . client_worker . SendReply ( <EOL> rdf_protodict . DataBlob ( ) , <EOL> session_id = rdfvalue . FlowSessionID ( flow_name = "<STR_LIT>" ) , <EOL> priority = rdf_flows . GrrMessage . Priority . LOW_PRIORITY , <EOL> require_fastpoll = False ) <EOL> def _FetchServerCertificate ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . server_certificate : <EOL> return True <EOL> response = self . http_manager . OpenServerEndpoint ( <EOL> "<STR_LIT>" , verify_cb = self . VerifyServerPEM ) <EOL> if response . Success ( ) : <EOL> self . server_certificate = response . data <EOL> return True <EOL> self . timer . SlowPoll ( ) <EOL> return False <EOL> def Run ( self ) : <EOL> """<STR_LIT>""" <EOL> while True : <EOL> if self . http_manager . consecutive_connection_errors > config_lib . CONFIG [ <EOL> "<STR_LIT>" ] : <EOL> return <EOL> self . client_worker . SendNannyMessage ( ) <EOL> now = time . time ( ) <EOL> if ( now > self . last_foreman_check + <EOL> config_lib . CONFIG [ "<STR_LIT>" ] ) : <EOL> try : <EOL> self . client_worker . SendReply ( <EOL> rdf_protodict . DataBlob ( ) , <EOL> session_id = rdfvalue . FlowSessionID ( flow_name = "<STR_LIT>" ) , <EOL> priority = rdf_flows . GrrMessage . Priority . LOW_PRIORITY , <EOL> require_fastpoll = False , blocking = False ) <EOL> self . last_foreman_check = now <EOL> except Queue . Full : <EOL> pass <EOL> try : <EOL> self . RunOnce ( ) <EOL> except Exception : <EOL> logging . warn ( "<STR_LIT>" , traceback . format_exc ( ) ) <EOL> if flags . FLAGS . debug : <EOL> pdb . post_mortem ( ) <EOL> if ( self . client_worker . MemoryExceeded ( ) and <EOL> not self . client_worker . IsActive ( ) and <EOL> self . client_worker . InQueueSize ( ) == <NUM_LIT:0> and <EOL> self . client_worker . OutQueueSize ( ) == <NUM_LIT:0> ) : <EOL> logging . warning ( "<STR_LIT>" ) <EOL> self . client_worker . SendClientAlert ( "<STR_LIT>" ) <EOL> self . client_worker . MemoryExceeded = lambda : True <EOL> self . RunOnce ( ) <EOL> sys . exit ( - <NUM_LIT:1> ) <EOL> self . timer . Wait ( ) <EOL> def InitiateEnrolment ( self ) : <EOL> """<STR_LIT>""" <EOL> logging . debug ( "<STR_LIT>" ) <EOL> now = time . time ( ) <EOL> if now > self . last_enrollment_time + <NUM_LIT:10> * <NUM_LIT> : <EOL> if not self . last_enrollment_time : <EOL> self . timer . FastPoll ( ) <EOL> self . last_enrollment_time = now <EOL> self . client_worker . SendReply ( <EOL> rdf_crypto . Certificate ( type = rdf_crypto . Certificate . Type . CSR , <EOL> pem = self . communicator . GetCSR ( ) ) , <EOL> session_id = rdfvalue . SessionID ( <EOL> queue = queues . ENROLLMENT , flow_name = "<STR_LIT>" ) ) <EOL> class ClientCommunicator ( communicator . Communicator ) : <EOL> """<STR_LIT>""" <EOL> def _ParseRSAKey ( self , rsa ) : <EOL> """<STR_LIT>""" <EOL> public_key = rsa . pub ( ) [ <NUM_LIT:1> ] <EOL> self . common_name = rdf_client . ClientURN ( "<STR_LIT>" % ( <EOL> hashlib . sha256 ( public_key ) . digest ( ) [ : <NUM_LIT:8> ] . encode ( "<STR_LIT>" ) ) ) <EOL> def _LoadOurCertificate ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . private_key : <EOL> try : <EOL> self . private_key . Validate ( ) <EOL> rsa = self . private_key . GetPrivateKey ( ) <EOL> self . _ParseRSAKey ( rsa ) <EOL> logging . info ( "<STR_LIT>" , self . common_name ) <EOL> return rsa <EOL> except type_info . TypeValueError : <EOL> pass <EOL> rsa = RSA . gen_key ( config_lib . CONFIG [ "<STR_LIT>" ] , <EOL> <NUM_LIT> , lambda : None ) <EOL> self . _ParseRSAKey ( rsa ) <EOL> logging . info ( "<STR_LIT>" , self . common_name ) <EOL> pk = EVP . PKey ( ) <EOL> pk . assign_rsa ( rsa ) <EOL> self . SavePrivateKey ( pk ) <EOL> return rsa <EOL> def GetCSR ( self ) : <EOL> """<STR_LIT>""" <EOL> csr = X509 . Request ( ) <EOL> pk = EVP . PKey ( ) <EOL> rsa = self . _LoadOurCertificate ( ) <EOL> pk . assign_rsa ( rsa ) <EOL> csr . set_pubkey ( pk ) <EOL> name = csr . get_subject ( ) <EOL> name . CN = str ( self . common_name ) <EOL> csr . sign ( pk , "<STR_LIT>" ) <EOL> return csr . as_pem ( ) <EOL> def SavePrivateKey ( self , pkey ) : <EOL> """<STR_LIT>""" <EOL> bio = BIO . MemoryBuffer ( ) <EOL> pkey . save_key_bio ( bio , cipher = None ) <EOL> self . private_key = rdf_crypto . PEMPrivateKey ( bio . read_all ( ) ) <EOL> config_lib . CONFIG . Set ( "<STR_LIT>" , self . private_key ) <EOL> config_lib . CONFIG . Write ( ) <EOL> def LoadServerCertificate ( self , server_certificate = None , <EOL> ca_certificate = None ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> server_cert = X509 . load_cert_string ( str ( server_certificate ) ) <EOL> ca_cert = X509 . load_cert_string ( str ( ca_certificate ) ) <EOL> if server_cert . verify ( ca_cert . get_pubkey ( ) ) != <NUM_LIT:1> : <EOL> self . server_name = None <EOL> raise IOError ( "<STR_LIT>" ) <EOL> server_cert_serial = server_cert . get_serial_number ( ) <EOL> if server_cert_serial < config_lib . CONFIG [ "<STR_LIT>" ] : <EOL> raise IOError ( "<STR_LIT>" ) <EOL> elif server_cert_serial > config_lib . CONFIG [ <EOL> "<STR_LIT>" ] : <EOL> logging . info ( "<STR_LIT>" , server_cert_serial ) <EOL> config_lib . CONFIG . Set ( "<STR_LIT>" , server_cert_serial ) <EOL> config_lib . CONFIG . Write ( ) <EOL> except X509 . X509Error : <EOL> raise IOError ( "<STR_LIT>" ) <EOL> self . server_name = self . pub_key_cache . GetCNFromCert ( server_cert ) <EOL> self . server_certificate = server_certificate <EOL> self . ca_certificate = ca_certificate <EOL> self . pub_key_cache . Put ( <EOL> self . server_name , self . pub_key_cache . PubKeyFromCert ( server_cert ) ) <EOL> def EncodeMessages ( self , message_list , result , ** kwargs ) : <EOL> kwargs [ "<STR_LIT>" ] = config_lib . CONFIG [ "<STR_LIT>" ] <EOL> return super ( ClientCommunicator , self ) . EncodeMessages ( <EOL> message_list , result , ** kwargs ) </s>
