<s> from hashlib import md5 <EOL> import logging <EOL> import math <EOL> import os <EOL> import platform <EOL> import re <EOL> import signal <EOL> import socket <EOL> import sys <EOL> import time <EOL> import types <EOL> import urllib2 <EOL> import uuid <EOL> import simplejson as json <EOL> import yaml <EOL> from tornado import ioloop <EOL> try : <EOL> from yaml import CLoader as yLoader <EOL> from yaml import CDumper as yDumper <EOL> except ImportError : <EOL> from yaml import Loader as yLoader <EOL> from yaml import Dumper as yDumper <EOL> from utils . pidfile import PidFile <EOL> from utils . platform import Platform <EOL> from utils . subprocess_output import subprocess <EOL> VALID_HOSTNAME_RFC_1123_PATTERN = re . compile ( r"<STR_LIT>" ) <EOL> MAX_HOSTNAME_LEN = <NUM_LIT:255> <EOL> COLON_NON_WIN_PATH = re . compile ( '<STR_LIT>' ) <EOL> log = logging . getLogger ( __name__ ) <EOL> NumericTypes = ( float , int , long ) <EOL> def plural ( count ) : <EOL> if count == <NUM_LIT:1> : <EOL> return "<STR_LIT>" <EOL> return "<STR_LIT:s>" <EOL> def get_tornado_ioloop ( ) : <EOL> return ioloop . IOLoop . current ( ) <EOL> def get_uuid ( ) : <EOL> return uuid . uuid5 ( uuid . NAMESPACE_DNS , platform . node ( ) + str ( uuid . getnode ( ) ) ) . hex <EOL> def get_os ( ) : <EOL> "<STR_LIT>" <EOL> if sys . platform == '<STR_LIT>' : <EOL> return '<STR_LIT>' <EOL> elif sys . platform . find ( '<STR_LIT>' ) != - <NUM_LIT:1> : <EOL> return '<STR_LIT>' <EOL> elif sys . platform . find ( '<STR_LIT>' ) != - <NUM_LIT:1> : <EOL> return '<STR_LIT>' <EOL> elif sys . platform . find ( '<STR_LIT:win32>' ) != - <NUM_LIT:1> : <EOL> return '<STR_LIT>' <EOL> elif sys . platform . find ( '<STR_LIT>' ) != - <NUM_LIT:1> : <EOL> return '<STR_LIT>' <EOL> else : <EOL> return sys . platform <EOL> def headers ( agentConfig ) : <EOL> return { <EOL> '<STR_LIT>' : '<STR_LIT>' % agentConfig [ '<STR_LIT:version>' ] , <EOL> '<STR_LIT:Content-Type>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> } <EOL> def windows_friendly_colon_split ( config_string ) : <EOL> '''<STR_LIT>''' <EOL> if Platform . is_win32 ( ) : <EOL> return COLON_NON_WIN_PATH . split ( config_string ) <EOL> else : <EOL> return config_string . split ( '<STR_LIT::>' ) <EOL> def getTopIndex ( ) : <EOL> macV = None <EOL> if sys . platform == '<STR_LIT>' : <EOL> macV = platform . mac_ver ( ) <EOL> if macV and macV [ <NUM_LIT:0> ] . startswith ( '<STR_LIT>' ) : <EOL> return <NUM_LIT:6> <EOL> else : <EOL> return <NUM_LIT:5> <EOL> def isnan ( val ) : <EOL> if hasattr ( math , '<STR_LIT>' ) : <EOL> return math . isnan ( val ) <EOL> return str ( val ) == str ( <NUM_LIT> * <NUM_LIT:0> ) <EOL> def cast_metric_val ( val ) : <EOL> if not isinstance ( val , NumericTypes ) : <EOL> for cast in [ int , float ] : <EOL> try : <EOL> val = cast ( val ) <EOL> return val <EOL> except ValueError : <EOL> continue <EOL> raise ValueError <EOL> return val <EOL> _IDS = { } <EOL> def get_next_id ( name ) : <EOL> global _IDS <EOL> current_id = _IDS . get ( name , <NUM_LIT:0> ) <EOL> current_id += <NUM_LIT:1> <EOL> _IDS [ name ] = current_id <EOL> return current_id <EOL> def is_valid_hostname ( hostname ) : <EOL> if hostname . lower ( ) in set ( [ <EOL> '<STR_LIT:localhost>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> ] ) : <EOL> log . warning ( "<STR_LIT>" % hostname ) <EOL> return False <EOL> if len ( hostname ) > MAX_HOSTNAME_LEN : <EOL> log . warning ( "<STR_LIT>" % ( hostname , MAX_HOSTNAME_LEN ) ) <EOL> return False <EOL> if VALID_HOSTNAME_RFC_1123_PATTERN . match ( hostname ) is None : <EOL> log . warning ( "<STR_LIT>" % hostname ) <EOL> return False <EOL> return True <EOL> def get_hostname ( config = None ) : <EOL> """<STR_LIT>""" <EOL> hostname = None <EOL> if config is None : <EOL> from config import get_config <EOL> config = get_config ( parse_args = True ) <EOL> config_hostname = config . get ( '<STR_LIT>' ) <EOL> if config_hostname and is_valid_hostname ( config_hostname ) : <EOL> return config_hostname <EOL> if hostname is None : <EOL> gce_hostname = GCE . get_hostname ( config ) <EOL> if gce_hostname is not None : <EOL> if is_valid_hostname ( gce_hostname ) : <EOL> return gce_hostname <EOL> if hostname is None : <EOL> def _get_hostname_unix ( ) : <EOL> try : <EOL> p = subprocess . Popen ( [ '<STR_LIT>' , '<STR_LIT>' ] , stdout = subprocess . PIPE ) <EOL> out , err = p . communicate ( ) <EOL> if p . returncode == <NUM_LIT:0> : <EOL> return out . strip ( ) <EOL> except Exception : <EOL> return None <EOL> os_name = get_os ( ) <EOL> if os_name in [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] : <EOL> unix_hostname = _get_hostname_unix ( ) <EOL> if unix_hostname and is_valid_hostname ( unix_hostname ) : <EOL> hostname = unix_hostname <EOL> if hostname is not None and True in [ hostname . lower ( ) . startswith ( p ) for p in [ u'<STR_LIT>' , u'<STR_LIT>' ] ] : <EOL> instanceid = EC2 . get_instance_id ( config ) <EOL> if instanceid : <EOL> hostname = instanceid <EOL> if hostname is None : <EOL> try : <EOL> socket_hostname = socket . gethostname ( ) <EOL> except socket . error , e : <EOL> socket_hostname = None <EOL> if socket_hostname and is_valid_hostname ( socket_hostname ) : <EOL> hostname = socket_hostname <EOL> if hostname is None : <EOL> log . critical ( '<STR_LIT>' ) <EOL> raise Exception ( '<STR_LIT>' ) <EOL> else : <EOL> return hostname <EOL> class GCE ( object ) : <EOL> URL = "<STR_LIT>" <EOL> TIMEOUT = <NUM_LIT:0.1> <EOL> SOURCE_TYPE_NAME = '<STR_LIT>' <EOL> metadata = None <EOL> EXCLUDED_ATTRIBUTES = [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] <EOL> @ staticmethod <EOL> def _get_metadata ( agentConfig ) : <EOL> if GCE . metadata is not None : <EOL> return GCE . metadata <EOL> if not agentConfig [ '<STR_LIT>' ] : <EOL> log . info ( "<STR_LIT>" ) <EOL> GCE . metadata = { } <EOL> return GCE . metadata <EOL> socket_to = None <EOL> try : <EOL> socket_to = socket . getdefaulttimeout ( ) <EOL> socket . setdefaulttimeout ( GCE . TIMEOUT ) <EOL> except Exception : <EOL> pass <EOL> try : <EOL> opener = urllib2 . build_opener ( ) <EOL> opener . addheaders = [ ( '<STR_LIT>' , '<STR_LIT:True>' ) ] <EOL> GCE . metadata = json . loads ( opener . open ( GCE . URL ) . read ( ) . strip ( ) ) <EOL> except Exception : <EOL> GCE . metadata = { } <EOL> try : <EOL> if socket_to is None : <EOL> socket_to = <NUM_LIT:3> <EOL> socket . setdefaulttimeout ( socket_to ) <EOL> except Exception : <EOL> pass <EOL> return GCE . metadata <EOL> @ staticmethod <EOL> def get_tags ( agentConfig ) : <EOL> if not agentConfig [ '<STR_LIT>' ] : <EOL> return None <EOL> try : <EOL> host_metadata = GCE . _get_metadata ( agentConfig ) <EOL> tags = [ ] <EOL> for key , value in host_metadata [ '<STR_LIT>' ] . get ( '<STR_LIT>' , { } ) . iteritems ( ) : <EOL> if key in GCE . EXCLUDED_ATTRIBUTES : <EOL> continue <EOL> tags . append ( "<STR_LIT>" % ( key , value ) ) <EOL> tags . extend ( host_metadata [ '<STR_LIT>' ] . get ( '<STR_LIT>' , [ ] ) ) <EOL> tags . append ( '<STR_LIT>' % host_metadata [ '<STR_LIT>' ] [ '<STR_LIT>' ] . split ( '<STR_LIT:/>' ) [ - <NUM_LIT:1> ] ) <EOL> tags . append ( '<STR_LIT>' % host_metadata [ '<STR_LIT>' ] [ '<STR_LIT>' ] . split ( '<STR_LIT:/>' ) [ - <NUM_LIT:1> ] ) <EOL> tags . append ( '<STR_LIT>' % host_metadata [ '<STR_LIT>' ] [ '<STR_LIT>' ] ) <EOL> tags . append ( '<STR_LIT>' % host_metadata [ '<STR_LIT>' ] [ '<STR_LIT:id>' ] ) <EOL> tags . append ( '<STR_LIT>' % host_metadata [ '<STR_LIT>' ] [ '<STR_LIT>' ] ) <EOL> tags . append ( '<STR_LIT>' % host_metadata [ '<STR_LIT>' ] [ '<STR_LIT>' ] ) <EOL> GCE . metadata [ '<STR_LIT>' ] = host_metadata [ '<STR_LIT>' ] [ '<STR_LIT>' ] . split ( '<STR_LIT:.>' ) [ <NUM_LIT:0> ] <EOL> return tags <EOL> except Exception : <EOL> return None <EOL> @ staticmethod <EOL> def get_hostname ( agentConfig ) : <EOL> try : <EOL> host_metadata = GCE . _get_metadata ( agentConfig ) <EOL> hostname = host_metadata [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> if agentConfig . get ( '<STR_LIT>' ) : <EOL> return hostname <EOL> else : <EOL> return hostname . split ( '<STR_LIT:.>' ) [ <NUM_LIT:0> ] <EOL> except Exception : <EOL> return None <EOL> @ staticmethod <EOL> def get_host_aliases ( agentConfig ) : <EOL> try : <EOL> host_metadata = GCE . _get_metadata ( agentConfig ) <EOL> project_id = host_metadata [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> instance_name = host_metadata [ '<STR_LIT>' ] [ '<STR_LIT>' ] . split ( '<STR_LIT:.>' ) [ <NUM_LIT:0> ] <EOL> return [ '<STR_LIT>' % ( instance_name , project_id ) ] <EOL> except Exception : <EOL> return None <EOL> class EC2 ( object ) : <EOL> """<STR_LIT>""" <EOL> EC2_METADATA_HOST = "<STR_LIT>" <EOL> METADATA_URL_BASE = EC2_METADATA_HOST + "<STR_LIT>" <EOL> INSTANCE_IDENTITY_URL = EC2_METADATA_HOST + "<STR_LIT>" <EOL> TIMEOUT = <NUM_LIT:0.1> <EOL> metadata = { } <EOL> @ staticmethod <EOL> def get_tags ( agentConfig ) : <EOL> if not agentConfig [ '<STR_LIT>' ] : <EOL> log . info ( "<STR_LIT>" ) <EOL> return [ ] <EOL> socket_to = None <EOL> try : <EOL> socket_to = socket . getdefaulttimeout ( ) <EOL> socket . setdefaulttimeout ( EC2 . TIMEOUT ) <EOL> except Exception : <EOL> pass <EOL> try : <EOL> iam_role = urllib2 . urlopen ( EC2 . METADATA_URL_BASE + "<STR_LIT>" ) . read ( ) . strip ( ) <EOL> iam_params = json . loads ( urllib2 . urlopen ( EC2 . METADATA_URL_BASE + "<STR_LIT>" + unicode ( iam_role ) ) . read ( ) . strip ( ) ) <EOL> instance_identity = json . loads ( urllib2 . urlopen ( EC2 . INSTANCE_IDENTITY_URL ) . read ( ) . strip ( ) ) <EOL> region = instance_identity [ '<STR_LIT>' ] <EOL> import boto . ec2 <EOL> connection = boto . ec2 . connect_to_region ( region , aws_access_key_id = iam_params [ '<STR_LIT>' ] , aws_secret_access_key = iam_params [ '<STR_LIT>' ] , security_token = iam_params [ '<STR_LIT>' ] ) <EOL> tag_object = connection . get_all_tags ( { '<STR_LIT>' : EC2 . metadata [ '<STR_LIT>' ] } ) <EOL> EC2_tags = [ u"<STR_LIT>" % ( tag . name , tag . value ) for tag in tag_object ] <EOL> except Exception : <EOL> log . exception ( "<STR_LIT>" ) <EOL> EC2_tags = [ ] <EOL> try : <EOL> if socket_to is None : <EOL> socket_to = <NUM_LIT:3> <EOL> socket . setdefaulttimeout ( socket_to ) <EOL> except Exception : <EOL> pass <EOL> return EC2_tags <EOL> @ staticmethod <EOL> def get_metadata ( agentConfig ) : <EOL> """<STR_LIT>""" <EOL> if not agentConfig [ '<STR_LIT>' ] : <EOL> log . info ( "<STR_LIT>" ) <EOL> return { } <EOL> socket_to = None <EOL> try : <EOL> socket_to = socket . getdefaulttimeout ( ) <EOL> socket . setdefaulttimeout ( EC2 . TIMEOUT ) <EOL> except Exception : <EOL> pass <EOL> for k in ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) : <EOL> try : <EOL> v = urllib2 . urlopen ( EC2 . METADATA_URL_BASE + "<STR_LIT:/>" + unicode ( k ) ) . read ( ) . strip ( ) <EOL> assert type ( v ) in ( types . StringType , types . UnicodeType ) and len ( v ) > <NUM_LIT:0> , "<STR_LIT>" % v <EOL> EC2 . metadata [ k . rstrip ( '<STR_LIT:/>' ) ] = v <EOL> except Exception : <EOL> pass <EOL> try : <EOL> if socket_to is None : <EOL> socket_to = <NUM_LIT:3> <EOL> socket . setdefaulttimeout ( socket_to ) <EOL> except Exception : <EOL> pass <EOL> return EC2 . metadata <EOL> @ staticmethod <EOL> def get_instance_id ( agentConfig ) : <EOL> try : <EOL> return EC2 . get_metadata ( agentConfig ) . get ( "<STR_LIT>" , None ) <EOL> except Exception : <EOL> return None <EOL> class Watchdog ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , duration , max_mem_mb = None ) : <EOL> import resource <EOL> self . _duration = int ( duration ) <EOL> signal . signal ( signal . SIGALRM , Watchdog . self_destruct ) <EOL> if max_mem_mb is not None : <EOL> self . _max_mem_kb = <NUM_LIT> * max_mem_mb <EOL> max_mem_bytes = <NUM_LIT> * self . _max_mem_kb <EOL> resource . setrlimit ( resource . RLIMIT_AS , ( max_mem_bytes , max_mem_bytes ) ) <EOL> self . memory_limit_enabled = True <EOL> else : <EOL> self . memory_limit_enabled = False <EOL> @ staticmethod <EOL> def self_destruct ( signum , frame ) : <EOL> try : <EOL> import traceback <EOL> log . error ( "<STR_LIT>" ) <EOL> log . error ( traceback . format_exc ( ) ) <EOL> finally : <EOL> os . kill ( os . getpid ( ) , signal . SIGKILL ) <EOL> def reset ( self ) : <EOL> if self . memory_limit_enabled : <EOL> mem_usage_kb = int ( os . popen ( '<STR_LIT>' % ( os . getpid ( ) , '<STR_LIT>' ) ) . read ( ) ) <EOL> if mem_usage_kb > ( <NUM_LIT> * self . _max_mem_kb ) : <EOL> Watchdog . self_destruct ( signal . SIGKILL , sys . _getframe ( <NUM_LIT:0> ) ) <EOL> log . debug ( "<STR_LIT>" % self . _duration ) <EOL> signal . alarm ( self . _duration ) <EOL> class LaconicFilter ( logging . Filter ) : <EOL> """<STR_LIT>""" <EOL> LACONIC_MEM_LIMIT = <NUM_LIT> <EOL> def __init__ ( self , name = "<STR_LIT>" ) : <EOL> logging . Filter . __init__ ( self , name ) <EOL> self . hashed_messages = { } <EOL> def hash ( self , msg ) : <EOL> return md5 ( msg ) . hexdigest ( ) <EOL> def filter ( self , record ) : <EOL> try : <EOL> h = self . hash ( record . getMessage ( ) ) <EOL> if h in self . hashed_messages : <EOL> return <NUM_LIT:0> <EOL> else : <EOL> if len ( self . hashed_messages ) >= LaconicFilter . LACONIC_MEM_LIMIT : <EOL> self . hashed_messages . clear ( ) <EOL> self . hashed_messages [ h ] = True <EOL> return <NUM_LIT:1> <EOL> except Exception : <EOL> return <NUM_LIT:1> <EOL> class Timer ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self ) : <EOL> self . start ( ) <EOL> def _now ( self ) : <EOL> return time . time ( ) <EOL> def start ( self ) : <EOL> self . started = self . _now ( ) <EOL> self . last = self . started <EOL> return self <EOL> def step ( self ) : <EOL> now = self . _now ( ) <EOL> step = now - self . last <EOL> self . last = now <EOL> return step <EOL> def total ( self , as_sec = True ) : <EOL> return self . _now ( ) - self . started <EOL> """<STR_LIT>""" <EOL> def chunks ( iterable , chunk_size ) : <EOL> """<STR_LIT>""" <EOL> iterable = iter ( iterable ) <EOL> while True : <EOL> chunk = [ None ] * chunk_size <EOL> count = <NUM_LIT:0> <EOL> try : <EOL> for _ in range ( chunk_size ) : <EOL> chunk [ count ] = iterable . next ( ) <EOL> count += <NUM_LIT:1> <EOL> yield chunk [ : count ] <EOL> except StopIteration : <EOL> if count : <EOL> yield chunk [ : count ] <EOL> break </s>
