<s> import sys <EOL> import os <EOL> import logging <EOL> from collections import namedtuple , OrderedDict <EOL> import time <EOL> import csv <EOL> import signal <EOL> import cPickle as pickle <EOL> import json <EOL> from ctypes import CDLL <EOL> try : <EOL> libc = CDLL ( '<STR_LIT>' ) <EOL> except Exception as e : <EOL> libc = None <EOL> import psutil <EOL> from emitter import Emitter <EOL> from features_crawler import FeaturesCrawler <EOL> from containers import get_filtered_list_of_containers <EOL> import defaults <EOL> import misc <EOL> from crawlmodes import Modes <EOL> logger = logging . getLogger ( '<STR_LIT>' ) <EOL> """<STR_LIT>""" <EOL> should_exit = False <EOL> def signal_handler_exit ( signum , stack ) : <EOL> global should_exit <EOL> logger . info ( '<STR_LIT>' ) <EOL> should_exit = True <EOL> def snapshot_single_frame ( <EOL> emitter , <EOL> features = defaults . DEFAULT_FEATURES_TO_CRAWL , <EOL> options = defaults . DEFAULT_CRAWL_OPTIONS , <EOL> crawler = None , <EOL> inputfile = "<STR_LIT>" , <EOL> ignore_exceptions = True , <EOL> ) : <EOL> global should_exit <EOL> crawlmode = crawler . crawl_mode <EOL> if crawlmode == Modes . FILE : <EOL> logger . debug ( <EOL> '<STR_LIT>' . format ( inputfile ) ) <EOL> FeatureFormat = namedtuple ( '<STR_LIT>' , [ '<STR_LIT:type>' , '<STR_LIT:key>' , <EOL> '<STR_LIT:value>' ] ) <EOL> if inputfile is None or not os . path . exists ( inputfile ) : <EOL> logger . error ( '<STR_LIT>' + inputfile + <EOL> '<STR_LIT>' ) <EOL> with open ( inputfile , '<STR_LIT:r>' ) as fd : <EOL> csv . field_size_limit ( sys . maxsize ) <EOL> csv_reader = csv . reader ( fd , delimiter = '<STR_LIT:\t>' , quotechar = "<STR_LIT:'>" ) <EOL> num_features = <NUM_LIT:0> <EOL> for row in csv_reader : <EOL> feature_data = FeatureFormat ( <EOL> row [ <NUM_LIT:0> ] , json . loads ( <EOL> row [ <NUM_LIT:1> ] ) , json . loads ( <EOL> row [ <NUM_LIT:2> ] , object_pairs_hook = OrderedDict ) ) <EOL> num_features += <NUM_LIT:1> <EOL> if feature_data . type in features : <EOL> emitter . emit ( feature_data . key , feature_data . value , <EOL> feature_data . type ) <EOL> logger . info ( '<STR_LIT>' % ( num_features , <EOL> inputfile ) ) <EOL> else : <EOL> for feature in features . split ( '<STR_LIT:U+002C>' ) : <EOL> feature_options = options . get ( <EOL> feature , defaults . DEFAULT_CRAWL_OPTIONS [ feature ] ) <EOL> if should_exit : <EOL> break <EOL> if feature_options is None : <EOL> continue <EOL> try : <EOL> _crawl_single_feature ( feature , <EOL> feature_options , <EOL> crawlmode , <EOL> crawler , <EOL> emitter ) <EOL> except Exception as e : <EOL> logger . exception ( e ) <EOL> if not ignore_exceptions : <EOL> raise e <EOL> def _crawl_single_feature ( feature , <EOL> feature_options , <EOL> crawlmode , <EOL> crawler , <EOL> emitter ) : <EOL> if feature == '<STR_LIT>' : <EOL> if crawlmode in [ Modes . INVM , Modes . OUTCONTAINER , Modes . MOUNTPOINT ] : <EOL> for ( key , feature ) in crawler . crawl_os ( ** feature_options ) : <EOL> emitter . emit ( key , feature ) <EOL> if feature == '<STR_LIT>' : <EOL> if crawlmode in [ Modes . INVM , Modes . OUTCONTAINER , Modes . MOUNTPOINT ] : <EOL> for ( key , feature ) in crawler . crawl_disk_partitions ( ** feature_options ) : <EOL> emitter . emit ( key , feature ) <EOL> else : <EOL> logger . warning ( '<STR_LIT>' + feature + <EOL> '<STR_LIT>' + crawlmode + <EOL> '<STR_LIT>' ) <EOL> if feature == '<STR_LIT>' : <EOL> if crawlmode in [ Modes . INVM , Modes . OUTCONTAINER ] : <EOL> for ( key , feature ) in crawler . crawl_metrics ( ** feature_options ) : <EOL> emitter . emit ( key , feature ) <EOL> else : <EOL> logger . warning ( '<STR_LIT>' + feature + <EOL> '<STR_LIT>' + crawlmode + <EOL> '<STR_LIT>' ) <EOL> if feature == '<STR_LIT>' : <EOL> if crawlmode in [ Modes . INVM , Modes . OUTVM , Modes . OUTCONTAINER ] : <EOL> for ( key , feature ) in crawler . crawl_processes ( ** feature_options ) : <EOL> emitter . emit ( key , feature ) <EOL> else : <EOL> logger . warning ( '<STR_LIT>' + feature + <EOL> '<STR_LIT>' + crawlmode + <EOL> '<STR_LIT>' ) <EOL> if feature == '<STR_LIT>' : <EOL> if crawlmode in [ Modes . INVM , Modes . OUTVM , Modes . OUTCONTAINER ] : <EOL> for ( key , feature ) in crawler . crawl_connections ( ** feature_options ) : <EOL> emitter . emit ( key , feature ) <EOL> else : <EOL> logger . warning ( '<STR_LIT>' + feature + <EOL> '<STR_LIT>' + crawlmode + <EOL> '<STR_LIT>' ) <EOL> if feature == '<STR_LIT>' : <EOL> if crawlmode in [ Modes . INVM , Modes . OUTCONTAINER , Modes . MOUNTPOINT ] : <EOL> for ( key , feature ) in crawler . crawl_packages ( ** feature_options ) : <EOL> emitter . emit ( key , feature ) <EOL> if feature == '<STR_LIT:file>' : <EOL> if crawlmode in [ Modes . INVM , Modes . OUTCONTAINER , Modes . MOUNTPOINT ] : <EOL> for ( key , feature ) in crawler . crawl_files ( ** feature_options ) : <EOL> emitter . emit ( key , feature ) <EOL> if feature == '<STR_LIT>' : <EOL> for ( key , feature ) in crawler . crawl_config_files ( ** feature_options ) : <EOL> emitter . emit ( key , feature ) <EOL> if feature == '<STR_LIT>' : <EOL> if crawlmode in [ Modes . INVM , Modes . OUTVM , Modes . OUTCONTAINER ] : <EOL> for ( key , feature ) in crawler . crawl_memory ( ** feature_options ) : <EOL> emitter . emit ( key , feature ) <EOL> else : <EOL> logger . warning ( '<STR_LIT>' + feature + <EOL> '<STR_LIT>' + crawlmode + <EOL> '<STR_LIT>' ) <EOL> if feature == '<STR_LIT>' : <EOL> if crawlmode in [ Modes . INVM , Modes . OUTVM ] : <EOL> feature_options [ '<STR_LIT>' ] = True <EOL> for ( key , feature ) in crawler . crawl_cpu ( ** feature_options ) : <EOL> emitter . emit ( key , feature ) <EOL> elif crawlmode in [ Modes . OUTCONTAINER ] : <EOL> feature_options [ '<STR_LIT>' ] = False <EOL> for ( key , feature ) in crawler . crawl_cpu ( ** feature_options ) : <EOL> emitter . emit ( key , feature ) <EOL> else : <EOL> logger . warning ( '<STR_LIT>' + feature + <EOL> '<STR_LIT>' + crawlmode + <EOL> '<STR_LIT>' ) <EOL> if feature == '<STR_LIT>' : <EOL> if crawlmode in [ Modes . INVM , Modes . OUTCONTAINER ] : <EOL> for ( key , feature ) in crawler . crawl_interface ( ** feature_options ) : <EOL> emitter . emit ( key , feature ) <EOL> else : <EOL> logger . warning ( '<STR_LIT>' + feature + <EOL> '<STR_LIT>' + crawlmode + <EOL> '<STR_LIT>' ) <EOL> if feature == '<STR_LIT>' : <EOL> if crawlmode in [ Modes . INVM , Modes . OUTCONTAINER ] : <EOL> for ( key , feature ) in crawler . crawl_load ( ** feature_options ) : <EOL> emitter . emit ( key , feature ) <EOL> else : <EOL> logger . warning ( '<STR_LIT>' + feature + <EOL> '<STR_LIT>' + crawlmode + <EOL> '<STR_LIT>' ) <EOL> if feature == '<STR_LIT>' : <EOL> if crawlmode in [ Modes . INVM ] : <EOL> for ( key , feature ) in crawler . crawl_dockerps ( ** feature_options ) : <EOL> emitter . emit ( key , feature , '<STR_LIT>' ) <EOL> else : <EOL> logger . warning ( '<STR_LIT>' + feature + <EOL> '<STR_LIT>' + crawlmode + <EOL> '<STR_LIT>' ) <EOL> if feature == '<STR_LIT>' : <EOL> if crawlmode in [ Modes . OUTCONTAINER ] : <EOL> for ( key , feature ) in crawler . crawl_dockerhistory ( ** feature_options ) : <EOL> emitter . emit ( key , feature , '<STR_LIT>' ) <EOL> else : <EOL> logger . warning ( '<STR_LIT>' + feature + <EOL> '<STR_LIT>' + crawlmode + <EOL> '<STR_LIT>' ) <EOL> if feature == '<STR_LIT>' : <EOL> if crawlmode in [ Modes . OUTCONTAINER ] : <EOL> for ( key , feature ) in crawler . crawl_dockerinspect ( ** feature_options ) : <EOL> emitter . emit ( key , feature , '<STR_LIT>' ) <EOL> else : <EOL> logger . warning ( '<STR_LIT>' + feature + <EOL> '<STR_LIT>' + crawlmode + <EOL> '<STR_LIT>' ) <EOL> if feature == '<STR_LIT>' : <EOL> for ( key , feature ) in crawler . crawl_test_infinite_loop ( ** feature_options ) : <EOL> emitter . emit ( key , feature ) <EOL> if feature == '<STR_LIT>' : <EOL> for ( key , feature ) in crawler . crawl_test_crash ( ** feature_options ) : <EOL> emitter . emit ( key , feature ) <EOL> def snapshot_generic ( <EOL> crawlmode = Modes . INVM , <EOL> urls = [ '<STR_LIT>' ] , <EOL> snapshot_num = <NUM_LIT:0> , <EOL> features = defaults . DEFAULT_FEATURES_TO_CRAWL , <EOL> options = defaults . DEFAULT_CRAWL_OPTIONS , <EOL> format = '<STR_LIT>' , <EOL> inputfile = '<STR_LIT>' , <EOL> overwrite = False , <EOL> namespace = '<STR_LIT>' , <EOL> since = '<STR_LIT>' , <EOL> since_timestamp = <NUM_LIT:0> , <EOL> ) : <EOL> crawler = FeaturesCrawler ( feature_epoch = since_timestamp , <EOL> crawl_mode = crawlmode , <EOL> namespace = namespace ) <EOL> compress = options [ '<STR_LIT>' ] <EOL> metadata = { <EOL> '<STR_LIT>' : namespace , <EOL> '<STR_LIT>' : features , <EOL> '<STR_LIT>' : int ( time . time ( ) ) , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : since , <EOL> '<STR_LIT>' : since_timestamp , <EOL> '<STR_LIT>' : compress , <EOL> '<STR_LIT>' : overwrite , <EOL> } <EOL> output_urls = [ ( '<STR_LIT>' . format ( u , snapshot_num ) <EOL> if u . startswith ( '<STR_LIT>' ) else u ) for u in urls ] <EOL> with Emitter ( <EOL> urls = output_urls , <EOL> emitter_args = metadata , <EOL> format = format , <EOL> ) as emitter : <EOL> snapshot_single_frame ( emitter , features , <EOL> options , crawler , inputfile ) <EOL> def snapshot_container ( <EOL> urls = [ '<STR_LIT>' ] , <EOL> snapshot_num = <NUM_LIT:0> , <EOL> features = defaults . DEFAULT_FEATURES_TO_CRAWL , <EOL> options = defaults . DEFAULT_CRAWL_OPTIONS , <EOL> format = '<STR_LIT>' , <EOL> inputfile = '<STR_LIT>' , <EOL> overwrite = False , <EOL> container = None , <EOL> since = '<STR_LIT>' , <EOL> since_timestamp = <NUM_LIT:0> , <EOL> ) : <EOL> crawler = FeaturesCrawler ( <EOL> feature_epoch = since_timestamp , <EOL> crawl_mode = Modes . OUTCONTAINER , <EOL> namespace = container . namespace , <EOL> container = container ) <EOL> compress = options [ '<STR_LIT>' ] <EOL> extra_metadata = options [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> extra_metadata_for_all = options [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> metadata = { <EOL> '<STR_LIT>' : container . namespace , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : features , <EOL> '<STR_LIT>' : int ( time . time ( ) ) , <EOL> '<STR_LIT>' : since , <EOL> '<STR_LIT>' : since_timestamp , <EOL> '<STR_LIT>' : compress , <EOL> '<STR_LIT>' : container . long_id , <EOL> '<STR_LIT>' : container . name , <EOL> '<STR_LIT>' : container . image , <EOL> '<STR_LIT>' : extra_metadata , <EOL> '<STR_LIT>' : extra_metadata_for_all , <EOL> } <EOL> output_urls = [ ] <EOL> for url in urls : <EOL> if url . startswith ( '<STR_LIT>' ) : <EOL> file_suffix = '<STR_LIT>' <EOL> if overwrite is True : <EOL> file_suffix = '<STR_LIT>' . format ( container . name ) <EOL> else : <EOL> file_suffix = '<STR_LIT>' . format ( container . short_id , snapshot_num ) <EOL> output_urls . append ( '<STR_LIT>' . format ( url , file_suffix ) ) <EOL> else : <EOL> output_urls . append ( url ) <EOL> with Emitter ( <EOL> urls = output_urls , <EOL> emitter_args = metadata , <EOL> format = format , <EOL> ) as emitter : <EOL> snapshot_single_frame ( emitter , features , options , <EOL> crawler , inputfile ) <EOL> def get_initial_since_values ( since ) : <EOL> last_snapshot_time = ( <EOL> psutil . boot_time ( ) if hasattr ( <EOL> psutil , '<STR_LIT>' ) else psutil . BOOT_TIME ) <EOL> if since == '<STR_LIT>' : <EOL> since_timestamp = <NUM_LIT:0> <EOL> elif since == '<STR_LIT>' : <EOL> since_timestamp = ( <EOL> psutil . boot_time ( ) if hasattr ( <EOL> psutil , '<STR_LIT>' ) else psutil . BOOT_TIME ) <EOL> elif since == '<STR_LIT>' : <EOL> since_timestamp = last_snapshot_time <EOL> else : <EOL> try : <EOL> since_timestamp = int ( since ) <EOL> except : <EOL> logger . error ( <EOL> '<STR_LIT>' . format ( since ) ) <EOL> since_timestamp = last_snapshot_time <EOL> return since_timestamp , last_snapshot_time <EOL> def snapshot ( <EOL> urls = [ '<STR_LIT>' ] , <EOL> namespace = misc . get_host_ipaddr ( ) , <EOL> features = defaults . DEFAULT_FEATURES_TO_CRAWL , <EOL> options = defaults . DEFAULT_CRAWL_OPTIONS , <EOL> since = '<STR_LIT>' , <EOL> frequency = - <NUM_LIT:1> , <EOL> crawlmode = Modes . INVM , <EOL> inputfile = '<STR_LIT>' , <EOL> format = '<STR_LIT>' , <EOL> overwrite = False , <EOL> ) : <EOL> """<STR_LIT>""" <EOL> global should_exit <EOL> saved_args = locals ( ) <EOL> logger . debug ( '<STR_LIT>' % ( saved_args ) ) <EOL> assert ( '<STR_LIT>' in options ) <EOL> environment = options . get ( '<STR_LIT>' , defaults . DEFAULT_ENVIRONMENT ) <EOL> since_timestamp , last_snapshot_time = get_initial_since_values ( since ) <EOL> snapshot_num = <NUM_LIT:0> <EOL> PR_SET_PDEATHSIG = <NUM_LIT:1> <EOL> libc . prctl ( PR_SET_PDEATHSIG , signal . SIGHUP ) <EOL> signal . signal ( signal . SIGHUP , signal_handler_exit ) <EOL> if crawlmode == Modes . OUTCONTAINER : <EOL> containers = get_filtered_list_of_containers ( options , namespace ) <EOL> while True : <EOL> snapshot_time = int ( time . time ( ) ) <EOL> if crawlmode == Modes . OUTCONTAINER : <EOL> curr_containers = get_filtered_list_of_containers ( <EOL> options , namespace ) <EOL> deleted = [ c for c in containers if c not in curr_containers ] <EOL> containers = curr_containers <EOL> for container in deleted : <EOL> if options . get ( '<STR_LIT>' , False ) : <EOL> container . unlink_logfiles ( options ) <EOL> logger . debug ( '<STR_LIT>' % ( len ( containers ) ) ) <EOL> for container in containers : <EOL> logger . info ( <EOL> '<STR_LIT>' % <EOL> ( container . pid , container . short_id , container . namespace ) ) <EOL> if options . get ( '<STR_LIT>' , False ) : <EOL> container . link_logfiles ( options = options ) <EOL> snapshot_container ( <EOL> urls = urls , <EOL> snapshot_num = snapshot_num , <EOL> features = features , <EOL> options = options , <EOL> format = format , <EOL> inputfile = inputfile , <EOL> container = container , <EOL> since = since , <EOL> since_timestamp = since_timestamp , <EOL> overwrite = overwrite <EOL> ) <EOL> elif crawlmode in ( Modes . INVM , <EOL> Modes . MOUNTPOINT , <EOL> Modes . DEVICE , <EOL> Modes . FILE , <EOL> Modes . ISCSI ) : <EOL> snapshot_generic ( <EOL> crawlmode = crawlmode , <EOL> urls = urls , <EOL> snapshot_num = snapshot_num , <EOL> features = features , <EOL> options = options , <EOL> format = format , <EOL> inputfile = inputfile , <EOL> namespace = namespace , <EOL> since = since , <EOL> since_timestamp = since_timestamp , <EOL> overwrite = overwrite <EOL> ) <EOL> else : <EOL> raise RuntimeError ( '<STR_LIT>' ) <EOL> if frequency < <NUM_LIT:0> or should_exit : <EOL> logger . info ( '<STR_LIT>' ) <EOL> break <EOL> if since == '<STR_LIT>' : <EOL> since_timestamp = snapshot_time <EOL> time . sleep ( frequency ) <EOL> snapshot_num += <NUM_LIT:1> </s>
