<s> import os <EOL> import subprocess <EOL> import logging <EOL> import shutil <EOL> from container import Container <EOL> import misc <EOL> import defaults <EOL> import json <EOL> from dockerutils import ( <EOL> exec_dockerps , <EOL> get_docker_container_json_logs_path , <EOL> get_docker_container_rootfs_path , <EOL> exec_dockerinspect ) <EOL> try : <EOL> import alchemy <EOL> except ImportError : <EOL> alchemy = None <EOL> from crawler_exceptions import ContainerInvalidEnvironment <EOL> logger = logging . getLogger ( '<STR_LIT>' ) <EOL> def list_docker_containers ( namespace_opts = { } ) : <EOL> """<STR_LIT>""" <EOL> for inspect in exec_dockerps ( ) : <EOL> long_id = inspect [ '<STR_LIT>' ] <EOL> try : <EOL> yield DockerContainer ( long_id , inspect , namespace_opts ) <EOL> except ContainerInvalidEnvironment as e : <EOL> logger . exception ( e ) <EOL> class DockerContainer ( Container ) : <EOL> def __init__ ( <EOL> self , <EOL> long_id , <EOL> inspect = None , <EOL> namespace_opts = { } , <EOL> ) : <EOL> if not inspect : <EOL> inspect = exec_dockerinspect ( long_id ) <EOL> state = inspect [ '<STR_LIT>' ] <EOL> self . image = inspect [ '<STR_LIT>' ] <EOL> assert ( long_id == inspect [ '<STR_LIT>' ] ) <EOL> self . long_id = long_id <EOL> self . pid = state [ '<STR_LIT>' ] <EOL> self . name = inspect [ '<STR_LIT:Name>' ] <EOL> self . running = state [ '<STR_LIT>' ] <EOL> self . created = inspect [ '<STR_LIT>' ] <EOL> self . network_settings = inspect [ '<STR_LIT>' ] <EOL> self . cmd = inspect [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> self . inspect = inspect <EOL> self . short_id = long_id [ : <NUM_LIT:12> ] <EOL> if self . name [ <NUM_LIT:0> ] == '<STR_LIT:/>' : <EOL> self . name = self . name [ <NUM_LIT:1> : ] <EOL> self . namespace = None <EOL> def is_docker_container ( self ) : <EOL> return True <EOL> def get_cgroup_dir ( self , dev = '<STR_LIT>' ) : <EOL> paths = [ os . path . join ( '<STR_LIT>' , dev ) , <EOL> os . path . join ( '<STR_LIT>' , dev ) ] <EOL> for path in paths : <EOL> if os . path . ismount ( path ) : <EOL> return path <EOL> try : <EOL> proc = subprocess . Popen ( <EOL> '<STR_LIT>' + <EOL> dev + <EOL> '<STR_LIT>' , <EOL> shell = True , <EOL> stdout = subprocess . PIPE , <EOL> stderr = subprocess . PIPE ) <EOL> return proc . stdout . read ( ) . strip ( ) <EOL> except Exception as e : <EOL> raise e <EOL> def get_memory_cgroup_path ( self , node = '<STR_LIT>' ) : <EOL> return os . path . join ( self . get_cgroup_dir ( '<STR_LIT>' ) , '<STR_LIT>' , <EOL> self . long_id , node ) <EOL> def get_cpu_cgroup_path ( self , node = '<STR_LIT>' ) : <EOL> return os . path . join ( self . get_cgroup_dir ( '<STR_LIT>' ) , '<STR_LIT>' , <EOL> self . long_id , node ) <EOL> def __str__ ( self ) : <EOL> return str ( self . __dict__ ) <EOL> def link_logfiles ( self , <EOL> options = defaults . DEFAULT_CRAWL_OPTIONS ) : <EOL> host_log_dir = self . _get_logfiles_links_dest ( <EOL> options [ '<STR_LIT>' ] [ '<STR_LIT>' ] , <EOL> options [ '<STR_LIT>' ] , <EOL> ) <EOL> logs_list = self . _get_logfiles_list ( host_log_dir , options ) <EOL> logger . debug ( '<STR_LIT>' % self . short_id ) <EOL> if not os . path . exists ( host_log_dir ) : <EOL> os . makedirs ( host_log_dir ) <EOL> for log in logs_list : <EOL> try : <EOL> if not os . path . exists ( log [ '<STR_LIT:source>' ] ) : <EOL> logger . debug ( <EOL> '<STR_LIT>' <EOL> % log [ '<STR_LIT:source>' ] ) <EOL> dest_dir = os . path . dirname ( log [ '<STR_LIT>' ] ) <EOL> if not os . path . exists ( dest_dir ) : <EOL> os . makedirs ( dest_dir ) <EOL> os . symlink ( log [ '<STR_LIT:source>' ] , log [ '<STR_LIT>' ] ) <EOL> logger . info ( <EOL> '<STR_LIT>' % <EOL> ( self . short_id , log [ '<STR_LIT:name>' ] , log [ '<STR_LIT:source>' ] , log [ '<STR_LIT>' ] ) ) <EOL> except ( OSError , IOError ) as e : <EOL> logger . debug ( '<STR_LIT>' <EOL> % ( log [ '<STR_LIT:source>' ] , log [ '<STR_LIT>' ] ) ) <EOL> except Exception as e : <EOL> logger . warning ( e ) <EOL> try : <EOL> log_types_file = options [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> types_host_log_path = os . path . join ( host_log_dir , <EOL> log_types_file ) <EOL> with open ( types_host_log_path , '<STR_LIT:w>' ) as outfile : <EOL> logs_dict = [ { '<STR_LIT:name>' : log [ '<STR_LIT:name>' ] , '<STR_LIT:type>' : log [ '<STR_LIT:type>' ] } <EOL> for log in logs_list ] <EOL> json . dump ( logs_dict , outfile ) <EOL> except ( OSError , IOError ) as e : <EOL> logger . exception ( e ) <EOL> def unlink_logfiles ( self , <EOL> options = defaults . DEFAULT_CRAWL_OPTIONS ) : <EOL> host_log_basedir = options [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> host_log_dir = self . _get_logfiles_links_dest ( <EOL> host_log_basedir , <EOL> options [ '<STR_LIT>' ] , <EOL> ) <EOL> logger . info ( '<STR_LIT>' <EOL> % self . short_id ) <EOL> try : <EOL> shutil . rmtree ( '<STR_LIT>' + self . namespace ) <EOL> except ( IOError , OSError ) as e : <EOL> pass <EOL> try : <EOL> shutil . move ( host_log_dir , '<STR_LIT>' + self . namespace ) <EOL> except ( IOError , OSError ) as e : <EOL> logger . exception ( e ) <EOL> pass <EOL> def _log_locations_json_sanity_check ( self , data ) : <EOL> """<STR_LIT>""" <EOL> max_data_len = <NUM_LIT> <EOL> max_files = <NUM_LIT:100> <EOL> if len ( data ) > max_data_len : <EOL> raise KeyError ( '<STR_LIT>' ) <EOL> if '<STR_LIT>' not in data : <EOL> raise KeyError ( '<STR_LIT>' ) <EOL> if len ( data . keys ( ) ) > <NUM_LIT:1> : <EOL> raise KeyError ( '<STR_LIT>' ) <EOL> log_files = data [ '<STR_LIT>' ] <EOL> if len ( log_files ) > max_files : <EOL> raise KeyError ( '<STR_LIT>' ) <EOL> for log in log_files : <EOL> if '<STR_LIT:name>' not in log . keys ( ) : <EOL> raise KeyError ( '<STR_LIT>' ) <EOL> if len ( log . keys ( ) ) > <NUM_LIT:2> : <EOL> raise KeyError ( '<STR_LIT>' ) <EOL> if len ( log . keys ( ) ) > <NUM_LIT:1> and '<STR_LIT:type>' not in log . keys ( ) : <EOL> raise KeyError ( '<STR_LIT>' ) <EOL> if not isinstance ( log [ '<STR_LIT:name>' ] , str ) : <EOL> raise KeyError ( '<STR_LIT>' ) <EOL> if '<STR_LIT:type>' in log and not isinstance ( log [ '<STR_LIT:type>' ] , str ) : <EOL> raise KeyError ( '<STR_LIT>' ) <EOL> def _parse_log_locations ( <EOL> self , <EOL> json_path = None , <EOL> var = None , <EOL> is_var = True , <EOL> isJson = True , <EOL> ) : <EOL> """<STR_LIT>""" <EOL> container = self <EOL> logs = [ ] <EOL> data = None <EOL> try : <EOL> if is_var : <EOL> data = misc . GetProcessEnv ( container . pid ) [ var ] <EOL> else : <EOL> with open ( json_path , '<STR_LIT:r>' ) as fp : <EOL> data = fp . read ( ) <EOL> if isJson : <EOL> if data . startswith ( '<STR_LIT:">' ) : <EOL> data = data [ <NUM_LIT:1> : ] <EOL> if data . endswith ( '<STR_LIT:">' ) : <EOL> data = data [ : - <NUM_LIT:1> ] <EOL> data = data . replace ( "<STR_LIT:'>" , '<STR_LIT:">' ) <EOL> data = json . loads ( data ) <EOL> self . _log_locations_json_sanity_check ( data ) <EOL> logs . extend ( data [ '<STR_LIT>' ] ) <EOL> else : <EOL> log_locations = data . split ( '<STR_LIT:U+002C>' ) <EOL> logs = [ { '<STR_LIT:name>' : name . strip ( ) , '<STR_LIT:type>' : None } <EOL> for name in log_locations ] <EOL> except ( KeyError , ValueError ) as e : <EOL> logger . debug ( '<STR_LIT>' % e ) <EOL> except ( IOError , OSError ) as e : <EOL> logger . debug ( e ) <EOL> return logs <EOL> def _get_container_log_files ( self , rootfs_path , <EOL> options = defaults . DEFAULT_CRAWL_OPTIONS , <EOL> ) : <EOL> """<STR_LIT>""" <EOL> container_logs = options [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> logs = self . _parse_log_locations ( <EOL> var = '<STR_LIT>' , <EOL> isJson = False ) <EOL> container_logs . extend ( logs ) <EOL> logs = self . _parse_log_locations ( <EOL> var = '<STR_LIT>' , <EOL> isJson = True ) <EOL> container_logs . extend ( logs ) <EOL> for log in container_logs : <EOL> name = log [ '<STR_LIT:name>' ] <EOL> if not os . path . isabs ( name ) or '<STR_LIT:..>' in name : <EOL> container_logs . remove ( log ) <EOL> logger . warning ( <EOL> '<STR_LIT>' % <EOL> name ) <EOL> return container_logs <EOL> def _get_logfiles_list ( self , <EOL> host_log_dir , <EOL> options = defaults . DEFAULT_CRAWL_OPTIONS ) : <EOL> """<STR_LIT>""" <EOL> rootfs_path = get_docker_container_rootfs_path ( <EOL> self . long_id , self . inspect ) <EOL> logs_list = [ ] <EOL> for logdict in self . _get_container_log_files ( rootfs_path , <EOL> options ) : <EOL> name = logdict [ '<STR_LIT:name>' ] <EOL> _type = logdict [ '<STR_LIT:type>' ] <EOL> log_source = rootfs_path + name <EOL> log_dest = host_log_dir + name <EOL> log = { <EOL> '<STR_LIT:name>' : name , <EOL> '<STR_LIT:type>' : _type , <EOL> '<STR_LIT:source>' : log_source , <EOL> '<STR_LIT>' : log_dest } <EOL> if log not in logs_list : <EOL> logs_list . append ( log ) <EOL> docker_log_source = get_docker_container_json_logs_path ( <EOL> self . long_id , self . inspect ) <EOL> name = '<STR_LIT>' <EOL> docker_log_dest = os . path . join ( host_log_dir , name ) <EOL> logs_list . append ( { '<STR_LIT:name>' : name , <EOL> '<STR_LIT:type>' : None , <EOL> '<STR_LIT:source>' : docker_log_source , <EOL> '<STR_LIT>' : docker_log_dest } ) <EOL> return logs_list <EOL> def _get_logfiles_links_dest ( <EOL> self , <EOL> host_log_basedir , <EOL> environment = '<STR_LIT>' , <EOL> ) : <EOL> """<STR_LIT>""" <EOL> container = self <EOL> ctr_namespace = container . namespace <EOL> pid = container . pid <EOL> short_id = container . short_id <EOL> long_id = container . long_id <EOL> if environment == '<STR_LIT>' : <EOL> host_log_dir = os . path . join ( host_log_basedir , ctr_namespace ) <EOL> return host_log_dir <EOL> elif environment == '<STR_LIT>' : <EOL> if not self . is_docker_container ( ) : <EOL> logger . info ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' % pid ) <EOL> return None <EOL> else : <EOL> if alchemy is None : <EOL> raise ImportError ( '<STR_LIT>' ) <EOL> host_dir = alchemy . get_logs_dir_on_host ( long_id , '<STR_LIT>' ) <EOL> if not host_dir : <EOL> logger . info ( '<STR_LIT>' <EOL> % short_id ) <EOL> host_log_dir = os . path . join ( host_log_basedir , host_dir ) <EOL> return host_log_dir <EOL> else : <EOL> logger . error ( '<STR_LIT>' % environment ) <EOL> return None </s>
