<s> import re <EOL> import logging <EOL> import os <EOL> import yaml <EOL> from collections import OrderedDict <EOL> from textsub import Textsub <EOL> from utils import CopyFilter <EOL> logger = logging . getLogger ( __name__ ) <EOL> DOTFILTERS_FILE = '<STR_LIT>' <EOL> DOTFILES_PATH = '<STR_LIT>' <EOL> TAG_SECRET_KEY = '<STR_LIT>' <EOL> TAG_SECRET_START = '<STR_LIT>' <EOL> TAG_SECRET_END = '<STR_LIT:$>' <EOL> keyword_dict = { <EOL> '<STR_LIT>' : r'<STR_LIT>' , <EOL> '<STR_LIT>' : r'<STR_LIT>' , <EOL> '<STR_LIT>' : r'<STR_LIT>' , <EOL> '<STR_LIT>' : r'<STR_LIT>' , <EOL> } <EOL> keyword_sub = Textsub ( keyword_dict ) <EOL> keyword_sub . compile ( ) <EOL> class CleanFilter ( object ) : <EOL> def __init__ ( self , name , rules = [ ] ) : <EOL> self . name = name <EOL> self . rules = rules <EOL> self . set_parent_rules ( ) <EOL> def __getstate__ ( self ) : <EOL> state = OrderedDict ( ) <EOL> state [ '<STR_LIT:name>' ] = self . name <EOL> state [ '<STR_LIT>' ] = self . rules <EOL> return state <EOL> def __setstate__ ( self , state ) : <EOL> self . name = state [ '<STR_LIT:name>' ] <EOL> self . rules = state [ '<STR_LIT>' ] <EOL> def set_parent_rules ( self ) : <EOL> for rule in self . rules : <EOL> rule . filter = self <EOL> def sub ( self , line ) : <EOL> for rule in self . rules : <EOL> line = rule . sub ( line ) <EOL> return line <EOL> def clean_filter_representer ( dumper , data ) : <EOL> return dumper . represent_mapping ( u'<STR_LIT>' , data . __getstate__ ( ) . items ( ) , False ) <EOL> def clean_filter_constructor ( loader , node ) : <EOL> mapping = loader . construct_mapping ( node ) <EOL> return CleanFilter ( ** mapping ) <EOL> class CleanSecret ( object ) : <EOL> def __init__ ( self , key , regex , substitute , description = '<STR_LIT>' , numbered = False ) : <EOL> self . key = key <EOL> self . description = description <EOL> self . numbered = numbered <EOL> self . regex = regex <EOL> self . substitute = substitute <EOL> self . n = <NUM_LIT:0> <EOL> self . filter = None <EOL> def __getstate__ ( self ) : <EOL> state = OrderedDict ( ) <EOL> state [ '<STR_LIT:key>' ] = self . key <EOL> state [ '<STR_LIT:description>' ] = self . description <EOL> state [ '<STR_LIT>' ] = self . numbered <EOL> state [ '<STR_LIT>' ] = self . _orig_regex <EOL> state [ '<STR_LIT>' ] = self . _substitute <EOL> return state <EOL> def __setstate__ ( self , state ) : <EOL> self . key = state [ '<STR_LIT:key>' ] <EOL> self . description = state [ '<STR_LIT:description>' ] <EOL> self . numbered = state [ '<STR_LIT>' ] <EOL> self . regex = state [ '<STR_LIT>' ] <EOL> self . substitute = state [ '<STR_LIT>' ] <EOL> self . n = <NUM_LIT:0> <EOL> self . filter = None <EOL> def get_regex ( self ) : <EOL> return self . _regex <EOL> def set_regex ( self , regex ) : <EOL> self . _orig_regex = regex <EOL> regex = keyword_sub . sub ( regex ) <EOL> self . _regex = re . compile ( regex ) <EOL> regex = property ( get_regex , set_regex ) <EOL> def sub ( self , line ) : <EOL> out = u'<STR_LIT>' <EOL> prev_start = - <NUM_LIT:1> <EOL> prev_end = - <NUM_LIT:1> <EOL> for m in self . regex . finditer ( line ) : <EOL> if prev_start == - <NUM_LIT:1> : <EOL> out += line [ : m . start ( ) ] <EOL> else : <EOL> if m . start ( ) > prev_end : <EOL> out += line [ prev_end : m . start ( ) ] <EOL> self . n += <NUM_LIT:1> <EOL> subs = m . expand ( self . substitute ) <EOL> logger . debug ( "<STR_LIT>" % ( <EOL> m . group ( <NUM_LIT:0> ) , <EOL> subs ) ) <EOL> out += subs <EOL> prev_start = m . start ( ) <EOL> prev_end = m . end ( ) <EOL> if prev_end != - <NUM_LIT:1> : <EOL> out += line [ prev_end : ] <EOL> return out <EOL> else : <EOL> return line <EOL> def get_substitute ( self ) : <EOL> key = '<STR_LIT>' <EOL> if self . numbered : <EOL> key += self . key + '<STR_LIT:_>' + unicode ( self . n ) <EOL> else : <EOL> key += self . key <EOL> return self . _substitute . replace ( TAG_SECRET_KEY , <EOL> TAG_SECRET_START + key + TAG_SECRET_END ) <EOL> def set_substitute ( self , substitute ) : <EOL> self . _substitute = substitute <EOL> substitute = property ( get_substitute , set_substitute ) <EOL> def clean_secret_representer ( dumper , data ) : <EOL> return dumper . represent_mapping ( u'<STR_LIT>' , data . __getstate__ ( ) . items ( ) , False ) <EOL> def clean_secret_constructor ( loader , node ) : <EOL> mapping = loader . construct_mapping ( node ) <EOL> return CleanSecret ( ** mapping ) <EOL> def create_filters ( ) : <EOL> s = [ ] <EOL> s . append ( CleanSecret ( "<STR_LIT>" , <EOL> r'<STR_LIT>' , <EOL> r'<STR_LIT>' , '<STR_LIT>' , True ) ) <EOL> f = [ ] <EOL> f . append ( CleanFilter ( '<STR_LIT>' , s ) ) <EOL> filters_file = open ( '<STR_LIT>' , '<STR_LIT:w>' ) <EOL> yaml . dump_all ( f , filters_file ) <EOL> filters_file . close ( ) <EOL> def load_filter ( name , filename ) : <EOL> if filename is None : <EOL> home_path = os . getenv ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> conf_path = os . getenv ( '<STR_LIT>' , <EOL> os . path . join ( home_path , DOTFILES_PATH ) ) <EOL> filename = os . path . join ( conf_path , DOTFILTERS_FILE ) <EOL> logger . debug ( "<STR_LIT>" % filename ) <EOL> with open ( filename , '<STR_LIT:r>' ) as filters_file : <EOL> for f in yaml . load_all ( filters_file ) : <EOL> if f . name == name : <EOL> f . set_parent_rules ( ) <EOL> return f <EOL> logger . warning ( "<STR_LIT>" % name ) <EOL> return CopyFilter ( ) <EOL> def clean ( args ) : <EOL> yaml . add_representer ( CleanFilter , clean_filter_representer ) <EOL> yaml . add_constructor ( u'<STR_LIT>' , clean_filter_constructor ) <EOL> yaml . add_representer ( CleanSecret , clean_secret_representer ) <EOL> yaml . add_constructor ( u'<STR_LIT>' , clean_secret_constructor ) <EOL> f = load_filter ( args . name , args . filters ) <EOL> if f is None : <EOL> logger . debug ( "<STR_LIT>" % args . name ) <EOL> return <EOL> while <NUM_LIT:1> : <EOL> try : <EOL> line = args . input . readline ( ) <EOL> except KeyboardInterrupt : <EOL> break <EOL> if not line : <EOL> break <EOL> args . output . write ( f . sub ( line ) ) </s>
