<s> import os <EOL> import re <EOL> def parse_url_rules ( urls_fp ) : <EOL> """<STR_LIT>""" <EOL> url_rules = [ ] <EOL> for line in urls_fp : <EOL> re_url = line . strip ( ) <EOL> if re_url : <EOL> url_rules . append ( { '<STR_LIT:str>' : re_url , '<STR_LIT>' : re . compile ( re_url ) } ) <EOL> return url_rules <EOL> def makedir_if_none_exists ( d ) : <EOL> if not os . path . exists ( d ) : <EOL> os . makedirs ( d ) <EOL> def total_seconds ( td ) : <EOL> """<STR_LIT>""" <EOL> return ( td . microseconds + ( td . seconds + td . days * <NUM_LIT> * <NUM_LIT> ) * <NUM_LIT> ) / <NUM_LIT> <EOL> def smart_str ( s , encoding = '<STR_LIT:utf-8>' ) : <EOL> if isinstance ( s , unicode ) : <EOL> return s . encode ( encoding ) <EOL> elif s and encoding != '<STR_LIT:utf-8>' : <EOL> return s . decode ( '<STR_LIT:utf-8>' , '<STR_LIT:ignore>' ) . encode ( encoding , '<STR_LIT:ignore>' ) <EOL> else : <EOL> return str ( s ) <EOL> def smart_unicode ( s , encoding = '<STR_LIT:utf-8>' ) : <EOL> if isinstance ( s , unicode ) : <EOL> return s <EOL> return s . decode ( encoding , '<STR_LIT:ignore>' ) </s>
