<s> """<STR_LIT>""" <EOL> from __future__ import unicode_literals , absolute_import <EOL> from collections import namedtuple <EOL> from optparse import OptionParser , OptionGroup <EOL> from pylinkchecker . compat import get_safe_str <EOL> from pylinkchecker . urlutil import get_clean_url_split <EOL> DEFAULT_TYPES = [ '<STR_LIT:a>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> TYPE_ATTRIBUTES = { <EOL> '<STR_LIT:a>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT:src>' , <EOL> '<STR_LIT>' : '<STR_LIT:src>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> } <EOL> DEFAULT_TIMEOUT = <NUM_LIT:10> <EOL> MODE_THREAD = "<STR_LIT>" <EOL> MODE_PROCESS = "<STR_LIT>" <EOL> MODE_GREEN = "<STR_LIT>" <EOL> DEFAULT_WORKERS = { <EOL> MODE_THREAD : <NUM_LIT:1> , <EOL> MODE_PROCESS : <NUM_LIT:1> , <EOL> MODE_GREEN : <NUM_LIT:1000> , <EOL> } <EOL> PARSER_STDLIB = "<STR_LIT>" <EOL> PARSER_LXML = "<STR_LIT>" <EOL> PARSER_HTML5 = "<STR_LIT>" <EOL> FORMAT_PLAIN = "<STR_LIT>" <EOL> FORMAT_HTML = "<STR_LIT:html>" <EOL> FORMAT_JSON = "<STR_LIT>" <EOL> WHEN_ALWAYS = "<STR_LIT>" <EOL> WHEN_ON_ERROR = "<STR_LIT:error>" <EOL> REPORT_TYPE_ERRORS = "<STR_LIT>" <EOL> REPORT_TYPE_SUMMARY = "<STR_LIT>" <EOL> REPORT_TYPE_ALL = "<STR_LIT:all>" <EOL> VERBOSE_QUIET = "<STR_LIT:0>" <EOL> VERBOSE_NORMAL = "<STR_LIT:1>" <EOL> VERBOSE_INFO = "<STR_LIT:2>" <EOL> HTML_MIME_TYPE = "<STR_LIT>" <EOL> PAGE_QUEUED = '<STR_LIT>' <EOL> PAGE_CRAWLED = '<STR_LIT>' <EOL> WorkerInit = namedtuple ( "<STR_LIT>" , [ "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" ] ) <EOL> WorkerConfig = namedtuple ( "<STR_LIT>" , [ "<STR_LIT:username>" , "<STR_LIT:password>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] ) <EOL> WorkerInput = namedtuple ( "<STR_LIT>" , [ "<STR_LIT>" , "<STR_LIT>" ] ) <EOL> Response = namedtuple ( "<STR_LIT>" , [ "<STR_LIT:content>" , "<STR_LIT:status>" , "<STR_LIT>" , <EOL> "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] ) <EOL> ExceptionStr = namedtuple ( "<STR_LIT>" , [ "<STR_LIT>" , "<STR_LIT:message>" ] ) <EOL> Link = namedtuple ( "<STR_LIT>" , [ "<STR_LIT:type>" , "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT>" ] ) <EOL> PageCrawl = namedtuple ( "<STR_LIT>" , [ "<STR_LIT>" , "<STR_LIT>" , <EOL> "<STR_LIT:status>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] ) <EOL> PageStatus = namedtuple ( "<STR_LIT>" , [ "<STR_LIT:status>" , "<STR_LIT>" ] ) <EOL> PageSource = namedtuple ( "<STR_LIT>" , [ "<STR_LIT>" , "<STR_LIT>" ] ) <EOL> class UTF8Class ( object ) : <EOL> """<STR_LIT>""" <EOL> def __str__ ( self ) : <EOL> return get_safe_str ( self . __unicode__ ( ) ) <EOL> def __repr__ ( self ) : <EOL> return get_safe_str ( self . __unicode__ ( ) ) <EOL> class LazyLogParam ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , func ) : <EOL> self . func = func <EOL> def __str__ ( self ) : <EOL> return str ( self . func ( ) ) <EOL> class Config ( UTF8Class ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self ) : <EOL> self . parser = self . _build_parser ( ) <EOL> self . options = None <EOL> self . start_urls = [ ] <EOL> self . worker_config = None <EOL> self . accepted_hosts = [ ] <EOL> self . ignored_prefixes = [ ] <EOL> self . worker_size = <NUM_LIT:0> <EOL> def should_crawl ( self , url_split ) : <EOL> """<STR_LIT>""" <EOL> return not self . options . run_once and self . is_local ( url_split ) <EOL> def is_local ( self , url_split ) : <EOL> """<STR_LIT>""" <EOL> return url_split . netloc in self . accepted_hosts <EOL> def should_download ( self , url_split ) : <EOL> """<STR_LIT>""" <EOL> local = self . is_local ( url_split ) <EOL> if not self . options . test_outside and not local : <EOL> return False <EOL> url = url_split . geturl ( ) <EOL> for ignored_prefix in self . ignored_prefixes : <EOL> if url . startswith ( ignored_prefix ) : <EOL> return False <EOL> return True <EOL> def parse_cli_config ( self ) : <EOL> """<STR_LIT>""" <EOL> ( self . options , self . start_urls ) = self . parser . parse_args ( ) <EOL> self . _parse_config ( ) <EOL> def parse_api_config ( self , start_urls , options_dict = None ) : <EOL> """<STR_LIT>""" <EOL> options = self . _get_options ( options_dict ) <EOL> ( self . options , self . start_urls ) = self . parser . parse_args ( options + start_urls ) <EOL> self . _parse_config ( ) <EOL> def _get_options ( self , options_dict ) : <EOL> if not options_dict : <EOL> options_dict = { } <EOL> options = [ ] <EOL> for key , value in options_dict . items ( ) : <EOL> if isinstance ( value , bool ) and value : <EOL> options . append ( "<STR_LIT>" . format ( key ) ) <EOL> else : <EOL> options . append ( "<STR_LIT>" . format ( key , value ) ) <EOL> return options <EOL> def _parse_config ( self ) : <EOL> self . worker_config = self . _build_worker_config ( self . options ) <EOL> self . accepted_hosts = self . _build_accepted_hosts ( self . options , <EOL> self . start_urls ) <EOL> if self . options . ignored_prefixes : <EOL> self . ignored_prefixes = self . options . ignored_prefixes . split ( '<STR_LIT:U+002C>' ) <EOL> if self . options . workers : <EOL> self . worker_size = self . options . workers <EOL> else : <EOL> self . worker_size = DEFAULT_WORKERS [ self . options . mode ] <EOL> def _build_worker_config ( self , options ) : <EOL> types = options . types . split ( '<STR_LIT:U+002C>' ) <EOL> for element_type in types : <EOL> if element_type not in DEFAULT_TYPES : <EOL> raise ValueError ( "<STR_LIT>" <EOL> . format ( element_type ) ) <EOL> return WorkerConfig ( options . username , options . password , types , <EOL> options . timeout , options . parser , options . strict_mode ) <EOL> def _build_accepted_hosts ( self , options , start_urls ) : <EOL> hosts = set ( ) <EOL> urls = [ ] <EOL> if self . options . accepted_hosts : <EOL> urls = self . options . accepted_hosts . split ( '<STR_LIT:U+002C>' ) <EOL> urls = urls + start_urls <EOL> for url in urls : <EOL> split_result = get_clean_url_split ( url ) <EOL> hosts . add ( split_result . netloc ) <EOL> return hosts <EOL> def _build_parser ( self ) : <EOL> import pylinkchecker <EOL> version = pylinkchecker . __version__ <EOL> parser = OptionParser ( usage = "<STR_LIT>" , <EOL> version = "<STR_LIT>" . format ( version ) ) <EOL> parser . add_option ( "<STR_LIT>" , "<STR_LIT>" , dest = "<STR_LIT>" , action = "<STR_LIT:store>" , <EOL> default = VERBOSE_QUIET , choices = [ VERBOSE_QUIET , VERBOSE_NORMAL , <EOL> VERBOSE_INFO ] ) <EOL> crawler_group = OptionGroup ( parser , "<STR_LIT>" , <EOL> "<STR_LIT>" ) <EOL> crawler_group . add_option ( "<STR_LIT>" , "<STR_LIT>" , dest = "<STR_LIT>" , <EOL> action = "<STR_LIT:store_true>" , default = False , <EOL> help = "<STR_LIT>" ) <EOL> crawler_group . add_option ( "<STR_LIT>" , "<STR_LIT>" , <EOL> dest = "<STR_LIT>" , action = "<STR_LIT:store>" , default = None , <EOL> help = "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> crawler_group . add_option ( "<STR_LIT>" , "<STR_LIT>" , dest = "<STR_LIT>" , <EOL> action = "<STR_LIT:store>" , default = None , <EOL> help = "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> crawler_group . add_option ( "<STR_LIT>" , "<STR_LIT>" , dest = "<STR_LIT:username>" , <EOL> action = "<STR_LIT:store>" , default = None , <EOL> help = "<STR_LIT>" ) <EOL> crawler_group . add_option ( "<STR_LIT>" , "<STR_LIT>" , dest = "<STR_LIT:password>" , <EOL> action = "<STR_LIT:store>" , default = None , <EOL> help = "<STR_LIT>" ) <EOL> crawler_group . add_option ( "<STR_LIT>" , "<STR_LIT>" , dest = "<STR_LIT>" , action = "<STR_LIT:store>" , <EOL> default = "<STR_LIT:U+002C>" . join ( DEFAULT_TYPES ) , <EOL> help = "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> crawler_group . add_option ( "<STR_LIT>" , "<STR_LIT>" , dest = "<STR_LIT>" , <EOL> type = "<STR_LIT:int>" , action = "<STR_LIT:store>" , default = DEFAULT_TIMEOUT , <EOL> help = "<STR_LIT>" ) <EOL> crawler_group . add_option ( "<STR_LIT>" , "<STR_LIT>" , dest = "<STR_LIT>" , <EOL> action = "<STR_LIT:store_true>" , default = False , <EOL> help = "<STR_LIT>" ) <EOL> crawler_group . add_option ( "<STR_LIT>" , "<STR_LIT>" , dest = "<STR_LIT>" , <EOL> action = "<STR_LIT:store_true>" , default = False , <EOL> help = "<STR_LIT>" ) <EOL> crawler_group . add_option ( "<STR_LIT>" , "<STR_LIT>" , dest = "<STR_LIT>" , <EOL> action = "<STR_LIT:store_true>" , default = False , <EOL> help = "<STR_LIT>" ) <EOL> parser . add_option_group ( crawler_group ) <EOL> perf_group = OptionGroup ( parser , "<STR_LIT>" , <EOL> "<STR_LIT>" ) <EOL> perf_group . add_option ( "<STR_LIT>" , "<STR_LIT>" , dest = "<STR_LIT>" , action = "<STR_LIT:store>" , <EOL> default = None , type = "<STR_LIT:int>" , <EOL> help = "<STR_LIT>" ) <EOL> perf_group . add_option ( "<STR_LIT>" , "<STR_LIT>" , dest = "<STR_LIT>" , action = "<STR_LIT:store>" , <EOL> default = MODE_THREAD , choices = [ MODE_THREAD , MODE_PROCESS , <EOL> MODE_GREEN ] , <EOL> help = "<STR_LIT>" ) <EOL> perf_group . add_option ( "<STR_LIT>" , "<STR_LIT>" , dest = "<STR_LIT>" , action = "<STR_LIT:store>" , <EOL> default = PARSER_STDLIB , choices = [ PARSER_STDLIB , PARSER_LXML , <EOL> PARSER_HTML5 ] , <EOL> help = "<STR_LIT>" ) <EOL> parser . add_option_group ( perf_group ) <EOL> output_group = OptionGroup ( parser , "<STR_LIT>" , <EOL> "<STR_LIT>" ) <EOL> output_group . add_option ( "<STR_LIT>" , "<STR_LIT>" , dest = "<STR_LIT>" , action = "<STR_LIT:store>" , <EOL> default = FORMAT_PLAIN , choices = [ FORMAT_PLAIN ] , <EOL> help = "<STR_LIT>" ) <EOL> output_group . add_option ( "<STR_LIT>" , "<STR_LIT>" , dest = "<STR_LIT>" , action = "<STR_LIT:store>" , <EOL> default = None , <EOL> help = "<STR_LIT>" ) <EOL> output_group . add_option ( "<STR_LIT>" , "<STR_LIT>" , dest = "<STR_LIT>" , action = "<STR_LIT:store>" , <EOL> default = WHEN_ALWAYS , choices = [ WHEN_ALWAYS , WHEN_ON_ERROR ] , <EOL> help = "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> output_group . add_option ( "<STR_LIT>" , "<STR_LIT>" , dest = "<STR_LIT>" , <EOL> action = "<STR_LIT:store>" , default = REPORT_TYPE_ERRORS , choices = [ <EOL> REPORT_TYPE_ERRORS , REPORT_TYPE_SUMMARY , REPORT_TYPE_ALL ] , <EOL> help = "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> output_group . add_option ( "<STR_LIT:-c>" , "<STR_LIT>" , dest = "<STR_LIT>" , <EOL> action = "<STR_LIT:store_true>" , default = False , <EOL> help = "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> crawler_group . add_option ( "<STR_LIT>" , "<STR_LIT>" , dest = "<STR_LIT>" , <EOL> action = "<STR_LIT:store_true>" , default = False , <EOL> help = "<STR_LIT>" ) <EOL> parser . add_option_group ( output_group ) <EOL> email_group = OptionGroup ( parser , "<STR_LIT>" , <EOL> "<STR_LIT>" ) <EOL> email_group . add_option ( "<STR_LIT>" , "<STR_LIT>" , dest = "<STR_LIT:address>" , action = "<STR_LIT:store>" , <EOL> default = None , <EOL> help = "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> email_group . add_option ( "<STR_LIT>" , dest = "<STR_LIT>" , action = "<STR_LIT:store>" , <EOL> default = None , <EOL> help = "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> email_group . add_option ( "<STR_LIT>" , "<STR_LIT>" , dest = "<STR_LIT>" , action = "<STR_LIT:store>" , <EOL> default = None , <EOL> help = "<STR_LIT>" ) <EOL> email_group . add_option ( "<STR_LIT>" , dest = "<STR_LIT:port>" , action = "<STR_LIT:store>" , <EOL> default = <NUM_LIT> , type = "<STR_LIT:int>" , <EOL> help = "<STR_LIT>" ) <EOL> email_group . add_option ( "<STR_LIT>" , dest = "<STR_LIT>" , action = "<STR_LIT:store_true>" , <EOL> default = False , <EOL> help = "<STR_LIT>" ) <EOL> email_group . add_option ( "<STR_LIT>" , dest = "<STR_LIT>" , action = "<STR_LIT:store>" , <EOL> default = None , <EOL> help = "<STR_LIT>" ) <EOL> email_group . add_option ( "<STR_LIT>" , dest = "<STR_LIT>" , <EOL> action = "<STR_LIT:store>" , default = None , <EOL> help = "<STR_LIT>" ) <EOL> email_group . add_option ( "<STR_LIT>" , dest = "<STR_LIT>" , <EOL> action = "<STR_LIT:store>" , default = None , <EOL> help = "<STR_LIT>" ) <EOL> parser . add_option_group ( email_group ) <EOL> return parser <EOL> def __unicode__ ( self ) : <EOL> return "<STR_LIT>" . format ( <EOL> self . start_urls , self . options ) <EOL> class SitePage ( UTF8Class ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , url_split , status = <NUM_LIT:200> , is_timeout = False , exception = None , <EOL> is_html = True , is_local = True ) : <EOL> self . url_split = url_split <EOL> self . original_source = None <EOL> self . sources = [ ] <EOL> self . type = type <EOL> self . status = status <EOL> self . is_timeout = is_timeout <EOL> self . exception = exception <EOL> self . is_html = is_html <EOL> self . is_local = is_local <EOL> self . is_ok = status and status < <NUM_LIT> <EOL> def add_sources ( self , page_sources ) : <EOL> self . sources . extend ( page_sources ) <EOL> def get_status_message ( self ) : <EOL> if self . status : <EOL> if self . status < <NUM_LIT> : <EOL> return "<STR_LIT>" . format ( self . status ) <EOL> elif self . status == <NUM_LIT> : <EOL> return "<STR_LIT>" <EOL> else : <EOL> return "<STR_LIT>" . format ( self . status ) <EOL> elif self . is_timeout : <EOL> return "<STR_LIT>" <EOL> elif self . exception : <EOL> return "<STR_LIT>" . format ( self . exception . type_name , <EOL> self . exception . message ) <EOL> else : <EOL> return "<STR_LIT:error>" <EOL> def __unicode__ ( self ) : <EOL> return "<STR_LIT>" . format ( self . url_split . geturl ( ) , self . status ) </s>
