<s> import urlparse <EOL> import httplib <EOL> import logging <EOL> import re <EOL> import threading <EOL> import Queue <EOL> from bs4 import BeautifulSoup <EOL> import multiprocessing <EOL> import time <EOL> from string import Template <EOL> import glob <EOL> import socket <EOL> import ipaddress <EOL> import os <EOL> import webbrowser <EOL> from lib . cmdline import parse_args <EOL> from lib . interface import InfoDisScannerBase <EOL> from lib . report import TEMPLATE_host , TEMPLATE_html , TEMPLATE_severity_high , TEMPLATE_severity_normal <EOL> class InfoDisScanner ( InfoDisScannerBase ) : <EOL> def __init__ ( self , url , lock , timeout = <NUM_LIT> , depth = <NUM_LIT:2> ) : <EOL> self . START_TIME = time . time ( ) <EOL> self . TIME_OUT = timeout <EOL> self . LINKS_LIMIT = <NUM_LIT:20> <EOL> self . final_severity = <NUM_LIT:0> <EOL> self . schema , self . host , self . path = self . _parse_url ( url ) <EOL> self . max_depth = self . _cal_depth ( self . path ) [ <NUM_LIT:1> ] + depth <EOL> self . check_404 ( ) <EOL> if self . _status == - <NUM_LIT:1> : <EOL> return None <EOL> if not self . has_404 : <EOL> print '<STR_LIT>' % self . host <EOL> self . _init_rules ( ) <EOL> self . url_queue = Queue . Queue ( ) <EOL> self . urls_in_queue = [ ] <EOL> _path , _depth = self . _cal_depth ( self . path ) <EOL> self . _enqueue ( _path ) <EOL> self . crawl_index ( _path ) <EOL> self . lock = threading . Lock ( ) <EOL> self . results = { } <EOL> @ staticmethod <EOL> def _parse_url ( url ) : <EOL> _ = urlparse . urlparse ( url , '<STR_LIT:http>' ) <EOL> if not _ . netloc : <EOL> _ = urlparse . urlparse ( '<STR_LIT>' + url , '<STR_LIT:http>' ) <EOL> assert ( _ . netloc != '<STR_LIT>' ) <EOL> return _ . scheme , _ . netloc , _ . path if _ . path else '<STR_LIT:/>' <EOL> def _cal_depth ( self , url ) : <EOL> """<STR_LIT>""" <EOL> if url . find ( '<STR_LIT:#>' ) >= <NUM_LIT:0> : url = url [ : url . find ( '<STR_LIT:#>' ) ] <EOL> if url . find ( '<STR_LIT:?>' ) >= <NUM_LIT:0> : url = url [ : url . find ( '<STR_LIT:?>' ) ] <EOL> if url . startswith ( '<STR_LIT>' ) : return '<STR_LIT>' , <NUM_LIT> <EOL> if url . startswith ( '<STR_LIT>' ) : return '<STR_LIT>' , <NUM_LIT> <EOL> if url . startswith ( '<STR_LIT:http>' ) : <EOL> _ = urlparse . urlparse ( url , '<STR_LIT:http>' ) <EOL> if _ . netloc == self . host : <EOL> url = _ . path <EOL> else : <EOL> return '<STR_LIT>' , <NUM_LIT> <EOL> url = url . replace ( '<STR_LIT>' , '<STR_LIT:/>' ) <EOL> if not url : return '<STR_LIT:/>' , <NUM_LIT:1> <EOL> if url [ <NUM_LIT:0> ] != '<STR_LIT:/>' : url = '<STR_LIT:/>' + url <EOL> url = url [ : url . rfind ( '<STR_LIT:/>' ) + <NUM_LIT:1> ] <EOL> depth = len ( url . split ( '<STR_LIT:/>' ) ) - <NUM_LIT:1> <EOL> return url , depth <EOL> def _init_rules ( self ) : <EOL> try : <EOL> self . url_dict = [ ] <EOL> p_severity = re . compile ( '<STR_LIT>' ) <EOL> p_tag = re . compile ( '<STR_LIT>' ) <EOL> p_status = re . compile ( '<STR_LIT>' ) <EOL> p_content_type = re . compile ( '<STR_LIT>' ) <EOL> p_content_type_no = re . compile ( '<STR_LIT>' ) <EOL> for rule_file in glob . glob ( '<STR_LIT>' ) : <EOL> infile = open ( rule_file , '<STR_LIT:r>' ) <EOL> for url in infile : <EOL> url = url . strip ( ) . replace ( '<STR_LIT>' , self . host ) <EOL> if url . startswith ( '<STR_LIT:/>' ) : <EOL> _ = p_severity . search ( url ) <EOL> severity = int ( _ . group ( <NUM_LIT:1> ) ) if _ else <NUM_LIT:3> <EOL> _ = p_tag . search ( url ) <EOL> tag = _ . group ( <NUM_LIT:1> ) if _ else '<STR_LIT>' <EOL> _ = p_status . search ( url ) <EOL> status = int ( _ . group ( <NUM_LIT:1> ) ) if _ else <NUM_LIT:0> <EOL> _ = p_content_type . search ( url ) <EOL> content_type = _ . group ( <NUM_LIT:1> ) if _ else '<STR_LIT>' <EOL> _ = p_content_type_no . search ( url ) <EOL> content_type_no = _ . group ( <NUM_LIT:1> ) if _ else '<STR_LIT>' <EOL> url = url . split ( ) [ <NUM_LIT:0> ] <EOL> self . url_dict . append ( ( url , severity , tag , status , content_type , content_type_no ) ) <EOL> infile . close ( ) <EOL> except Exception , e : <EOL> logging . error ( '<STR_LIT>' % e ) <EOL> def _http_request ( self , url , timeout = <NUM_LIT> ) : <EOL> try : <EOL> if not url : url = '<STR_LIT:/>' <EOL> conn_fuc = httplib . HTTPSConnection if self . schema == '<STR_LIT>' else httplib . HTTPConnection <EOL> conn = conn_fuc ( self . host , timeout = timeout ) <EOL> conn . request ( method = '<STR_LIT:GET>' , url = url , <EOL> headers = { '<STR_LIT>' : '<STR_LIT>' } <EOL> ) <EOL> resp = conn . getresponse ( ) <EOL> resp_headers = dict ( resp . getheaders ( ) ) <EOL> status = resp . status <EOL> if resp_headers . get ( '<STR_LIT>' , '<STR_LIT>' ) . find ( '<STR_LIT:text>' ) >= <NUM_LIT:0> or resp_headers . get ( '<STR_LIT>' , '<STR_LIT>' ) . find ( '<STR_LIT:html>' ) >= <NUM_LIT:0> or int ( resp_headers . get ( '<STR_LIT>' , '<STR_LIT:0>' ) ) <= <NUM_LIT> : <EOL> html_doc = self . _decode_response_text ( resp . read ( ) ) <EOL> else : <EOL> html_doc = '<STR_LIT>' <EOL> conn . close ( ) <EOL> return status , resp_headers , html_doc <EOL> except Exception , e : <EOL> return - <NUM_LIT:1> , { } , '<STR_LIT>' <EOL> @ staticmethod <EOL> def _decode_response_text ( rtxt , charset = None ) : <EOL> if charset : <EOL> try : <EOL> return rtxt . decode ( charset ) <EOL> except : <EOL> pass <EOL> encodings = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> for _ in encodings : <EOL> try : <EOL> return rtxt . decode ( _ ) <EOL> except : <EOL> pass <EOL> try : <EOL> return rtxt . decode ( '<STR_LIT:ascii>' , '<STR_LIT:ignore>' ) <EOL> except : <EOL> pass <EOL> raise Exception ( '<STR_LIT>' ) <EOL> def get_status ( self , url ) : <EOL> return self . _http_request ( url ) [ <NUM_LIT:0> ] <EOL> def check_404 ( self ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> self . _status , headers , html_doc = self . _http_request ( '<STR_LIT>' ) <EOL> if self . _status == - <NUM_LIT:1> : <EOL> print '<STR_LIT>' % self . host <EOL> self . has_404 = ( self . _status == <NUM_LIT> ) <EOL> return self . has_404 <EOL> except Exception , e : <EOL> logging . error ( "<STR_LIT>" % e ) <EOL> def _enqueue ( self , url ) : <EOL> if url in self . urls_in_queue : <EOL> return False <EOL> elif len ( self . urls_in_queue ) >= self . LINKS_LIMIT : <EOL> return False <EOL> else : <EOL> self . urls_in_queue . append ( url ) <EOL> for _ in self . url_dict : <EOL> full_url = url . rstrip ( '<STR_LIT:/>' ) + _ [ <NUM_LIT:0> ] <EOL> url_description = { '<STR_LIT>' : url . rstrip ( '<STR_LIT:/>' ) , '<STR_LIT>' : full_url } <EOL> item = ( url_description , _ [ <NUM_LIT:1> ] , _ [ <NUM_LIT:2> ] , _ [ <NUM_LIT:3> ] , _ [ <NUM_LIT:4> ] , _ [ <NUM_LIT:5> ] ) <EOL> self . url_queue . put ( item ) <EOL> return True <EOL> def crawl_index ( self , path ) : <EOL> try : <EOL> status , headers , html_doc = self . _http_request ( path ) <EOL> if status != <NUM_LIT:200> : <EOL> return <EOL> soup = BeautifulSoup ( html_doc , "<STR_LIT>" ) <EOL> links = soup . find_all ( '<STR_LIT:a>' ) <EOL> for l in links : <EOL> url = l . get ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> url , depth = self . _cal_depth ( url ) <EOL> if depth <= self . max_depth : <EOL> self . _enqueue ( url ) <EOL> except Exception , e : <EOL> logging . error ( '<STR_LIT>' % e ) <EOL> def _get_url ( self ) : <EOL> """<STR_LIT>""" <EOL> self . lock . acquire ( ) <EOL> if self . url_index_offset < self . len_urls : <EOL> url = self . urls [ self . url_index_offset ] <EOL> else : <EOL> url = None , None , None <EOL> self . url_index_offset += <NUM_LIT:1> <EOL> self . lock . release ( ) <EOL> return url <EOL> def _update_severity ( self , severity ) : <EOL> if severity > self . final_severity : <EOL> self . final_severity = severity <EOL> def _scan_worker ( self ) : <EOL> while self . url_queue . qsize ( ) > <NUM_LIT:0> : <EOL> if time . time ( ) - self . START_TIME > self . TIME_OUT : <EOL> print '<STR_LIT>' % self . host <EOL> return <EOL> try : <EOL> item = self . url_queue . get ( timeout = <NUM_LIT:1.0> ) <EOL> except : <EOL> return <EOL> try : <EOL> url_description , severity , tag , code , content_type , content_type_no = item <EOL> url = url_description [ '<STR_LIT>' ] <EOL> prefix = url_description [ '<STR_LIT>' ] <EOL> except Exception , e : <EOL> logging . error ( '<STR_LIT>' % e ) <EOL> continue <EOL> if not item or not url : <EOL> break <EOL> try : <EOL> status , headers , html_doc = self . _http_request ( url ) <EOL> if ( status in [ <NUM_LIT:200> , <NUM_LIT> , <NUM_LIT> , <NUM_LIT> ] ) and ( self . has_404 or status != self . _status ) : <EOL> if code and status != code : <EOL> continue <EOL> if not tag or html_doc . find ( tag ) >= <NUM_LIT:0> : <EOL> if content_type and headers . get ( '<STR_LIT>' , '<STR_LIT>' ) . find ( content_type ) < <NUM_LIT:0> or content_type_no and headers . get ( '<STR_LIT>' , '<STR_LIT>' ) . find ( content_type_no ) >= <NUM_LIT:0> : <EOL> continue <EOL> self . lock . acquire ( ) <EOL> if not prefix in self . results : <EOL> self . results [ prefix ] = [ ] <EOL> self . results [ prefix ] . append ( { '<STR_LIT:status>' : status , '<STR_LIT:url>' : '<STR_LIT>' % ( self . schema , self . host , url ) } ) <EOL> self . _update_severity ( severity ) <EOL> self . lock . release ( ) <EOL> if len ( self . results ) >= <NUM_LIT:30> : <EOL> print '<STR_LIT>' % self . host <EOL> return <EOL> except Exception , e : <EOL> logging . error ( '<STR_LIT>' % ( url , e ) ) <EOL> def scan ( self , threads = <NUM_LIT:20> ) : <EOL> if self . _status == - <NUM_LIT:1> : <EOL> return False <EOL> threads_list = [ ] <EOL> for i in range ( threads ) : <EOL> t = threading . Thread ( target = self . _scan_worker ) <EOL> threads_list . append ( t ) <EOL> t . start ( ) <EOL> for t in threads_list : <EOL> t . join ( ) <EOL> for key in self . results . keys ( ) : <EOL> if len ( self . results [ key ] ) > <NUM_LIT:10> : <EOL> del self . results [ key ] <EOL> return self . host , self . results , self . final_severity <EOL> def batch_scan ( url , q_results , lock , threads_num , timeout ) : <EOL> print '<STR_LIT>' , url <EOL> a = InfoDisScanner ( url , lock , timeout * <NUM_LIT> ) <EOL> host , results , severity = a . scan ( threads = threads_num ) <EOL> if results : <EOL> q_results . put ( ( host , results , severity ) ) <EOL> if results : <EOL> for key in results . keys ( ) : <EOL> for url in results [ key ] : <EOL> print '<STR_LIT>' % ( url [ '<STR_LIT:status>' ] , url [ '<STR_LIT:url>' ] ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> args = parse_args ( ) <EOL> if args . d : <EOL> all_files = glob . glob ( args . d + '<STR_LIT>' ) <EOL> elif args . f : <EOL> all_files = [ args . f ] <EOL> else : <EOL> all_files = [ '<STR_LIT>' ] <EOL> for file in all_files : <EOL> start_time = time . time ( ) <EOL> if args . host : <EOL> lines = [ args . host ] <EOL> else : <EOL> with open ( file ) as inFile : <EOL> lines = inFile . readlines ( ) <EOL> q_results = multiprocessing . Manager ( ) . Queue ( ) <EOL> lock = multiprocessing . Manager ( ) . Lock ( ) <EOL> pool = multiprocessing . Pool ( args . p ) <EOL> scanned_ips = [ ] <EOL> for line in lines : <EOL> if line . strip ( ) : <EOL> hosts = line . split ( ) <EOL> for host in hosts : <EOL> host = host . strip ( '<STR_LIT:U+002C>' ) <EOL> _schema , host , _path = InfoDisScanner . _parse_url ( host ) <EOL> try : <EOL> ip = socket . gethostbyname ( host . split ( '<STR_LIT::>' ) [ <NUM_LIT:0> ] ) <EOL> if ip : <EOL> scanned_ips . append ( ip ) <EOL> pool . apply_async ( func = batch_scan , args = ( host , q_results , lock , args . t , args . timeout ) ) <EOL> except Exception , e : <EOL> print e <EOL> if args . network != <NUM_LIT:32> : <EOL> for ip in scanned_ips : <EOL> _ips = ipaddress . IPv4Network ( u'<STR_LIT>' % ( ip , args . network ) , strict = False ) . hosts ( ) <EOL> for _ip in _ips : <EOL> _ip = str ( _ip ) <EOL> if _ip not in scanned_ips : <EOL> scanned_ips . append ( _ip ) <EOL> pool . apply_async ( func = batch_scan , args = ( _ip , q_results , lock , args . t , args . timeout ) ) <EOL> pool . close ( ) <EOL> pool . join ( ) <EOL> t_html = Template ( TEMPLATE_html ) <EOL> t_host = Template ( TEMPLATE_host ) <EOL> t_high = Template ( TEMPLATE_severity_high ) <EOL> t_normal = Template ( TEMPLATE_severity_normal ) <EOL> html_doc = "<STR_LIT>" <EOL> if q_results . qsize ( ) == <NUM_LIT:0> : <EOL> lock . acquire ( ) <EOL> print '<STR_LIT>' % file <EOL> lock . release ( ) <EOL> continue <EOL> while q_results . qsize ( ) > <NUM_LIT:0> : <EOL> host , results , severity = q_results . get ( ) <EOL> _str = "<STR_LIT>" <EOL> t_lst = t_high if severity == <NUM_LIT:3> else t_normal <EOL> for key in results . keys ( ) : <EOL> for _ in results [ key ] : <EOL> _str += t_lst . substitute ( { '<STR_LIT:status>' : _ [ '<STR_LIT:status>' ] , '<STR_LIT:url>' : _ [ '<STR_LIT:url>' ] } ) <EOL> _str = t_host . substitute ( { '<STR_LIT:host>' : host , '<STR_LIT:list>' : _str } ) <EOL> html_doc += _str <EOL> cost_time = time . time ( ) - start_time <EOL> cost_min = int ( cost_time / <NUM_LIT> ) <EOL> cost_seconds = '<STR_LIT>' % ( cost_time % <NUM_LIT> ) <EOL> html_doc = t_html . substitute ( { '<STR_LIT>' : cost_min , '<STR_LIT>' : cost_seconds , '<STR_LIT:content>' : html_doc } ) <EOL> report_name = os . path . basename ( file ) . lower ( ) . replace ( '<STR_LIT>' , '<STR_LIT>' ) + '<STR_LIT:_>' + time . strftime ( '<STR_LIT>' , time . localtime ( ) ) + '<STR_LIT>' <EOL> with open ( '<STR_LIT>' % report_name , '<STR_LIT:w>' ) as outFile : <EOL> outFile . write ( html_doc ) <EOL> print '<STR_LIT>' % report_name <EOL> if args . browser : <EOL> try : <EOL> webbrowser . open_new_tab ( os . path . abspath ( '<STR_LIT>' % report_name ) ) <EOL> except : <EOL> print '<STR_LIT>' % report_name </s>
