<s> import re <EOL> import urllib2 <EOL> from core . parser import Parser <EOL> from core . display import Display , ProgressBar <EOL> class Gather ( ) : <EOL> def __init__ ( self , domain , display = None ) : <EOL> self . domain = domain <EOL> self . display = display <EOL> self . results = "<STR_LIT>" <EOL> self . user_agent = "<STR_LIT>" <EOL> self . p = ProgressBar ( display = self . display ) <EOL> self . gather ( ) <EOL> self . parser = Parser ( self . results , self . domain ) <EOL> def hosts ( self ) : <EOL> return self . parser . hosts ( ) <EOL> def emails ( self ) : <EOL> return self . parser . emails ( ) <EOL> @ staticmethod <EOL> def get_sources ( ) : <EOL> return "<STR_LIT>" <EOL> def search ( self , url , offset = <NUM_LIT:1> , maxoffset = <NUM_LIT:0> , title = "<STR_LIT>" ) : <EOL> current_offset = <NUM_LIT:0> <EOL> data = "<STR_LIT>" <EOL> self . p . reset ( title = title ) <EOL> while current_offset <= maxoffset : <EOL> self . p . rotate ( ) <EOL> temp_url = re . sub ( r'<STR_LIT>' , str ( current_offset ) , url ) <EOL> try : <EOL> headers = { '<STR_LIT>' : self . user_agent } <EOL> req = urllib2 . Request ( temp_url , None , headers ) <EOL> data += urllib2 . urlopen ( req ) . read ( ) <EOL> except urllib2 . URLError as e : <EOL> self . display . error ( "<STR_LIT>" % ( title ) ) <EOL> return data <EOL> except Exception as e : <EOL> print e <EOL> current_offset += offset <EOL> self . p . done ( ) <EOL> return data <EOL> def gather ( self , maxoffset = <NUM_LIT> ) : <EOL> self . results += self . search ( title = "<STR_LIT>" , url = "<STR_LIT>" + self . domain + "<STR_LIT>" , offset = <NUM_LIT:100> , maxoffset = maxoffset ) <EOL> self . results += self . search ( title = "<STR_LIT>" , url = "<STR_LIT>" + self . domain + "<STR_LIT>" , offset = <NUM_LIT:50> , maxoffset = maxoffset ) <EOL> self . results += self . search ( title = "<STR_LIT>" , url = "<STR_LIT>" + self . domain + "<STR_LIT>" , offset = <NUM_LIT:100> , maxoffset = maxoffset ) <EOL> self . results += self . search ( title = "<STR_LIT>" , url = "<STR_LIT>" + self . domain + "<STR_LIT>" , offset = <NUM_LIT:10> , maxoffset = maxoffset / <NUM_LIT:10> ) <EOL> self . results += self . search ( title = "<STR_LIT>" , url = "<STR_LIT>" + self . domain + "<STR_LIT>" , offset = <NUM_LIT:50> , maxoffset = maxoffset ) <EOL> self . results += self . search ( title = "<STR_LIT>" , url = "<STR_LIT>" + self . domain + "<STR_LIT>" , offset = <NUM_LIT:10> , maxoffset = maxoffset / <NUM_LIT:10> ) <EOL> self . results += self . search ( title = "<STR_LIT>" , url = "<STR_LIT>" + self . domain + "<STR_LIT>" , offset = <NUM_LIT:10> , maxoffset = maxoffset / <NUM_LIT:10> ) <EOL> self . results += self . search ( title = "<STR_LIT>" , url = "<STR_LIT>" + self . domain + "<STR_LIT>" ) </s>
