lib/wmap/url_crawler.rb
# Fast crawling by utilizing fork manager parallel to spawn numbers of child processes at the same time
# each child process will continuously work on the target pool until all the works are done
Fast crawling by utilizing fork manager parallel to spawn numbers of child processes at the same time
each child process will continuously work on the target pool until all the works are done
Fast crawling by utilizing fork manager parallel to spawn numbers of child processes at the same time each child process will continuously work on the target pool until all the works are done
def crawl_workers (targets,num=@max_parallel)
		raise "Input error - expecting targets in an array format: #{targets}" unless targets.kind_of? Array
		puts "Sanitize the URL seeds to eliminate the unnecessary duplication(s) ..." if @verbose
		targets -= ["", nil]
		uniq_sites=Hash.new
		targets.dup.map do |target|
			if is_url?(target)
				host=url_2_host(target)
				ip=host_2_ip(host).to_s
				next if ip.nil?
				port=url_2_port(target).to_s
				next if port.nil?
				site_key=ip+":"+port
				unless uniq_sites.key?(site_key)
					uniq_sites[site_key]=target
				end
			end
		end
		puts "Sanitization done! " if @verbose
		puts "Start the parallel engine on the normalized crawling list:\n #{targets} "
		puts "Maximum number of web crawling sessions allowed: #{num}" 
		raise "Error: target list is empty!" if targets.size < 1
		Parallel.map(uniq_sites.values, :in_processes => num) { |target|
			puts "Working on #{target} ..." if @verbose
			crawl(target)
		}.dup.each do |process|
			puts "process.inspect: #{process}" if @verbose
			urls=process
			urls-=["",nil] unless urls.nil?
			if urls.nil?
				next
			elsif urls.empty?
				next
			else
				urls.map do |url|
					url.strip!
					@discovered_urls_by_crawler[url]=true unless @discovered_urls_by_crawler.key?(url)
				end
			end
		end
		return @discovered_urls_by_crawler.keys
	rescue Exception => ee
		puts "Exception on method #{__method__}: #{ee}" if @verbose
		return nil
	end
def crawl_workers ( targets , num = @max_parallel ) raise "Input error - expecting targets in an array format: #{targets}" unless targets . kind_of? Array puts "Sanitize the URL seeds to eliminate the unnecessary duplication(s) ..." if @verbose targets -= [ "" , nil ] uniq_sites = Hash . new targets . dup . map do | target | if is_url? ( target ) host = url_2_host ( target ) ip = host_2_ip ( host ) . to_s next if ip . nil? port = url_2_port ( target ) . to_s next if port . nil? site_key = ip + ":" + port unless uniq_sites . key? ( site_key ) uniq_sites [ site_key ] = target end end end puts "Sanitization done! " if @verbose puts "Start the parallel engine on the normalized crawling list:\n #{targets} " puts "Maximum number of web crawling sessions allowed: #{num}" raise "Error: target list is empty!" if targets . size < 1 Parallel . map ( uniq_sites . values , :in_processes => num ) { | target | puts "Working on #{target} ..." if @verbose crawl ( target ) } . dup . each do | process | puts "process.inspect: #{process}" if @verbose urls = process urls -= [ "" , nil ] unless urls . nil? if urls . nil? next elsif urls . empty? next else urls . map do | url | url . strip! @discovered_urls_by_crawler [ url ] = true unless @discovered_urls_by_crawler . key? ( url ) end end end return @discovered_urls_by_crawler . keys rescue Exception => ee puts "Exception on method #{__method__}: #{ee}" if @verbose return nil end
Fast crawling by utilizing fork manager parallel to spawn numbers of child processes at the same time
each child process will continuously work on the target pool until all the works are done
Fast crawling by utilizing fork manager parallel to spawn numbers of child processes at the same time each child process will continuously work on the target pool until all the works are done
#puts "This could be awhile depending on the list size. Please be patient ..." # 09/30/2013 Add additional logic to eliminate the duplicate target site(s) before the crawlers are invoked. #if @verbose #do nothing #$discovered_urls[url]=true unless $discovered_urls.key?(url) #return sites
targets
<not_specific>
num
<not_specific>