lib/wmap/url_crawler.rb
# The worker instance of crawler who perform the labour work
The worker instance of crawler who perform the labour work
The worker instance of crawler who perform the labour work
def crawl_worker(url0)
		puts "Please be aware that it may take a while to crawl #{url0}, depending on the site's responsiveness and discovery contents."
		if is_url?(url0)
			host=url_2_host(url0)
			ip=host_2_ip(host).to_s
			raise "Invalid IP address: #{url0}" if ip.nil?
			port=url_2_port(url0).to_s
			raise "Invalid port number: #{url0}" if port.nil?
		else
			raise "Invalid URL: #{url0}. Please check it out with your browser again."
		end
		log_info=Hash.new
		log_info[1]="Start working on #{url0}"
		url_stores=Hash.new
		url_stores[url0]=true unless url_stores.key?(url0)
		@discovered_urls_by_crawler[url0]=true unless @discovered_urls_by_crawler.key?(url0)
		@crawl_start[url0]=true unless @crawl_start.key?(url0)
		@crawl_depth.times do
			url_stores.keys.each do |url|
				next if @visited_urls_by_crawler.key?(url)
				url_object = open_url(url)
				next if url_object == nil
				url = update_url_if_redirected(url, url_object)
				url_body = read_url(url)
				next if url_body.nil? or url_body.empty?
				url_stores[url]=true unless url_stores.key?(url)
				@discovered_urls_by_crawler[url]=true unless @discovered_urls_by_crawler.key?(url)
				doc = Nokogiri::HTML(url_body)
				next if doc == nil
				if url_stores.size >= @crawl_page_limit
					@discovered_urls_by_crawler.merge!(url_stores)
					puts "Finish web crawling the url: #{url0}"
					return url_stores
				end
				page_urls = find_urls_on_page(doc, url)
				page_urls.uniq!
				page_urls.map do |y|
					y=normalize_url(y)
					url_stores[y]=true unless url_stores.key?(y)
					@discovered_urls_by_crawler[y]=true unless @discovered_urls_by_crawler.key?(y)
				end
			end
		end
		puts "Finish web crawling on: #{url0}"
		log_info[2]="Finish working on: #{url0}"
		wlog(log_info, "UrlCrawler", @log_file)
		@crawl_done[url0]=true unless @crawl_done.key?(url0)
		return url_stores
	rescue => ee
		puts "Exception on method #{__method__} for URL #{url0}: #{ee}" if @verbose
		log_info[3]="Exception on #{url0}"
		wlog(log_info,"UrlCrawler",@log_file)
		return url_stores
	end
def crawl_worker ( url0 ) puts "Please be aware that it may take a while to crawl #{url0}, depending on the site's responsiveness and discovery contents." if is_url? ( url0 ) host = url_2_host ( url0 ) ip = host_2_ip ( host ) . to_s raise "Invalid IP address: #{url0}" if ip . nil? port = url_2_port ( url0 ) . to_s raise "Invalid port number: #{url0}" if port . nil? else raise "Invalid URL: #{url0}. Please check it out with your browser again." end log_info = Hash . new log_info [ 1 ] = "Start working on #{url0}" url_stores = Hash . new url_stores [ url0 ] = true unless url_stores . key? ( url0 ) @discovered_urls_by_crawler [ url0 ] = true unless @discovered_urls_by_crawler . key? ( url0 ) @crawl_start [ url0 ] = true unless @crawl_start . key? ( url0 ) @crawl_depth . times do url_stores . keys . each do | url | next if @visited_urls_by_crawler . key? ( url ) url_object = open_url ( url ) next if url_object == nil url = update_url_if_redirected ( url , url_object ) url_body = read_url ( url ) next if url_body . nil? or url_body . empty? url_stores [ url ] = true unless url_stores . key? ( url ) @discovered_urls_by_crawler [ url ] = true unless @discovered_urls_by_crawler . key? ( url ) doc = Nokogiri :: HTML ( url_body ) next if doc == nil if url_stores . size >= @crawl_page_limit @discovered_urls_by_crawler . merge! ( url_stores ) puts "Finish web crawling the url: #{url0}" return url_stores end page_urls = find_urls_on_page ( doc , url ) page_urls . uniq! page_urls . map do | y | y = normalize_url ( y ) url_stores [ y ] = true unless url_stores . key? ( y ) @discovered_urls_by_crawler [ y ] = true unless @discovered_urls_by_crawler . key? ( y ) end end end puts "Finish web crawling on: #{url0}" log_info [ 2 ] = "Finish working on: #{url0}" wlog ( log_info , "UrlCrawler" , @log_file ) @crawl_done [ url0 ] = true unless @crawl_done . key? ( url0 ) return url_stores rescue => ee puts "Exception on method #{__method__} for URL #{url0}: #{ee}" if @verbose log_info [ 3 ] = "Exception on #{url0}" wlog ( log_info , "UrlCrawler" , @log_file ) return url_stores end
The worker instance of crawler who perform the labour work
The worker instance of crawler who perform the labour work
# Input URL sanity check first #			$discovered_urls[url0]=true unless $discovered_urls.key?(url0) # 10/01/2013 add logic to avoid unnecessary crawling within the same child instance # Protection code - to avoid parsing failure on the empty or nil object #					$discovered_urls[url]=true unless $discovered_urls.key?(url) #@visited_urls_by_crawler.merge!(url_stores) #						$discovered_urls.merge!(url_stores) #						$discovered_urls[y]=true unless $discovered_urls.key?(y)
url0
<not_specific>