lib/wmap/url_crawler.rb
# A web crawler to crawl a known website and search for html links within the same root domain. For example,
# by crawling 'http://www.yahoo.com/' it could discover 'http://login.yahoo.com/'
A web crawler to crawl a known website and search for html links within the same root domain.
A web crawler to crawl a known website and search for html links within the same root domain .
def crawl(url)
		puts "Start web crawling on #{url}"
		result=Array.new
		url=url.chomp.strip
		result.push(url_2_site(url))
		raise "Error! Invalid url format: #{urls}" unless is_url?(url)
		pre_crawl(url)
		status = Timeout::timeout(Crawl_timeout/1000) {
			result+=crawl_worker(url).keys
		}
		puts "Web crawling time-out on #{url}: #{status}" if @verbose
		return result
	rescue => ee
		puts "Exception on method #{__method__} for URL #{url}: #{ee}"
		return result
	end
def crawl ( url ) puts "Start web crawling on #{url}" result = Array . new url = url . chomp . strip result . push ( url_2_site ( url ) ) raise "Error! Invalid url format: #{urls}" unless is_url? ( url ) pre_crawl ( url ) status = Timeout :: timeout ( Crawl_timeout / 1000 ) { result += crawl_worker ( url ) . keys } puts "Web crawling time-out on #{url}: #{status}" if @verbose return result rescue => ee puts "Exception on method #{__method__} for URL #{url}: #{ee}" return result end
A web crawler to crawl a known website and search for html links within the same root domain.
A web crawler to crawl a known website and search for html links within the same root domain .
# Add logic to profile the web server before crawling; this is used to optimize the crawling speed
url
<not_specific>