lib/wmap/url_crawler.rb
# Search 'current_url' and return found URLs under the same domain
Search 'current_url' and return found URLs under the same domain
Search ' current_url ' and return found URLs under the same domain
def find_urls_on_page(doc, current_url)
		puts "Search and return URLs within the doc: #{doc}" if @verbose
		urls_list = []
		links=doc.css('a')
		links.map do |x|
			new_url = x.attribute('href').to_s
			unless new_url == nil
				if new_url.match("http")
						urls_list.push(new_url)
				else
					new_url = make_absolute(current_url, new_url)
					urls_list.push(new_url)
				end
			end
		end
		elements=doc.css("meta[http-equiv]")
		unless elements.size == 0
			link=elements.attr("content").value.split(/url\=/i)[1]
			unless link.nil?
				new_url = make_absolute(current_url, link)
				urls_list.push(new_url) unless new_url.nil?
			end
		end
		return urls_list.uniq-["",nil]
  rescue => ee
    puts "Exception on method #{__method__}: #{ee}" if @verbose
    return nil
  end
def find_urls_on_page ( doc , current_url ) puts "Search and return URLs within the doc: #{doc}" if @verbose urls_list = [ ] links = doc . css ( 'a' ) links . map do | x | new_url = x . attribute ( 'href' ) . to_s unless new_url == nil if new_url . match ( "http" ) urls_list . push ( new_url ) else new_url = make_absolute ( current_url , new_url ) urls_list . push ( new_url ) end end end elements = doc . css ( "meta[http-equiv]" ) unless elements . size == 0 link = elements . attr ( "content" ) . value . split ( / url \= /i ) [ 1 ] unless link . nil? new_url = make_absolute ( current_url , link ) urls_list . push ( new_url ) unless new_url . nil? end end return urls_list . uniq - [ "" , nil ] rescue => ee puts "Exception on method #{__method__}: #{ee}" if @verbose return nil end
Search 'current_url' and return found URLs under the same domain
Search ' current_url ' and return found URLs under the same domain
# case 1 - search embedded HTML tag <a href='url'> for the url elements #puts "x: #{x}" #if urls_on_same_domain?(new_url,current_url) #end # case 2 - search client side redirect - <meta http-equiv="refresh" content="5;URL='http://example.com/'"> #puts "Found URLs under page #{current_url}:\n#{urls_list}" if @verbose
doc
<not_specific>
current_url
<not_specific>