lib/wmap/google_search_scraper.rb
# Search for nodes by css, and extract the hyper links
Search for nodes by css, and extract the hyper links
Search for nodes by css and extract the hyper links
def extract_links (doc)
		begin
			puts "Extract the meaningful links from the DOC." if @verbose
			links=Array.new
			doc.css('a').each do |link|
				ref=link.attribute('href').to_s
				if ref =~ /\/url\?/
					my_key=ref.sub(/\/url\?q\=/,'')
					my_site=url_2_site(my_key)
					links.push(my_key)
					@discovered_urls_from_scraper[my_key]=true unless @discovered_urls_from_scraper.key?(my_key)
					@discovered_sites_from_scraper[my_site]=true unless @discovered_sites_from_scraper.key?(my_site)
				end
			end
			return links
		rescue Exception => ee
			puts "Exception on method extract_links: #{ee}" if @verbose
			return nil
		end 
	end
def extract_links ( doc ) begin puts "Extract the meaningful links from the DOC." if @verbose links = Array . new doc . css ( 'a' ) . each do | link | ref = link . attribute ( 'href' ) . to_s if ref =~ / \/ url \? / my_key = ref . sub ( / \/ url \? q \= / , '' ) my_site = url_2_site ( my_key ) links . push ( my_key ) @discovered_urls_from_scraper [ my_key ] = true unless @discovered_urls_from_scraper . key? ( my_key ) @discovered_sites_from_scraper [ my_site ] = true unless @discovered_sites_from_scraper . key? ( my_site ) end end return links rescue Exception => ee puts "Exception on method extract_links: #{ee}" if @verbose return nil end end
Search for nodes by css, and extract the hyper links
Search for nodes by css and extract the hyper links
doc
<not_specific>