lib/wmap/url_crawler.rb
# Wrapper for the OpenURI open method - create an open_uri object and return the reference upon success
Wrapper for the OpenURI open method - create an open_uri object and return the reference upon success
Wrapper for the OpenURI open method - create an open_uri object and return the reference upon success
def open_url(url,user_agent=@user_agent)
		puts "Open url #{url} by creating an open_uri object. Return the reference upon success." if @verbose
		if url =~ /http\:/i
			url_object = open(url, :allow_redirections=>:safe, :read_timeout=>Max_http_timeout/1000, "User-Agent"=>user_agent)
		elsif url =~ /https\:/i
			url_object = open(url, :ssl_verify_mode=>0, :allow_redirections=>:safe, :read_timeout=>Max_http_timeout/1000, "User-Agent"=>user_agent)
		else
			raise "Invalid URL format - please specify the protocol prefix http(s) in the URL: #{url}"
		end
		return url_object
  rescue => ee
    puts "Exception on method #{__method__} for #{url}: #{ee}" if @verbose
    return nil
  end
def open_url ( url , user_agent = @user_agent ) puts "Open url #{url} by creating an open_uri object. Return the reference upon success." if @verbose if url =~ / http \: /i url_object = open ( url , :allow_redirections => :safe , :read_timeout => Max_http_timeout / 1000 , "User-Agent" => user_agent ) elsif url =~ / https \: /i url_object = open ( url , :ssl_verify_mode => 0 , :allow_redirections => :safe , :read_timeout => Max_http_timeout / 1000 , "User-Agent" => user_agent ) else raise "Invalid URL format - please specify the protocol prefix http(s) in the URL: #{url}" end return url_object rescue => ee puts "Exception on method #{__method__} for #{url}: #{ee}" if @verbose return nil end
Wrapper for the OpenURI open method - create an open_uri object and return the reference upon success
Wrapper for the OpenURI open method - create an open_uri object and return the reference upon success
# patch for allow the 'un-safe' URL redirection i.e. https://www.example.com -> http://www.example.com #url_object = open(url) #url_object = open(url,:ssl_verify_mode => 0)
url
<not_specific>
user_agent
<not_specific>