lib/wmap/utils/url_magic.rb
# Given an URL, open the page, then return the DOM text from a normal user perspective
Given an URL, open the page, then return the DOM text from a normal user perspective
Given an URL open the page then return the DOM text from a normal user perspective
def open_page(url)
    args = {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE, allow_redirections: :safe, \
      read_timeout: Max_http_timeout/1000, "User-Agent"=>User_agent}
    doc = Nokogiri::HTML(open(url, args))
    if doc.text.include?("Please enable JavaScript to view the page content")
      puts "Invoke headless chrome through webdriver ..." if @verbose
      args = ['--ignore-certificate-errors', '--disable-popup-blocking', '--disable-translate', '--disk-cache-size 8192']
      browser = Watir::Browser.new :chrome, headless: true, switches: %w[--user-agent=OWASP\ WMAP\ Spider]
      browser.goto(url)
      sleep(2) 
      doc = Nokogiri::HTML(browser.html)
      browser.close
    end
    puts doc.text if @verbose
    return doc
  rescue => ee
    puts "Exception on method #{__method__} for #{url}: #{ee}"
    browser.close unless browser.nil?
    return doc.text
  end
def open_page ( url ) args = { ssl_verify_mode : OpenSSL :: SSL :: VERIFY_NONE , allow_redirections : :safe , read_timeout : Max_http_timeout / 1000 , "User-Agent" => User_agent } doc = Nokogiri :: HTML ( open ( url , args ) ) if doc . text . include? ( "Please enable JavaScript to view the page content" ) puts "Invoke headless chrome through webdriver ..." if @verbose args = [ '--ignore-certificate-errors' , '--disable-popup-blocking' , '--disable-translate' , '--disk-cache-size 8192' ] browser = Watir :: Browser . new :chrome , headless : true , switches : %w[ --user-agent=OWASP\ WMAP\ Spider ] browser . goto ( url ) sleep ( 2 ) doc = Nokogiri :: HTML ( browser . html ) browser . close end puts doc . text if @verbose return doc rescue => ee puts "Exception on method #{__method__} for #{url}: #{ee}" browser . close unless browser . nil? return doc . text end
Given an URL, open the page, then return the DOM text from a normal user perspective
Given an URL open the page then return the DOM text from a normal user perspective
#Selenium::WebDriver::Chrome.path = "/usr/local/bin/chromedriver" #driver = Selenium::WebDriver.for :chrome # http://watir.com/guides/chrome/ # wait for the loading
url
<not_specific>