lib/anemone/http.rb
#
# Create new Pages from the response of an HTTP request to *url*,
# including redirects
#
Create new Pages from the response of an HTTP request to *url*,
including redirects
Create new Pages from the response of an HTTP request to * url * including redirects
def fetch_pages(link)
      begin
        url = URI(link.url) 
        get(url, link.referer) do |response, code, location, redirect_to, response_time|
          page = Page.new(:url           => location, 
                          :body          => response.body.dup,
                          :code          => code,
                          :headers       => response.to_hash,
                          :referer       => link.referer,
                          :depth         => link.depth,
                          :redirect_to   => redirect_to,
                          :response_time => response_time,
                          :fetched_at    => Time.now)
          page.links 
          page.enq
        end
      rescue Exception => e
        if verbose?
          puts e.inspect
          puts e.backtrace
        end
        if @opts[:link_fetch_attemps] > link.fetch_attempts
          link.attributes = {:state => Link::NEW, :error => e.message, :fetch_attempts => (link.fetch_attempts + 1)}
          link.save
        end
      end
    end
def fetch_pages ( link ) begin url = URI ( link . url ) get ( url , link . referer ) do | response , code , location , redirect_to , response_time | page = Page . new ( :url => location , :body => response . body . dup , :code => code , :headers => response . to_hash , :referer => link . referer , :depth => link . depth , :redirect_to => redirect_to , :response_time => response_time , :fetched_at => Time . now ) page . links page . enq end rescue Exception => e if verbose? puts e . inspect puts e . backtrace end if @opts [ :link_fetch_attemps ] > link . fetch_attempts link . attributes = { :state => Link :: NEW , :error => e . message , :fetch_attempts => ( link . fetch_attempts + 1 ) } link . save end end end
Create new Pages from the response of an HTTP request to *url*,
including redirects
Create new Pages from the response of an HTTP request to * url * including redirects
#unless link.url.is_a?(URI) #Process links #Enq link if it not reach to max fetch treshold
link
<not_specific>