app/services/post_scraper.rb
# works as an alternative to scrape! when you want to scrape particular
# top-level threads of a post sequentially
# "threads" are URL permalinks to the threads to scrape, which it will scrape
# in the given order
works as an alternative to scrape. when you want to scrape particular
top-level threads of a post sequentially
"threads" are URL permalinks to the threads to scrape, which it will scrape
in the given order
works as an alternative to scrape . when you want to scrape particular top - level threads of a post sequentially " threads " are URL permalinks to the threads to scrape which it will scrape in the given order
def scrape_threads!(threads)
    raise RuntimeError.new('threaded_import must be true to use scrape_threads!') unless @threaded_import
    @html_doc = doc_from_url(@url)
    Post.transaction do
      import_post_from_doc(@html_doc)
      threads.each do |thread|
        @html_doc = doc_from_url(thread)
        import_replies_from_doc(@html_doc)
        links = page_links
        links.each_with_index do |link, i|
          logger.debug "Scraping '#{@post.subject}': page #{i + 1}/#{links.count}"
          doc = doc_from_url(link)
          import_replies_from_doc(doc)
        end
      end
      finalize_post_data
    end
    GenerateFlatPostJob.perform_later(@post.id)
    @post
  end
def scrape_threads! ( threads ) raise RuntimeError . new ( 'threaded_import must be true to use scrape_threads!' ) unless @threaded_import @html_doc = doc_from_url ( @url ) Post . transaction do import_post_from_doc ( @html_doc ) threads . each do | thread | @html_doc = doc_from_url ( thread ) import_replies_from_doc ( @html_doc ) links = page_links links . each_with_index do | link , i | logger . debug "Scraping '#{@post.subject}': page #{i + 1}/#{links.count}" doc = doc_from_url ( link ) import_replies_from_doc ( doc ) end end finalize_post_data end GenerateFlatPostJob . perform_later ( @post . id ) @post end
works as an alternative to scrape!
works as an alternative to scrape!
# if threads.blank? #   threads = top_level_comments(@html_doc) # end
threads
<not_specific>