app/jobs/github_repo_scraper.rb
# Retrieves the open issues and comments for each repository
Retrieves the open issues and comments for each repository
Retrieves the open issues and comments for each repository
def issues(scrape_limit_opts={}, get_repo_meta=false)
      handle_scrape_limits(scrape_limit_opts)
      @comments_cache = []
      @repositories.each do |repo|
        break unless get_repo_doc(repo, "/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc")
        update_repo_meta if get_repo_meta
        puts "Scraping issues for #{repo.name}"
        issues = [] 
        loop do
          raw_issues = @github_doc.doc.css("div.issues-listing ul li div.d-table")
          raw_issues.each do |raw_issue|
            built_issue = build_issue(raw_issue)
            next if built_issue.nil?
            issue = Issue.find_or_create_by(
              repository_id: built_issue['repository_id'],
              issue_number: built_issue['issue_number']
            ) do |i|
              i.name = built_issue['name']
              i.creator = built_issue['creator']
              i.url = built_issue['url']
              i.open_date = built_issue['open_date']
            end
            issues << issue
          end
          next_url_anchor = @github_doc.doc.css("a.next_page")
          if next_url_anchor.present?
            next_url_rel_path = next_url_anchor.attribute("href").value
            break unless @github_doc.new_doc(BASE_URL + next_url_rel_path)
          else
            break
          end
        end
        issues.each do |issue|
          doc_path = BASE_URL + issue.url
          next unless @github_doc.new_doc(doc_path)
          raw_comments = @github_doc.doc.css("div.timeline-comment-wrapper")
          freshest_comment = issue.issue_comments.order("github_created_at DESC").first
          raw_comments.each do |raw_comment|
            unless freshest_comment.nil?
              next if comment_outside_time_range?(raw_comment, freshest_comment)
            end
            comment_hash = build_comment(raw_comment)
            comment_hash['issue_id'] = issue.id
            @comments_cache << IssueComment.new(comment_hash)
          end
          if @comments_cache.count > 30
            IssueComment.import(@comments_cache, on_duplicate_key_ignore: true)
            @comments_cache.clear
          end
        end
      end
    end
def issues ( scrape_limit_opts = { } , get_repo_meta = false ) handle_scrape_limits ( scrape_limit_opts ) @comments_cache = [ ] @repositories . each do | repo | break unless get_repo_doc ( repo , "/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc" ) update_repo_meta if get_repo_meta puts "Scraping issues for #{repo.name}" issues = [ ] loop do raw_issues = @github_doc . doc . css ( "div.issues-listing ul li div.d-table" ) raw_issues . each do | raw_issue | built_issue = build_issue ( raw_issue ) next if built_issue . nil? issue = Issue . find_or_create_by ( repository_id : built_issue [ 'repository_id' ] , issue_number : built_issue [ 'issue_number' ] ) do | i | i . name = built_issue [ 'name' ] i . creator = built_issue [ 'creator' ] i . url = built_issue [ 'url' ] i . open_date = built_issue [ 'open_date' ] end issues << issue end next_url_anchor = @github_doc . doc . css ( "a.next_page" ) if next_url_anchor . present? next_url_rel_path = next_url_anchor . attribute ( "href" ) . value break unless @github_doc . new_doc ( BASE_URL + next_url_rel_path ) else break end end issues . each do | issue | doc_path = BASE_URL + issue . url next unless @github_doc . new_doc ( doc_path ) raw_comments = @github_doc . doc . css ( "div.timeline-comment-wrapper" ) freshest_comment = issue . issue_comments . order ( "github_created_at DESC" ) . first raw_comments . each do | raw_comment | unless freshest_comment . nil? next if comment_outside_time_range? ( raw_comment , freshest_comment ) end comment_hash = build_comment ( raw_comment ) comment_hash [ 'issue_id' ] = issue . id @comments_cache << IssueComment . new ( comment_hash ) end if @comments_cache . count > 30 IssueComment . import ( @comments_cache , on_duplicate_key_ignore : true ) @comments_cache . clear end end end end
Retrieves the open issues and comments for each repository
Retrieves the open issues and comments for each repository
# cache issues so we can cycle through without hitting the db # Get all the issues from page # Get all the comments for each issue
scrape_limit_opts
<not_specific>
get_repo_meta
<not_specific>