{"action":"created","comment":{"url":"https://api.github.com/repos/linkedin/gobblin/pulls/comments/93505338","pull_request_review_id":14034724,"id":93505338,"diff_hunk":"@@ -0,0 +1,366 @@\n+package gobblin.ingestion.google.webmaster;\n+\n+import com.google.api.client.auth.oauth2.Credential;\n+import com.google.api.client.googleapis.batch.BatchRequest;\n+import com.google.api.client.googleapis.batch.json.JsonBatchCallback;\n+import com.google.api.client.repackaged.com.google.common.base.Preconditions;\n+import com.google.api.services.webmasters.model.ApiDimensionFilter;\n+import com.google.api.services.webmasters.model.SearchAnalyticsQueryResponse;\n+import java.io.IOException;\n+import java.util.ArrayDeque;\n+import java.util.ArrayList;\n+import java.util.Arrays;\n+import java.util.Collection;\n+import java.util.LinkedList;\n+import java.util.List;\n+import java.util.Queue;\n+import java.util.Random;\n+import java.util.concurrent.Callable;\n+import java.util.concurrent.ConcurrentLinkedDeque;\n+import java.util.concurrent.ExecutionException;\n+import java.util.concurrent.ExecutorService;\n+import java.util.concurrent.Executors;\n+import java.util.concurrent.Future;\n+import java.util.concurrent.ThreadFactory;\n+import java.util.concurrent.TimeUnit;\n+import java.util.concurrent.TimeoutException;\n+import org.apache.commons.lang3.tuple.Pair;\n+import org.slf4j.Logger;\n+import org.slf4j.LoggerFactory;\n+\n+import static gobblin.ingestion.google.webmaster.GoogleWebmasterFilter.*;\n+\n+\n+public class GoogleWebmasterDataFetcherImpl extends GoogleWebmasterDataFetcher {\n+\n+  private final static Logger LOG = LoggerFactory.getLogger(GoogleWebmasterDataFetcherImpl.class);\n+  private final String _siteProperty;\n+  private final GoogleWebmasterClient _client;\n+  private final List<ProducerJob> _jobs;\n+\n+  public GoogleWebmasterDataFetcherImpl(String siteProperty, Credential credential, String appName,\n+      List<ProducerJob> jobs) throws IOException {\n+    this(siteProperty, new GoogleWebmasterClientImpl(credential, appName), jobs);\n+  }\n+\n+  /**\n+   * For test only\n+   */\n+  GoogleWebmasterDataFetcherImpl(String siteProperty, GoogleWebmasterClient client, List<ProducerJob> jobs)\n+      throws IOException {\n+    Preconditions.checkArgument(siteProperty.endsWith(\"/\"), \"The site property must end in \\\"/\\\"\");\n+    _siteProperty = siteProperty;\n+    _client = client;\n+    _jobs = jobs;\n+  }\n+\n+  /**\n+   * Due to the limitation of the API, we can get a maximum of 5000 rows at a time. Another limitation is that, results are sorted by click count descending. If two rows have the same click count, they are sorted in an arbitrary way. (Read more at https://developers.google.com/webmaster-tools/v3/searchanalytics). So we try to get all pages by partitions, if a partition has 5000 rows returned. We try partition current partition into more granular levels.\n+   *\n+   */\n+  @Override\n+  public Collection<ProducerJob> getAllPages(String startDate, String endDate, String country, int rowLimit)\n+      throws IOException {\n+    if (!_jobs.isEmpty()) {\n+      LOG.info(\"Service got hot started.\");\n+      return _jobs;\n+    }\n+\n+    ApiDimensionFilter countryFilter = GoogleWebmasterFilter.countryEqFilter(country);\n+\n+    List<GoogleWebmasterFilter.Dimension> requestedDimensions = new ArrayList<>();\n+    requestedDimensions.add(GoogleWebmasterFilter.Dimension.PAGE);\n+\n+    Collection<String> allPages =\n+        _client.getPages(_siteProperty, startDate, endDate, country, rowLimit, requestedDimensions,\n+            Arrays.asList(countryFilter), 0);\n+    int actualSize = allPages.size();\n+\n+    if (rowLimit < GoogleWebmasterClient.API_ROW_LIMIT || actualSize < GoogleWebmasterClient.API_ROW_LIMIT) {\n+      LOG.info(String.format(\"A total of %d pages fetched for property %s at country-%s from %s to %s\", actualSize,\n+          _siteProperty, country, startDate, endDate));\n+    } else {\n+      int expectedSize = getPagesSize(startDate, endDate, country, requestedDimensions, Arrays.asList(countryFilter));\n+      LOG.info(String.format(\"Total number of pages is %d for market-%s from %s to %s\", expectedSize,\n+          GoogleWebmasterFilter.countryFilterToString(countryFilter), startDate, endDate));\n+      Queue<Pair<String, FilterOperator>> jobs = new ArrayDeque<>();\n+      expandJobs(jobs, _siteProperty);\n+\n+      allPages = getPages(startDate, endDate, requestedDimensions, countryFilter, jobs);\n+      allPages.add(_siteProperty);\n+      actualSize = allPages.size();\n+      if (actualSize != expectedSize) {\n+        LOG.warn(\n+            String.format(\"Expected page size for country-%s is %d, but only able to get %d\", country, expectedSize,\n+                actualSize));\n+      }\n+      LOG.info(String.format(\"A total of %d pages fetched for property %s at country-%s from %s to %s\", actualSize,\n+          _siteProperty, country, startDate, endDate));\n+    }\n+\n+    ArrayDeque<ProducerJob> jobs = new ArrayDeque<>(actualSize);\n+    for (String page : allPages) {\n+      jobs.add(new SimpleProducerJob(page, startDate, endDate));\n+    }\n+    return jobs;\n+  }\n+\n+  private int getPagesSize(final String startDate, final String endDate, final String country,\n+      final List<Dimension> requestedDimensions, final List<ApiDimensionFilter> apiDimensionFilters)\n+      throws IOException {\n+\n+    int startRow = 0;\n+    int count = 4;\n+    final ExecutorService es = Executors.newFixedThreadPool(count);\n+    int r = 0;\n+    while (r < 50) {\n+      r++;\n+      List<Future<Integer>> results = new ArrayList<>(count);\n+\n+      for (int i = 0; i < count; ++i) {\n+        startRow += GoogleWebmasterClient.API_ROW_LIMIT;\n+        final int start = startRow;\n+        Future<Integer> submit = es.submit(new Callable<Integer>() {\n+          @Override\n+          public Integer call() {\n+            LOG.info(String.format(\"Getting page size from %s...\", start));\n+            while (true) {\n+              try {\n+                List<String> pages =\n+                    _client.getPages(_siteProperty, startDate, endDate, country, GoogleWebmasterClient.API_ROW_LIMIT,\n+                        requestedDimensions, apiDimensionFilters, start);\n+                if (pages.size() < GoogleWebmasterClient.API_ROW_LIMIT) {\n+                  //Figured out the size\n+                  return pages.size() + start;\n+                } else {\n+                  //The size is not determined.\n+                  return -1;\n+                }\n+              } catch (IOException e) {\n+                LOG.info(String.format(\"Getting page size from %s failed. Retrying...\", start));\n+              }\n+              try {\n+                Thread.sleep(200);","path":"gobblin-modules/google-ingestion/src/main/java/gobblin/ingestion/google/webmaster/GoogleWebmasterDataFetcherImpl.java","position":154,"original_position":143,"commit_id":"ac8e5b63158099e5396defedf9b36d326d8fed91","original_commit_id":"7a5fe1f96af278948f79fee92fc94afdfef96a59","user":{"login":"enjoyear","id":13011503,"avatar_url":"https://avatars.githubusercontent.com/u/13011503?v=3","gravatar_id":"","url":"https://api.github.com/users/enjoyear","html_url":"https://github.com/enjoyear","followers_url":"https://api.github.com/users/enjoyear/followers","following_url":"https://api.github.com/users/enjoyear/following{/other_user}","gists_url":"https://api.github.com/users/enjoyear/gists{/gist_id}","starred_url":"https://api.github.com/users/enjoyear/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/enjoyear/subscriptions","organizations_url":"https://api.github.com/users/enjoyear/orgs","repos_url":"https://api.github.com/users/enjoyear/repos","events_url":"https://api.github.com/users/enjoyear/events{/privacy}","received_events_url":"https://api.github.com/users/enjoyear/received_events","type":"User","site_admin":false},"body":"After I apply the retryer, I feel like the code becomes a little bit harder to read. Please compare the following.\r\nWith retryer:\r\n`List<Future<Integer>> results = new ArrayList<>((int) groupSize);\r\n\r\n    Map<String, Object> configMap = ImmutableMap.<String, Object>builder()\r\n        .put(RetryerFactory.RETRY_TIME_OUT_MS, TimeUnit.MINUTES.toMillis(TIME_OUT))\r\n        .put(RetryerFactory.RETRY_INTERVAL_MS, Math.round(1000 / REQUESTS_PER_SECOND))\r\n        .put(RetryerFactory.RETRY_TYPE, RetryerFactory.RetryType.FIXED.name()).build();\r\n    Config retryConfig = ConfigFactory.parseMap(configMap);\r\n\r\n    final Retryer<Integer> retryer = RetryerFactory.newInstance(retryConfig);\r\n\r\n    while (true) {\r\n      for (int i = 0; i < groupSize; ++i) {\r\n        startRow += GoogleWebmasterClient.API_ROW_LIMIT;\r\n        final int start = startRow;\r\n        final String interruptedMsg = String\r\n            .format(\"Interrupted while trying to get the size of all pages for %s. Current start row is %d.\", country,\r\n                start);\r\n\r\n        Future<Integer> submit = es.submit(new Callable<Integer>() {\r\n          @Override\r\n          public Integer call() {\r\n            log.info(String.format(\"Getting page size from %s...\", start));\r\n\r\n            Callable<Integer> retryCallable = new Callable<Integer>() {\r\n              @Override\r\n              public Integer call()\r\n                  throws Exception {\r\n                try {\r\n                  LIMITER.acquirePermits(1);\r\n                  List<String> pages = _client\r\n                      .getPages(_siteProperty, startDate, endDate, country, GoogleWebmasterClient.API_ROW_LIMIT,\r\n                          requestedDimensions, apiDimensionFilters, start);\r\n                  if (pages.size() < GoogleWebmasterClient.API_ROW_LIMIT) {\r\n                    return pages.size() + start;  //Figured out the size\r\n                  } else {\r\n                    return -1;\r\n                  }\r\n                } catch (InterruptedException e) {\r\n                  log.error(\"RateBasedLimiter: \" + interruptedMsg, e);\r\n                  return -1;\r\n                }\r\n              }\r\n            };\r\n\r\n            try {\r\n              return retryer.call(retryCallable);\r\n            } catch (ExecutionException e) {\r\n              throw new RuntimeException(e);\r\n            } catch (RetryException e) {\r\n              throw new RuntimeException(String\r\n                  .format(\"Exceeding the timeout of %d minutes while getting the total size of all pages.\", TIME_OUT),\r\n                  e);\r\n            }\r\n          }\r\n        });\r\n        results.add(submit);\r\n      }\r\n      //Check the results group in order. The first non-negative count indicates the size of total pages.\r\n      for (Future<Integer> result : results) {\r\n        try {\r\n          Integer integer = result.get();\r\n          if (integer >= 0) {\r\n            es.shutdownNow();\r\n            return integer;\r\n          }\r\n        } catch (InterruptedException | ExecutionException e) {\r\n          throw new RuntimeException(e);\r\n        }\r\n      }\r\n      results.clear();\r\n    }` \r\n\r\nPrevious version without retryer:\r\n`List<Future<Integer>> results = new ArrayList<>((int) groupSize);\r\n\r\n    while (true) {\r\n      for (int i = 0; i < groupSize; ++i) {\r\n        startRow += GoogleWebmasterClient.API_ROW_LIMIT;\r\n        final int start = startRow;\r\n        final String interruptedMsg = String\r\n            .format(\"Interrupted while trying to get the size of all pages for %s. Current start row is %d.\", country,\r\n                start);\r\n\r\n        Future<Integer> submit = es.submit(new Callable<Integer>() {\r\n          @Override\r\n          public Integer call() {\r\n            log.info(String.format(\"Getting page size from %s...\", start));\r\n            while (true) {\r\n              if (Thread.interrupted()) {\r\n                log.error(interruptedMsg);\r\n                return -1;\r\n              }\r\n              try {\r\n                LIMITER.acquirePermits(1);\r\n                List<String> pages = _client\r\n                    .getPages(_siteProperty, startDate, endDate, country, GoogleWebmasterClient.API_ROW_LIMIT,\r\n                        requestedDimensions, apiDimensionFilters, start);\r\n                if (pages.size() < GoogleWebmasterClient.API_ROW_LIMIT) {\r\n                  return pages.size() + start;  //Figured out the size\r\n                } else {\r\n                  return -1;\r\n                }\r\n              } catch (IOException e) {\r\n                log.info(String.format(\"Getting page size from %s failed. Retrying...\", start));\r\n              } catch (InterruptedException e) {\r\n                log.error(\"RateBasedLimiter: \" + interruptedMsg, e);\r\n                return -1;\r\n              }\r\n            }\r\n          }\r\n        });\r\n        results.add(submit);\r\n      }\r\n      //Check the results group in order. The first non-negative count indicates the size of total pages.\r\n      for (Future<Integer> result : results) {\r\n        try {\r\n          Integer integer = result.get(GET_PAGE_SIZE_TIME_OUT, TimeUnit.MINUTES);\r\n          if (integer >= 0) {\r\n            es.shutdownNow();\r\n            return integer;\r\n          }\r\n        } catch (InterruptedException | ExecutionException e) {\r\n          throw new RuntimeException(e);\r\n        } catch (TimeoutException e) {\r\n          throw new RuntimeException(String\r\n              .format(\"Exceeding the timeout of %d minutes while getting the total size of all pages.\",\r\n                  GET_PAGE_SIZE_TIME_OUT), e);\r\n        }\r\n      }\r\n      results.clear();\r\n    }`\r\n\r\nI actually feel that the version without retryer is better in this case.\r\n\r\nFor other cases, I cannot use the retryer because the job can evolve based on the results, but the retryer only supports retrying static jobs.\r\n","created_at":"2016-12-21T19:38:21Z","updated_at":"2016-12-21T19:38:21Z","html_url":"https://github.com/linkedin/gobblin/pull/1422#discussion_r93505338","pull_request_url":"https://api.github.com/repos/linkedin/gobblin/pulls/1422","_links":{"self":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/comments/93505338"},"html":{"href":"https://github.com/linkedin/gobblin/pull/1422#discussion_r93505338"},"pull_request":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/1422"}}},"pull_request":{"url":"https://api.github.com/repos/linkedin/gobblin/pulls/1422","id":95059641,"html_url":"https://github.com/linkedin/gobblin/pull/1422","diff_url":"https://github.com/linkedin/gobblin/pull/1422.diff","patch_url":"https://github.com/linkedin/gobblin/pull/1422.patch","issue_url":"https://api.github.com/repos/linkedin/gobblin/issues/1422","number":1422,"state":"closed","locked":false,"title":"Chen/google webmaster copy","user":{"login":"enjoyear","id":13011503,"avatar_url":"https://avatars.githubusercontent.com/u/13011503?v=3","gravatar_id":"","url":"https://api.github.com/users/enjoyear","html_url":"https://github.com/enjoyear","followers_url":"https://api.github.com/users/enjoyear/followers","following_url":"https://api.github.com/users/enjoyear/following{/other_user}","gists_url":"https://api.github.com/users/enjoyear/gists{/gist_id}","starred_url":"https://api.github.com/users/enjoyear/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/enjoyear/subscriptions","organizations_url":"https://api.github.com/users/enjoyear/orgs","repos_url":"https://api.github.com/users/enjoyear/repos","events_url":"https://api.github.com/users/enjoyear/events{/privacy}","received_events_url":"https://api.github.com/users/enjoyear/received_events","type":"User","site_admin":false},"body":"This pull request is a copy of my branch chen/google-webmaster. Previously it was merged using \"git rebase\" and got duplicated commits. Now I am merging with \"git merge\", so recreate a new pull request here.","created_at":"2016-11-23T18:46:04Z","updated_at":"2016-12-21T19:38:21Z","closed_at":"2016-12-15T23:00:36Z","merged_at":"2016-12-15T23:00:36Z","merge_commit_sha":"4513e4a773bd4b69a44c154c8160074ec8b5041d","assignee":{"login":"chavdar","id":928795,"avatar_url":"https://avatars.githubusercontent.com/u/928795?v=3","gravatar_id":"","url":"https://api.github.com/users/chavdar","html_url":"https://github.com/chavdar","followers_url":"https://api.github.com/users/chavdar/followers","following_url":"https://api.github.com/users/chavdar/following{/other_user}","gists_url":"https://api.github.com/users/chavdar/gists{/gist_id}","starred_url":"https://api.github.com/users/chavdar/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/chavdar/subscriptions","organizations_url":"https://api.github.com/users/chavdar/orgs","repos_url":"https://api.github.com/users/chavdar/repos","events_url":"https://api.github.com/users/chavdar/events{/privacy}","received_events_url":"https://api.github.com/users/chavdar/received_events","type":"User","site_admin":false},"assignees":[{"login":"chavdar","id":928795,"avatar_url":"https://avatars.githubusercontent.com/u/928795?v=3","gravatar_id":"","url":"https://api.github.com/users/chavdar","html_url":"https://github.com/chavdar","followers_url":"https://api.github.com/users/chavdar/followers","following_url":"https://api.github.com/users/chavdar/following{/other_user}","gists_url":"https://api.github.com/users/chavdar/gists{/gist_id}","starred_url":"https://api.github.com/users/chavdar/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/chavdar/subscriptions","organizations_url":"https://api.github.com/users/chavdar/orgs","repos_url":"https://api.github.com/users/chavdar/repos","events_url":"https://api.github.com/users/chavdar/events{/privacy}","received_events_url":"https://api.github.com/users/chavdar/received_events","type":"User","site_admin":false}],"milestone":null,"commits_url":"https://api.github.com/repos/linkedin/gobblin/pulls/1422/commits","review_comments_url":"https://api.github.com/repos/linkedin/gobblin/pulls/1422/comments","review_comment_url":"https://api.github.com/repos/linkedin/gobblin/pulls/comments{/number}","comments_url":"https://api.github.com/repos/linkedin/gobblin/issues/1422/comments","statuses_url":"https://api.github.com/repos/linkedin/gobblin/statuses/ac8e5b63158099e5396defedf9b36d326d8fed91","head":{"label":"enjoyear:chen/google-webmaster-copy","ref":"chen/google-webmaster-copy","sha":"ac8e5b63158099e5396defedf9b36d326d8fed91","user":{"login":"enjoyear","id":13011503,"avatar_url":"https://avatars.githubusercontent.com/u/13011503?v=3","gravatar_id":"","url":"https://api.github.com/users/enjoyear","html_url":"https://github.com/enjoyear","followers_url":"https://api.github.com/users/enjoyear/followers","following_url":"https://api.github.com/users/enjoyear/following{/other_user}","gists_url":"https://api.github.com/users/enjoyear/gists{/gist_id}","starred_url":"https://api.github.com/users/enjoyear/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/enjoyear/subscriptions","organizations_url":"https://api.github.com/users/enjoyear/orgs","repos_url":"https://api.github.com/users/enjoyear/repos","events_url":"https://api.github.com/users/enjoyear/events{/privacy}","received_events_url":"https://api.github.com/users/enjoyear/received_events","type":"User","site_admin":false},"repo":{"id":73136259,"name":"gobblin","full_name":"enjoyear/gobblin","owner":{"login":"enjoyear","id":13011503,"avatar_url":"https://avatars.githubusercontent.com/u/13011503?v=3","gravatar_id":"","url":"https://api.github.com/users/enjoyear","html_url":"https://github.com/enjoyear","followers_url":"https://api.github.com/users/enjoyear/followers","following_url":"https://api.github.com/users/enjoyear/following{/other_user}","gists_url":"https://api.github.com/users/enjoyear/gists{/gist_id}","starred_url":"https://api.github.com/users/enjoyear/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/enjoyear/subscriptions","organizations_url":"https://api.github.com/users/enjoyear/orgs","repos_url":"https://api.github.com/users/enjoyear/repos","events_url":"https://api.github.com/users/enjoyear/events{/privacy}","received_events_url":"https://api.github.com/users/enjoyear/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/enjoyear/gobblin","description":"Universal data ingestion framework for Hadoop.","fork":true,"url":"https://api.github.com/repos/enjoyear/gobblin","forks_url":"https://api.github.com/repos/enjoyear/gobblin/forks","keys_url":"https://api.github.com/repos/enjoyear/gobblin/keys{/key_id}","collaborators_url":"https://api.github.com/repos/enjoyear/gobblin/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/enjoyear/gobblin/teams","hooks_url":"https://api.github.com/repos/enjoyear/gobblin/hooks","issue_events_url":"https://api.github.com/repos/enjoyear/gobblin/issues/events{/number}","events_url":"https://api.github.com/repos/enjoyear/gobblin/events","assignees_url":"https://api.github.com/repos/enjoyear/gobblin/assignees{/user}","branches_url":"https://api.github.com/repos/enjoyear/gobblin/branches{/branch}","tags_url":"https://api.github.com/repos/enjoyear/gobblin/tags","blobs_url":"https://api.github.com/repos/enjoyear/gobblin/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/enjoyear/gobblin/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/enjoyear/gobblin/git/refs{/sha}","trees_url":"https://api.github.com/repos/enjoyear/gobblin/git/trees{/sha}","statuses_url":"https://api.github.com/repos/enjoyear/gobblin/statuses/{sha}","languages_url":"https://api.github.com/repos/enjoyear/gobblin/languages","stargazers_url":"https://api.github.com/repos/enjoyear/gobblin/stargazers","contributors_url":"https://api.github.com/repos/enjoyear/gobblin/contributors","subscribers_url":"https://api.github.com/repos/enjoyear/gobblin/subscribers","subscription_url":"https://api.github.com/repos/enjoyear/gobblin/subscription","commits_url":"https://api.github.com/repos/enjoyear/gobblin/commits{/sha}","git_commits_url":"https://api.github.com/repos/enjoyear/gobblin/git/commits{/sha}","comments_url":"https://api.github.com/repos/enjoyear/gobblin/comments{/number}","issue_comment_url":"https://api.github.com/repos/enjoyear/gobblin/issues/comments{/number}","contents_url":"https://api.github.com/repos/enjoyear/gobblin/contents/{+path}","compare_url":"https://api.github.com/repos/enjoyear/gobblin/compare/{base}...{head}","merges_url":"https://api.github.com/repos/enjoyear/gobblin/merges","archive_url":"https://api.github.com/repos/enjoyear/gobblin/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/enjoyear/gobblin/downloads","issues_url":"https://api.github.com/repos/enjoyear/gobblin/issues{/number}","pulls_url":"https://api.github.com/repos/enjoyear/gobblin/pulls{/number}","milestones_url":"https://api.github.com/repos/enjoyear/gobblin/milestones{/number}","notifications_url":"https://api.github.com/repos/enjoyear/gobblin/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/enjoyear/gobblin/labels{/name}","releases_url":"https://api.github.com/repos/enjoyear/gobblin/releases{/id}","deployments_url":"https://api.github.com/repos/enjoyear/gobblin/deployments","created_at":"2016-11-08T01:17:04Z","updated_at":"2016-11-08T01:17:09Z","pushed_at":"2016-12-21T00:58:37Z","git_url":"git://github.com/enjoyear/gobblin.git","ssh_url":"git@github.com:enjoyear/gobblin.git","clone_url":"https://github.com/enjoyear/gobblin.git","svn_url":"https://github.com/enjoyear/gobblin","homepage":"https://github.com/linkedin/gobblin/wiki","size":102931,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"linkedin:master","ref":"master","sha":"6cc8335ee845460e7e0c1e37662f4a101b1ba69e","user":{"login":"linkedin","id":357098,"avatar_url":"https://avatars.githubusercontent.com/u/357098?v=3","gravatar_id":"","url":"https://api.github.com/users/linkedin","html_url":"https://github.com/linkedin","followers_url":"https://api.github.com/users/linkedin/followers","following_url":"https://api.github.com/users/linkedin/following{/other_user}","gists_url":"https://api.github.com/users/linkedin/gists{/gist_id}","starred_url":"https://api.github.com/users/linkedin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/linkedin/subscriptions","organizations_url":"https://api.github.com/users/linkedin/orgs","repos_url":"https://api.github.com/users/linkedin/repos","events_url":"https://api.github.com/users/linkedin/events{/privacy}","received_events_url":"https://api.github.com/users/linkedin/received_events","type":"Organization","site_admin":false},"repo":{"id":27392543,"name":"gobblin","full_name":"linkedin/gobblin","owner":{"login":"linkedin","id":357098,"avatar_url":"https://avatars.githubusercontent.com/u/357098?v=3","gravatar_id":"","url":"https://api.github.com/users/linkedin","html_url":"https://github.com/linkedin","followers_url":"https://api.github.com/users/linkedin/followers","following_url":"https://api.github.com/users/linkedin/following{/other_user}","gists_url":"https://api.github.com/users/linkedin/gists{/gist_id}","starred_url":"https://api.github.com/users/linkedin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/linkedin/subscriptions","organizations_url":"https://api.github.com/users/linkedin/orgs","repos_url":"https://api.github.com/users/linkedin/repos","events_url":"https://api.github.com/users/linkedin/events{/privacy}","received_events_url":"https://api.github.com/users/linkedin/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/linkedin/gobblin","description":"Universal data ingestion framework for Hadoop.","fork":false,"url":"https://api.github.com/repos/linkedin/gobblin","forks_url":"https://api.github.com/repos/linkedin/gobblin/forks","keys_url":"https://api.github.com/repos/linkedin/gobblin/keys{/key_id}","collaborators_url":"https://api.github.com/repos/linkedin/gobblin/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/linkedin/gobblin/teams","hooks_url":"https://api.github.com/repos/linkedin/gobblin/hooks","issue_events_url":"https://api.github.com/repos/linkedin/gobblin/issues/events{/number}","events_url":"https://api.github.com/repos/linkedin/gobblin/events","assignees_url":"https://api.github.com/repos/linkedin/gobblin/assignees{/user}","branches_url":"https://api.github.com/repos/linkedin/gobblin/branches{/branch}","tags_url":"https://api.github.com/repos/linkedin/gobblin/tags","blobs_url":"https://api.github.com/repos/linkedin/gobblin/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/linkedin/gobblin/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/linkedin/gobblin/git/refs{/sha}","trees_url":"https://api.github.com/repos/linkedin/gobblin/git/trees{/sha}","statuses_url":"https://api.github.com/repos/linkedin/gobblin/statuses/{sha}","languages_url":"https://api.github.com/repos/linkedin/gobblin/languages","stargazers_url":"https://api.github.com/repos/linkedin/gobblin/stargazers","contributors_url":"https://api.github.com/repos/linkedin/gobblin/contributors","subscribers_url":"https://api.github.com/repos/linkedin/gobblin/subscribers","subscription_url":"https://api.github.com/repos/linkedin/gobblin/subscription","commits_url":"https://api.github.com/repos/linkedin/gobblin/commits{/sha}","git_commits_url":"https://api.github.com/repos/linkedin/gobblin/git/commits{/sha}","comments_url":"https://api.github.com/repos/linkedin/gobblin/comments{/number}","issue_comment_url":"https://api.github.com/repos/linkedin/gobblin/issues/comments{/number}","contents_url":"https://api.github.com/repos/linkedin/gobblin/contents/{+path}","compare_url":"https://api.github.com/repos/linkedin/gobblin/compare/{base}...{head}","merges_url":"https://api.github.com/repos/linkedin/gobblin/merges","archive_url":"https://api.github.com/repos/linkedin/gobblin/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/linkedin/gobblin/downloads","issues_url":"https://api.github.com/repos/linkedin/gobblin/issues{/number}","pulls_url":"https://api.github.com/repos/linkedin/gobblin/pulls{/number}","milestones_url":"https://api.github.com/repos/linkedin/gobblin/milestones{/number}","notifications_url":"https://api.github.com/repos/linkedin/gobblin/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/linkedin/gobblin/labels{/name}","releases_url":"https://api.github.com/repos/linkedin/gobblin/releases{/id}","deployments_url":"https://api.github.com/repos/linkedin/gobblin/deployments","created_at":"2014-12-01T18:10:50Z","updated_at":"2016-12-21T05:26:33Z","pushed_at":"2016-12-21T18:37:27Z","git_url":"git://github.com/linkedin/gobblin.git","ssh_url":"git@github.com:linkedin/gobblin.git","clone_url":"https://github.com/linkedin/gobblin.git","svn_url":"https://github.com/linkedin/gobblin","homepage":"https://github.com/linkedin/gobblin/wiki","size":105466,"stargazers_count":795,"watchers_count":795,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":335,"mirror_url":null,"open_issues_count":138,"forks":335,"open_issues":138,"watchers":795,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/1422"},"html":{"href":"https://github.com/linkedin/gobblin/pull/1422"},"issue":{"href":"https://api.github.com/repos/linkedin/gobblin/issues/1422"},"comments":{"href":"https://api.github.com/repos/linkedin/gobblin/issues/1422/comments"},"review_comments":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/1422/comments"},"review_comment":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/1422/commits"},"statuses":{"href":"https://api.github.com/repos/linkedin/gobblin/statuses/ac8e5b63158099e5396defedf9b36d326d8fed91"}}}}
{"actor":{"display_login":"enjoyear"}}