{"action":"created","comment":{"url":"https://api.github.com/repos/linkedin/gobblin/pulls/comments/88125079","pull_request_review_id":8708061,"id":88125079,"diff_hunk":"@@ -0,0 +1,291 @@\n+package gobblin.source.extractor.extract.google.webmaster;\n+\n+import com.google.api.client.repackaged.com.google.common.base.Preconditions;\n+import com.google.api.services.webmasters.model.ApiDataRow;\n+import com.google.api.services.webmasters.model.ApiDimensionFilter;\n+import com.google.api.services.webmasters.model.SearchAnalyticsQueryResponse;\n+import java.io.IOException;\n+import java.util.ArrayList;\n+import java.util.Arrays;\n+import java.util.Deque;\n+import java.util.HashSet;\n+import java.util.LinkedList;\n+import java.util.List;\n+import java.util.Set;\n+import org.apache.commons.lang3.tuple.ImmutableTriple;\n+import org.apache.commons.lang3.tuple.Triple;\n+import org.slf4j.Logger;\n+import org.slf4j.LoggerFactory;\n+\n+import static gobblin.source.extractor.extract.google.webmaster.GoogleWebmasterFilter.*;\n+\n+\n+public class GoogleWebmasterDataFetcherImpl implements GoogleWebmasterDataFetcher {\n+\n+  private final static Logger LOG = LoggerFactory.getLogger(GoogleWebmasterDataFetcherImpl.class);\n+  private final String _siteProperty; //Also used as a prefix\n+  private final GoogleWebmasterClient _client;\n+  private boolean _isHypercritical;\n+\n+  public GoogleWebmasterDataFetcherImpl(String siteProperty, boolean isHypercritical, String credentialFile,\n+      String appName, String scope) throws IOException {\n+    this(siteProperty, isHypercritical, new GoogleWebmasterClientImpl(credentialFile, appName, scope));\n+  }\n+\n+  public GoogleWebmasterDataFetcherImpl(String siteProperty, boolean isHypercritical, GoogleWebmasterClient client)\n+      throws IOException {\n+    //Missing \"/\" in the end will affect the getPagePrefixes logic\n+    Preconditions.checkArgument(siteProperty.endsWith(\"/\"), \"The site property must end in \\\"/\\\"\");\n+    _siteProperty = siteProperty;\n+    _isHypercritical = isHypercritical;\n+    _client = client;\n+  }\n+\n+  /**\n+   * Due to the limitation of the API, we can get a maximum of 5000 rows at a time. Another limitation is that, results are sorted by click count descending. If two rows have the same click count, they are sorted in an arbitrary way. (Read more at https://developers.google.com/webmaster-tools/v3/searchanalytics)\n+   *\n+   * In order to get all pages in a given date and country, we need to first figure out what is the total size of the data set. We achieve this by utilizing the feature of configuring the starting row when sending an API request. Keep asking for the maximum number of lines, which is 5000, allowed by the API until it returns fewer than 5000 lines, which indicates that you are at the last block of your data set. So we can sum up all lines to get the total count. But because the order is not guaranteed, only the count is reliable. The actual result will contain duplicates. Then it comes to the next step.\n+   *\n+   * In short, the steps of this algorithm are:\n+   *\n+   * 1. Request for all pages. If the request is less than 5000 or the response has fewer than 5000 rows, we've got all pages to return. Otherwise, go to next step for special handling.\n+   * 2. Now we need to know a full list of unique page prefixes/root paths. We will keep asking for the maximum number(5000) of rows until it comes to the last block, which is indicated by response giving less than 5000 rows.\n+   * 3. Remove duplicated pages got in last step.\n+   * 4. Parse out the list of keywords from last step.\n+   * 5. Send another request by adding the filters to exclude any pages/keywords in last step.\n+   * 6. Repeat steps 4 to 5 until we get a full union list of keywords, or prefixes, or root paths(e.g. www.property.com/path1/, www.property.com/path2/)\n+   * 7. For each page root path, apply the logic in pagesWithRootPaths to get all pages with that page root path/prefix.\n+   *    In detail, for example, you want all pages for your https://www.linkedin.com/ property. This property has several page prefixes such as https://www.linkedin.com/topic/, https://www.linkedin.com/jobs/, https://www.linkedin.com/groups/ etc.. For each prefix, you request for all pages starting with that prefix.\n+   * 8. Union all pages from last step and return.\n+   *\n+   * @param date\n+   * @param country\n+   * @param rowLimit\n+   * @return\n+   * @throws IOException\n+   */\n+  @Override\n+  public Set<String> getAllPages(String date, GoogleWebmasterFilter.Country country, int rowLimit) throws IOException {\n+    ApiDimensionFilter countryFilter = GoogleWebmasterFilter.countryFilter(country);\n+\n+    List<GoogleWebmasterFilter.Dimension> requestedDimensions = new ArrayList<>();\n+    requestedDimensions.add(GoogleWebmasterFilter.Dimension.PAGE);\n+\n+    List<String> response =\n+        _client.getPages(_siteProperty, date, country, rowLimit, requestedDimensions, Arrays.asList(countryFilter), 0);\n+    if (rowLimit < GoogleWebmasterClient.API_ROW_LIMIT || response.size() < 5000) {\n+      return new HashSet<>(response);\n+    }\n+\n+    Triple<Set<String>, Set<String>, Integer> results = getPagePrefixes(date, requestedDimensions, countryFilter);\n+\n+    //A page w/o root path can be https://www.linkedin.com/no_root_path\n+    Set<String> pagesNoRoots = results.getLeft();\n+    //A page with root path can be https://www.linkedin.com/root_path_here/\n+    Set<String> pagesRoots = results.getMiddle();\n+    Integer expectedSize = results.getRight();\n+\n+    LOG.info(String.format(\"Total number of pages w/o root paths is %d.\", pagesNoRoots.size()));\n+    LOG.info(String.format(\"Total of %d prefixes found. They are:\", pagesRoots.size()));\n+    for (String prefix : pagesRoots) {\n+      LOG.info(prefix);\n+    }\n+\n+    Set<String> allPages = pagesWithRootPaths(date, requestedDimensions, countryFilter, pagesRoots);\n+    allPages.addAll(pagesNoRoots);\n+    int size = allPages.size();\n+    if (size < expectedSize) {\n+      String msg =\n+          String.format(\"Size of the whole data set is %d, but only able to get %d lines. Date is %s and Market is %s.\",\n+              expectedSize, size, date, country);\n+      if (_isHypercritical) {\n+        LOG.error(msg);\n+        throw new RuntimeException(\n+            \"Cannot get all pages kept by Google Webmaster Service. There seems to be a defect with this algorithm.\");\n+      } else {\n+        LOG.warn(msg);\n+      }\n+    }\n+    LOG.info(String.format(\"A total of %d pages fetched for market-%s on %s\", allPages.size(), country, date));\n+    return allPages;\n+  }\n+\n+  /**\n+   *\n+   * @param date\n+   * @param requestedDimensions\n+   * @param countryFilter\n+   * @return return <pages w/o root paths, unique page root paths, size of the data set>\n+   * @throws IOException\n+   */\n+  ImmutableTriple<Set<String>, Set<String>, Integer> getPagePrefixes(String date,\n+      List<GoogleWebmasterFilter.Dimension> requestedDimensions, ApiDimensionFilter countryFilter) throws IOException {\n+    Country country = GoogleWebmasterFilter.countryFilterToEnum(countryFilter);\n+    Set<String> pagesNoRoots = new HashSet<>();  //Accumulating all pages w/o roots\n+    Set<String> pagesRoots = new HashSet<>(); //Accumulating all pages with roots\n+\n+    int entireSize = -1;\n+    List<ApiDimensionFilter> filters = new ArrayList<>();\n+    filters.add(countryFilter);\n+    int startRow = 0;\n+    while (true) {\n+      LOG.info(String.format(\"Start fetching from row %d with %d filters...\", startRow, filters.size()));\n+\n+      List<String> responsePages =\n+          _client.getPages(_siteProperty, date, country, GoogleWebmasterClient.API_ROW_LIMIT, requestedDimensions,\n+              filters, startRow);\n+      for (String p : responsePages) {\n+        //_siteProperty URL ends with \"/\". And we care about the first slash after property URL string\n+        int fourthSlashIndex = p.indexOf(\"/\", _siteProperty.length());\n+        if (fourthSlashIndex == -1) {\n+          pagesNoRoots.add(p); //Add all pages without the forth slash, it's a page w/o root path\n+        } else {\n+          pagesRoots.add(p.substring(0, fourthSlashIndex + 1));\n+        }\n+      }\n+\n+      int fetchedSize = responsePages.size();\n+      LOG.info(\"Number of pages fetched is \" + fetchedSize);\n+      startRow += fetchedSize;\n+      if (fetchedSize < GoogleWebmasterClient.API_ROW_LIMIT) {\n+        //fetchedSize < rowLimit indicates that we are at the last block of the data set.\n+        if (entireSize != -1) {\n+          //only break after we've known the total size of the data set.\n+          break;\n+        }\n+\n+        if (entireSize == -1) {\n+          LOG.info(String.format(\"Total size of the data set for market-%s on %s is %d\", country, date, startRow));\n+          entireSize = startRow;\n+        }\n+\n+        //update starting row and filters before starting\n+        startRow = 0;\n+        filters = new ArrayList<>();\n+        filters.add(countryFilter);\n+        for (String p : pagesRoots) {\n+          filters.add(GoogleWebmasterFilter.pageFilter(GoogleWebmasterFilter.FilterOperator.NOTCONTAINS, p));\n+        }\n+        for (String p : pagesNoRoots) {\n+          if (_siteProperty.equals(p)) {\n+            //Must NOT add site property to the exclusion list, otherwise no pages will be returned.\n+            continue;\n+          }\n+          filters.add(GoogleWebmasterFilter.pageFilter(GoogleWebmasterFilter.FilterOperator.NOTCONTAINS, p));\n+        }\n+        LOG.info(String.format(\n+            \"Starting another round of fetching for all pages. Currently there are %d pages w/o roots, and %d pages with roots\",\n+            pagesNoRoots.size(), pagesRoots.size()));\n+      }\n+    }\n+\n+    return new ImmutableTriple<>(pagesNoRoots, pagesRoots, entireSize);\n+  }\n+\n+  /**\n+   * This method gives you all pages given the page prefixes/keywords/root paths.\n+   *\n+   * The detailed steps are as follows:\n+   *\n+   * 1. For each page prefix, construct a filter group with two filters. One for country; the other for containing that page prefix.\n+   * 2. Request for all pages with the filter group. If the count is 5000, go to step 3; otherwise, end of the process, we've already got all pages.\n+   * 3. Expand current prefix by appending a char from the list [a-z,0-9,%,_,-,/].\n+   * 4. Perform another request for each new prefix at last step. If the number of pages is 5000, repeat steps 3~4; otherwise, end of the process, we've already got all pages.\n+   * 5. Union all pages as the return value\n+   *\n+   * @param date\n+   * @param dimensions requested dimensions\n+   * @param pagePrefixes a pagePrefix must end with \"/\"\n+   * @return all pages given the page prefixes/keywords, or page root paths.\n+   * @throws IOException\n+   */\n+  private Set<String> pagesWithRootPaths(String date, List<Dimension> dimensions, ApiDimensionFilter countryFilter,\n+      Set<String> pagePrefixes) throws IOException {\n+\n+    for (String prefix : pagePrefixes) {\n+      Preconditions.checkArgument(prefix.endsWith(\"/\"), \"Starting prefix must end with '/'\");\n+    }\n+    Country countryString = countryFilterToEnum(countryFilter);\n+\n+    List<ApiDimensionFilter> filters = new LinkedList<>();\n+    filters.add(countryFilter);\n+    filters.add(null); //placeholder for page filter\n+\n+    Set<String> uniquePages = new HashSet<>();\n+    Deque<String> toProcess = new LinkedList<>(pagePrefixes);\n+    while (!toProcess.isEmpty()) {\n+      String prefix = toProcess.remove();\n+      LOG.info(\"Current page prefix is \" + prefix);\n+      filters.set(1, GoogleWebmasterFilter.pageFilter(GoogleWebmasterFilter.FilterOperator.CONTAINS, prefix));","path":"gobblin-core/src/main/java/gobblin/source/extractor/extract/google/webmaster/GoogleWebmasterDataFetcherImpl.java","position":220,"original_position":219,"commit_id":"ecd4551ff0962acb9c772fc7232dd9006a323ff5","original_commit_id":"805575298fe043fabb282b1e381d47eb64b92e9d","user":{"login":"enjoyear","id":13011503,"avatar_url":"https://avatars.githubusercontent.com/u/13011503?v=3","gravatar_id":"","url":"https://api.github.com/users/enjoyear","html_url":"https://github.com/enjoyear","followers_url":"https://api.github.com/users/enjoyear/followers","following_url":"https://api.github.com/users/enjoyear/following{/other_user}","gists_url":"https://api.github.com/users/enjoyear/gists{/gist_id}","starred_url":"https://api.github.com/users/enjoyear/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/enjoyear/subscriptions","organizations_url":"https://api.github.com/users/enjoyear/orgs","repos_url":"https://api.github.com/users/enjoyear/repos","events_url":"https://api.github.com/users/enjoyear/events{/privacy}","received_events_url":"https://api.github.com/users/enjoyear/received_events","type":"User","site_admin":false},"body":"Good point. Fixed.","created_at":"2016-11-15T22:07:57Z","updated_at":"2016-11-15T22:07:57Z","html_url":"https://github.com/linkedin/gobblin/pull/1389#discussion_r88125079","pull_request_url":"https://api.github.com/repos/linkedin/gobblin/pulls/1389","_links":{"self":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/comments/88125079"},"html":{"href":"https://github.com/linkedin/gobblin/pull/1389#discussion_r88125079"},"pull_request":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/1389"}}},"pull_request":{"url":"https://api.github.com/repos/linkedin/gobblin/pulls/1389","id":93421850,"html_url":"https://github.com/linkedin/gobblin/pull/1389","diff_url":"https://github.com/linkedin/gobblin/pull/1389.diff","patch_url":"https://github.com/linkedin/gobblin/pull/1389.patch","issue_url":"https://api.github.com/repos/linkedin/gobblin/issues/1389","number":1389,"state":"open","locked":false,"title":"Implemented the source and extractor for Google Webmaster connector","user":{"login":"enjoyear","id":13011503,"avatar_url":"https://avatars.githubusercontent.com/u/13011503?v=3","gravatar_id":"","url":"https://api.github.com/users/enjoyear","html_url":"https://github.com/enjoyear","followers_url":"https://api.github.com/users/enjoyear/followers","following_url":"https://api.github.com/users/enjoyear/following{/other_user}","gists_url":"https://api.github.com/users/enjoyear/gists{/gist_id}","starred_url":"https://api.github.com/users/enjoyear/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/enjoyear/subscriptions","organizations_url":"https://api.github.com/users/enjoyear/orgs","repos_url":"https://api.github.com/users/enjoyear/repos","events_url":"https://api.github.com/users/enjoyear/events{/privacy}","received_events_url":"https://api.github.com/users/enjoyear/received_events","type":"User","site_admin":false},"body":"Implemented a connector for Google webmaster API. Google Webmaster API enables you to download data from Google Search Console for search analytics of the verified sites. See more here https://developers.google.com/webmaster-tools/. This gobblin job partitions the whole task into sub-tasks for each day. Each sub-task is handled by a GoogleWebmasterExtractor for that date, and each GoogleWebmasterExtractor holds a queue of GoogleWebmasterExtractorIterators, each of which does the query task for each filter(Currently, only the country filter is supported.) on that date. GoogleWebmasterClientImpl implements the business logic of performing API calls to download data from Google Search console.","created_at":"2016-11-12T01:16:19Z","updated_at":"2016-11-15T22:07:57Z","closed_at":null,"merged_at":null,"merge_commit_sha":"30d1a27aaf45ffd163de76ae95ea69eae01af617","assignee":{"login":"abti","id":69499,"avatar_url":"https://avatars.githubusercontent.com/u/69499?v=3","gravatar_id":"","url":"https://api.github.com/users/abti","html_url":"https://github.com/abti","followers_url":"https://api.github.com/users/abti/followers","following_url":"https://api.github.com/users/abti/following{/other_user}","gists_url":"https://api.github.com/users/abti/gists{/gist_id}","starred_url":"https://api.github.com/users/abti/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/abti/subscriptions","organizations_url":"https://api.github.com/users/abti/orgs","repos_url":"https://api.github.com/users/abti/repos","events_url":"https://api.github.com/users/abti/events{/privacy}","received_events_url":"https://api.github.com/users/abti/received_events","type":"User","site_admin":false},"assignees":[{"login":"abti","id":69499,"avatar_url":"https://avatars.githubusercontent.com/u/69499?v=3","gravatar_id":"","url":"https://api.github.com/users/abti","html_url":"https://github.com/abti","followers_url":"https://api.github.com/users/abti/followers","following_url":"https://api.github.com/users/abti/following{/other_user}","gists_url":"https://api.github.com/users/abti/gists{/gist_id}","starred_url":"https://api.github.com/users/abti/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/abti/subscriptions","organizations_url":"https://api.github.com/users/abti/orgs","repos_url":"https://api.github.com/users/abti/repos","events_url":"https://api.github.com/users/abti/events{/privacy}","received_events_url":"https://api.github.com/users/abti/received_events","type":"User","site_admin":false}],"milestone":null,"commits_url":"https://api.github.com/repos/linkedin/gobblin/pulls/1389/commits","review_comments_url":"https://api.github.com/repos/linkedin/gobblin/pulls/1389/comments","review_comment_url":"https://api.github.com/repos/linkedin/gobblin/pulls/comments{/number}","comments_url":"https://api.github.com/repos/linkedin/gobblin/issues/1389/comments","statuses_url":"https://api.github.com/repos/linkedin/gobblin/statuses/ecd4551ff0962acb9c772fc7232dd9006a323ff5","head":{"label":"enjoyear:chen/google-webmaster","ref":"chen/google-webmaster","sha":"ecd4551ff0962acb9c772fc7232dd9006a323ff5","user":{"login":"enjoyear","id":13011503,"avatar_url":"https://avatars.githubusercontent.com/u/13011503?v=3","gravatar_id":"","url":"https://api.github.com/users/enjoyear","html_url":"https://github.com/enjoyear","followers_url":"https://api.github.com/users/enjoyear/followers","following_url":"https://api.github.com/users/enjoyear/following{/other_user}","gists_url":"https://api.github.com/users/enjoyear/gists{/gist_id}","starred_url":"https://api.github.com/users/enjoyear/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/enjoyear/subscriptions","organizations_url":"https://api.github.com/users/enjoyear/orgs","repos_url":"https://api.github.com/users/enjoyear/repos","events_url":"https://api.github.com/users/enjoyear/events{/privacy}","received_events_url":"https://api.github.com/users/enjoyear/received_events","type":"User","site_admin":false},"repo":{"id":73136259,"name":"gobblin","full_name":"enjoyear/gobblin","owner":{"login":"enjoyear","id":13011503,"avatar_url":"https://avatars.githubusercontent.com/u/13011503?v=3","gravatar_id":"","url":"https://api.github.com/users/enjoyear","html_url":"https://github.com/enjoyear","followers_url":"https://api.github.com/users/enjoyear/followers","following_url":"https://api.github.com/users/enjoyear/following{/other_user}","gists_url":"https://api.github.com/users/enjoyear/gists{/gist_id}","starred_url":"https://api.github.com/users/enjoyear/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/enjoyear/subscriptions","organizations_url":"https://api.github.com/users/enjoyear/orgs","repos_url":"https://api.github.com/users/enjoyear/repos","events_url":"https://api.github.com/users/enjoyear/events{/privacy}","received_events_url":"https://api.github.com/users/enjoyear/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/enjoyear/gobblin","description":"Universal data ingestion framework for Hadoop.","fork":true,"url":"https://api.github.com/repos/enjoyear/gobblin","forks_url":"https://api.github.com/repos/enjoyear/gobblin/forks","keys_url":"https://api.github.com/repos/enjoyear/gobblin/keys{/key_id}","collaborators_url":"https://api.github.com/repos/enjoyear/gobblin/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/enjoyear/gobblin/teams","hooks_url":"https://api.github.com/repos/enjoyear/gobblin/hooks","issue_events_url":"https://api.github.com/repos/enjoyear/gobblin/issues/events{/number}","events_url":"https://api.github.com/repos/enjoyear/gobblin/events","assignees_url":"https://api.github.com/repos/enjoyear/gobblin/assignees{/user}","branches_url":"https://api.github.com/repos/enjoyear/gobblin/branches{/branch}","tags_url":"https://api.github.com/repos/enjoyear/gobblin/tags","blobs_url":"https://api.github.com/repos/enjoyear/gobblin/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/enjoyear/gobblin/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/enjoyear/gobblin/git/refs{/sha}","trees_url":"https://api.github.com/repos/enjoyear/gobblin/git/trees{/sha}","statuses_url":"https://api.github.com/repos/enjoyear/gobblin/statuses/{sha}","languages_url":"https://api.github.com/repos/enjoyear/gobblin/languages","stargazers_url":"https://api.github.com/repos/enjoyear/gobblin/stargazers","contributors_url":"https://api.github.com/repos/enjoyear/gobblin/contributors","subscribers_url":"https://api.github.com/repos/enjoyear/gobblin/subscribers","subscription_url":"https://api.github.com/repos/enjoyear/gobblin/subscription","commits_url":"https://api.github.com/repos/enjoyear/gobblin/commits{/sha}","git_commits_url":"https://api.github.com/repos/enjoyear/gobblin/git/commits{/sha}","comments_url":"https://api.github.com/repos/enjoyear/gobblin/comments{/number}","issue_comment_url":"https://api.github.com/repos/enjoyear/gobblin/issues/comments{/number}","contents_url":"https://api.github.com/repos/enjoyear/gobblin/contents/{+path}","compare_url":"https://api.github.com/repos/enjoyear/gobblin/compare/{base}...{head}","merges_url":"https://api.github.com/repos/enjoyear/gobblin/merges","archive_url":"https://api.github.com/repos/enjoyear/gobblin/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/enjoyear/gobblin/downloads","issues_url":"https://api.github.com/repos/enjoyear/gobblin/issues{/number}","pulls_url":"https://api.github.com/repos/enjoyear/gobblin/pulls{/number}","milestones_url":"https://api.github.com/repos/enjoyear/gobblin/milestones{/number}","notifications_url":"https://api.github.com/repos/enjoyear/gobblin/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/enjoyear/gobblin/labels{/name}","releases_url":"https://api.github.com/repos/enjoyear/gobblin/releases{/id}","deployments_url":"https://api.github.com/repos/enjoyear/gobblin/deployments","created_at":"2016-11-08T01:17:04Z","updated_at":"2016-11-08T01:17:09Z","pushed_at":"2016-11-15T19:35:40Z","git_url":"git://github.com/enjoyear/gobblin.git","ssh_url":"git@github.com:enjoyear/gobblin.git","clone_url":"https://github.com/enjoyear/gobblin.git","svn_url":"https://github.com/enjoyear/gobblin","homepage":"https://github.com/linkedin/gobblin/wiki","size":99766,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"linkedin:master","ref":"master","sha":"63bbebae46ed77b9dc4c0e0b5e9cfb4918412602","user":{"login":"linkedin","id":357098,"avatar_url":"https://avatars.githubusercontent.com/u/357098?v=3","gravatar_id":"","url":"https://api.github.com/users/linkedin","html_url":"https://github.com/linkedin","followers_url":"https://api.github.com/users/linkedin/followers","following_url":"https://api.github.com/users/linkedin/following{/other_user}","gists_url":"https://api.github.com/users/linkedin/gists{/gist_id}","starred_url":"https://api.github.com/users/linkedin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/linkedin/subscriptions","organizations_url":"https://api.github.com/users/linkedin/orgs","repos_url":"https://api.github.com/users/linkedin/repos","events_url":"https://api.github.com/users/linkedin/events{/privacy}","received_events_url":"https://api.github.com/users/linkedin/received_events","type":"Organization","site_admin":false},"repo":{"id":27392543,"name":"gobblin","full_name":"linkedin/gobblin","owner":{"login":"linkedin","id":357098,"avatar_url":"https://avatars.githubusercontent.com/u/357098?v=3","gravatar_id":"","url":"https://api.github.com/users/linkedin","html_url":"https://github.com/linkedin","followers_url":"https://api.github.com/users/linkedin/followers","following_url":"https://api.github.com/users/linkedin/following{/other_user}","gists_url":"https://api.github.com/users/linkedin/gists{/gist_id}","starred_url":"https://api.github.com/users/linkedin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/linkedin/subscriptions","organizations_url":"https://api.github.com/users/linkedin/orgs","repos_url":"https://api.github.com/users/linkedin/repos","events_url":"https://api.github.com/users/linkedin/events{/privacy}","received_events_url":"https://api.github.com/users/linkedin/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/linkedin/gobblin","description":"Universal data ingestion framework for Hadoop.","fork":false,"url":"https://api.github.com/repos/linkedin/gobblin","forks_url":"https://api.github.com/repos/linkedin/gobblin/forks","keys_url":"https://api.github.com/repos/linkedin/gobblin/keys{/key_id}","collaborators_url":"https://api.github.com/repos/linkedin/gobblin/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/linkedin/gobblin/teams","hooks_url":"https://api.github.com/repos/linkedin/gobblin/hooks","issue_events_url":"https://api.github.com/repos/linkedin/gobblin/issues/events{/number}","events_url":"https://api.github.com/repos/linkedin/gobblin/events","assignees_url":"https://api.github.com/repos/linkedin/gobblin/assignees{/user}","branches_url":"https://api.github.com/repos/linkedin/gobblin/branches{/branch}","tags_url":"https://api.github.com/repos/linkedin/gobblin/tags","blobs_url":"https://api.github.com/repos/linkedin/gobblin/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/linkedin/gobblin/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/linkedin/gobblin/git/refs{/sha}","trees_url":"https://api.github.com/repos/linkedin/gobblin/git/trees{/sha}","statuses_url":"https://api.github.com/repos/linkedin/gobblin/statuses/{sha}","languages_url":"https://api.github.com/repos/linkedin/gobblin/languages","stargazers_url":"https://api.github.com/repos/linkedin/gobblin/stargazers","contributors_url":"https://api.github.com/repos/linkedin/gobblin/contributors","subscribers_url":"https://api.github.com/repos/linkedin/gobblin/subscribers","subscription_url":"https://api.github.com/repos/linkedin/gobblin/subscription","commits_url":"https://api.github.com/repos/linkedin/gobblin/commits{/sha}","git_commits_url":"https://api.github.com/repos/linkedin/gobblin/git/commits{/sha}","comments_url":"https://api.github.com/repos/linkedin/gobblin/comments{/number}","issue_comment_url":"https://api.github.com/repos/linkedin/gobblin/issues/comments{/number}","contents_url":"https://api.github.com/repos/linkedin/gobblin/contents/{+path}","compare_url":"https://api.github.com/repos/linkedin/gobblin/compare/{base}...{head}","merges_url":"https://api.github.com/repos/linkedin/gobblin/merges","archive_url":"https://api.github.com/repos/linkedin/gobblin/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/linkedin/gobblin/downloads","issues_url":"https://api.github.com/repos/linkedin/gobblin/issues{/number}","pulls_url":"https://api.github.com/repos/linkedin/gobblin/pulls{/number}","milestones_url":"https://api.github.com/repos/linkedin/gobblin/milestones{/number}","notifications_url":"https://api.github.com/repos/linkedin/gobblin/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/linkedin/gobblin/labels{/name}","releases_url":"https://api.github.com/repos/linkedin/gobblin/releases{/id}","deployments_url":"https://api.github.com/repos/linkedin/gobblin/deployments","created_at":"2014-12-01T18:10:50Z","updated_at":"2016-11-15T02:00:30Z","pushed_at":"2016-11-15T20:42:28Z","git_url":"git://github.com/linkedin/gobblin.git","ssh_url":"git@github.com:linkedin/gobblin.git","clone_url":"https://github.com/linkedin/gobblin.git","svn_url":"https://github.com/linkedin/gobblin","homepage":"https://github.com/linkedin/gobblin/wiki","size":182580,"stargazers_count":766,"watchers_count":766,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":313,"mirror_url":null,"open_issues_count":134,"forks":313,"open_issues":134,"watchers":766,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/1389"},"html":{"href":"https://github.com/linkedin/gobblin/pull/1389"},"issue":{"href":"https://api.github.com/repos/linkedin/gobblin/issues/1389"},"comments":{"href":"https://api.github.com/repos/linkedin/gobblin/issues/1389/comments"},"review_comments":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/1389/comments"},"review_comment":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/1389/commits"},"statuses":{"href":"https://api.github.com/repos/linkedin/gobblin/statuses/ecd4551ff0962acb9c772fc7232dd9006a323ff5"}}}}
{"actor":{"display_login":"enjoyear"}}