{"action":"created","comment":{"url":"https://api.github.com/repos/linkedin/gobblin/pulls/comments/82830276","pull_request_review_id":3705366,"id":82830276,"diff_hunk":"@@ -190,20 +215,55 @@ public int compare(URI c1, URI c2) {\n       return ImmutableList.of();\n     }\n \n-    List<ConfigBasedDataset> result = new ArrayList<ConfigBasedDataset>();\n-    for (URI leaf : leafDatasets) {\n-      try {\n-        result.add(new ConfigBasedDataset(configClient.getConfig(leaf)));\n-      } catch (VersionDoesNotExistException | ConfigStoreFactoryDoesNotExistsException\n-          | ConfigStoreCreationException e) {\n-        log.error(\"Caught error while retrieve config for \" + leaf + \", skipping.\");\n-      }\n-    }\n+    final List<CopyableDatasetBase> result = new CopyOnWriteArrayList<>();\n+\n+    Iterator<Callable<Void>> callableIterator =\n+        Iterators.transform(leafDatasets.iterator(), new Function<URI, Callable<Void>>() {\n+          @Override\n+          public Callable<Void> apply(final URI datasetURI) {\n+            return findDatasetsCallable(configClient, datasetURI, props, result);\n+          }\n+        });\n+\n+    this.executeItertorExecutor(callableIterator);\n+    log.info(String.format(\"found %s datasets in ConfigBasedDatasetsFinder \", result.size()));\n     return result;\n   }\n \n   @Override\n   public Path commonDatasetRoot() {\n     return this.replicationDatasetCommonRoot;\n   }\n+\n+  private void executeItertorExecutor(Iterator<Callable<Void>> callableIterator) throws IOException {\n+    try {\n+      List<Future<Void>> futures = new IteratorExecutor<>(callableIterator, this.threadPoolSize,","path":"gobblin-data-management/src/main/java/gobblin/data/management/copy/replication/ConfigBasedDatasetsFinder.java","position":153,"original_position":153,"commit_id":"baef82d1d048ebad3ca136576000c8e19cf5d3d7","original_commit_id":"baef82d1d048ebad3ca136576000c8e19cf5d3d7","user":{"login":"ibuenros","id":1364112,"avatar_url":"https://avatars.githubusercontent.com/u/1364112?v=3","gravatar_id":"","url":"https://api.github.com/users/ibuenros","html_url":"https://github.com/ibuenros","followers_url":"https://api.github.com/users/ibuenros/followers","following_url":"https://api.github.com/users/ibuenros/following{/other_user}","gists_url":"https://api.github.com/users/ibuenros/gists{/gist_id}","starred_url":"https://api.github.com/users/ibuenros/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ibuenros/subscriptions","organizations_url":"https://api.github.com/users/ibuenros/orgs","repos_url":"https://api.github.com/users/ibuenros/repos","events_url":"https://api.github.com/users/ibuenros/events{/privacy}","received_events_url":"https://api.github.com/users/ibuenros/received_events","type":"User","site_admin":false},"body":"If you do go this route it is easier to do `IteratorExecutor.logFailures(iteratorExecutor.executeAndGetResults(), logger, 100)`","created_at":"2016-10-11T16:14:01Z","updated_at":"2016-10-11T16:14:01Z","html_url":"https://github.com/linkedin/gobblin/pull/1306#discussion_r82830276","pull_request_url":"https://api.github.com/repos/linkedin/gobblin/pulls/1306","_links":{"self":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/comments/82830276"},"html":{"href":"https://github.com/linkedin/gobblin/pull/1306#discussion_r82830276"},"pull_request":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/1306"}}},"pull_request":{"url":"https://api.github.com/repos/linkedin/gobblin/pulls/1306","id":88513904,"html_url":"https://github.com/linkedin/gobblin/pull/1306","diff_url":"https://github.com/linkedin/gobblin/pull/1306.diff","patch_url":"https://github.com/linkedin/gobblin/pull/1306.patch","issue_url":"https://api.github.com/repos/linkedin/gobblin/issues/1306","number":1306,"state":"open","locked":false,"title":"Data replication for pull mode","user":{"login":"tuGithub","id":9965624,"avatar_url":"https://avatars.githubusercontent.com/u/9965624?v=3","gravatar_id":"","url":"https://api.github.com/users/tuGithub","html_url":"https://github.com/tuGithub","followers_url":"https://api.github.com/users/tuGithub/followers","following_url":"https://api.github.com/users/tuGithub/following{/other_user}","gists_url":"https://api.github.com/users/tuGithub/gists{/gist_id}","starred_url":"https://api.github.com/users/tuGithub/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tuGithub/subscriptions","organizations_url":"https://api.github.com/users/tuGithub/orgs","repos_url":"https://api.github.com/users/tuGithub/repos","events_url":"https://api.github.com/users/tuGithub/events{/privacy}","received_events_url":"https://api.github.com/users/tuGithub/received_events","type":"User","site_admin":false},"body":"","created_at":"2016-10-07T21:52:59Z","updated_at":"2016-10-11T16:38:10Z","closed_at":null,"merged_at":null,"merge_commit_sha":"cb8ad653d07bd7b76747956595d48c1bdfe88e83","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/linkedin/gobblin/pulls/1306/commits","review_comments_url":"https://api.github.com/repos/linkedin/gobblin/pulls/1306/comments","review_comment_url":"https://api.github.com/repos/linkedin/gobblin/pulls/comments{/number}","comments_url":"https://api.github.com/repos/linkedin/gobblin/issues/1306/comments","statuses_url":"https://api.github.com/repos/linkedin/gobblin/statuses/baef82d1d048ebad3ca136576000c8e19cf5d3d7","head":{"label":"tuGithub:dataReplication_10_07","ref":"dataReplication_10_07","sha":"baef82d1d048ebad3ca136576000c8e19cf5d3d7","user":{"login":"tuGithub","id":9965624,"avatar_url":"https://avatars.githubusercontent.com/u/9965624?v=3","gravatar_id":"","url":"https://api.github.com/users/tuGithub","html_url":"https://github.com/tuGithub","followers_url":"https://api.github.com/users/tuGithub/followers","following_url":"https://api.github.com/users/tuGithub/following{/other_user}","gists_url":"https://api.github.com/users/tuGithub/gists{/gist_id}","starred_url":"https://api.github.com/users/tuGithub/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tuGithub/subscriptions","organizations_url":"https://api.github.com/users/tuGithub/orgs","repos_url":"https://api.github.com/users/tuGithub/repos","events_url":"https://api.github.com/users/tuGithub/events{/privacy}","received_events_url":"https://api.github.com/users/tuGithub/received_events","type":"User","site_admin":false},"repo":{"id":34822460,"name":"gobblin-1","full_name":"tuGithub/gobblin-1","owner":{"login":"tuGithub","id":9965624,"avatar_url":"https://avatars.githubusercontent.com/u/9965624?v=3","gravatar_id":"","url":"https://api.github.com/users/tuGithub","html_url":"https://github.com/tuGithub","followers_url":"https://api.github.com/users/tuGithub/followers","following_url":"https://api.github.com/users/tuGithub/following{/other_user}","gists_url":"https://api.github.com/users/tuGithub/gists{/gist_id}","starred_url":"https://api.github.com/users/tuGithub/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tuGithub/subscriptions","organizations_url":"https://api.github.com/users/tuGithub/orgs","repos_url":"https://api.github.com/users/tuGithub/repos","events_url":"https://api.github.com/users/tuGithub/events{/privacy}","received_events_url":"https://api.github.com/users/tuGithub/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/tuGithub/gobblin-1","description":"","fork":true,"url":"https://api.github.com/repos/tuGithub/gobblin-1","forks_url":"https://api.github.com/repos/tuGithub/gobblin-1/forks","keys_url":"https://api.github.com/repos/tuGithub/gobblin-1/keys{/key_id}","collaborators_url":"https://api.github.com/repos/tuGithub/gobblin-1/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/tuGithub/gobblin-1/teams","hooks_url":"https://api.github.com/repos/tuGithub/gobblin-1/hooks","issue_events_url":"https://api.github.com/repos/tuGithub/gobblin-1/issues/events{/number}","events_url":"https://api.github.com/repos/tuGithub/gobblin-1/events","assignees_url":"https://api.github.com/repos/tuGithub/gobblin-1/assignees{/user}","branches_url":"https://api.github.com/repos/tuGithub/gobblin-1/branches{/branch}","tags_url":"https://api.github.com/repos/tuGithub/gobblin-1/tags","blobs_url":"https://api.github.com/repos/tuGithub/gobblin-1/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/tuGithub/gobblin-1/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/tuGithub/gobblin-1/git/refs{/sha}","trees_url":"https://api.github.com/repos/tuGithub/gobblin-1/git/trees{/sha}","statuses_url":"https://api.github.com/repos/tuGithub/gobblin-1/statuses/{sha}","languages_url":"https://api.github.com/repos/tuGithub/gobblin-1/languages","stargazers_url":"https://api.github.com/repos/tuGithub/gobblin-1/stargazers","contributors_url":"https://api.github.com/repos/tuGithub/gobblin-1/contributors","subscribers_url":"https://api.github.com/repos/tuGithub/gobblin-1/subscribers","subscription_url":"https://api.github.com/repos/tuGithub/gobblin-1/subscription","commits_url":"https://api.github.com/repos/tuGithub/gobblin-1/commits{/sha}","git_commits_url":"https://api.github.com/repos/tuGithub/gobblin-1/git/commits{/sha}","comments_url":"https://api.github.com/repos/tuGithub/gobblin-1/comments{/number}","issue_comment_url":"https://api.github.com/repos/tuGithub/gobblin-1/issues/comments{/number}","contents_url":"https://api.github.com/repos/tuGithub/gobblin-1/contents/{+path}","compare_url":"https://api.github.com/repos/tuGithub/gobblin-1/compare/{base}...{head}","merges_url":"https://api.github.com/repos/tuGithub/gobblin-1/merges","archive_url":"https://api.github.com/repos/tuGithub/gobblin-1/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/tuGithub/gobblin-1/downloads","issues_url":"https://api.github.com/repos/tuGithub/gobblin-1/issues{/number}","pulls_url":"https://api.github.com/repos/tuGithub/gobblin-1/pulls{/number}","milestones_url":"https://api.github.com/repos/tuGithub/gobblin-1/milestones{/number}","notifications_url":"https://api.github.com/repos/tuGithub/gobblin-1/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/tuGithub/gobblin-1/labels{/name}","releases_url":"https://api.github.com/repos/tuGithub/gobblin-1/releases{/id}","deployments_url":"https://api.github.com/repos/tuGithub/gobblin-1/deployments","created_at":"2015-04-29T22:53:44Z","updated_at":"2016-06-01T17:27:17Z","pushed_at":"2016-10-11T03:14:52Z","git_url":"git://github.com/tuGithub/gobblin-1.git","ssh_url":"git@github.com:tuGithub/gobblin-1.git","clone_url":"https://github.com/tuGithub/gobblin-1.git","svn_url":"https://github.com/tuGithub/gobblin-1","homepage":null,"size":91381,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"linkedin:master","ref":"master","sha":"a2a6d94c14c13a36fd33d4695ec8c6f791f8d73d","user":{"login":"linkedin","id":357098,"avatar_url":"https://avatars.githubusercontent.com/u/357098?v=3","gravatar_id":"","url":"https://api.github.com/users/linkedin","html_url":"https://github.com/linkedin","followers_url":"https://api.github.com/users/linkedin/followers","following_url":"https://api.github.com/users/linkedin/following{/other_user}","gists_url":"https://api.github.com/users/linkedin/gists{/gist_id}","starred_url":"https://api.github.com/users/linkedin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/linkedin/subscriptions","organizations_url":"https://api.github.com/users/linkedin/orgs","repos_url":"https://api.github.com/users/linkedin/repos","events_url":"https://api.github.com/users/linkedin/events{/privacy}","received_events_url":"https://api.github.com/users/linkedin/received_events","type":"Organization","site_admin":false},"repo":{"id":27392543,"name":"gobblin","full_name":"linkedin/gobblin","owner":{"login":"linkedin","id":357098,"avatar_url":"https://avatars.githubusercontent.com/u/357098?v=3","gravatar_id":"","url":"https://api.github.com/users/linkedin","html_url":"https://github.com/linkedin","followers_url":"https://api.github.com/users/linkedin/followers","following_url":"https://api.github.com/users/linkedin/following{/other_user}","gists_url":"https://api.github.com/users/linkedin/gists{/gist_id}","starred_url":"https://api.github.com/users/linkedin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/linkedin/subscriptions","organizations_url":"https://api.github.com/users/linkedin/orgs","repos_url":"https://api.github.com/users/linkedin/repos","events_url":"https://api.github.com/users/linkedin/events{/privacy}","received_events_url":"https://api.github.com/users/linkedin/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/linkedin/gobblin","description":"Universal data ingestion framework for Hadoop.","fork":false,"url":"https://api.github.com/repos/linkedin/gobblin","forks_url":"https://api.github.com/repos/linkedin/gobblin/forks","keys_url":"https://api.github.com/repos/linkedin/gobblin/keys{/key_id}","collaborators_url":"https://api.github.com/repos/linkedin/gobblin/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/linkedin/gobblin/teams","hooks_url":"https://api.github.com/repos/linkedin/gobblin/hooks","issue_events_url":"https://api.github.com/repos/linkedin/gobblin/issues/events{/number}","events_url":"https://api.github.com/repos/linkedin/gobblin/events","assignees_url":"https://api.github.com/repos/linkedin/gobblin/assignees{/user}","branches_url":"https://api.github.com/repos/linkedin/gobblin/branches{/branch}","tags_url":"https://api.github.com/repos/linkedin/gobblin/tags","blobs_url":"https://api.github.com/repos/linkedin/gobblin/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/linkedin/gobblin/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/linkedin/gobblin/git/refs{/sha}","trees_url":"https://api.github.com/repos/linkedin/gobblin/git/trees{/sha}","statuses_url":"https://api.github.com/repos/linkedin/gobblin/statuses/{sha}","languages_url":"https://api.github.com/repos/linkedin/gobblin/languages","stargazers_url":"https://api.github.com/repos/linkedin/gobblin/stargazers","contributors_url":"https://api.github.com/repos/linkedin/gobblin/contributors","subscribers_url":"https://api.github.com/repos/linkedin/gobblin/subscribers","subscription_url":"https://api.github.com/repos/linkedin/gobblin/subscription","commits_url":"https://api.github.com/repos/linkedin/gobblin/commits{/sha}","git_commits_url":"https://api.github.com/repos/linkedin/gobblin/git/commits{/sha}","comments_url":"https://api.github.com/repos/linkedin/gobblin/comments{/number}","issue_comment_url":"https://api.github.com/repos/linkedin/gobblin/issues/comments{/number}","contents_url":"https://api.github.com/repos/linkedin/gobblin/contents/{+path}","compare_url":"https://api.github.com/repos/linkedin/gobblin/compare/{base}...{head}","merges_url":"https://api.github.com/repos/linkedin/gobblin/merges","archive_url":"https://api.github.com/repos/linkedin/gobblin/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/linkedin/gobblin/downloads","issues_url":"https://api.github.com/repos/linkedin/gobblin/issues{/number}","pulls_url":"https://api.github.com/repos/linkedin/gobblin/pulls{/number}","milestones_url":"https://api.github.com/repos/linkedin/gobblin/milestones{/number}","notifications_url":"https://api.github.com/repos/linkedin/gobblin/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/linkedin/gobblin/labels{/name}","releases_url":"https://api.github.com/repos/linkedin/gobblin/releases{/id}","deployments_url":"https://api.github.com/repos/linkedin/gobblin/deployments","created_at":"2014-12-01T18:10:50Z","updated_at":"2016-10-08T20:10:36Z","pushed_at":"2016-10-11T03:14:52Z","git_url":"git://github.com/linkedin/gobblin.git","ssh_url":"git@github.com:linkedin/gobblin.git","clone_url":"https://github.com/linkedin/gobblin.git","svn_url":"https://github.com/linkedin/gobblin","homepage":"https://github.com/linkedin/gobblin/wiki","size":182026,"stargazers_count":734,"watchers_count":734,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":294,"mirror_url":null,"open_issues_count":121,"forks":294,"open_issues":121,"watchers":734,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/1306"},"html":{"href":"https://github.com/linkedin/gobblin/pull/1306"},"issue":{"href":"https://api.github.com/repos/linkedin/gobblin/issues/1306"},"comments":{"href":"https://api.github.com/repos/linkedin/gobblin/issues/1306/comments"},"review_comments":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/1306/comments"},"review_comment":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/1306/commits"},"statuses":{"href":"https://api.github.com/repos/linkedin/gobblin/statuses/baef82d1d048ebad3ca136576000c8e19cf5d3d7"}}}}
{"actor":{"display_login":"ibuenros"}}