{"action":"created","comment":{"url":"https://api.github.com/repos/twitter/scalding/pulls/comments/74092078","id":74092078,"diff_hunk":"@@ -0,0 +1,122 @@\n+package com.twitter.scalding;\n+\n+import java.util.Set;\n+\n+import cascading.flow.FlowElement;\n+import cascading.flow.planner.iso.expression.ElementCapture;\n+import cascading.flow.planner.iso.expression.ExpressionGraph;\n+import cascading.flow.planner.iso.expression.FlowElementExpression;\n+import cascading.flow.planner.iso.expression.PathScopeExpression;\n+import cascading.flow.planner.iso.finder.Match;\n+import cascading.flow.planner.iso.transformer.ElementFactory;\n+import cascading.flow.planner.iso.transformer.GraphTransformer;\n+import cascading.flow.planner.iso.transformer.ReplaceGraphTransformer;\n+import cascading.flow.planner.iso.transformer.Transformed;\n+import cascading.flow.planner.graph.ElementGraph;\n+import cascading.flow.planner.graph.ElementGraphs;\n+import cascading.flow.planner.rule.PlanPhase;\n+import cascading.flow.planner.rule.RuleExpression;\n+import cascading.flow.planner.rule.RuleTransformer;\n+import cascading.flow.planner.rule.expressiongraph.SyncPipeExpressionGraph;\n+import cascading.pipe.HashJoin;\n+import cascading.tap.hadoop.Hfs;\n+import cascading.tap.hadoop.util.TempHfs;\n+\n+import static cascading.flow.planner.rule.PlanPhase.BalanceAssembly;\n+\n+class HashJoinExpression extends RuleExpression {\n+\n+  public HashJoinExpression() {\n+    super(\n+      new SyncPipeExpressionGraph(),\n+      new ExpressionGraph()\n+        .arc(\n+          new FlowElementExpression(ElementCapture.Primary, Hfs.class),\n+          PathScopeExpression.BLOCKING,\n+          new FlowElementExpression(HashJoin.class)\n+        )","path":"scalding-core/src/main/java/com/twitter/scalding/cascading/HashJoinDistCacheTransformer.java","position":37,"original_position":37,"commit_id":"e56dd1e36390c4ac1745079f2938d08b2c2422cb","original_commit_id":"e56dd1e36390c4ac1745079f2938d08b2c2422cb","user":{"login":"rubanm","id":2401144,"avatar_url":"https://avatars.githubusercontent.com/u/2401144?v=3","gravatar_id":"","url":"https://api.github.com/users/rubanm","html_url":"https://github.com/rubanm","followers_url":"https://api.github.com/users/rubanm/followers","following_url":"https://api.github.com/users/rubanm/following{/other_user}","gists_url":"https://api.github.com/users/rubanm/gists{/gist_id}","starred_url":"https://api.github.com/users/rubanm/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rubanm/subscriptions","organizations_url":"https://api.github.com/users/rubanm/orgs","repos_url":"https://api.github.com/users/rubanm/repos","events_url":"https://api.github.com/users/rubanm/events{/privacy}","received_events_url":"https://api.github.com/users/rubanm/received_events","type":"User","site_admin":false},"body":"This currently does not ignore HashJoins that are self-joins iiuc.","created_at":"2016-08-09T16:20:03Z","updated_at":"2016-08-09T16:20:03Z","html_url":"https://github.com/twitter/scalding/pull/1585#discussion_r74092078","pull_request_url":"https://api.github.com/repos/twitter/scalding/pulls/1585","_links":{"self":{"href":"https://api.github.com/repos/twitter/scalding/pulls/comments/74092078"},"html":{"href":"https://github.com/twitter/scalding/pull/1585#discussion_r74092078"},"pull_request":{"href":"https://api.github.com/repos/twitter/scalding/pulls/1585"}}},"pull_request":{"url":"https://api.github.com/repos/twitter/scalding/pulls/1585","id":80650449,"html_url":"https://github.com/twitter/scalding/pull/1585","diff_url":"https://github.com/twitter/scalding/pull/1585.diff","patch_url":"https://github.com/twitter/scalding/pull/1585.patch","issue_url":"https://api.github.com/repos/twitter/scalding/issues/1585","number":1585,"state":"open","locked":false,"title":"[cascading3] use DistCacheTap for HashJoins","user":{"login":"rubanm","id":2401144,"avatar_url":"https://avatars.githubusercontent.com/u/2401144?v=3","gravatar_id":"","url":"https://api.github.com/users/rubanm","html_url":"https://github.com/rubanm","followers_url":"https://api.github.com/users/rubanm/followers","following_url":"https://api.github.com/users/rubanm/following{/other_user}","gists_url":"https://api.github.com/users/rubanm/gists{/gist_id}","starred_url":"https://api.github.com/users/rubanm/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rubanm/subscriptions","organizations_url":"https://api.github.com/users/rubanm/orgs","repos_url":"https://api.github.com/users/rubanm/repos","events_url":"https://api.github.com/users/rubanm/events{/privacy}","received_events_url":"https://api.github.com/users/rubanm/received_events","type":"User","site_admin":false},"body":"Based on the discussion on https://groups.google.com/forum/#!topic/cascading-user/2zhharIYTjk\r\n\r\nThis adds a Cascading rule for wrapping the rhs of any HashJoin in a DistCacheTap.\r\n\r\nNote this is implemented via a separate FlowConnector and the default behavior has not been changed.\r\n\r\n@cwensel a review would be great, thanks.","created_at":"2016-08-09T16:18:15Z","updated_at":"2016-08-09T16:20:03Z","closed_at":null,"merged_at":null,"merge_commit_sha":"cc6d64f77a3efe9f98813480d1f81583f704d085","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/twitter/scalding/pulls/1585/commits","review_comments_url":"https://api.github.com/repos/twitter/scalding/pulls/1585/comments","review_comment_url":"https://api.github.com/repos/twitter/scalding/pulls/comments{/number}","comments_url":"https://api.github.com/repos/twitter/scalding/issues/1585/comments","statuses_url":"https://api.github.com/repos/twitter/scalding/statuses/e56dd1e36390c4ac1745079f2938d08b2c2422cb","head":{"label":"twitter:rubanm/cascading3/distcachetap_support","ref":"rubanm/cascading3/distcachetap_support","sha":"e56dd1e36390c4ac1745079f2938d08b2c2422cb","user":{"login":"twitter","id":50278,"avatar_url":"https://avatars.githubusercontent.com/u/50278?v=3","gravatar_id":"","url":"https://api.github.com/users/twitter","html_url":"https://github.com/twitter","followers_url":"https://api.github.com/users/twitter/followers","following_url":"https://api.github.com/users/twitter/following{/other_user}","gists_url":"https://api.github.com/users/twitter/gists{/gist_id}","starred_url":"https://api.github.com/users/twitter/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/twitter/subscriptions","organizations_url":"https://api.github.com/users/twitter/orgs","repos_url":"https://api.github.com/users/twitter/repos","events_url":"https://api.github.com/users/twitter/events{/privacy}","received_events_url":"https://api.github.com/users/twitter/received_events","type":"Organization","site_admin":false},"repo":{"id":3146558,"name":"scalding","full_name":"twitter/scalding","owner":{"login":"twitter","id":50278,"avatar_url":"https://avatars.githubusercontent.com/u/50278?v=3","gravatar_id":"","url":"https://api.github.com/users/twitter","html_url":"https://github.com/twitter","followers_url":"https://api.github.com/users/twitter/followers","following_url":"https://api.github.com/users/twitter/following{/other_user}","gists_url":"https://api.github.com/users/twitter/gists{/gist_id}","starred_url":"https://api.github.com/users/twitter/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/twitter/subscriptions","organizations_url":"https://api.github.com/users/twitter/orgs","repos_url":"https://api.github.com/users/twitter/repos","events_url":"https://api.github.com/users/twitter/events{/privacy}","received_events_url":"https://api.github.com/users/twitter/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/twitter/scalding","description":"A Scala API for Cascading","fork":false,"url":"https://api.github.com/repos/twitter/scalding","forks_url":"https://api.github.com/repos/twitter/scalding/forks","keys_url":"https://api.github.com/repos/twitter/scalding/keys{/key_id}","collaborators_url":"https://api.github.com/repos/twitter/scalding/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/twitter/scalding/teams","hooks_url":"https://api.github.com/repos/twitter/scalding/hooks","issue_events_url":"https://api.github.com/repos/twitter/scalding/issues/events{/number}","events_url":"https://api.github.com/repos/twitter/scalding/events","assignees_url":"https://api.github.com/repos/twitter/scalding/assignees{/user}","branches_url":"https://api.github.com/repos/twitter/scalding/branches{/branch}","tags_url":"https://api.github.com/repos/twitter/scalding/tags","blobs_url":"https://api.github.com/repos/twitter/scalding/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/twitter/scalding/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/twitter/scalding/git/refs{/sha}","trees_url":"https://api.github.com/repos/twitter/scalding/git/trees{/sha}","statuses_url":"https://api.github.com/repos/twitter/scalding/statuses/{sha}","languages_url":"https://api.github.com/repos/twitter/scalding/languages","stargazers_url":"https://api.github.com/repos/twitter/scalding/stargazers","contributors_url":"https://api.github.com/repos/twitter/scalding/contributors","subscribers_url":"https://api.github.com/repos/twitter/scalding/subscribers","subscription_url":"https://api.github.com/repos/twitter/scalding/subscription","commits_url":"https://api.github.com/repos/twitter/scalding/commits{/sha}","git_commits_url":"https://api.github.com/repos/twitter/scalding/git/commits{/sha}","comments_url":"https://api.github.com/repos/twitter/scalding/comments{/number}","issue_comment_url":"https://api.github.com/repos/twitter/scalding/issues/comments{/number}","contents_url":"https://api.github.com/repos/twitter/scalding/contents/{+path}","compare_url":"https://api.github.com/repos/twitter/scalding/compare/{base}...{head}","merges_url":"https://api.github.com/repos/twitter/scalding/merges","archive_url":"https://api.github.com/repos/twitter/scalding/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/twitter/scalding/downloads","issues_url":"https://api.github.com/repos/twitter/scalding/issues{/number}","pulls_url":"https://api.github.com/repos/twitter/scalding/pulls{/number}","milestones_url":"https://api.github.com/repos/twitter/scalding/milestones{/number}","notifications_url":"https://api.github.com/repos/twitter/scalding/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/twitter/scalding/labels{/name}","releases_url":"https://api.github.com/repos/twitter/scalding/releases{/id}","deployments_url":"https://api.github.com/repos/twitter/scalding/deployments","created_at":"2012-01-10T16:22:08Z","updated_at":"2016-08-08T20:33:16Z","pushed_at":"2016-08-09T16:18:16Z","git_url":"git://github.com/twitter/scalding.git","ssh_url":"git@github.com:twitter/scalding.git","clone_url":"https://github.com/twitter/scalding.git","svn_url":"https://github.com/twitter/scalding","homepage":"http://twitter.com/scalding","size":21850,"stargazers_count":2598,"watchers_count":2598,"language":"Scala","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":612,"mirror_url":null,"open_issues_count":270,"forks":612,"open_issues":270,"watchers":2598,"default_branch":"develop"}},"base":{"label":"twitter:cascading3","ref":"cascading3","sha":"59a1abcf9cdc791c9927b32317134743917da5ae","user":{"login":"twitter","id":50278,"avatar_url":"https://avatars.githubusercontent.com/u/50278?v=3","gravatar_id":"","url":"https://api.github.com/users/twitter","html_url":"https://github.com/twitter","followers_url":"https://api.github.com/users/twitter/followers","following_url":"https://api.github.com/users/twitter/following{/other_user}","gists_url":"https://api.github.com/users/twitter/gists{/gist_id}","starred_url":"https://api.github.com/users/twitter/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/twitter/subscriptions","organizations_url":"https://api.github.com/users/twitter/orgs","repos_url":"https://api.github.com/users/twitter/repos","events_url":"https://api.github.com/users/twitter/events{/privacy}","received_events_url":"https://api.github.com/users/twitter/received_events","type":"Organization","site_admin":false},"repo":{"id":3146558,"name":"scalding","full_name":"twitter/scalding","owner":{"login":"twitter","id":50278,"avatar_url":"https://avatars.githubusercontent.com/u/50278?v=3","gravatar_id":"","url":"https://api.github.com/users/twitter","html_url":"https://github.com/twitter","followers_url":"https://api.github.com/users/twitter/followers","following_url":"https://api.github.com/users/twitter/following{/other_user}","gists_url":"https://api.github.com/users/twitter/gists{/gist_id}","starred_url":"https://api.github.com/users/twitter/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/twitter/subscriptions","organizations_url":"https://api.github.com/users/twitter/orgs","repos_url":"https://api.github.com/users/twitter/repos","events_url":"https://api.github.com/users/twitter/events{/privacy}","received_events_url":"https://api.github.com/users/twitter/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/twitter/scalding","description":"A Scala API for Cascading","fork":false,"url":"https://api.github.com/repos/twitter/scalding","forks_url":"https://api.github.com/repos/twitter/scalding/forks","keys_url":"https://api.github.com/repos/twitter/scalding/keys{/key_id}","collaborators_url":"https://api.github.com/repos/twitter/scalding/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/twitter/scalding/teams","hooks_url":"https://api.github.com/repos/twitter/scalding/hooks","issue_events_url":"https://api.github.com/repos/twitter/scalding/issues/events{/number}","events_url":"https://api.github.com/repos/twitter/scalding/events","assignees_url":"https://api.github.com/repos/twitter/scalding/assignees{/user}","branches_url":"https://api.github.com/repos/twitter/scalding/branches{/branch}","tags_url":"https://api.github.com/repos/twitter/scalding/tags","blobs_url":"https://api.github.com/repos/twitter/scalding/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/twitter/scalding/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/twitter/scalding/git/refs{/sha}","trees_url":"https://api.github.com/repos/twitter/scalding/git/trees{/sha}","statuses_url":"https://api.github.com/repos/twitter/scalding/statuses/{sha}","languages_url":"https://api.github.com/repos/twitter/scalding/languages","stargazers_url":"https://api.github.com/repos/twitter/scalding/stargazers","contributors_url":"https://api.github.com/repos/twitter/scalding/contributors","subscribers_url":"https://api.github.com/repos/twitter/scalding/subscribers","subscription_url":"https://api.github.com/repos/twitter/scalding/subscription","commits_url":"https://api.github.com/repos/twitter/scalding/commits{/sha}","git_commits_url":"https://api.github.com/repos/twitter/scalding/git/commits{/sha}","comments_url":"https://api.github.com/repos/twitter/scalding/comments{/number}","issue_comment_url":"https://api.github.com/repos/twitter/scalding/issues/comments{/number}","contents_url":"https://api.github.com/repos/twitter/scalding/contents/{+path}","compare_url":"https://api.github.com/repos/twitter/scalding/compare/{base}...{head}","merges_url":"https://api.github.com/repos/twitter/scalding/merges","archive_url":"https://api.github.com/repos/twitter/scalding/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/twitter/scalding/downloads","issues_url":"https://api.github.com/repos/twitter/scalding/issues{/number}","pulls_url":"https://api.github.com/repos/twitter/scalding/pulls{/number}","milestones_url":"https://api.github.com/repos/twitter/scalding/milestones{/number}","notifications_url":"https://api.github.com/repos/twitter/scalding/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/twitter/scalding/labels{/name}","releases_url":"https://api.github.com/repos/twitter/scalding/releases{/id}","deployments_url":"https://api.github.com/repos/twitter/scalding/deployments","created_at":"2012-01-10T16:22:08Z","updated_at":"2016-08-08T20:33:16Z","pushed_at":"2016-08-09T16:18:16Z","git_url":"git://github.com/twitter/scalding.git","ssh_url":"git@github.com:twitter/scalding.git","clone_url":"https://github.com/twitter/scalding.git","svn_url":"https://github.com/twitter/scalding","homepage":"http://twitter.com/scalding","size":21850,"stargazers_count":2598,"watchers_count":2598,"language":"Scala","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":612,"mirror_url":null,"open_issues_count":270,"forks":612,"open_issues":270,"watchers":2598,"default_branch":"develop"}},"_links":{"self":{"href":"https://api.github.com/repos/twitter/scalding/pulls/1585"},"html":{"href":"https://github.com/twitter/scalding/pull/1585"},"issue":{"href":"https://api.github.com/repos/twitter/scalding/issues/1585"},"comments":{"href":"https://api.github.com/repos/twitter/scalding/issues/1585/comments"},"review_comments":{"href":"https://api.github.com/repos/twitter/scalding/pulls/1585/comments"},"review_comment":{"href":"https://api.github.com/repos/twitter/scalding/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/twitter/scalding/pulls/1585/commits"},"statuses":{"href":"https://api.github.com/repos/twitter/scalding/statuses/e56dd1e36390c4ac1745079f2938d08b2c2422cb"}}}}
{"actor":{"display_login":"rubanm"}}