{"action":"created","comment":{"url":"https://api.github.com/repos/amplab/keystone/pulls/comments/51510733","id":51510733,"diff_hunk":"@@ -0,0 +1,94 @@\n+package workflow\n+\n+import org.apache.spark.rdd.RDD\n+\n+/**\n+ * Node-level optimization, such as selecting a Linear Solver\n+ *\n+ * @param sampleFraction The fraction of the RDD to use for operations\n+ */\n+class NodeOptimizationRule(sampleFraction: Double = 0.01, seed: Long = 0) extends Rule {\n+  override def apply[A, B](plan: Pipeline[A, B]): Pipeline[A, B] = {\n+    val instructions = WorkflowUtils.pipelineToInstructions(plan)\n+\n+    // First, figure out which instructions we should actually execute.\n+    // Take the set of all instructions that are parents of an optimizable node transform or fit instruction.\n+    // And, keep those that do not depend on the runtime input of a fit pipeline\n+    val instructionsToExecute = instructions.zipWithIndex.map {\n+      case (TransformerApplyNode(tIndex, _), index) =>\n+        instructions(tIndex) match {\n+          case _: OptimizableTransformer[_, _] => WorkflowUtils.getParents(index, instructions) + index\n+          case _ => Set[Int]()\n+        }\n+\n+      case (EstimatorFitNode(estIndex, _), index) =>\n+        instructions(estIndex) match {\n+          case _: OptimizableEstimator[_, _] => WorkflowUtils.getParents(index, instructions) + index\n+          case _: OptimizableLabelEstimator[_, _, _] => WorkflowUtils.getParents(index, instructions) + index\n+          case _ => Set[Int]()\n+        }\n+\n+      case _ => Set[Int]()\n+    }.reduce(_ union _) -- WorkflowUtils.getChildren(Pipeline.SOURCE, instructions) - Pipeline.SOURCE\n+\n+    // Execute the minimal amount necessary of the pipeline on sampled nodes, and optimize the optimizable nodes\n+    val optimizedInstructions = instructions.toArray\n+    val registers = new Array[Any](instructions.length)","path":"src/main/scala/workflow/NodeOptimizationRule.scala","position":36,"original_position":36,"commit_id":"b9a01657ac1acd17e7e85de9553147fdadac8a64","original_commit_id":"b9a01657ac1acd17e7e85de9553147fdadac8a64","user":{"login":"etrain","id":1326181,"avatar_url":"https://avatars.githubusercontent.com/u/1326181?v=3","gravatar_id":"","url":"https://api.github.com/users/etrain","html_url":"https://github.com/etrain","followers_url":"https://api.github.com/users/etrain/followers","following_url":"https://api.github.com/users/etrain/following{/other_user}","gists_url":"https://api.github.com/users/etrain/gists{/gist_id}","starred_url":"https://api.github.com/users/etrain/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/etrain/subscriptions","organizations_url":"https://api.github.com/users/etrain/orgs","repos_url":"https://api.github.com/users/etrain/repos","events_url":"https://api.github.com/users/etrain/events{/privacy}","received_events_url":"https://api.github.com/users/etrain/received_events","type":"User","site_admin":false},"body":"Sorry - I'm not following. What are the registers?","created_at":"2016-02-02T01:41:13Z","updated_at":"2016-02-02T01:41:13Z","html_url":"https://github.com/amplab/keystone/pull/222#discussion_r51510733","pull_request_url":"https://api.github.com/repos/amplab/keystone/pulls/222","_links":{"self":{"href":"https://api.github.com/repos/amplab/keystone/pulls/comments/51510733"},"html":{"href":"https://github.com/amplab/keystone/pull/222#discussion_r51510733"},"pull_request":{"href":"https://api.github.com/repos/amplab/keystone/pulls/222"}}},"pull_request":{"url":"https://api.github.com/repos/amplab/keystone/pulls/222","id":57912485,"html_url":"https://github.com/amplab/keystone/pull/222","diff_url":"https://github.com/amplab/keystone/pull/222.diff","patch_url":"https://github.com/amplab/keystone/pull/222.patch","issue_url":"https://api.github.com/repos/amplab/keystone/issues/222","number":222,"state":"open","locked":false,"title":"Added a node-level optimization rule","user":{"login":"tomerk","id":676357,"avatar_url":"https://avatars.githubusercontent.com/u/676357?v=3","gravatar_id":"","url":"https://api.github.com/users/tomerk","html_url":"https://github.com/tomerk","followers_url":"https://api.github.com/users/tomerk/followers","following_url":"https://api.github.com/users/tomerk/following{/other_user}","gists_url":"https://api.github.com/users/tomerk/gists{/gist_id}","starred_url":"https://api.github.com/users/tomerk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tomerk/subscriptions","organizations_url":"https://api.github.com/users/tomerk/orgs","repos_url":"https://api.github.com/users/tomerk/repos","events_url":"https://api.github.com/users/tomerk/events{/privacy}","received_events_url":"https://api.github.com/users/tomerk/received_events","type":"User","site_admin":false},"body":"@etrain @shivaram ","created_at":"2016-02-01T22:43:22Z","updated_at":"2016-02-02T01:41:13Z","closed_at":null,"merged_at":null,"merge_commit_sha":"b4ab348c78a8c5f34e81f316eec692b1f9f2a1b7","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/amplab/keystone/pulls/222/commits","review_comments_url":"https://api.github.com/repos/amplab/keystone/pulls/222/comments","review_comment_url":"https://api.github.com/repos/amplab/keystone/pulls/comments{/number}","comments_url":"https://api.github.com/repos/amplab/keystone/issues/222/comments","statuses_url":"https://api.github.com/repos/amplab/keystone/statuses/b9a01657ac1acd17e7e85de9553147fdadac8a64","head":{"label":"tomerk:node-level-optimize","ref":"node-level-optimize","sha":"b9a01657ac1acd17e7e85de9553147fdadac8a64","user":{"login":"tomerk","id":676357,"avatar_url":"https://avatars.githubusercontent.com/u/676357?v=3","gravatar_id":"","url":"https://api.github.com/users/tomerk","html_url":"https://github.com/tomerk","followers_url":"https://api.github.com/users/tomerk/followers","following_url":"https://api.github.com/users/tomerk/following{/other_user}","gists_url":"https://api.github.com/users/tomerk/gists{/gist_id}","starred_url":"https://api.github.com/users/tomerk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tomerk/subscriptions","organizations_url":"https://api.github.com/users/tomerk/orgs","repos_url":"https://api.github.com/users/tomerk/repos","events_url":"https://api.github.com/users/tomerk/events{/privacy}","received_events_url":"https://api.github.com/users/tomerk/received_events","type":"User","site_admin":false},"repo":{"id":36905160,"name":"keystone","full_name":"tomerk/keystone","owner":{"login":"tomerk","id":676357,"avatar_url":"https://avatars.githubusercontent.com/u/676357?v=3","gravatar_id":"","url":"https://api.github.com/users/tomerk","html_url":"https://github.com/tomerk","followers_url":"https://api.github.com/users/tomerk/followers","following_url":"https://api.github.com/users/tomerk/following{/other_user}","gists_url":"https://api.github.com/users/tomerk/gists{/gist_id}","starred_url":"https://api.github.com/users/tomerk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tomerk/subscriptions","organizations_url":"https://api.github.com/users/tomerk/orgs","repos_url":"https://api.github.com/users/tomerk/repos","events_url":"https://api.github.com/users/tomerk/events{/privacy}","received_events_url":"https://api.github.com/users/tomerk/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/tomerk/keystone","description":"Simplifying robust end-to-end machine learning on Apache Spark.","fork":true,"url":"https://api.github.com/repos/tomerk/keystone","forks_url":"https://api.github.com/repos/tomerk/keystone/forks","keys_url":"https://api.github.com/repos/tomerk/keystone/keys{/key_id}","collaborators_url":"https://api.github.com/repos/tomerk/keystone/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/tomerk/keystone/teams","hooks_url":"https://api.github.com/repos/tomerk/keystone/hooks","issue_events_url":"https://api.github.com/repos/tomerk/keystone/issues/events{/number}","events_url":"https://api.github.com/repos/tomerk/keystone/events","assignees_url":"https://api.github.com/repos/tomerk/keystone/assignees{/user}","branches_url":"https://api.github.com/repos/tomerk/keystone/branches{/branch}","tags_url":"https://api.github.com/repos/tomerk/keystone/tags","blobs_url":"https://api.github.com/repos/tomerk/keystone/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/tomerk/keystone/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/tomerk/keystone/git/refs{/sha}","trees_url":"https://api.github.com/repos/tomerk/keystone/git/trees{/sha}","statuses_url":"https://api.github.com/repos/tomerk/keystone/statuses/{sha}","languages_url":"https://api.github.com/repos/tomerk/keystone/languages","stargazers_url":"https://api.github.com/repos/tomerk/keystone/stargazers","contributors_url":"https://api.github.com/repos/tomerk/keystone/contributors","subscribers_url":"https://api.github.com/repos/tomerk/keystone/subscribers","subscription_url":"https://api.github.com/repos/tomerk/keystone/subscription","commits_url":"https://api.github.com/repos/tomerk/keystone/commits{/sha}","git_commits_url":"https://api.github.com/repos/tomerk/keystone/git/commits{/sha}","comments_url":"https://api.github.com/repos/tomerk/keystone/comments{/number}","issue_comment_url":"https://api.github.com/repos/tomerk/keystone/issues/comments{/number}","contents_url":"https://api.github.com/repos/tomerk/keystone/contents/{+path}","compare_url":"https://api.github.com/repos/tomerk/keystone/compare/{base}...{head}","merges_url":"https://api.github.com/repos/tomerk/keystone/merges","archive_url":"https://api.github.com/repos/tomerk/keystone/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/tomerk/keystone/downloads","issues_url":"https://api.github.com/repos/tomerk/keystone/issues{/number}","pulls_url":"https://api.github.com/repos/tomerk/keystone/pulls{/number}","milestones_url":"https://api.github.com/repos/tomerk/keystone/milestones{/number}","notifications_url":"https://api.github.com/repos/tomerk/keystone/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/tomerk/keystone/labels{/name}","releases_url":"https://api.github.com/repos/tomerk/keystone/releases{/id}","deployments_url":"https://api.github.com/repos/tomerk/keystone/deployments","created_at":"2015-06-05T01:29:58Z","updated_at":"2016-01-21T20:52:37Z","pushed_at":"2016-02-01T23:04:05Z","git_url":"git://github.com/tomerk/keystone.git","ssh_url":"git@github.com:tomerk/keystone.git","clone_url":"https://github.com/tomerk/keystone.git","svn_url":"https://github.com/tomerk/keystone","homepage":"http://keystone-ml.org/","size":16987,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":1,"mirror_url":null,"open_issues_count":0,"forks":1,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"amplab:master","ref":"master","sha":"f37aa9f8026f67a8544f2dd0c8ed4ac8fbf7e5c6","user":{"login":"amplab","id":1438454,"avatar_url":"https://avatars.githubusercontent.com/u/1438454?v=3","gravatar_id":"","url":"https://api.github.com/users/amplab","html_url":"https://github.com/amplab","followers_url":"https://api.github.com/users/amplab/followers","following_url":"https://api.github.com/users/amplab/following{/other_user}","gists_url":"https://api.github.com/users/amplab/gists{/gist_id}","starred_url":"https://api.github.com/users/amplab/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/amplab/subscriptions","organizations_url":"https://api.github.com/users/amplab/orgs","repos_url":"https://api.github.com/users/amplab/repos","events_url":"https://api.github.com/users/amplab/events{/privacy}","received_events_url":"https://api.github.com/users/amplab/received_events","type":"Organization","site_admin":false},"repo":{"id":33748726,"name":"keystone","full_name":"amplab/keystone","owner":{"login":"amplab","id":1438454,"avatar_url":"https://avatars.githubusercontent.com/u/1438454?v=3","gravatar_id":"","url":"https://api.github.com/users/amplab","html_url":"https://github.com/amplab","followers_url":"https://api.github.com/users/amplab/followers","following_url":"https://api.github.com/users/amplab/following{/other_user}","gists_url":"https://api.github.com/users/amplab/gists{/gist_id}","starred_url":"https://api.github.com/users/amplab/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/amplab/subscriptions","organizations_url":"https://api.github.com/users/amplab/orgs","repos_url":"https://api.github.com/users/amplab/repos","events_url":"https://api.github.com/users/amplab/events{/privacy}","received_events_url":"https://api.github.com/users/amplab/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/amplab/keystone","description":"Simplifying robust end-to-end machine learning on Apache Spark.","fork":false,"url":"https://api.github.com/repos/amplab/keystone","forks_url":"https://api.github.com/repos/amplab/keystone/forks","keys_url":"https://api.github.com/repos/amplab/keystone/keys{/key_id}","collaborators_url":"https://api.github.com/repos/amplab/keystone/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/amplab/keystone/teams","hooks_url":"https://api.github.com/repos/amplab/keystone/hooks","issue_events_url":"https://api.github.com/repos/amplab/keystone/issues/events{/number}","events_url":"https://api.github.com/repos/amplab/keystone/events","assignees_url":"https://api.github.com/repos/amplab/keystone/assignees{/user}","branches_url":"https://api.github.com/repos/amplab/keystone/branches{/branch}","tags_url":"https://api.github.com/repos/amplab/keystone/tags","blobs_url":"https://api.github.com/repos/amplab/keystone/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/amplab/keystone/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/amplab/keystone/git/refs{/sha}","trees_url":"https://api.github.com/repos/amplab/keystone/git/trees{/sha}","statuses_url":"https://api.github.com/repos/amplab/keystone/statuses/{sha}","languages_url":"https://api.github.com/repos/amplab/keystone/languages","stargazers_url":"https://api.github.com/repos/amplab/keystone/stargazers","contributors_url":"https://api.github.com/repos/amplab/keystone/contributors","subscribers_url":"https://api.github.com/repos/amplab/keystone/subscribers","subscription_url":"https://api.github.com/repos/amplab/keystone/subscription","commits_url":"https://api.github.com/repos/amplab/keystone/commits{/sha}","git_commits_url":"https://api.github.com/repos/amplab/keystone/git/commits{/sha}","comments_url":"https://api.github.com/repos/amplab/keystone/comments{/number}","issue_comment_url":"https://api.github.com/repos/amplab/keystone/issues/comments{/number}","contents_url":"https://api.github.com/repos/amplab/keystone/contents/{+path}","compare_url":"https://api.github.com/repos/amplab/keystone/compare/{base}...{head}","merges_url":"https://api.github.com/repos/amplab/keystone/merges","archive_url":"https://api.github.com/repos/amplab/keystone/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/amplab/keystone/downloads","issues_url":"https://api.github.com/repos/amplab/keystone/issues{/number}","pulls_url":"https://api.github.com/repos/amplab/keystone/pulls{/number}","milestones_url":"https://api.github.com/repos/amplab/keystone/milestones{/number}","notifications_url":"https://api.github.com/repos/amplab/keystone/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/amplab/keystone/labels{/name}","releases_url":"https://api.github.com/repos/amplab/keystone/releases{/id}","deployments_url":"https://api.github.com/repos/amplab/keystone/deployments","created_at":"2015-04-10T20:16:11Z","updated_at":"2016-01-29T08:24:42Z","pushed_at":"2016-02-01T23:04:17Z","git_url":"git://github.com/amplab/keystone.git","ssh_url":"git@github.com:amplab/keystone.git","clone_url":"https://github.com/amplab/keystone.git","svn_url":"https://github.com/amplab/keystone","homepage":"http://keystone-ml.org/","size":17186,"stargazers_count":287,"watchers_count":287,"language":"Scala","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":57,"mirror_url":null,"open_issues_count":43,"forks":57,"open_issues":43,"watchers":287,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/amplab/keystone/pulls/222"},"html":{"href":"https://github.com/amplab/keystone/pull/222"},"issue":{"href":"https://api.github.com/repos/amplab/keystone/issues/222"},"comments":{"href":"https://api.github.com/repos/amplab/keystone/issues/222/comments"},"review_comments":{"href":"https://api.github.com/repos/amplab/keystone/pulls/222/comments"},"review_comment":{"href":"https://api.github.com/repos/amplab/keystone/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/amplab/keystone/pulls/222/commits"},"statuses":{"href":"https://api.github.com/repos/amplab/keystone/statuses/b9a01657ac1acd17e7e85de9553147fdadac8a64"}}}}