{"action":"created","comment":{"url":"https://api.github.com/repos/dmlc/xgboost/pulls/comments/55143983","id":55143983,"diff_hunk":"@@ -37,42 +40,33 @@ object XGBoost extends Serializable {\n       xgBoostConfMap: Map[String, AnyRef],\n       numWorkers: Int, round: Int, obj: ObjectiveTrait, eval: EvalTrait): RDD[Booster] = {\n     import DataUtils._\n-    val sc = trainingData.sparkContext\n-    val tracker = new RabitTracker(numWorkers)\n-    if (tracker.start()) {\n-      trainingData.repartition(numWorkers).mapPartitions {\n-        trainingSamples =>\n-          Rabit.init(new java.util.HashMap[String, String]() {\n-            put(\"DMLC_TASK_ID\", TaskContext.getPartitionId().toString)\n-          })\n-          val dMatrix = new DMatrix(new JDMatrix(trainingSamples, null))\n-          val booster = SXGBoost.train(xgBoostConfMap, dMatrix, round,\n-            watches = new HashMap[String, DMatrix], obj, eval)\n-          Rabit.shutdown()\n-          Iterator(booster)\n-      }.cache()\n-    } else {\n-      null\n-    }\n+    trainingData.repartition(numWorkers).mapPartitions {\n+      trainingSamples =>\n+        Rabit.init(new java.util.HashMap[String, String]() {\n+          put(\"DMLC_TASK_ID\", TaskContext.getPartitionId().toString)\n+        })\n+        val dMatrix = new DMatrix(new JDMatrix(trainingSamples, null))\n+        val booster = SXGBoost.train(xgBoostConfMap, dMatrix, round,\n+          watches = new mutable.HashMap[String, DMatrix]{put(\"train\", dMatrix)}.toMap, obj, eval)\n+        Rabit.shutdown()\n+        Iterator(booster)\n+    }.cache()\n   }\n \n-  def train(config: Config, trainingData: RDD[LabeledPoint], obj: ObjectiveTrait = null,\n-      eval: EvalTrait = null): Option[XGBoostModel] = {\n-    import DataUtils._\n-    val numWorkers = config.getInt(\"numWorkers\")\n-    val round = config.getInt(\"round\")\n+  def train(trainingData: RDD[LabeledPoint], configMap: Map[String, AnyRef], round: Int,\n+       obj: ObjectiveTrait = null, eval: EvalTrait = null): Option[XGBoostModel] = {\n+    val numWorkers = trainingData.partitions.length\n     val sc = trainingData.sparkContext\n     val tracker = new RabitTracker(numWorkers)\n-    if (tracker.start()) {\n-      // TODO: build configuration map from config\n-      val xgBoostConfigMap = new HashMap[String, AnyRef]()\n-      val boosters = buildDistributedBoosters(trainingData, xgBoostConfigMap, numWorkers, round,\n-        obj, eval)\n-      // force the job\n-      sc.runJob(boosters, (boosters: Iterator[Booster]) => boosters)\n-      tracker.waitFor()\n-      // TODO: how to choose best model\n-      Some(boosters.first())\n+    require(tracker.start(), \"FAULT: Failed to start tracker\")\n+    boosters = buildDistributedBoosters(trainingData, configMap, numWorkers, round, obj, eval)\n+    // force the job\n+    sc.runJob(boosters, (boosters: Iterator[Booster]) => boosters)\n+    val booster = boosters.first()\n+    val returnVal = tracker.waitFor()","path":"jvm-packages/xgboost4j-spark/src/main/scala/ml/dmlc/xgboost4j/scala/spark/XGBoost.scala","position":83,"original_position":83,"commit_id":"f768edfede06906d53e30ddebccf3cdef3291ce1","original_commit_id":"f768edfede06906d53e30ddebccf3cdef3291ce1","user":{"login":"CodingCat","id":678008,"avatar_url":"https://avatars.githubusercontent.com/u/678008?v=3","gravatar_id":"","url":"https://api.github.com/users/CodingCat","html_url":"https://github.com/CodingCat","followers_url":"https://api.github.com/users/CodingCat/followers","following_url":"https://api.github.com/users/CodingCat/following{/other_user}","gists_url":"https://api.github.com/users/CodingCat/gists{/gist_id}","starred_url":"https://api.github.com/users/CodingCat/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/CodingCat/subscriptions","organizations_url":"https://api.github.com/users/CodingCat/orgs","repos_url":"https://api.github.com/users/CodingCat/repos","events_url":"https://api.github.com/users/CodingCat/events{/privacy}","received_events_url":"https://api.github.com/users/CodingCat/received_events","type":"User","site_admin":false},"body":"it is somewhat uncertain for me, \r\n\r\nin what case tracker will return 1?","created_at":"2016-03-06T16:50:03Z","updated_at":"2016-03-06T16:50:03Z","html_url":"https://github.com/dmlc/xgboost/pull/925#discussion_r55143983","pull_request_url":"https://api.github.com/repos/dmlc/xgboost/pulls/925","_links":{"self":{"href":"https://api.github.com/repos/dmlc/xgboost/pulls/comments/55143983"},"html":{"href":"https://github.com/dmlc/xgboost/pull/925#discussion_r55143983"},"pull_request":{"href":"https://api.github.com/repos/dmlc/xgboost/pulls/925"}}},"pull_request":{"url":"https://api.github.com/repos/dmlc/xgboost/pulls/925","id":61843042,"html_url":"https://github.com/dmlc/xgboost/pull/925","diff_url":"https://github.com/dmlc/xgboost/pull/925.diff","patch_url":"https://github.com/dmlc/xgboost/pull/925.patch","issue_url":"https://api.github.com/repos/dmlc/xgboost/issues/925","number":925,"state":"open","locked":false,"title":"adjust the return values of RabitTracker.waitFor(), remove typesafe.Config","user":{"login":"CodingCat","id":678008,"avatar_url":"https://avatars.githubusercontent.com/u/678008?v=3","gravatar_id":"","url":"https://api.github.com/users/CodingCat","html_url":"https://github.com/CodingCat","followers_url":"https://api.github.com/users/CodingCat/followers","following_url":"https://api.github.com/users/CodingCat/following{/other_user}","gists_url":"https://api.github.com/users/CodingCat/gists{/gist_id}","starred_url":"https://api.github.com/users/CodingCat/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/CodingCat/subscriptions","organizations_url":"https://api.github.com/users/CodingCat/orgs","repos_url":"https://api.github.com/users/CodingCat/repos","events_url":"https://api.github.com/users/CodingCat/events{/privacy}","received_events_url":"https://api.github.com/users/CodingCat/received_events","type":"User","site_admin":false},"body":"","created_at":"2016-03-06T13:43:57Z","updated_at":"2016-03-06T16:50:03Z","closed_at":null,"merged_at":null,"merge_commit_sha":"c80f6fd3616309a5b457574cc3c2f68fb2f1e3e3","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/dmlc/xgboost/pulls/925/commits","review_comments_url":"https://api.github.com/repos/dmlc/xgboost/pulls/925/comments","review_comment_url":"https://api.github.com/repos/dmlc/xgboost/pulls/comments{/number}","comments_url":"https://api.github.com/repos/dmlc/xgboost/issues/925/comments","statuses_url":"https://api.github.com/repos/dmlc/xgboost/statuses/f768edfede06906d53e30ddebccf3cdef3291ce1","head":{"label":"CodingCat:master","ref":"master","sha":"f768edfede06906d53e30ddebccf3cdef3291ce1","user":{"login":"CodingCat","id":678008,"avatar_url":"https://avatars.githubusercontent.com/u/678008?v=3","gravatar_id":"","url":"https://api.github.com/users/CodingCat","html_url":"https://github.com/CodingCat","followers_url":"https://api.github.com/users/CodingCat/followers","following_url":"https://api.github.com/users/CodingCat/following{/other_user}","gists_url":"https://api.github.com/users/CodingCat/gists{/gist_id}","starred_url":"https://api.github.com/users/CodingCat/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/CodingCat/subscriptions","organizations_url":"https://api.github.com/users/CodingCat/orgs","repos_url":"https://api.github.com/users/CodingCat/repos","events_url":"https://api.github.com/users/CodingCat/events{/privacy}","received_events_url":"https://api.github.com/users/CodingCat/received_events","type":"User","site_admin":false},"repo":{"id":47598194,"name":"xgboost","full_name":"CodingCat/xgboost","owner":{"login":"CodingCat","id":678008,"avatar_url":"https://avatars.githubusercontent.com/u/678008?v=3","gravatar_id":"","url":"https://api.github.com/users/CodingCat","html_url":"https://github.com/CodingCat","followers_url":"https://api.github.com/users/CodingCat/followers","following_url":"https://api.github.com/users/CodingCat/following{/other_user}","gists_url":"https://api.github.com/users/CodingCat/gists{/gist_id}","starred_url":"https://api.github.com/users/CodingCat/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/CodingCat/subscriptions","organizations_url":"https://api.github.com/users/CodingCat/orgs","repos_url":"https://api.github.com/users/CodingCat/repos","events_url":"https://api.github.com/users/CodingCat/events{/privacy}","received_events_url":"https://api.github.com/users/CodingCat/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/CodingCat/xgboost","description":"Large-scale and Distributed Gradient Boosting (GBDT, GBRT or GBM) Library, on single node, hadoop yarn and more.","fork":true,"url":"https://api.github.com/repos/CodingCat/xgboost","forks_url":"https://api.github.com/repos/CodingCat/xgboost/forks","keys_url":"https://api.github.com/repos/CodingCat/xgboost/keys{/key_id}","collaborators_url":"https://api.github.com/repos/CodingCat/xgboost/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/CodingCat/xgboost/teams","hooks_url":"https://api.github.com/repos/CodingCat/xgboost/hooks","issue_events_url":"https://api.github.com/repos/CodingCat/xgboost/issues/events{/number}","events_url":"https://api.github.com/repos/CodingCat/xgboost/events","assignees_url":"https://api.github.com/repos/CodingCat/xgboost/assignees{/user}","branches_url":"https://api.github.com/repos/CodingCat/xgboost/branches{/branch}","tags_url":"https://api.github.com/repos/CodingCat/xgboost/tags","blobs_url":"https://api.github.com/repos/CodingCat/xgboost/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/CodingCat/xgboost/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/CodingCat/xgboost/git/refs{/sha}","trees_url":"https://api.github.com/repos/CodingCat/xgboost/git/trees{/sha}","statuses_url":"https://api.github.com/repos/CodingCat/xgboost/statuses/{sha}","languages_url":"https://api.github.com/repos/CodingCat/xgboost/languages","stargazers_url":"https://api.github.com/repos/CodingCat/xgboost/stargazers","contributors_url":"https://api.github.com/repos/CodingCat/xgboost/contributors","subscribers_url":"https://api.github.com/repos/CodingCat/xgboost/subscribers","subscription_url":"https://api.github.com/repos/CodingCat/xgboost/subscription","commits_url":"https://api.github.com/repos/CodingCat/xgboost/commits{/sha}","git_commits_url":"https://api.github.com/repos/CodingCat/xgboost/git/commits{/sha}","comments_url":"https://api.github.com/repos/CodingCat/xgboost/comments{/number}","issue_comment_url":"https://api.github.com/repos/CodingCat/xgboost/issues/comments{/number}","contents_url":"https://api.github.com/repos/CodingCat/xgboost/contents/{+path}","compare_url":"https://api.github.com/repos/CodingCat/xgboost/compare/{base}...{head}","merges_url":"https://api.github.com/repos/CodingCat/xgboost/merges","archive_url":"https://api.github.com/repos/CodingCat/xgboost/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/CodingCat/xgboost/downloads","issues_url":"https://api.github.com/repos/CodingCat/xgboost/issues{/number}","pulls_url":"https://api.github.com/repos/CodingCat/xgboost/pulls{/number}","milestones_url":"https://api.github.com/repos/CodingCat/xgboost/milestones{/number}","notifications_url":"https://api.github.com/repos/CodingCat/xgboost/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/CodingCat/xgboost/labels{/name}","releases_url":"https://api.github.com/repos/CodingCat/xgboost/releases{/id}","deployments_url":"https://api.github.com/repos/CodingCat/xgboost/deployments","created_at":"2015-12-08T04:18:55Z","updated_at":"2016-01-14T17:35:10Z","pushed_at":"2016-03-06T15:53:28Z","git_url":"git://github.com/CodingCat/xgboost.git","ssh_url":"git@github.com:CodingCat/xgboost.git","clone_url":"https://github.com/CodingCat/xgboost.git","svn_url":"https://github.com/CodingCat/xgboost","homepage":"","size":5225,"stargazers_count":0,"watchers_count":0,"language":"C++","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"dmlc:master","ref":"master","sha":"457ff82e33d880275f9c66a971044deaecdfb4b4","user":{"login":"dmlc","id":11508361,"avatar_url":"https://avatars.githubusercontent.com/u/11508361?v=3","gravatar_id":"","url":"https://api.github.com/users/dmlc","html_url":"https://github.com/dmlc","followers_url":"https://api.github.com/users/dmlc/followers","following_url":"https://api.github.com/users/dmlc/following{/other_user}","gists_url":"https://api.github.com/users/dmlc/gists{/gist_id}","starred_url":"https://api.github.com/users/dmlc/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dmlc/subscriptions","organizations_url":"https://api.github.com/users/dmlc/orgs","repos_url":"https://api.github.com/users/dmlc/repos","events_url":"https://api.github.com/users/dmlc/events{/privacy}","received_events_url":"https://api.github.com/users/dmlc/received_events","type":"Organization","site_admin":false},"repo":{"id":16587283,"name":"xgboost","full_name":"dmlc/xgboost","owner":{"login":"dmlc","id":11508361,"avatar_url":"https://avatars.githubusercontent.com/u/11508361?v=3","gravatar_id":"","url":"https://api.github.com/users/dmlc","html_url":"https://github.com/dmlc","followers_url":"https://api.github.com/users/dmlc/followers","following_url":"https://api.github.com/users/dmlc/following{/other_user}","gists_url":"https://api.github.com/users/dmlc/gists{/gist_id}","starred_url":"https://api.github.com/users/dmlc/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dmlc/subscriptions","organizations_url":"https://api.github.com/users/dmlc/orgs","repos_url":"https://api.github.com/users/dmlc/repos","events_url":"https://api.github.com/users/dmlc/events{/privacy}","received_events_url":"https://api.github.com/users/dmlc/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/dmlc/xgboost","description":"Scalable, Portable and Distributed Gradient Boosting (GBDT, GBRT or GBM) Library,  for Python, R, Java, C++ and more","fork":false,"url":"https://api.github.com/repos/dmlc/xgboost","forks_url":"https://api.github.com/repos/dmlc/xgboost/forks","keys_url":"https://api.github.com/repos/dmlc/xgboost/keys{/key_id}","collaborators_url":"https://api.github.com/repos/dmlc/xgboost/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/dmlc/xgboost/teams","hooks_url":"https://api.github.com/repos/dmlc/xgboost/hooks","issue_events_url":"https://api.github.com/repos/dmlc/xgboost/issues/events{/number}","events_url":"https://api.github.com/repos/dmlc/xgboost/events","assignees_url":"https://api.github.com/repos/dmlc/xgboost/assignees{/user}","branches_url":"https://api.github.com/repos/dmlc/xgboost/branches{/branch}","tags_url":"https://api.github.com/repos/dmlc/xgboost/tags","blobs_url":"https://api.github.com/repos/dmlc/xgboost/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/dmlc/xgboost/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/dmlc/xgboost/git/refs{/sha}","trees_url":"https://api.github.com/repos/dmlc/xgboost/git/trees{/sha}","statuses_url":"https://api.github.com/repos/dmlc/xgboost/statuses/{sha}","languages_url":"https://api.github.com/repos/dmlc/xgboost/languages","stargazers_url":"https://api.github.com/repos/dmlc/xgboost/stargazers","contributors_url":"https://api.github.com/repos/dmlc/xgboost/contributors","subscribers_url":"https://api.github.com/repos/dmlc/xgboost/subscribers","subscription_url":"https://api.github.com/repos/dmlc/xgboost/subscription","commits_url":"https://api.github.com/repos/dmlc/xgboost/commits{/sha}","git_commits_url":"https://api.github.com/repos/dmlc/xgboost/git/commits{/sha}","comments_url":"https://api.github.com/repos/dmlc/xgboost/comments{/number}","issue_comment_url":"https://api.github.com/repos/dmlc/xgboost/issues/comments{/number}","contents_url":"https://api.github.com/repos/dmlc/xgboost/contents/{+path}","compare_url":"https://api.github.com/repos/dmlc/xgboost/compare/{base}...{head}","merges_url":"https://api.github.com/repos/dmlc/xgboost/merges","archive_url":"https://api.github.com/repos/dmlc/xgboost/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/dmlc/xgboost/downloads","issues_url":"https://api.github.com/repos/dmlc/xgboost/issues{/number}","pulls_url":"https://api.github.com/repos/dmlc/xgboost/pulls{/number}","milestones_url":"https://api.github.com/repos/dmlc/xgboost/milestones{/number}","notifications_url":"https://api.github.com/repos/dmlc/xgboost/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/dmlc/xgboost/labels{/name}","releases_url":"https://api.github.com/repos/dmlc/xgboost/releases{/id}","deployments_url":"https://api.github.com/repos/dmlc/xgboost/deployments","created_at":"2014-02-06T17:28:03Z","updated_at":"2016-03-06T15:50:59Z","pushed_at":"2016-03-06T13:44:14Z","git_url":"git://github.com/dmlc/xgboost.git","ssh_url":"git@github.com:dmlc/xgboost.git","clone_url":"https://github.com/dmlc/xgboost.git","svn_url":"https://github.com/dmlc/xgboost","homepage":"","size":19535,"stargazers_count":2636,"watchers_count":2636,"language":"C++","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1511,"mirror_url":null,"open_issues_count":77,"forks":1511,"open_issues":77,"watchers":2636,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/dmlc/xgboost/pulls/925"},"html":{"href":"https://github.com/dmlc/xgboost/pull/925"},"issue":{"href":"https://api.github.com/repos/dmlc/xgboost/issues/925"},"comments":{"href":"https://api.github.com/repos/dmlc/xgboost/issues/925/comments"},"review_comments":{"href":"https://api.github.com/repos/dmlc/xgboost/pulls/925/comments"},"review_comment":{"href":"https://api.github.com/repos/dmlc/xgboost/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/dmlc/xgboost/pulls/925/commits"},"statuses":{"href":"https://api.github.com/repos/dmlc/xgboost/statuses/f768edfede06906d53e30ddebccf3cdef3291ce1"}}}}