{"action":"created","comment":{"url":"https://api.github.com/repos/linkedin/photon-ml/pulls/comments/63236578","id":63236578,"diff_hunk":"@@ -169,19 +169,26 @@ protected[ml] class Driver(\n   }\n \n   protected def prepareTrainingData(): Unit = {\n+    // Verify that selected features files exist\n     params.selectedFeaturesFile.foreach(file => {\n       val path = new Path(file)\n       if (! path.getFileSystem(sc.hadoopConfiguration).exists(path)) {\n         throw new IOException(\"Could not find [\" + file + \"]. Check that the file exists\")\n       }\n     })\n+\n+    // Read training data\n     trainingData = suite.readLabeledPointsFromAvro(\n       sc, params.trainDir, params.selectedFeaturesFile, params.minNumPartitions)\n       .persist(trainDataStorageLevel).setName(\"training data\")\n-    featureNum = trainingData.first().features.size\n+\n     trainingDataNum = trainingData.count().toInt\n+    require(trainingDataNum > 0,\n+      \"No training data found. Ensure that training data exists and feature vectors are not empty.\")","path":"photon-ml/src/main/scala/com/linkedin/photon/ml/Driver.scala","position":20,"original_position":20,"commit_id":"d541bdf0773a3bed6b49996807f006fce19234c3","original_commit_id":"d541bdf0773a3bed6b49996807f006fce19234c3","user":{"login":"joshvfleming","id":212140,"avatar_url":"https://avatars.githubusercontent.com/u/212140?v=3","gravatar_id":"","url":"https://api.github.com/users/joshvfleming","html_url":"https://github.com/joshvfleming","followers_url":"https://api.github.com/users/joshvfleming/followers","following_url":"https://api.github.com/users/joshvfleming/following{/other_user}","gists_url":"https://api.github.com/users/joshvfleming/gists{/gist_id}","starred_url":"https://api.github.com/users/joshvfleming/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/joshvfleming/subscriptions","organizations_url":"https://api.github.com/users/joshvfleming/orgs","repos_url":"https://api.github.com/users/joshvfleming/repos","events_url":"https://api.github.com/users/joshvfleming/events{/privacy}","received_events_url":"https://api.github.com/users/joshvfleming/received_events","type":"User","site_admin":false},"body":"That's right, @XianXing. We map over the records and return ```None``` if the feature vector is empty, and then we ```flatMap``` over the result. This means that a dataset of empty feature vectors becomes an empty collection.","created_at":"2016-05-13T19:13:44Z","updated_at":"2016-05-13T19:13:44Z","html_url":"https://github.com/linkedin/photon-ml/pull/74#discussion_r63236578","pull_request_url":"https://api.github.com/repos/linkedin/photon-ml/pulls/74","_links":{"self":{"href":"https://api.github.com/repos/linkedin/photon-ml/pulls/comments/63236578"},"html":{"href":"https://github.com/linkedin/photon-ml/pull/74#discussion_r63236578"},"pull_request":{"href":"https://api.github.com/repos/linkedin/photon-ml/pulls/74"}}},"pull_request":{"url":"https://api.github.com/repos/linkedin/photon-ml/pulls/74","id":70028008,"html_url":"https://github.com/linkedin/photon-ml/pull/74","diff_url":"https://github.com/linkedin/photon-ml/pull/74.diff","patch_url":"https://github.com/linkedin/photon-ml/pull/74.patch","issue_url":"https://api.github.com/repos/linkedin/photon-ml/issues/74","number":74,"state":"closed","locked":false,"title":"Improve error reporting for empty training set","user":{"login":"joshvfleming","id":212140,"avatar_url":"https://avatars.githubusercontent.com/u/212140?v=3","gravatar_id":"","url":"https://api.github.com/users/joshvfleming","html_url":"https://github.com/joshvfleming","followers_url":"https://api.github.com/users/joshvfleming/followers","following_url":"https://api.github.com/users/joshvfleming/following{/other_user}","gists_url":"https://api.github.com/users/joshvfleming/gists{/gist_id}","starred_url":"https://api.github.com/users/joshvfleming/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/joshvfleming/subscriptions","organizations_url":"https://api.github.com/users/joshvfleming/orgs","repos_url":"https://api.github.com/users/joshvfleming/repos","events_url":"https://api.github.com/users/joshvfleming/events{/privacy}","received_events_url":"https://api.github.com/users/joshvfleming/received_events","type":"User","site_admin":false},"body":"This arose in response to a recent user issue. The feature vectors in\r\ntheir training set were empty, and we failed on the call to\r\ntrainingData.first() with an unhelpful error message. This change\r\nimproves the error reporting for this case.","created_at":"2016-05-13T17:05:33Z","updated_at":"2016-05-13T19:13:44Z","closed_at":"2016-05-13T19:08:55Z","merged_at":"2016-05-13T19:08:55Z","merge_commit_sha":"298a6a3949443468099c8833695e75215214e129","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/linkedin/photon-ml/pulls/74/commits","review_comments_url":"https://api.github.com/repos/linkedin/photon-ml/pulls/74/comments","review_comment_url":"https://api.github.com/repos/linkedin/photon-ml/pulls/comments{/number}","comments_url":"https://api.github.com/repos/linkedin/photon-ml/issues/74/comments","statuses_url":"https://api.github.com/repos/linkedin/photon-ml/statuses/d541bdf0773a3bed6b49996807f006fce19234c3","head":{"label":"joshvfleming:master","ref":"master","sha":"d541bdf0773a3bed6b49996807f006fce19234c3","user":{"login":"joshvfleming","id":212140,"avatar_url":"https://avatars.githubusercontent.com/u/212140?v=3","gravatar_id":"","url":"https://api.github.com/users/joshvfleming","html_url":"https://github.com/joshvfleming","followers_url":"https://api.github.com/users/joshvfleming/followers","following_url":"https://api.github.com/users/joshvfleming/following{/other_user}","gists_url":"https://api.github.com/users/joshvfleming/gists{/gist_id}","starred_url":"https://api.github.com/users/joshvfleming/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/joshvfleming/subscriptions","organizations_url":"https://api.github.com/users/joshvfleming/orgs","repos_url":"https://api.github.com/users/joshvfleming/repos","events_url":"https://api.github.com/users/joshvfleming/events{/privacy}","received_events_url":"https://api.github.com/users/joshvfleming/received_events","type":"User","site_admin":false},"repo":{"id":56254851,"name":"photon-ml","full_name":"joshvfleming/photon-ml","owner":{"login":"joshvfleming","id":212140,"avatar_url":"https://avatars.githubusercontent.com/u/212140?v=3","gravatar_id":"","url":"https://api.github.com/users/joshvfleming","html_url":"https://github.com/joshvfleming","followers_url":"https://api.github.com/users/joshvfleming/followers","following_url":"https://api.github.com/users/joshvfleming/following{/other_user}","gists_url":"https://api.github.com/users/joshvfleming/gists{/gist_id}","starred_url":"https://api.github.com/users/joshvfleming/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/joshvfleming/subscriptions","organizations_url":"https://api.github.com/users/joshvfleming/orgs","repos_url":"https://api.github.com/users/joshvfleming/repos","events_url":"https://api.github.com/users/joshvfleming/events{/privacy}","received_events_url":"https://api.github.com/users/joshvfleming/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/joshvfleming/photon-ml","description":"A scalable machine learning library on Apache Spark","fork":true,"url":"https://api.github.com/repos/joshvfleming/photon-ml","forks_url":"https://api.github.com/repos/joshvfleming/photon-ml/forks","keys_url":"https://api.github.com/repos/joshvfleming/photon-ml/keys{/key_id}","collaborators_url":"https://api.github.com/repos/joshvfleming/photon-ml/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/joshvfleming/photon-ml/teams","hooks_url":"https://api.github.com/repos/joshvfleming/photon-ml/hooks","issue_events_url":"https://api.github.com/repos/joshvfleming/photon-ml/issues/events{/number}","events_url":"https://api.github.com/repos/joshvfleming/photon-ml/events","assignees_url":"https://api.github.com/repos/joshvfleming/photon-ml/assignees{/user}","branches_url":"https://api.github.com/repos/joshvfleming/photon-ml/branches{/branch}","tags_url":"https://api.github.com/repos/joshvfleming/photon-ml/tags","blobs_url":"https://api.github.com/repos/joshvfleming/photon-ml/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/joshvfleming/photon-ml/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/joshvfleming/photon-ml/git/refs{/sha}","trees_url":"https://api.github.com/repos/joshvfleming/photon-ml/git/trees{/sha}","statuses_url":"https://api.github.com/repos/joshvfleming/photon-ml/statuses/{sha}","languages_url":"https://api.github.com/repos/joshvfleming/photon-ml/languages","stargazers_url":"https://api.github.com/repos/joshvfleming/photon-ml/stargazers","contributors_url":"https://api.github.com/repos/joshvfleming/photon-ml/contributors","subscribers_url":"https://api.github.com/repos/joshvfleming/photon-ml/subscribers","subscription_url":"https://api.github.com/repos/joshvfleming/photon-ml/subscription","commits_url":"https://api.github.com/repos/joshvfleming/photon-ml/commits{/sha}","git_commits_url":"https://api.github.com/repos/joshvfleming/photon-ml/git/commits{/sha}","comments_url":"https://api.github.com/repos/joshvfleming/photon-ml/comments{/number}","issue_comment_url":"https://api.github.com/repos/joshvfleming/photon-ml/issues/comments{/number}","contents_url":"https://api.github.com/repos/joshvfleming/photon-ml/contents/{+path}","compare_url":"https://api.github.com/repos/joshvfleming/photon-ml/compare/{base}...{head}","merges_url":"https://api.github.com/repos/joshvfleming/photon-ml/merges","archive_url":"https://api.github.com/repos/joshvfleming/photon-ml/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/joshvfleming/photon-ml/downloads","issues_url":"https://api.github.com/repos/joshvfleming/photon-ml/issues{/number}","pulls_url":"https://api.github.com/repos/joshvfleming/photon-ml/pulls{/number}","milestones_url":"https://api.github.com/repos/joshvfleming/photon-ml/milestones{/number}","notifications_url":"https://api.github.com/repos/joshvfleming/photon-ml/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/joshvfleming/photon-ml/labels{/name}","releases_url":"https://api.github.com/repos/joshvfleming/photon-ml/releases{/id}","deployments_url":"https://api.github.com/repos/joshvfleming/photon-ml/deployments","created_at":"2016-04-14T16:55:25Z","updated_at":"2016-04-14T16:55:27Z","pushed_at":"2016-05-13T17:04:17Z","git_url":"git://github.com/joshvfleming/photon-ml.git","ssh_url":"git@github.com:joshvfleming/photon-ml.git","clone_url":"https://github.com/joshvfleming/photon-ml.git","svn_url":"https://github.com/joshvfleming/photon-ml","homepage":"","size":16504,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"linkedin:master","ref":"master","sha":"d43b1ad58f3c7ad9f2eb92d377d29ba8b1a1085d","user":{"login":"linkedin","id":357098,"avatar_url":"https://avatars.githubusercontent.com/u/357098?v=3","gravatar_id":"","url":"https://api.github.com/users/linkedin","html_url":"https://github.com/linkedin","followers_url":"https://api.github.com/users/linkedin/followers","following_url":"https://api.github.com/users/linkedin/following{/other_user}","gists_url":"https://api.github.com/users/linkedin/gists{/gist_id}","starred_url":"https://api.github.com/users/linkedin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/linkedin/subscriptions","organizations_url":"https://api.github.com/users/linkedin/orgs","repos_url":"https://api.github.com/users/linkedin/repos","events_url":"https://api.github.com/users/linkedin/events{/privacy}","received_events_url":"https://api.github.com/users/linkedin/received_events","type":"Organization","site_admin":false},"repo":{"id":50964104,"name":"photon-ml","full_name":"linkedin/photon-ml","owner":{"login":"linkedin","id":357098,"avatar_url":"https://avatars.githubusercontent.com/u/357098?v=3","gravatar_id":"","url":"https://api.github.com/users/linkedin","html_url":"https://github.com/linkedin","followers_url":"https://api.github.com/users/linkedin/followers","following_url":"https://api.github.com/users/linkedin/following{/other_user}","gists_url":"https://api.github.com/users/linkedin/gists{/gist_id}","starred_url":"https://api.github.com/users/linkedin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/linkedin/subscriptions","organizations_url":"https://api.github.com/users/linkedin/orgs","repos_url":"https://api.github.com/users/linkedin/repos","events_url":"https://api.github.com/users/linkedin/events{/privacy}","received_events_url":"https://api.github.com/users/linkedin/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/linkedin/photon-ml","description":"A scalable machine learning library on Apache Spark","fork":false,"url":"https://api.github.com/repos/linkedin/photon-ml","forks_url":"https://api.github.com/repos/linkedin/photon-ml/forks","keys_url":"https://api.github.com/repos/linkedin/photon-ml/keys{/key_id}","collaborators_url":"https://api.github.com/repos/linkedin/photon-ml/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/linkedin/photon-ml/teams","hooks_url":"https://api.github.com/repos/linkedin/photon-ml/hooks","issue_events_url":"https://api.github.com/repos/linkedin/photon-ml/issues/events{/number}","events_url":"https://api.github.com/repos/linkedin/photon-ml/events","assignees_url":"https://api.github.com/repos/linkedin/photon-ml/assignees{/user}","branches_url":"https://api.github.com/repos/linkedin/photon-ml/branches{/branch}","tags_url":"https://api.github.com/repos/linkedin/photon-ml/tags","blobs_url":"https://api.github.com/repos/linkedin/photon-ml/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/linkedin/photon-ml/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/linkedin/photon-ml/git/refs{/sha}","trees_url":"https://api.github.com/repos/linkedin/photon-ml/git/trees{/sha}","statuses_url":"https://api.github.com/repos/linkedin/photon-ml/statuses/{sha}","languages_url":"https://api.github.com/repos/linkedin/photon-ml/languages","stargazers_url":"https://api.github.com/repos/linkedin/photon-ml/stargazers","contributors_url":"https://api.github.com/repos/linkedin/photon-ml/contributors","subscribers_url":"https://api.github.com/repos/linkedin/photon-ml/subscribers","subscription_url":"https://api.github.com/repos/linkedin/photon-ml/subscription","commits_url":"https://api.github.com/repos/linkedin/photon-ml/commits{/sha}","git_commits_url":"https://api.github.com/repos/linkedin/photon-ml/git/commits{/sha}","comments_url":"https://api.github.com/repos/linkedin/photon-ml/comments{/number}","issue_comment_url":"https://api.github.com/repos/linkedin/photon-ml/issues/comments{/number}","contents_url":"https://api.github.com/repos/linkedin/photon-ml/contents/{+path}","compare_url":"https://api.github.com/repos/linkedin/photon-ml/compare/{base}...{head}","merges_url":"https://api.github.com/repos/linkedin/photon-ml/merges","archive_url":"https://api.github.com/repos/linkedin/photon-ml/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/linkedin/photon-ml/downloads","issues_url":"https://api.github.com/repos/linkedin/photon-ml/issues{/number}","pulls_url":"https://api.github.com/repos/linkedin/photon-ml/pulls{/number}","milestones_url":"https://api.github.com/repos/linkedin/photon-ml/milestones{/number}","notifications_url":"https://api.github.com/repos/linkedin/photon-ml/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/linkedin/photon-ml/labels{/name}","releases_url":"https://api.github.com/repos/linkedin/photon-ml/releases{/id}","deployments_url":"https://api.github.com/repos/linkedin/photon-ml/deployments","created_at":"2016-02-03T01:12:07Z","updated_at":"2016-05-09T22:29:26Z","pushed_at":"2016-05-13T19:08:55Z","git_url":"git://github.com/linkedin/photon-ml.git","ssh_url":"git@github.com:linkedin/photon-ml.git","clone_url":"https://github.com/linkedin/photon-ml.git","svn_url":"https://github.com/linkedin/photon-ml","homepage":"","size":16626,"stargazers_count":25,"watchers_count":25,"language":"Scala","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":8,"mirror_url":null,"open_issues_count":20,"forks":8,"open_issues":20,"watchers":25,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/linkedin/photon-ml/pulls/74"},"html":{"href":"https://github.com/linkedin/photon-ml/pull/74"},"issue":{"href":"https://api.github.com/repos/linkedin/photon-ml/issues/74"},"comments":{"href":"https://api.github.com/repos/linkedin/photon-ml/issues/74/comments"},"review_comments":{"href":"https://api.github.com/repos/linkedin/photon-ml/pulls/74/comments"},"review_comment":{"href":"https://api.github.com/repos/linkedin/photon-ml/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/linkedin/photon-ml/pulls/74/commits"},"statuses":{"href":"https://api.github.com/repos/linkedin/photon-ml/statuses/d541bdf0773a3bed6b49996807f006fce19234c3"}}}}