{"action":"created","comment":{"url":"https://api.github.com/repos/caskdata/cdap/pulls/comments/61176100","id":61176100,"diff_hunk":"@@ -0,0 +1,117 @@\n+/*\n+ * Copyright © 2016 Cask Data, Inc.\n+ *\n+ * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not\n+ * use this file except in compliance with the License. You may obtain a copy of\n+ * the License at\n+ *\n+ * http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n+ * License for the specific language governing permissions and limitations under\n+ * the License.\n+ */\n+package co.cask.cdap.examples.sparkstreaming\n+\n+import co.cask.cdap.api.common.Bytes\n+import co.cask.cdap.api.spark.{SparkExecutionContext, SparkMain}\n+import com.google.common.base.Strings\n+import kafka.serializer.{DefaultDecoder, StringDecoder}\n+import org.apache.spark.SparkContext\n+import org.apache.spark.mllib.classification.{NaiveBayes, NaiveBayesModel}\n+import org.apache.spark.mllib.feature.HashingTF\n+import org.apache.spark.mllib.linalg.Vector\n+import org.apache.spark.mllib.regression.LabeledPoint\n+import org.apache.spark.rdd.RDD\n+import org.apache.spark.streaming.kafka.KafkaUtils\n+import org.apache.spark.streaming.{Seconds, StreamingContext}\n+import org.slf4j.{Logger, LoggerFactory}\n+\n+/**\n+  * Consumes messages from kafka topics and classify them as spam or ham based on NaiveBayes model.\n+  * A comma separated list of brokers and topics must be provided as runtime arguments to this program.\n+  */\n+class SpamClassifierProgram extends SparkMain {\n+\n+  private final val LOG: Logger = LoggerFactory.getLogger(classOf[SpamClassifierProgram])\n+\n+  override def run(implicit sec: SparkExecutionContext) {\n+\n+    val sparkContext: SparkContext = new SparkContext\n+\n+    LOG.info(\"Reading stream {} to build classification model\", SpamClassifier.STREAM)\n+\n+    // read the training data stream\n+    val trainingData: RDD[String] = sparkContext.fromStream(SpamClassifier.STREAM, 0, Long.MaxValue)","path":"cdap-examples/SpamClassifier/src/main/scala/co/cask/cdap/examples/sparkstreaming/SpamClassifierProgram.scala","position":47,"original_position":47,"commit_id":"04993dd9811c73c9ea61c9941345347145c6bbab","original_commit_id":"04993dd9811c73c9ea61c9941345347145c6bbab","user":{"login":"chtyim","id":885032,"avatar_url":"https://avatars.githubusercontent.com/u/885032?v=3","gravatar_id":"","url":"https://api.github.com/users/chtyim","html_url":"https://github.com/chtyim","followers_url":"https://api.github.com/users/chtyim/followers","following_url":"https://api.github.com/users/chtyim/following{/other_user}","gists_url":"https://api.github.com/users/chtyim/gists{/gist_id}","starred_url":"https://api.github.com/users/chtyim/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/chtyim/subscriptions","organizations_url":"https://api.github.com/users/chtyim/orgs","repos_url":"https://api.github.com/users/chtyim/repos","events_url":"https://api.github.com/users/chtyim/events{/privacy}","received_events_url":"https://api.github.com/users/chtyim/received_events","type":"User","site_admin":false},"body":"You can do `val trainingData = sc.fromStream[String](...)`","created_at":"2016-04-26T22:30:15Z","updated_at":"2016-04-26T22:30:15Z","html_url":"https://github.com/caskdata/cdap/pull/5671#discussion_r61176100","pull_request_url":"https://api.github.com/repos/caskdata/cdap/pulls/5671","_links":{"self":{"href":"https://api.github.com/repos/caskdata/cdap/pulls/comments/61176100"},"html":{"href":"https://github.com/caskdata/cdap/pull/5671#discussion_r61176100"},"pull_request":{"href":"https://api.github.com/repos/caskdata/cdap/pulls/5671"}}},"pull_request":{"url":"https://api.github.com/repos/caskdata/cdap/pulls/5671","id":67965802,"html_url":"https://github.com/caskdata/cdap/pull/5671","diff_url":"https://github.com/caskdata/cdap/pull/5671.diff","patch_url":"https://github.com/caskdata/cdap/pull/5671.patch","issue_url":"https://api.github.com/repos/caskdata/cdap/issues/5671","number":5671,"state":"open","locked":false,"title":"CDAP-5446 SpamClassifier: A Spark Streaming Example","user":{"login":"rohitsinha54","id":1472867,"avatar_url":"https://avatars.githubusercontent.com/u/1472867?v=3","gravatar_id":"","url":"https://api.github.com/users/rohitsinha54","html_url":"https://github.com/rohitsinha54","followers_url":"https://api.github.com/users/rohitsinha54/followers","following_url":"https://api.github.com/users/rohitsinha54/following{/other_user}","gists_url":"https://api.github.com/users/rohitsinha54/gists{/gist_id}","starred_url":"https://api.github.com/users/rohitsinha54/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rohitsinha54/subscriptions","organizations_url":"https://api.github.com/users/rohitsinha54/orgs","repos_url":"https://api.github.com/users/rohitsinha54/repos","events_url":"https://api.github.com/users/rohitsinha54/events{/privacy}","received_events_url":"https://api.github.com/users/rohitsinha54/received_events","type":"User","site_admin":false},"body":"# SpamClassifier\r\n\r\n- A Spark Streaming example application\r\n- Reads training data of messages labeled as spam or ham. Dataset: https://archive.ics.uci.edu/ml/datasets/SMS+Spam+Collection\r\n- Builds a Spark MlLib NaiveBayes model for spam classification\r\n- Reads kafka message in realtime through spark streaming and classifies them as spam/ham by predicting based on the above model.\r\n- Writes the message id and spam/ham label to dataset  which is queries through a service.\r\n\r\nTo Do:\r\n\r\n- [ ] Test on Cluster\r\n- [ ] Documentation\r\n\r\nIssue: https://issues.cask.co/browse/CDAP-5446","created_at":"2016-04-26T22:02:35Z","updated_at":"2016-04-26T22:30:15Z","closed_at":null,"merged_at":null,"merge_commit_sha":"c7cb5a40df3eff0c44d224321807524cd7641d2f","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/caskdata/cdap/pulls/5671/commits","review_comments_url":"https://api.github.com/repos/caskdata/cdap/pulls/5671/comments","review_comment_url":"https://api.github.com/repos/caskdata/cdap/pulls/comments{/number}","comments_url":"https://api.github.com/repos/caskdata/cdap/issues/5671/comments","statuses_url":"https://api.github.com/repos/caskdata/cdap/statuses/04993dd9811c73c9ea61c9941345347145c6bbab","head":{"label":"caskdata:feature/CDAP-5446-spark-streaming-exp","ref":"feature/CDAP-5446-spark-streaming-exp","sha":"04993dd9811c73c9ea61c9941345347145c6bbab","user":{"login":"caskdata","id":8123023,"avatar_url":"https://avatars.githubusercontent.com/u/8123023?v=3","gravatar_id":"","url":"https://api.github.com/users/caskdata","html_url":"https://github.com/caskdata","followers_url":"https://api.github.com/users/caskdata/followers","following_url":"https://api.github.com/users/caskdata/following{/other_user}","gists_url":"https://api.github.com/users/caskdata/gists{/gist_id}","starred_url":"https://api.github.com/users/caskdata/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/caskdata/subscriptions","organizations_url":"https://api.github.com/users/caskdata/orgs","repos_url":"https://api.github.com/users/caskdata/repos","events_url":"https://api.github.com/users/caskdata/events{/privacy}","received_events_url":"https://api.github.com/users/caskdata/received_events","type":"Organization","site_admin":false},"repo":{"id":22542759,"name":"cdap","full_name":"caskdata/cdap","owner":{"login":"caskdata","id":8123023,"avatar_url":"https://avatars.githubusercontent.com/u/8123023?v=3","gravatar_id":"","url":"https://api.github.com/users/caskdata","html_url":"https://github.com/caskdata","followers_url":"https://api.github.com/users/caskdata/followers","following_url":"https://api.github.com/users/caskdata/following{/other_user}","gists_url":"https://api.github.com/users/caskdata/gists{/gist_id}","starred_url":"https://api.github.com/users/caskdata/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/caskdata/subscriptions","organizations_url":"https://api.github.com/users/caskdata/orgs","repos_url":"https://api.github.com/users/caskdata/repos","events_url":"https://api.github.com/users/caskdata/events{/privacy}","received_events_url":"https://api.github.com/users/caskdata/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/caskdata/cdap","description":"Cask Data Application Platform (CDAP)","fork":false,"url":"https://api.github.com/repos/caskdata/cdap","forks_url":"https://api.github.com/repos/caskdata/cdap/forks","keys_url":"https://api.github.com/repos/caskdata/cdap/keys{/key_id}","collaborators_url":"https://api.github.com/repos/caskdata/cdap/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/caskdata/cdap/teams","hooks_url":"https://api.github.com/repos/caskdata/cdap/hooks","issue_events_url":"https://api.github.com/repos/caskdata/cdap/issues/events{/number}","events_url":"https://api.github.com/repos/caskdata/cdap/events","assignees_url":"https://api.github.com/repos/caskdata/cdap/assignees{/user}","branches_url":"https://api.github.com/repos/caskdata/cdap/branches{/branch}","tags_url":"https://api.github.com/repos/caskdata/cdap/tags","blobs_url":"https://api.github.com/repos/caskdata/cdap/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/caskdata/cdap/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/caskdata/cdap/git/refs{/sha}","trees_url":"https://api.github.com/repos/caskdata/cdap/git/trees{/sha}","statuses_url":"https://api.github.com/repos/caskdata/cdap/statuses/{sha}","languages_url":"https://api.github.com/repos/caskdata/cdap/languages","stargazers_url":"https://api.github.com/repos/caskdata/cdap/stargazers","contributors_url":"https://api.github.com/repos/caskdata/cdap/contributors","subscribers_url":"https://api.github.com/repos/caskdata/cdap/subscribers","subscription_url":"https://api.github.com/repos/caskdata/cdap/subscription","commits_url":"https://api.github.com/repos/caskdata/cdap/commits{/sha}","git_commits_url":"https://api.github.com/repos/caskdata/cdap/git/commits{/sha}","comments_url":"https://api.github.com/repos/caskdata/cdap/comments{/number}","issue_comment_url":"https://api.github.com/repos/caskdata/cdap/issues/comments{/number}","contents_url":"https://api.github.com/repos/caskdata/cdap/contents/{+path}","compare_url":"https://api.github.com/repos/caskdata/cdap/compare/{base}...{head}","merges_url":"https://api.github.com/repos/caskdata/cdap/merges","archive_url":"https://api.github.com/repos/caskdata/cdap/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/caskdata/cdap/downloads","issues_url":"https://api.github.com/repos/caskdata/cdap/issues{/number}","pulls_url":"https://api.github.com/repos/caskdata/cdap/pulls{/number}","milestones_url":"https://api.github.com/repos/caskdata/cdap/milestones{/number}","notifications_url":"https://api.github.com/repos/caskdata/cdap/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/caskdata/cdap/labels{/name}","releases_url":"https://api.github.com/repos/caskdata/cdap/releases{/id}","deployments_url":"https://api.github.com/repos/caskdata/cdap/deployments","created_at":"2014-08-02T09:00:10Z","updated_at":"2016-04-19T22:42:01Z","pushed_at":"2016-04-26T22:24:27Z","git_url":"git://github.com/caskdata/cdap.git","ssh_url":"git@github.com:caskdata/cdap.git","clone_url":"https://github.com/caskdata/cdap.git","svn_url":"https://github.com/caskdata/cdap","homepage":"http://cdap.io","size":605663,"stargazers_count":115,"watchers_count":115,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":true,"forks_count":40,"mirror_url":null,"open_issues_count":11,"forks":40,"open_issues":11,"watchers":115,"default_branch":"release/3.4"}},"base":{"label":"caskdata:release/3.4","ref":"release/3.4","sha":"524a15c2e052acd4c10b07503ef9f8890d945826","user":{"login":"caskdata","id":8123023,"avatar_url":"https://avatars.githubusercontent.com/u/8123023?v=3","gravatar_id":"","url":"https://api.github.com/users/caskdata","html_url":"https://github.com/caskdata","followers_url":"https://api.github.com/users/caskdata/followers","following_url":"https://api.github.com/users/caskdata/following{/other_user}","gists_url":"https://api.github.com/users/caskdata/gists{/gist_id}","starred_url":"https://api.github.com/users/caskdata/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/caskdata/subscriptions","organizations_url":"https://api.github.com/users/caskdata/orgs","repos_url":"https://api.github.com/users/caskdata/repos","events_url":"https://api.github.com/users/caskdata/events{/privacy}","received_events_url":"https://api.github.com/users/caskdata/received_events","type":"Organization","site_admin":false},"repo":{"id":22542759,"name":"cdap","full_name":"caskdata/cdap","owner":{"login":"caskdata","id":8123023,"avatar_url":"https://avatars.githubusercontent.com/u/8123023?v=3","gravatar_id":"","url":"https://api.github.com/users/caskdata","html_url":"https://github.com/caskdata","followers_url":"https://api.github.com/users/caskdata/followers","following_url":"https://api.github.com/users/caskdata/following{/other_user}","gists_url":"https://api.github.com/users/caskdata/gists{/gist_id}","starred_url":"https://api.github.com/users/caskdata/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/caskdata/subscriptions","organizations_url":"https://api.github.com/users/caskdata/orgs","repos_url":"https://api.github.com/users/caskdata/repos","events_url":"https://api.github.com/users/caskdata/events{/privacy}","received_events_url":"https://api.github.com/users/caskdata/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/caskdata/cdap","description":"Cask Data Application Platform (CDAP)","fork":false,"url":"https://api.github.com/repos/caskdata/cdap","forks_url":"https://api.github.com/repos/caskdata/cdap/forks","keys_url":"https://api.github.com/repos/caskdata/cdap/keys{/key_id}","collaborators_url":"https://api.github.com/repos/caskdata/cdap/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/caskdata/cdap/teams","hooks_url":"https://api.github.com/repos/caskdata/cdap/hooks","issue_events_url":"https://api.github.com/repos/caskdata/cdap/issues/events{/number}","events_url":"https://api.github.com/repos/caskdata/cdap/events","assignees_url":"https://api.github.com/repos/caskdata/cdap/assignees{/user}","branches_url":"https://api.github.com/repos/caskdata/cdap/branches{/branch}","tags_url":"https://api.github.com/repos/caskdata/cdap/tags","blobs_url":"https://api.github.com/repos/caskdata/cdap/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/caskdata/cdap/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/caskdata/cdap/git/refs{/sha}","trees_url":"https://api.github.com/repos/caskdata/cdap/git/trees{/sha}","statuses_url":"https://api.github.com/repos/caskdata/cdap/statuses/{sha}","languages_url":"https://api.github.com/repos/caskdata/cdap/languages","stargazers_url":"https://api.github.com/repos/caskdata/cdap/stargazers","contributors_url":"https://api.github.com/repos/caskdata/cdap/contributors","subscribers_url":"https://api.github.com/repos/caskdata/cdap/subscribers","subscription_url":"https://api.github.com/repos/caskdata/cdap/subscription","commits_url":"https://api.github.com/repos/caskdata/cdap/commits{/sha}","git_commits_url":"https://api.github.com/repos/caskdata/cdap/git/commits{/sha}","comments_url":"https://api.github.com/repos/caskdata/cdap/comments{/number}","issue_comment_url":"https://api.github.com/repos/caskdata/cdap/issues/comments{/number}","contents_url":"https://api.github.com/repos/caskdata/cdap/contents/{+path}","compare_url":"https://api.github.com/repos/caskdata/cdap/compare/{base}...{head}","merges_url":"https://api.github.com/repos/caskdata/cdap/merges","archive_url":"https://api.github.com/repos/caskdata/cdap/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/caskdata/cdap/downloads","issues_url":"https://api.github.com/repos/caskdata/cdap/issues{/number}","pulls_url":"https://api.github.com/repos/caskdata/cdap/pulls{/number}","milestones_url":"https://api.github.com/repos/caskdata/cdap/milestones{/number}","notifications_url":"https://api.github.com/repos/caskdata/cdap/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/caskdata/cdap/labels{/name}","releases_url":"https://api.github.com/repos/caskdata/cdap/releases{/id}","deployments_url":"https://api.github.com/repos/caskdata/cdap/deployments","created_at":"2014-08-02T09:00:10Z","updated_at":"2016-04-19T22:42:01Z","pushed_at":"2016-04-26T22:24:27Z","git_url":"git://github.com/caskdata/cdap.git","ssh_url":"git@github.com:caskdata/cdap.git","clone_url":"https://github.com/caskdata/cdap.git","svn_url":"https://github.com/caskdata/cdap","homepage":"http://cdap.io","size":605663,"stargazers_count":115,"watchers_count":115,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":true,"forks_count":40,"mirror_url":null,"open_issues_count":11,"forks":40,"open_issues":11,"watchers":115,"default_branch":"release/3.4"}},"_links":{"self":{"href":"https://api.github.com/repos/caskdata/cdap/pulls/5671"},"html":{"href":"https://github.com/caskdata/cdap/pull/5671"},"issue":{"href":"https://api.github.com/repos/caskdata/cdap/issues/5671"},"comments":{"href":"https://api.github.com/repos/caskdata/cdap/issues/5671/comments"},"review_comments":{"href":"https://api.github.com/repos/caskdata/cdap/pulls/5671/comments"},"review_comment":{"href":"https://api.github.com/repos/caskdata/cdap/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/caskdata/cdap/pulls/5671/commits"},"statuses":{"href":"https://api.github.com/repos/caskdata/cdap/statuses/04993dd9811c73c9ea61c9941345347145c6bbab"}}}}