{"action":"created","comment":{"url":"https://api.github.com/repos/mozilla/telemetry-batch-view/pulls/comments/68299096","id":68299096,"diff_hunk":"@@ -1,117 +1,139 @@\n-package com.mozilla.telemetry.streams\n+package com.mozilla.telemetry.views\n \n-import awscala.s3._\n import org.apache.avro.{Schema, SchemaBuilder}\n import org.apache.avro.generic.{GenericData, GenericRecord, GenericRecordBuilder}\n-import org.apache.spark.{Partitioner, SparkContext}\n+import org.apache.spark.{Partitioner, SparkContext, SparkConf}\n import org.apache.spark.rdd.RDD\n import org.json4s._\n import org.json4s.jackson.JsonMethods._\n+import org.rogach.scallop._\n+import scala.collection.JavaConversions._\n import scala.collection.JavaConverters._\n import scala.collection.mutable.ListBuffer\n import scala.math.abs\n import scala.reflect.ClassTag\n import com.mozilla.telemetry.parquet.ParquetFile\n import com.mozilla.telemetry.avro\n-import com.mozilla.telemetry.heka.HekaFrame\n+import com.mozilla.telemetry.heka.{Dataset, Message}\n import com.mozilla.telemetry.histograms._\n import com.mozilla.telemetry.utils._\n-import com.mozilla.telemetry.{DerivedStream, ObjectSummary}\n-import com.mozilla.telemetry.DerivedStream.s3\n-\n-private class ClientIdPartitioner(size: Int) extends Partitioner{\n-  def numPartitions: Int = size\n-  def getPartition(key: Any): Int = key match {\n-    case (clientId: String, startDate: String, counter: Int) => abs(clientId.hashCode) % size\n-    case _ => throw new Exception(\"Invalid key\")\n-  }\n-}\n \n-class ClientIterator(it: Iterator[(String, Map[String, Any])], maxHistorySize: Int = 1000) extends Iterator[List[Map[String, Any]]]{\n-  // Less than 1% of clients in a sampled dataset over a 3 months period has more than 1000 fragments.\n-  var buffer = ListBuffer[Map[String, Any]]()\n-  var currentKey =\n-    if (it.hasNext) {\n-      val (key, value) = it.next()\n-      buffer += value\n-      key\n-    } else {\n-      null\n+object LongitudinalView {\n+  private class ClientIdPartitioner(size: Int) extends Partitioner {\n+    def numPartitions: Int = size\n+\n+    def getPartition(key: Any): Int = key match {\n+      case (clientId: String, startDate: String, counter: Int) => abs(clientId.hashCode) % size\n+      case _ => throw new Exception(\"Invalid key\")\n     }\n+  }\n+\n+  private class ClientIterator(it: Iterator[(String, Map[String, Any])], maxHistorySize: Int = 1000) extends Iterator[List[Map[String, Any]]] {\n+    // Less than 1% of clients in a sampled dataset over a 3 months period has more than 1000 fragments.\n+    var buffer = ListBuffer[Map[String, Any]]()\n+    var currentKey =\n+      if (it.hasNext) {\n+        val (key, value) = it.next()\n+        buffer += value\n+        key\n+      } else {\n+        null\n+      }\n \n   override def hasNext() = buffer.nonEmpty\n \n-  override def next(): List[Map[String, Any]] = {\n-    while (it.hasNext) {\n-      val (key, value) = it.next()\n-\n-      if (key == currentKey && buffer.size == maxHistorySize) {\n-        // Trim long histories\n-        val result = buffer.toList\n-        buffer.clear\n-\n-        // Fast forward to next client\n-        while (it.hasNext) {\n-          val (k, _) = it.next()\n-          if (k != currentKey) {\n-            buffer += value\n-            return result\n+    override def next(): List[Map[String, Any]] = {\n+      while (it.hasNext) {\n+        val (key, value) = it.next()\n+\n+        if (key == currentKey && buffer.size == maxHistorySize) {\n+          // Trim long histories\n+          val result = buffer.toList\n+          buffer.clear\n+\n+          // Fast forward to next client\n+          while (it.hasNext) {\n+            val (k, _) = it.next()\n+            if (k != currentKey) {\n+              buffer += value\n+              return result\n+            }\n           }\n-        }\n \n-        return result\n-      } else if (key == currentKey){\n-        buffer += value\n-      } else {\n-        val result = buffer.toList\n-        buffer.clear\n-        buffer += value\n-        currentKey = key\n-        return result\n+          return result\n+        } else if (key == currentKey) {\n+          buffer += value\n+        } else {\n+          val result = buffer.toList\n+          buffer.clear\n+          buffer += value\n+          currentKey = key\n+          return result\n+        }\n       }\n+\n+      val result = buffer.toList\n+      buffer.clear\n+      result\n     }\n+  }\n \n-    val result = buffer.toList\n-    buffer.clear\n-    result\n+  private class Opts(args: Array[String]) extends ScallopConf(args) {\n+    val from = opt[String](\"from\", descr = \"From submission date\", required = true)\n+    val to = opt[String](\"to\", descr = \"To submission date\", required = true)\n+    val outputBucket = opt[String](\"bucket\", descr = \"bucket\", required = true)\n+    verify()\n   }\n-}\n \n-case class Longitudinal() extends DerivedStream {\n-  override def streamName: String = \"telemetry-sample\"\n-  override def filterPrefix: String = \"telemetry/4/main/*/*/*/42/\"\n+  def main(args: Array[String]): Unit = {\n+    val opts = new Opts(args)\n+    val from = opts.from()\n+    val to = opts.to()\n+\n+    val sparkConf = new SparkConf().setAppName(\"E10sExperiment\")","path":"src/main/scala/com/mozilla/telemetry/views/Longitudinal.scala","position":149,"original_position":149,"commit_id":"30bcf2b7f7136e2f9769c3b69dc8895d4fe2fe8b","original_commit_id":"be80cb845549292db8e00fd16cf2d344cb859213","user":{"login":"mreid-moz","id":969479,"avatar_url":"https://avatars.githubusercontent.com/u/969479?v=3","gravatar_id":"","url":"https://api.github.com/users/mreid-moz","html_url":"https://github.com/mreid-moz","followers_url":"https://api.github.com/users/mreid-moz/followers","following_url":"https://api.github.com/users/mreid-moz/following{/other_user}","gists_url":"https://api.github.com/users/mreid-moz/gists{/gist_id}","starred_url":"https://api.github.com/users/mreid-moz/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mreid-moz/subscriptions","organizations_url":"https://api.github.com/users/mreid-moz/orgs","repos_url":"https://api.github.com/users/mreid-moz/repos","events_url":"https://api.github.com/users/mreid-moz/events{/privacy}","received_events_url":"https://api.github.com/users/mreid-moz/received_events","type":"User","site_admin":false},"body":"s/E10sExperiment/LongitudinalView/\r\n\r\nThis keeps happening, maybe the pattern should be to grab the current class name.","created_at":"2016-06-23T19:27:26Z","updated_at":"2016-06-23T19:27:26Z","html_url":"https://github.com/mozilla/telemetry-batch-view/pull/85#discussion_r68299096","pull_request_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/pulls/85","_links":{"self":{"href":"https://api.github.com/repos/mozilla/telemetry-batch-view/pulls/comments/68299096"},"html":{"href":"https://github.com/mozilla/telemetry-batch-view/pull/85#discussion_r68299096"},"pull_request":{"href":"https://api.github.com/repos/mozilla/telemetry-batch-view/pulls/85"}}},"pull_request":{"url":"https://api.github.com/repos/mozilla/telemetry-batch-view/pulls/85","id":74773962,"html_url":"https://github.com/mozilla/telemetry-batch-view/pull/85","diff_url":"https://github.com/mozilla/telemetry-batch-view/pull/85.diff","patch_url":"https://github.com/mozilla/telemetry-batch-view/pull/85.patch","issue_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/issues/85","number":85,"state":"open","locked":false,"title":"Various improvements.","user":{"login":"vitillo","id":2061747,"avatar_url":"https://avatars.githubusercontent.com/u/2061747?v=3","gravatar_id":"","url":"https://api.github.com/users/vitillo","html_url":"https://github.com/vitillo","followers_url":"https://api.github.com/users/vitillo/followers","following_url":"https://api.github.com/users/vitillo/following{/other_user}","gists_url":"https://api.github.com/users/vitillo/gists{/gist_id}","starred_url":"https://api.github.com/users/vitillo/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vitillo/subscriptions","organizations_url":"https://api.github.com/users/vitillo/orgs","repos_url":"https://api.github.com/users/vitillo/repos","events_url":"https://api.github.com/users/vitillo/events{/privacy}","received_events_url":"https://api.github.com/users/vitillo/received_events","type":"User","site_admin":false},"body":"\n\n<!-- Reviewable:start -->\nThis change is [<img src=\"https://reviewable.io/review_button.svg\" height=\"35\" align=\"absmiddle\" alt=\"Reviewable\"/>](https://reviewable.io/reviews/mozilla/telemetry-batch-view/85)\n<!-- Reviewable:end -->\n","created_at":"2016-06-22T12:21:13Z","updated_at":"2016-06-23T19:27:26Z","closed_at":null,"merged_at":null,"merge_commit_sha":"5f3536582ac6954a8285633f05b17e55c2d5d20d","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/pulls/85/commits","review_comments_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/pulls/85/comments","review_comment_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/pulls/comments{/number}","comments_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/issues/85/comments","statuses_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/statuses/30bcf2b7f7136e2f9769c3b69dc8895d4fe2fe8b","head":{"label":"mozilla:deprecate_stream","ref":"deprecate_stream","sha":"30bcf2b7f7136e2f9769c3b69dc8895d4fe2fe8b","user":{"login":"mozilla","id":131524,"avatar_url":"https://avatars.githubusercontent.com/u/131524?v=3","gravatar_id":"","url":"https://api.github.com/users/mozilla","html_url":"https://github.com/mozilla","followers_url":"https://api.github.com/users/mozilla/followers","following_url":"https://api.github.com/users/mozilla/following{/other_user}","gists_url":"https://api.github.com/users/mozilla/gists{/gist_id}","starred_url":"https://api.github.com/users/mozilla/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mozilla/subscriptions","organizations_url":"https://api.github.com/users/mozilla/orgs","repos_url":"https://api.github.com/users/mozilla/repos","events_url":"https://api.github.com/users/mozilla/events{/privacy}","received_events_url":"https://api.github.com/users/mozilla/received_events","type":"Organization","site_admin":false},"repo":{"id":44998549,"name":"telemetry-batch-view","full_name":"mozilla/telemetry-batch-view","owner":{"login":"mozilla","id":131524,"avatar_url":"https://avatars.githubusercontent.com/u/131524?v=3","gravatar_id":"","url":"https://api.github.com/users/mozilla","html_url":"https://github.com/mozilla","followers_url":"https://api.github.com/users/mozilla/followers","following_url":"https://api.github.com/users/mozilla/following{/other_user}","gists_url":"https://api.github.com/users/mozilla/gists{/gist_id}","starred_url":"https://api.github.com/users/mozilla/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mozilla/subscriptions","organizations_url":"https://api.github.com/users/mozilla/orgs","repos_url":"https://api.github.com/users/mozilla/repos","events_url":"https://api.github.com/users/mozilla/events{/privacy}","received_events_url":"https://api.github.com/users/mozilla/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/mozilla/telemetry-batch-view","description":"A Scala framework to build derived datasets, aka batch views, of Telemetry data.","fork":false,"url":"https://api.github.com/repos/mozilla/telemetry-batch-view","forks_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/forks","keys_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/keys{/key_id}","collaborators_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/teams","hooks_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/hooks","issue_events_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/issues/events{/number}","events_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/events","assignees_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/assignees{/user}","branches_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/branches{/branch}","tags_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/tags","blobs_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/git/refs{/sha}","trees_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/git/trees{/sha}","statuses_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/statuses/{sha}","languages_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/languages","stargazers_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/stargazers","contributors_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/contributors","subscribers_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/subscribers","subscription_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/subscription","commits_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/commits{/sha}","git_commits_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/git/commits{/sha}","comments_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/comments{/number}","issue_comment_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/issues/comments{/number}","contents_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/contents/{+path}","compare_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/compare/{base}...{head}","merges_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/merges","archive_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/downloads","issues_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/issues{/number}","pulls_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/pulls{/number}","milestones_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/milestones{/number}","notifications_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/labels{/name}","releases_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/releases{/id}","deployments_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/deployments","created_at":"2015-10-26T21:29:57Z","updated_at":"2016-06-08T06:51:59Z","pushed_at":"2016-06-23T13:22:34Z","git_url":"git://github.com/mozilla/telemetry-batch-view.git","ssh_url":"git@github.com:mozilla/telemetry-batch-view.git","clone_url":"https://github.com/mozilla/telemetry-batch-view.git","svn_url":"https://github.com/mozilla/telemetry-batch-view","homepage":"","size":674,"stargazers_count":3,"watchers_count":3,"language":"Scala","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":6,"mirror_url":null,"open_issues_count":5,"forks":6,"open_issues":5,"watchers":3,"default_branch":"master"}},"base":{"label":"mozilla:master","ref":"master","sha":"b076610859ae59bf9208c7cdbb65f1beb399983c","user":{"login":"mozilla","id":131524,"avatar_url":"https://avatars.githubusercontent.com/u/131524?v=3","gravatar_id":"","url":"https://api.github.com/users/mozilla","html_url":"https://github.com/mozilla","followers_url":"https://api.github.com/users/mozilla/followers","following_url":"https://api.github.com/users/mozilla/following{/other_user}","gists_url":"https://api.github.com/users/mozilla/gists{/gist_id}","starred_url":"https://api.github.com/users/mozilla/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mozilla/subscriptions","organizations_url":"https://api.github.com/users/mozilla/orgs","repos_url":"https://api.github.com/users/mozilla/repos","events_url":"https://api.github.com/users/mozilla/events{/privacy}","received_events_url":"https://api.github.com/users/mozilla/received_events","type":"Organization","site_admin":false},"repo":{"id":44998549,"name":"telemetry-batch-view","full_name":"mozilla/telemetry-batch-view","owner":{"login":"mozilla","id":131524,"avatar_url":"https://avatars.githubusercontent.com/u/131524?v=3","gravatar_id":"","url":"https://api.github.com/users/mozilla","html_url":"https://github.com/mozilla","followers_url":"https://api.github.com/users/mozilla/followers","following_url":"https://api.github.com/users/mozilla/following{/other_user}","gists_url":"https://api.github.com/users/mozilla/gists{/gist_id}","starred_url":"https://api.github.com/users/mozilla/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mozilla/subscriptions","organizations_url":"https://api.github.com/users/mozilla/orgs","repos_url":"https://api.github.com/users/mozilla/repos","events_url":"https://api.github.com/users/mozilla/events{/privacy}","received_events_url":"https://api.github.com/users/mozilla/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/mozilla/telemetry-batch-view","description":"A Scala framework to build derived datasets, aka batch views, of Telemetry data.","fork":false,"url":"https://api.github.com/repos/mozilla/telemetry-batch-view","forks_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/forks","keys_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/keys{/key_id}","collaborators_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/teams","hooks_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/hooks","issue_events_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/issues/events{/number}","events_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/events","assignees_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/assignees{/user}","branches_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/branches{/branch}","tags_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/tags","blobs_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/git/refs{/sha}","trees_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/git/trees{/sha}","statuses_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/statuses/{sha}","languages_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/languages","stargazers_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/stargazers","contributors_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/contributors","subscribers_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/subscribers","subscription_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/subscription","commits_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/commits{/sha}","git_commits_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/git/commits{/sha}","comments_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/comments{/number}","issue_comment_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/issues/comments{/number}","contents_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/contents/{+path}","compare_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/compare/{base}...{head}","merges_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/merges","archive_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/downloads","issues_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/issues{/number}","pulls_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/pulls{/number}","milestones_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/milestones{/number}","notifications_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/labels{/name}","releases_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/releases{/id}","deployments_url":"https://api.github.com/repos/mozilla/telemetry-batch-view/deployments","created_at":"2015-10-26T21:29:57Z","updated_at":"2016-06-08T06:51:59Z","pushed_at":"2016-06-23T13:22:34Z","git_url":"git://github.com/mozilla/telemetry-batch-view.git","ssh_url":"git@github.com:mozilla/telemetry-batch-view.git","clone_url":"https://github.com/mozilla/telemetry-batch-view.git","svn_url":"https://github.com/mozilla/telemetry-batch-view","homepage":"","size":674,"stargazers_count":3,"watchers_count":3,"language":"Scala","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":6,"mirror_url":null,"open_issues_count":5,"forks":6,"open_issues":5,"watchers":3,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/mozilla/telemetry-batch-view/pulls/85"},"html":{"href":"https://github.com/mozilla/telemetry-batch-view/pull/85"},"issue":{"href":"https://api.github.com/repos/mozilla/telemetry-batch-view/issues/85"},"comments":{"href":"https://api.github.com/repos/mozilla/telemetry-batch-view/issues/85/comments"},"review_comments":{"href":"https://api.github.com/repos/mozilla/telemetry-batch-view/pulls/85/comments"},"review_comment":{"href":"https://api.github.com/repos/mozilla/telemetry-batch-view/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/mozilla/telemetry-batch-view/pulls/85/commits"},"statuses":{"href":"https://api.github.com/repos/mozilla/telemetry-batch-view/statuses/30bcf2b7f7136e2f9769c3b69dc8895d4fe2fe8b"}}}}
{"actor":{"display_login":"mreid-moz"}}