{"action":"created","comment":{"url":"https://api.github.com/repos/vitillo/telemetry-batch-view/pulls/comments/50716786","id":50716786,"diff_hunk":"@@ -18,16 +18,19 @@ import scala.collection.JavaConverters._\n import scala.io.Source\n import telemetry.streams.{E10sExperiment, ExecutiveStream, Churn}\n \n+// key is the S3 filename, size is the object size in bytes.\n case class ObjectSummary(key: String, size: Long) // S3ObjectSummary can't be serialized\n \n abstract class DerivedStream extends java.io.Serializable{\n   private val appConf = ConfigFactory.load()\n   private val parquetBucket = Bucket(appConf.getString(\"app.parquetBucket\"))\n   private val metadataBucket = Bucket(\"net-mozaws-prod-us-west-2-pipeline-metadata\")\n-  private val metaPrefix = {\n+  protected val metadataSources = {","path":"src/main/scala/DerivedStream.scala","position":12,"original_position":12,"commit_id":"c8863d94e73cd418594a3ec332463105fe7113ad","original_commit_id":"c8863d94e73cd418594a3ec332463105fe7113ad","user":{"login":"vitillo","id":2061747,"avatar_url":"https://avatars.githubusercontent.com/u/2061747?v=3","gravatar_id":"","url":"https://api.github.com/users/vitillo","html_url":"https://github.com/vitillo","followers_url":"https://api.github.com/users/vitillo/followers","following_url":"https://api.github.com/users/vitillo/following{/other_user}","gists_url":"https://api.github.com/users/vitillo/gists{/gist_id}","starred_url":"https://api.github.com/users/vitillo/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vitillo/subscriptions","organizations_url":"https://api.github.com/users/vitillo/orgs","repos_url":"https://api.github.com/users/vitillo/repos","events_url":"https://api.github.com/users/vitillo/events{/privacy}","received_events_url":"https://api.github.com/users/vitillo/received_events","type":"User","site_admin":false},"body":"Yes please.","created_at":"2016-01-25T16:26:28Z","updated_at":"2016-01-25T16:26:28Z","html_url":"https://github.com/vitillo/telemetry-batch-view/pull/10#discussion_r50716786","pull_request_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/pulls/10","_links":{"self":{"href":"https://api.github.com/repos/vitillo/telemetry-batch-view/pulls/comments/50716786"},"html":{"href":"https://github.com/vitillo/telemetry-batch-view/pull/10#discussion_r50716786"},"pull_request":{"href":"https://api.github.com/repos/vitillo/telemetry-batch-view/pulls/10"}}},"pull_request":{"url":"https://api.github.com/repos/vitillo/telemetry-batch-view/pulls/10","id":56897600,"html_url":"https://github.com/vitillo/telemetry-batch-view/pull/10","diff_url":"https://github.com/vitillo/telemetry-batch-view/pull/10.diff","patch_url":"https://github.com/vitillo/telemetry-batch-view/pull/10.patch","issue_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/issues/10","number":10,"state":"open","locked":false,"title":"Modify Churn to use fewer partitions","user":{"login":"mreid-moz","id":969479,"avatar_url":"https://avatars.githubusercontent.com/u/969479?v=3","gravatar_id":"","url":"https://api.github.com/users/mreid-moz","html_url":"https://github.com/mreid-moz","followers_url":"https://api.github.com/users/mreid-moz/followers","following_url":"https://api.github.com/users/mreid-moz/following{/other_user}","gists_url":"https://api.github.com/users/mreid-moz/gists{/gist_id}","starred_url":"https://api.github.com/users/mreid-moz/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mreid-moz/subscriptions","organizations_url":"https://api.github.com/users/mreid-moz/orgs","repos_url":"https://api.github.com/users/mreid-moz/repos","events_url":"https://api.github.com/users/mreid-moz/events{/privacy}","received_events_url":"https://api.github.com/users/mreid-moz/received_events","type":"User","site_admin":false},"body":"Instead of extending `SimpleDerivedStream`, directly extend `DerivedStream`, and ignore the supplied list of S3 objects, instead fetching the file list for each day in the specified range.\r\n\r\nAlso repartition into 100 pieces to avoid huge numbers of tiny files on S3 which causes bad performance.\r\n\r\nThere are a few ugly hacks in here (including the hard-coding of `100` as the number of partitions). Feedback welcome.","created_at":"2016-01-22T14:59:42Z","updated_at":"2016-01-25T16:26:28Z","closed_at":null,"merged_at":null,"merge_commit_sha":"b88a444bde4dae6fdc4fecc10f0dc40969d39abb","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/pulls/10/commits","review_comments_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/pulls/10/comments","review_comment_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/pulls/comments{/number}","comments_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/issues/10/comments","statuses_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/statuses/c8863d94e73cd418594a3ec332463105fe7113ad","head":{"label":"mreid-moz:churn_non_simple","ref":"churn_non_simple","sha":"c8863d94e73cd418594a3ec332463105fe7113ad","user":{"login":"mreid-moz","id":969479,"avatar_url":"https://avatars.githubusercontent.com/u/969479?v=3","gravatar_id":"","url":"https://api.github.com/users/mreid-moz","html_url":"https://github.com/mreid-moz","followers_url":"https://api.github.com/users/mreid-moz/followers","following_url":"https://api.github.com/users/mreid-moz/following{/other_user}","gists_url":"https://api.github.com/users/mreid-moz/gists{/gist_id}","starred_url":"https://api.github.com/users/mreid-moz/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mreid-moz/subscriptions","organizations_url":"https://api.github.com/users/mreid-moz/orgs","repos_url":"https://api.github.com/users/mreid-moz/repos","events_url":"https://api.github.com/users/mreid-moz/events{/privacy}","received_events_url":"https://api.github.com/users/mreid-moz/received_events","type":"User","site_admin":false},"repo":{"id":49297614,"name":"telemetry-batch-view","full_name":"mreid-moz/telemetry-batch-view","owner":{"login":"mreid-moz","id":969479,"avatar_url":"https://avatars.githubusercontent.com/u/969479?v=3","gravatar_id":"","url":"https://api.github.com/users/mreid-moz","html_url":"https://github.com/mreid-moz","followers_url":"https://api.github.com/users/mreid-moz/followers","following_url":"https://api.github.com/users/mreid-moz/following{/other_user}","gists_url":"https://api.github.com/users/mreid-moz/gists{/gist_id}","starred_url":"https://api.github.com/users/mreid-moz/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mreid-moz/subscriptions","organizations_url":"https://api.github.com/users/mreid-moz/orgs","repos_url":"https://api.github.com/users/mreid-moz/repos","events_url":"https://api.github.com/users/mreid-moz/events{/privacy}","received_events_url":"https://api.github.com/users/mreid-moz/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/mreid-moz/telemetry-batch-view","description":"A Scala \"framework\" to build derived datasets, aka batch views, of Telemetry data.","fork":true,"url":"https://api.github.com/repos/mreid-moz/telemetry-batch-view","forks_url":"https://api.github.com/repos/mreid-moz/telemetry-batch-view/forks","keys_url":"https://api.github.com/repos/mreid-moz/telemetry-batch-view/keys{/key_id}","collaborators_url":"https://api.github.com/repos/mreid-moz/telemetry-batch-view/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/mreid-moz/telemetry-batch-view/teams","hooks_url":"https://api.github.com/repos/mreid-moz/telemetry-batch-view/hooks","issue_events_url":"https://api.github.com/repos/mreid-moz/telemetry-batch-view/issues/events{/number}","events_url":"https://api.github.com/repos/mreid-moz/telemetry-batch-view/events","assignees_url":"https://api.github.com/repos/mreid-moz/telemetry-batch-view/assignees{/user}","branches_url":"https://api.github.com/repos/mreid-moz/telemetry-batch-view/branches{/branch}","tags_url":"https://api.github.com/repos/mreid-moz/telemetry-batch-view/tags","blobs_url":"https://api.github.com/repos/mreid-moz/telemetry-batch-view/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/mreid-moz/telemetry-batch-view/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/mreid-moz/telemetry-batch-view/git/refs{/sha}","trees_url":"https://api.github.com/repos/mreid-moz/telemetry-batch-view/git/trees{/sha}","statuses_url":"https://api.github.com/repos/mreid-moz/telemetry-batch-view/statuses/{sha}","languages_url":"https://api.github.com/repos/mreid-moz/telemetry-batch-view/languages","stargazers_url":"https://api.github.com/repos/mreid-moz/telemetry-batch-view/stargazers","contributors_url":"https://api.github.com/repos/mreid-moz/telemetry-batch-view/contributors","subscribers_url":"https://api.github.com/repos/mreid-moz/telemetry-batch-view/subscribers","subscription_url":"https://api.github.com/repos/mreid-moz/telemetry-batch-view/subscription","commits_url":"https://api.github.com/repos/mreid-moz/telemetry-batch-view/commits{/sha}","git_commits_url":"https://api.github.com/repos/mreid-moz/telemetry-batch-view/git/commits{/sha}","comments_url":"https://api.github.com/repos/mreid-moz/telemetry-batch-view/comments{/number}","issue_comment_url":"https://api.github.com/repos/mreid-moz/telemetry-batch-view/issues/comments{/number}","contents_url":"https://api.github.com/repos/mreid-moz/telemetry-batch-view/contents/{+path}","compare_url":"https://api.github.com/repos/mreid-moz/telemetry-batch-view/compare/{base}...{head}","merges_url":"https://api.github.com/repos/mreid-moz/telemetry-batch-view/merges","archive_url":"https://api.github.com/repos/mreid-moz/telemetry-batch-view/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/mreid-moz/telemetry-batch-view/downloads","issues_url":"https://api.github.com/repos/mreid-moz/telemetry-batch-view/issues{/number}","pulls_url":"https://api.github.com/repos/mreid-moz/telemetry-batch-view/pulls{/number}","milestones_url":"https://api.github.com/repos/mreid-moz/telemetry-batch-view/milestones{/number}","notifications_url":"https://api.github.com/repos/mreid-moz/telemetry-batch-view/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/mreid-moz/telemetry-batch-view/labels{/name}","releases_url":"https://api.github.com/repos/mreid-moz/telemetry-batch-view/releases{/id}","deployments_url":"https://api.github.com/repos/mreid-moz/telemetry-batch-view/deployments","created_at":"2016-01-08T21:22:59Z","updated_at":"2016-01-08T21:23:00Z","pushed_at":"2016-01-22T14:36:23Z","git_url":"git://github.com/mreid-moz/telemetry-batch-view.git","ssh_url":"git@github.com:mreid-moz/telemetry-batch-view.git","clone_url":"https://github.com/mreid-moz/telemetry-batch-view.git","svn_url":"https://github.com/mreid-moz/telemetry-batch-view","homepage":"","size":123,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"vitillo:master","ref":"master","sha":"8b2ceb6772ebf8c8070925c4ef79fb89e75bd028","user":{"login":"vitillo","id":2061747,"avatar_url":"https://avatars.githubusercontent.com/u/2061747?v=3","gravatar_id":"","url":"https://api.github.com/users/vitillo","html_url":"https://github.com/vitillo","followers_url":"https://api.github.com/users/vitillo/followers","following_url":"https://api.github.com/users/vitillo/following{/other_user}","gists_url":"https://api.github.com/users/vitillo/gists{/gist_id}","starred_url":"https://api.github.com/users/vitillo/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vitillo/subscriptions","organizations_url":"https://api.github.com/users/vitillo/orgs","repos_url":"https://api.github.com/users/vitillo/repos","events_url":"https://api.github.com/users/vitillo/events{/privacy}","received_events_url":"https://api.github.com/users/vitillo/received_events","type":"User","site_admin":false},"repo":{"id":44998549,"name":"telemetry-batch-view","full_name":"vitillo/telemetry-batch-view","owner":{"login":"vitillo","id":2061747,"avatar_url":"https://avatars.githubusercontent.com/u/2061747?v=3","gravatar_id":"","url":"https://api.github.com/users/vitillo","html_url":"https://github.com/vitillo","followers_url":"https://api.github.com/users/vitillo/followers","following_url":"https://api.github.com/users/vitillo/following{/other_user}","gists_url":"https://api.github.com/users/vitillo/gists{/gist_id}","starred_url":"https://api.github.com/users/vitillo/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vitillo/subscriptions","organizations_url":"https://api.github.com/users/vitillo/orgs","repos_url":"https://api.github.com/users/vitillo/repos","events_url":"https://api.github.com/users/vitillo/events{/privacy}","received_events_url":"https://api.github.com/users/vitillo/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/vitillo/telemetry-batch-view","description":"A Scala \"framework\" to build derived datasets, aka batch views, of Telemetry data.","fork":false,"url":"https://api.github.com/repos/vitillo/telemetry-batch-view","forks_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/forks","keys_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/keys{/key_id}","collaborators_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/teams","hooks_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/hooks","issue_events_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/issues/events{/number}","events_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/events","assignees_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/assignees{/user}","branches_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/branches{/branch}","tags_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/tags","blobs_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/git/refs{/sha}","trees_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/git/trees{/sha}","statuses_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/statuses/{sha}","languages_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/languages","stargazers_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/stargazers","contributors_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/contributors","subscribers_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/subscribers","subscription_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/subscription","commits_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/commits{/sha}","git_commits_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/git/commits{/sha}","comments_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/comments{/number}","issue_comment_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/issues/comments{/number}","contents_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/contents/{+path}","compare_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/compare/{base}...{head}","merges_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/merges","archive_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/downloads","issues_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/issues{/number}","pulls_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/pulls{/number}","milestones_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/milestones{/number}","notifications_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/labels{/name}","releases_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/releases{/id}","deployments_url":"https://api.github.com/repos/vitillo/telemetry-batch-view/deployments","created_at":"2015-10-26T21:29:57Z","updated_at":"2016-01-22T09:43:46Z","pushed_at":"2016-01-22T21:35:12Z","git_url":"git://github.com/vitillo/telemetry-batch-view.git","ssh_url":"git@github.com:vitillo/telemetry-batch-view.git","clone_url":"https://github.com/vitillo/telemetry-batch-view.git","svn_url":"https://github.com/vitillo/telemetry-batch-view","homepage":"","size":135,"stargazers_count":2,"watchers_count":2,"language":"Scala","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":1,"forks":1,"open_issues":1,"watchers":2,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/vitillo/telemetry-batch-view/pulls/10"},"html":{"href":"https://github.com/vitillo/telemetry-batch-view/pull/10"},"issue":{"href":"https://api.github.com/repos/vitillo/telemetry-batch-view/issues/10"},"comments":{"href":"https://api.github.com/repos/vitillo/telemetry-batch-view/issues/10/comments"},"review_comments":{"href":"https://api.github.com/repos/vitillo/telemetry-batch-view/pulls/10/comments"},"review_comment":{"href":"https://api.github.com/repos/vitillo/telemetry-batch-view/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/vitillo/telemetry-batch-view/pulls/10/commits"},"statuses":{"href":"https://api.github.com/repos/vitillo/telemetry-batch-view/statuses/c8863d94e73cd418594a3ec332463105fe7113ad"}}}}