{"action":"created","comment":{"url":"https://api.github.com/repos/twitter/summingbird/pulls/comments/75568037","id":75568037,"diff_hunk":"@@ -80,27 +90,36 @@ class AggregatorOutputCollector[K, V: Semigroup](\n   Method wraps the adding the tuple to the spoutCache along with adding the corresponding\n   messageId to the messageId Tracker.\n    */\n-  private def add(tuple: (K, V), streamid: String, messageId: Option[AnyRef] = None) = {\n+  private def add(tuple: (K, V), streamid: String, messageId: Option[AnyRef] = None): Future[Map[K, V]] = {\n     if (messageId.isDefined)\n       trackMessageId(tuple, messageId.get, streamid)\n     addToCache(tuple, streamid)\n   }\n \n-  private def addToCache(tuple: (K, V), streamid: String) = {\n+  /*\n+   * As there are separate summers for each stream. This method takes care of the stream lookup\n+   * and adding the tuple to the corresponding cache.\n+   */\n+  private def addToCache(tuple: (K, V), streamid: String): Future[Map[K, V]] = {\n     spoutCaches.get(streamid) match {\n-      case Some(cac) => cac.add(tuple)\n+      case Some(cache) => cache.add(tuple)\n       case None => {\n         spoutCaches(streamid) = summerBuilder.getSummer[K, V](implicitly[Semigroup[V]])\n         spoutCaches(streamid).add(tuple)\n       }\n     }\n   }\n \n+  /**\n+   * The lookup of stream is followed by the lookup on summerShard happens.\n+   * The messageId is added to the corresponding group.\n+   * All the messageIds are sent along with the crushed tuple.\n+   */\n   private def trackMessageId(tuple: (K, V), o: AnyRef, s: String): Unit = {\n-    val messageIdTracker = streamMessageIdTracker.getOrElse(s, MMap[Int, MList[Object]]())\n+    val messageIdTracker = cachesByStreamId.getOrElse(s, MMap[Int, MList[Object]]())","path":"summingbird-storm/src/main/scala/com/twitter/summingbird/storm/collector/AggregatorOutputCollector.scala","position":null,"original_position":121,"commit_id":"ddfa6062467c361c284872eadab47decc20882a9","original_commit_id":"ddfa6062467c361c284872eadab47decc20882a9","user":{"login":"johnynek","id":67958,"avatar_url":"https://avatars.githubusercontent.com/u/67958?v=3","gravatar_id":"","url":"https://api.github.com/users/johnynek","html_url":"https://github.com/johnynek","followers_url":"https://api.github.com/users/johnynek/followers","following_url":"https://api.github.com/users/johnynek/following{/other_user}","gists_url":"https://api.github.com/users/johnynek/gists{/gist_id}","starred_url":"https://api.github.com/users/johnynek/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/johnynek/subscriptions","organizations_url":"https://api.github.com/users/johnynek/orgs","repos_url":"https://api.github.com/users/johnynek/repos","events_url":"https://api.github.com/users/johnynek/events{/privacy}","received_events_url":"https://api.github.com/users/johnynek/received_events","type":"User","site_admin":false},"body":"if you use `.getOrElseUpdate` you don't need to reinsert at the end. Can we do that?","created_at":"2016-08-20T00:23:44Z","updated_at":"2016-08-20T00:23:44Z","html_url":"https://github.com/twitter/summingbird/pull/682#discussion_r75568037","pull_request_url":"https://api.github.com/repos/twitter/summingbird/pulls/682","_links":{"self":{"href":"https://api.github.com/repos/twitter/summingbird/pulls/comments/75568037"},"html":{"href":"https://github.com/twitter/summingbird/pull/682#discussion_r75568037"},"pull_request":{"href":"https://api.github.com/repos/twitter/summingbird/pulls/682"}}},"pull_request":{"url":"https://api.github.com/repos/twitter/summingbird/pulls/682","id":81182439,"html_url":"https://github.com/twitter/summingbird/pull/682","diff_url":"https://github.com/twitter/summingbird/pull/682.diff","patch_url":"https://github.com/twitter/summingbird/pull/682.patch","issue_url":"https://api.github.com/repos/twitter/summingbird/issues/682","number":682,"state":"open","locked":false,"title":"Adding crush down capability into the SourceNode","user":{"login":"NPraneeth","id":15038254,"avatar_url":"https://avatars.githubusercontent.com/u/15038254?v=3","gravatar_id":"","url":"https://api.github.com/users/NPraneeth","html_url":"https://github.com/NPraneeth","followers_url":"https://api.github.com/users/NPraneeth/followers","following_url":"https://api.github.com/users/NPraneeth/following{/other_user}","gists_url":"https://api.github.com/users/NPraneeth/gists{/gist_id}","starred_url":"https://api.github.com/users/NPraneeth/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/NPraneeth/subscriptions","organizations_url":"https://api.github.com/users/NPraneeth/orgs","repos_url":"https://api.github.com/users/NPraneeth/repos","events_url":"https://api.github.com/users/NPraneeth/events{/privacy}","received_events_url":"https://api.github.com/users/NPraneeth/received_events","type":"User","site_admin":false},"body":"Closes #679\r\nThis PR mainly adds the summing capability to the spout. When FMMergeableWithSource is opted in, the flatMap operation in the spout creates a fanOut. The fanOut is being crushed down by using the capability of AsyncSummers inside the spout and emitting only crushed tuples. The spout-side aggregation metrics are added to the spout for performance analysis. The messageIds are also aggregated along with the tuples and ack/fail back is sent to all the aggregated messageIds. Tuples from different streams are aggregated individually.  ","created_at":"2016-08-12T21:00:11Z","updated_at":"2016-08-20T00:23:44Z","closed_at":null,"merged_at":null,"merge_commit_sha":"1e3d88eb0e8a19f71e64f185be65a49e0288cdd9","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/twitter/summingbird/pulls/682/commits","review_comments_url":"https://api.github.com/repos/twitter/summingbird/pulls/682/comments","review_comment_url":"https://api.github.com/repos/twitter/summingbird/pulls/comments{/number}","comments_url":"https://api.github.com/repos/twitter/summingbird/issues/682/comments","statuses_url":"https://api.github.com/repos/twitter/summingbird/statuses/ddfa6062467c361c284872eadab47decc20882a9","head":{"label":"NPraneeth:pnaramsetti/add-aggregation-spout","ref":"pnaramsetti/add-aggregation-spout","sha":"ddfa6062467c361c284872eadab47decc20882a9","user":{"login":"NPraneeth","id":15038254,"avatar_url":"https://avatars.githubusercontent.com/u/15038254?v=3","gravatar_id":"","url":"https://api.github.com/users/NPraneeth","html_url":"https://github.com/NPraneeth","followers_url":"https://api.github.com/users/NPraneeth/followers","following_url":"https://api.github.com/users/NPraneeth/following{/other_user}","gists_url":"https://api.github.com/users/NPraneeth/gists{/gist_id}","starred_url":"https://api.github.com/users/NPraneeth/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/NPraneeth/subscriptions","organizations_url":"https://api.github.com/users/NPraneeth/orgs","repos_url":"https://api.github.com/users/NPraneeth/repos","events_url":"https://api.github.com/users/NPraneeth/events{/privacy}","received_events_url":"https://api.github.com/users/NPraneeth/received_events","type":"User","site_admin":false},"repo":{"id":65408713,"name":"summingbird","full_name":"NPraneeth/summingbird","owner":{"login":"NPraneeth","id":15038254,"avatar_url":"https://avatars.githubusercontent.com/u/15038254?v=3","gravatar_id":"","url":"https://api.github.com/users/NPraneeth","html_url":"https://github.com/NPraneeth","followers_url":"https://api.github.com/users/NPraneeth/followers","following_url":"https://api.github.com/users/NPraneeth/following{/other_user}","gists_url":"https://api.github.com/users/NPraneeth/gists{/gist_id}","starred_url":"https://api.github.com/users/NPraneeth/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/NPraneeth/subscriptions","organizations_url":"https://api.github.com/users/NPraneeth/orgs","repos_url":"https://api.github.com/users/NPraneeth/repos","events_url":"https://api.github.com/users/NPraneeth/events{/privacy}","received_events_url":"https://api.github.com/users/NPraneeth/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/NPraneeth/summingbird","description":"Streaming MapReduce with Scalding and Storm","fork":true,"url":"https://api.github.com/repos/NPraneeth/summingbird","forks_url":"https://api.github.com/repos/NPraneeth/summingbird/forks","keys_url":"https://api.github.com/repos/NPraneeth/summingbird/keys{/key_id}","collaborators_url":"https://api.github.com/repos/NPraneeth/summingbird/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/NPraneeth/summingbird/teams","hooks_url":"https://api.github.com/repos/NPraneeth/summingbird/hooks","issue_events_url":"https://api.github.com/repos/NPraneeth/summingbird/issues/events{/number}","events_url":"https://api.github.com/repos/NPraneeth/summingbird/events","assignees_url":"https://api.github.com/repos/NPraneeth/summingbird/assignees{/user}","branches_url":"https://api.github.com/repos/NPraneeth/summingbird/branches{/branch}","tags_url":"https://api.github.com/repos/NPraneeth/summingbird/tags","blobs_url":"https://api.github.com/repos/NPraneeth/summingbird/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/NPraneeth/summingbird/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/NPraneeth/summingbird/git/refs{/sha}","trees_url":"https://api.github.com/repos/NPraneeth/summingbird/git/trees{/sha}","statuses_url":"https://api.github.com/repos/NPraneeth/summingbird/statuses/{sha}","languages_url":"https://api.github.com/repos/NPraneeth/summingbird/languages","stargazers_url":"https://api.github.com/repos/NPraneeth/summingbird/stargazers","contributors_url":"https://api.github.com/repos/NPraneeth/summingbird/contributors","subscribers_url":"https://api.github.com/repos/NPraneeth/summingbird/subscribers","subscription_url":"https://api.github.com/repos/NPraneeth/summingbird/subscription","commits_url":"https://api.github.com/repos/NPraneeth/summingbird/commits{/sha}","git_commits_url":"https://api.github.com/repos/NPraneeth/summingbird/git/commits{/sha}","comments_url":"https://api.github.com/repos/NPraneeth/summingbird/comments{/number}","issue_comment_url":"https://api.github.com/repos/NPraneeth/summingbird/issues/comments{/number}","contents_url":"https://api.github.com/repos/NPraneeth/summingbird/contents/{+path}","compare_url":"https://api.github.com/repos/NPraneeth/summingbird/compare/{base}...{head}","merges_url":"https://api.github.com/repos/NPraneeth/summingbird/merges","archive_url":"https://api.github.com/repos/NPraneeth/summingbird/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/NPraneeth/summingbird/downloads","issues_url":"https://api.github.com/repos/NPraneeth/summingbird/issues{/number}","pulls_url":"https://api.github.com/repos/NPraneeth/summingbird/pulls{/number}","milestones_url":"https://api.github.com/repos/NPraneeth/summingbird/milestones{/number}","notifications_url":"https://api.github.com/repos/NPraneeth/summingbird/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/NPraneeth/summingbird/labels{/name}","releases_url":"https://api.github.com/repos/NPraneeth/summingbird/releases{/id}","deployments_url":"https://api.github.com/repos/NPraneeth/summingbird/deployments","created_at":"2016-08-10T18:58:51Z","updated_at":"2016-08-10T18:58:53Z","pushed_at":"2016-08-19T18:45:08Z","git_url":"git://github.com/NPraneeth/summingbird.git","ssh_url":"git@github.com:NPraneeth/summingbird.git","clone_url":"https://github.com/NPraneeth/summingbird.git","svn_url":"https://github.com/NPraneeth/summingbird","homepage":"https://twitter.com/summingbird","size":6237,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":1,"forks":0,"open_issues":1,"watchers":0,"default_branch":"develop"}},"base":{"label":"twitter:develop","ref":"develop","sha":"22db7b0a65f472c42c8f283a7b7773eb68f3d973","user":{"login":"twitter","id":50278,"avatar_url":"https://avatars.githubusercontent.com/u/50278?v=3","gravatar_id":"","url":"https://api.github.com/users/twitter","html_url":"https://github.com/twitter","followers_url":"https://api.github.com/users/twitter/followers","following_url":"https://api.github.com/users/twitter/following{/other_user}","gists_url":"https://api.github.com/users/twitter/gists{/gist_id}","starred_url":"https://api.github.com/users/twitter/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/twitter/subscriptions","organizations_url":"https://api.github.com/users/twitter/orgs","repos_url":"https://api.github.com/users/twitter/repos","events_url":"https://api.github.com/users/twitter/events{/privacy}","received_events_url":"https://api.github.com/users/twitter/received_events","type":"Organization","site_admin":false},"repo":{"id":5957708,"name":"summingbird","full_name":"twitter/summingbird","owner":{"login":"twitter","id":50278,"avatar_url":"https://avatars.githubusercontent.com/u/50278?v=3","gravatar_id":"","url":"https://api.github.com/users/twitter","html_url":"https://github.com/twitter","followers_url":"https://api.github.com/users/twitter/followers","following_url":"https://api.github.com/users/twitter/following{/other_user}","gists_url":"https://api.github.com/users/twitter/gists{/gist_id}","starred_url":"https://api.github.com/users/twitter/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/twitter/subscriptions","organizations_url":"https://api.github.com/users/twitter/orgs","repos_url":"https://api.github.com/users/twitter/repos","events_url":"https://api.github.com/users/twitter/events{/privacy}","received_events_url":"https://api.github.com/users/twitter/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/twitter/summingbird","description":"Streaming MapReduce with Scalding and Storm","fork":false,"url":"https://api.github.com/repos/twitter/summingbird","forks_url":"https://api.github.com/repos/twitter/summingbird/forks","keys_url":"https://api.github.com/repos/twitter/summingbird/keys{/key_id}","collaborators_url":"https://api.github.com/repos/twitter/summingbird/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/twitter/summingbird/teams","hooks_url":"https://api.github.com/repos/twitter/summingbird/hooks","issue_events_url":"https://api.github.com/repos/twitter/summingbird/issues/events{/number}","events_url":"https://api.github.com/repos/twitter/summingbird/events","assignees_url":"https://api.github.com/repos/twitter/summingbird/assignees{/user}","branches_url":"https://api.github.com/repos/twitter/summingbird/branches{/branch}","tags_url":"https://api.github.com/repos/twitter/summingbird/tags","blobs_url":"https://api.github.com/repos/twitter/summingbird/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/twitter/summingbird/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/twitter/summingbird/git/refs{/sha}","trees_url":"https://api.github.com/repos/twitter/summingbird/git/trees{/sha}","statuses_url":"https://api.github.com/repos/twitter/summingbird/statuses/{sha}","languages_url":"https://api.github.com/repos/twitter/summingbird/languages","stargazers_url":"https://api.github.com/repos/twitter/summingbird/stargazers","contributors_url":"https://api.github.com/repos/twitter/summingbird/contributors","subscribers_url":"https://api.github.com/repos/twitter/summingbird/subscribers","subscription_url":"https://api.github.com/repos/twitter/summingbird/subscription","commits_url":"https://api.github.com/repos/twitter/summingbird/commits{/sha}","git_commits_url":"https://api.github.com/repos/twitter/summingbird/git/commits{/sha}","comments_url":"https://api.github.com/repos/twitter/summingbird/comments{/number}","issue_comment_url":"https://api.github.com/repos/twitter/summingbird/issues/comments{/number}","contents_url":"https://api.github.com/repos/twitter/summingbird/contents/{+path}","compare_url":"https://api.github.com/repos/twitter/summingbird/compare/{base}...{head}","merges_url":"https://api.github.com/repos/twitter/summingbird/merges","archive_url":"https://api.github.com/repos/twitter/summingbird/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/twitter/summingbird/downloads","issues_url":"https://api.github.com/repos/twitter/summingbird/issues{/number}","pulls_url":"https://api.github.com/repos/twitter/summingbird/pulls{/number}","milestones_url":"https://api.github.com/repos/twitter/summingbird/milestones{/number}","notifications_url":"https://api.github.com/repos/twitter/summingbird/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/twitter/summingbird/labels{/name}","releases_url":"https://api.github.com/repos/twitter/summingbird/releases{/id}","deployments_url":"https://api.github.com/repos/twitter/summingbird/deployments","created_at":"2012-09-25T22:38:35Z","updated_at":"2016-08-19T14:30:19Z","pushed_at":"2016-08-19T18:45:10Z","git_url":"git://github.com/twitter/summingbird.git","ssh_url":"git@github.com:twitter/summingbird.git","clone_url":"https://github.com/twitter/summingbird.git","svn_url":"https://github.com/twitter/summingbird","homepage":"https://twitter.com/summingbird","size":8565,"stargazers_count":1772,"watchers_count":1772,"language":"Scala","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":230,"mirror_url":null,"open_issues_count":135,"forks":230,"open_issues":135,"watchers":1772,"default_branch":"develop"}},"_links":{"self":{"href":"https://api.github.com/repos/twitter/summingbird/pulls/682"},"html":{"href":"https://github.com/twitter/summingbird/pull/682"},"issue":{"href":"https://api.github.com/repos/twitter/summingbird/issues/682"},"comments":{"href":"https://api.github.com/repos/twitter/summingbird/issues/682/comments"},"review_comments":{"href":"https://api.github.com/repos/twitter/summingbird/pulls/682/comments"},"review_comment":{"href":"https://api.github.com/repos/twitter/summingbird/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/twitter/summingbird/pulls/682/commits"},"statuses":{"href":"https://api.github.com/repos/twitter/summingbird/statuses/ddfa6062467c361c284872eadab47decc20882a9"}}}}
{"actor":{"display_login":"johnynek"}}