{"action":"created","comment":{"url":"https://api.github.com/repos/hail-is/hail/pulls/comments/311696952","pull_request_review_id":272145547,"id":311696952,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDMxMTY5Njk1Mg==","diff_hunk":"@@ -0,0 +1,236 @@\n+package is.hail.expr.ir.agg\n+\n+import is.hail.annotations.{Region, StagedRegionValueBuilder}\n+import is.hail.asm4s._\n+import is.hail.expr.ir.{EmitTriplet, EmitRegion, EmitMethodBuilder}\n+import is.hail.expr.types.physical._\n+import is.hail.utils._\n+\n+object StagedBlockLinkedList {\n+  val defaultBlockCap: Int = 64\n+\n+  // src : Code[PointerTo[T]]\n+  // dst : Code[PointerTo[T]]\n+  private def deepCopy(er: EmitRegion, typ: PType, src: Code[Long], dst: Code[Long]): Code[Unit] =\n+    storeDeepCopy(er, typ, Region.loadIRIntermediate(typ)(src), dst)\n+\n+  // v : Code[T]\n+  // dst : Code[PointerTo[T]]\n+  private def storeDeepCopy(er: EmitRegion, typ: PType, v: Code[_], dst: Code[Long]): Code[Unit] = {\n+    val ftyp = typ.fundamentalType\n+    if (ftyp.isPrimitive)\n+      Region.storePrimitive(ftyp, dst)(v)\n+    else\n+      StagedRegionValueBuilder.deepCopy(er, typ, coerce[Long](v), dst)\n+  }\n+}\n+\n+class StagedBlockLinkedList(val elemType: PType, val mb: EmitMethodBuilder) {\n+  import StagedBlockLinkedList._\n+\n+\n+  /*** State ***/\n+\n+  val firstNode = mb.newField[Long]\n+  val lastNode = mb.newField[Long]\n+  val totalCount = mb.newField[Int]\n+\n+  val storageType = PStruct(\n+    \"firstNode\" -> PInt64Required,\n+    \"lastNode\" -> PInt64Required,\n+    \"totalCount\" -> PInt32Required)\n+\n+  def load(src: Code[Long]): Code[Unit] = Code(\n+    firstNode := Region.loadAddress(storageType.fieldOffset(src, 0)),\n+    lastNode := Region.loadAddress(storageType.fieldOffset(src, 1)),\n+    totalCount := Region.loadInt(storageType.fieldOffset(src, 2)))\n+\n+  def store(dst: Code[Long]): Code[Unit] = Code(\n+    Region.storeAddress(storageType.fieldOffset(dst, 0), firstNode),\n+    Region.storeAddress(storageType.fieldOffset(dst, 1), lastNode),\n+    Region.storeInt(storageType.fieldOffset(dst, 2), totalCount))\n+\n+\n+  /*** Temporaries ***\n+   * 'i': used to iterate over block buffers in foreachElemAddress\n+   * 'p': used to track missingness in foreachBlockNode\n+   * 'tmpNode': used to hold pointers to newly allocated nodes */\n+  val i = mb.newField[Int]\n+  val p = mb.newField[Boolean]\n+  val tmpNode = mb.newField[Long]\n+\n+\n+  /*** Node management ***/\n+\n+  type Node = Code[Long]\n+\n+  val bufferType = PArray(elemType.setRequired(false), required = true)\n+\n+  val nodeType = PStruct(\n+    \"buf\" -> bufferType,\n+    \"count\" -> PInt32Required,\n+    \"next\" -> PInt64Optional)\n+\n+  private def buffer(n: Node): Code[Long] =\n+    Region.loadAddress(nodeType.fieldOffset(n, 0))\n+\n+  private[agg] def capacity(n: Node): Code[Int] =\n+    bufferType.loadLength(buffer(n))\n+\n+  private def count(n: Node): Code[Int] =\n+    Region.loadInt(nodeType.fieldOffset(n, 1))\n+\n+  private def incrCount(n: Node): Code[Unit] =\n+    Region.storeInt(nodeType.fieldOffset(n, 1), count(n) + 1)\n+\n+  private def next(n: Node): Node =\n+    Region.loadAddress(nodeType.fieldOffset(n, 2))\n+\n+  private def hasNext(n: Node): Code[Boolean] =\n+    nodeType.isFieldDefined(n, 2)\n+\n+  private def setNext(n: Node, nNext: Node): Code[Unit] = Code(\n+    nodeType.setFieldPresent(n, 2),\n+    Region.storeAddress(nodeType.fieldOffset(n, 2), nNext))\n+\n+  private def initNode(n: Node, buf: Code[Long], count: Code[Int]): Code[Unit] =\n+    Code(\n+      Region.storeAddress(nodeType.fieldOffset(n, 0), buf),\n+      Region.storeInt(nodeType.fieldOffset(n, 1), count),\n+      nodeType.setFieldMissing(n, 2))\n+\n+  private def elemAddress(n: Node, i: Code[Int]): EmitTriplet =","path":"hail/src/main/scala/is/hail/expr/ir/agg/StagedBlockLinkedList.scala","position":93,"original_position":102,"commit_id":"b99800be6cf6025bed04b1aeef0bd7fb1db8ea62","original_commit_id":"7a4943561b84ce8192f4b95a9b7c875348157423","user":{"login":"iitalics","id":1772927,"node_id":"MDQ6VXNlcjE3NzI5Mjc=","avatar_url":"https://avatars2.githubusercontent.com/u/1772927?v=4","gravatar_id":"","url":"https://api.github.com/users/iitalics","html_url":"https://github.com/iitalics","followers_url":"https://api.github.com/users/iitalics/followers","following_url":"https://api.github.com/users/iitalics/following{/other_user}","gists_url":"https://api.github.com/users/iitalics/gists{/gist_id}","starred_url":"https://api.github.com/users/iitalics/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/iitalics/subscriptions","organizations_url":"https://api.github.com/users/iitalics/orgs","repos_url":"https://api.github.com/users/iitalics/repos","events_url":"https://api.github.com/users/iitalics/events{/privacy}","received_events_url":"https://api.github.com/users/iitalics/received_events","type":"User","site_admin":false},"body":"yeah I think that's a good idea","created_at":"2019-08-07T18:21:38Z","updated_at":"2019-08-07T18:21:38Z","html_url":"https://github.com/hail-is/hail/pull/6822#discussion_r311696952","pull_request_url":"https://api.github.com/repos/hail-is/hail/pulls/6822","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/hail-is/hail/pulls/comments/311696952"},"html":{"href":"https://github.com/hail-is/hail/pull/6822#discussion_r311696952"},"pull_request":{"href":"https://api.github.com/repos/hail-is/hail/pulls/6822"}},"in_reply_to_id":311695425},"pull_request":{"url":"https://api.github.com/repos/hail-is/hail/pulls/6822","id":304862187,"node_id":"MDExOlB1bGxSZXF1ZXN0MzA0ODYyMTg3","html_url":"https://github.com/hail-is/hail/pull/6822","diff_url":"https://github.com/hail-is/hail/pull/6822.diff","patch_url":"https://github.com/hail-is/hail/pull/6822.patch","issue_url":"https://api.github.com/repos/hail-is/hail/issues/6822","number":6822,"state":"open","locked":false,"title":"Staged CollectAggregator w/ block linked list","user":{"login":"iitalics","id":1772927,"node_id":"MDQ6VXNlcjE3NzI5Mjc=","avatar_url":"https://avatars2.githubusercontent.com/u/1772927?v=4","gravatar_id":"","url":"https://api.github.com/users/iitalics","html_url":"https://github.com/iitalics","followers_url":"https://api.github.com/users/iitalics/followers","following_url":"https://api.github.com/users/iitalics/following{/other_user}","gists_url":"https://api.github.com/users/iitalics/gists{/gist_id}","starred_url":"https://api.github.com/users/iitalics/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/iitalics/subscriptions","organizations_url":"https://api.github.com/users/iitalics/orgs","repos_url":"https://api.github.com/users/iitalics/repos","events_url":"https://api.github.com/users/iitalics/events{/privacy}","received_events_url":"https://api.github.com/users/iitalics/received_events","type":"User","site_admin":false},"body":"implements a staged list-of-buffers datastructure and uses it to implement the collect aggregator\r\n\r\nnot sure if this is needed if tim's arraybuilder version works better","created_at":"2019-08-06T19:42:36Z","updated_at":"2019-08-07T18:21:38Z","closed_at":null,"merged_at":null,"merge_commit_sha":"f83e3cc815ed3604d0225ef64ad6d761b5ea86b6","assignee":{"login":"catoverdrive","id":19789871,"node_id":"MDQ6VXNlcjE5Nzg5ODcx","avatar_url":"https://avatars3.githubusercontent.com/u/19789871?v=4","gravatar_id":"","url":"https://api.github.com/users/catoverdrive","html_url":"https://github.com/catoverdrive","followers_url":"https://api.github.com/users/catoverdrive/followers","following_url":"https://api.github.com/users/catoverdrive/following{/other_user}","gists_url":"https://api.github.com/users/catoverdrive/gists{/gist_id}","starred_url":"https://api.github.com/users/catoverdrive/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/catoverdrive/subscriptions","organizations_url":"https://api.github.com/users/catoverdrive/orgs","repos_url":"https://api.github.com/users/catoverdrive/repos","events_url":"https://api.github.com/users/catoverdrive/events{/privacy}","received_events_url":"https://api.github.com/users/catoverdrive/received_events","type":"User","site_admin":false},"assignees":[{"login":"catoverdrive","id":19789871,"node_id":"MDQ6VXNlcjE5Nzg5ODcx","avatar_url":"https://avatars3.githubusercontent.com/u/19789871?v=4","gravatar_id":"","url":"https://api.github.com/users/catoverdrive","html_url":"https://github.com/catoverdrive","followers_url":"https://api.github.com/users/catoverdrive/followers","following_url":"https://api.github.com/users/catoverdrive/following{/other_user}","gists_url":"https://api.github.com/users/catoverdrive/gists{/gist_id}","starred_url":"https://api.github.com/users/catoverdrive/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/catoverdrive/subscriptions","organizations_url":"https://api.github.com/users/catoverdrive/orgs","repos_url":"https://api.github.com/users/catoverdrive/repos","events_url":"https://api.github.com/users/catoverdrive/events{/privacy}","received_events_url":"https://api.github.com/users/catoverdrive/received_events","type":"User","site_admin":false}],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/hail-is/hail/pulls/6822/commits","review_comments_url":"https://api.github.com/repos/hail-is/hail/pulls/6822/comments","review_comment_url":"https://api.github.com/repos/hail-is/hail/pulls/comments{/number}","comments_url":"https://api.github.com/repos/hail-is/hail/issues/6822/comments","statuses_url":"https://api.github.com/repos/hail-is/hail/statuses/b99800be6cf6025bed04b1aeef0bd7fb1db8ea62","head":{"label":"iitalics:staged-collect-agg","ref":"staged-collect-agg","sha":"b99800be6cf6025bed04b1aeef0bd7fb1db8ea62","user":{"login":"iitalics","id":1772927,"node_id":"MDQ6VXNlcjE3NzI5Mjc=","avatar_url":"https://avatars2.githubusercontent.com/u/1772927?v=4","gravatar_id":"","url":"https://api.github.com/users/iitalics","html_url":"https://github.com/iitalics","followers_url":"https://api.github.com/users/iitalics/followers","following_url":"https://api.github.com/users/iitalics/following{/other_user}","gists_url":"https://api.github.com/users/iitalics/gists{/gist_id}","starred_url":"https://api.github.com/users/iitalics/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/iitalics/subscriptions","organizations_url":"https://api.github.com/users/iitalics/orgs","repos_url":"https://api.github.com/users/iitalics/repos","events_url":"https://api.github.com/users/iitalics/events{/privacy}","received_events_url":"https://api.github.com/users/iitalics/received_events","type":"User","site_admin":false},"repo":{"id":196061045,"node_id":"MDEwOlJlcG9zaXRvcnkxOTYwNjEwNDU=","name":"hail","full_name":"iitalics/hail","private":false,"owner":{"login":"iitalics","id":1772927,"node_id":"MDQ6VXNlcjE3NzI5Mjc=","avatar_url":"https://avatars2.githubusercontent.com/u/1772927?v=4","gravatar_id":"","url":"https://api.github.com/users/iitalics","html_url":"https://github.com/iitalics","followers_url":"https://api.github.com/users/iitalics/followers","following_url":"https://api.github.com/users/iitalics/following{/other_user}","gists_url":"https://api.github.com/users/iitalics/gists{/gist_id}","starred_url":"https://api.github.com/users/iitalics/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/iitalics/subscriptions","organizations_url":"https://api.github.com/users/iitalics/orgs","repos_url":"https://api.github.com/users/iitalics/repos","events_url":"https://api.github.com/users/iitalics/events{/privacy}","received_events_url":"https://api.github.com/users/iitalics/received_events","type":"User","site_admin":false},"html_url":"https://github.com/iitalics/hail","description":"Scalable genomic data analysis.","fork":true,"url":"https://api.github.com/repos/iitalics/hail","forks_url":"https://api.github.com/repos/iitalics/hail/forks","keys_url":"https://api.github.com/repos/iitalics/hail/keys{/key_id}","collaborators_url":"https://api.github.com/repos/iitalics/hail/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/iitalics/hail/teams","hooks_url":"https://api.github.com/repos/iitalics/hail/hooks","issue_events_url":"https://api.github.com/repos/iitalics/hail/issues/events{/number}","events_url":"https://api.github.com/repos/iitalics/hail/events","assignees_url":"https://api.github.com/repos/iitalics/hail/assignees{/user}","branches_url":"https://api.github.com/repos/iitalics/hail/branches{/branch}","tags_url":"https://api.github.com/repos/iitalics/hail/tags","blobs_url":"https://api.github.com/repos/iitalics/hail/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/iitalics/hail/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/iitalics/hail/git/refs{/sha}","trees_url":"https://api.github.com/repos/iitalics/hail/git/trees{/sha}","statuses_url":"https://api.github.com/repos/iitalics/hail/statuses/{sha}","languages_url":"https://api.github.com/repos/iitalics/hail/languages","stargazers_url":"https://api.github.com/repos/iitalics/hail/stargazers","contributors_url":"https://api.github.com/repos/iitalics/hail/contributors","subscribers_url":"https://api.github.com/repos/iitalics/hail/subscribers","subscription_url":"https://api.github.com/repos/iitalics/hail/subscription","commits_url":"https://api.github.com/repos/iitalics/hail/commits{/sha}","git_commits_url":"https://api.github.com/repos/iitalics/hail/git/commits{/sha}","comments_url":"https://api.github.com/repos/iitalics/hail/comments{/number}","issue_comment_url":"https://api.github.com/repos/iitalics/hail/issues/comments{/number}","contents_url":"https://api.github.com/repos/iitalics/hail/contents/{+path}","compare_url":"https://api.github.com/repos/iitalics/hail/compare/{base}...{head}","merges_url":"https://api.github.com/repos/iitalics/hail/merges","archive_url":"https://api.github.com/repos/iitalics/hail/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/iitalics/hail/downloads","issues_url":"https://api.github.com/repos/iitalics/hail/issues{/number}","pulls_url":"https://api.github.com/repos/iitalics/hail/pulls{/number}","milestones_url":"https://api.github.com/repos/iitalics/hail/milestones{/number}","notifications_url":"https://api.github.com/repos/iitalics/hail/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/iitalics/hail/labels{/name}","releases_url":"https://api.github.com/repos/iitalics/hail/releases{/id}","deployments_url":"https://api.github.com/repos/iitalics/hail/deployments","created_at":"2019-07-09T18:20:59Z","updated_at":"2019-08-06T19:45:41Z","pushed_at":"2019-08-07T18:20:32Z","git_url":"git://github.com/iitalics/hail.git","ssh_url":"git@github.com:iitalics/hail.git","clone_url":"https://github.com/iitalics/hail.git","svn_url":"https://github.com/iitalics/hail","homepage":"https://hail.is","size":72073,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"disabled":false,"open_issues_count":0,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit","node_id":"MDc6TGljZW5zZTEz"},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"hail-is:master","ref":"master","sha":"9dd435ce3c7242e682f402b8b904477740b296db","user":{"login":"hail-is","id":20861792,"node_id":"MDEyOk9yZ2FuaXphdGlvbjIwODYxNzky","avatar_url":"https://avatars1.githubusercontent.com/u/20861792?v=4","gravatar_id":"","url":"https://api.github.com/users/hail-is","html_url":"https://github.com/hail-is","followers_url":"https://api.github.com/users/hail-is/followers","following_url":"https://api.github.com/users/hail-is/following{/other_user}","gists_url":"https://api.github.com/users/hail-is/gists{/gist_id}","starred_url":"https://api.github.com/users/hail-is/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hail-is/subscriptions","organizations_url":"https://api.github.com/users/hail-is/orgs","repos_url":"https://api.github.com/users/hail-is/repos","events_url":"https://api.github.com/users/hail-is/events{/privacy}","received_events_url":"https://api.github.com/users/hail-is/received_events","type":"Organization","site_admin":false},"repo":{"id":45069467,"node_id":"MDEwOlJlcG9zaXRvcnk0NTA2OTQ2Nw==","name":"hail","full_name":"hail-is/hail","private":false,"owner":{"login":"hail-is","id":20861792,"node_id":"MDEyOk9yZ2FuaXphdGlvbjIwODYxNzky","avatar_url":"https://avatars1.githubusercontent.com/u/20861792?v=4","gravatar_id":"","url":"https://api.github.com/users/hail-is","html_url":"https://github.com/hail-is","followers_url":"https://api.github.com/users/hail-is/followers","following_url":"https://api.github.com/users/hail-is/following{/other_user}","gists_url":"https://api.github.com/users/hail-is/gists{/gist_id}","starred_url":"https://api.github.com/users/hail-is/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hail-is/subscriptions","organizations_url":"https://api.github.com/users/hail-is/orgs","repos_url":"https://api.github.com/users/hail-is/repos","events_url":"https://api.github.com/users/hail-is/events{/privacy}","received_events_url":"https://api.github.com/users/hail-is/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/hail-is/hail","description":"Scalable genomic data analysis.","fork":false,"url":"https://api.github.com/repos/hail-is/hail","forks_url":"https://api.github.com/repos/hail-is/hail/forks","keys_url":"https://api.github.com/repos/hail-is/hail/keys{/key_id}","collaborators_url":"https://api.github.com/repos/hail-is/hail/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/hail-is/hail/teams","hooks_url":"https://api.github.com/repos/hail-is/hail/hooks","issue_events_url":"https://api.github.com/repos/hail-is/hail/issues/events{/number}","events_url":"https://api.github.com/repos/hail-is/hail/events","assignees_url":"https://api.github.com/repos/hail-is/hail/assignees{/user}","branches_url":"https://api.github.com/repos/hail-is/hail/branches{/branch}","tags_url":"https://api.github.com/repos/hail-is/hail/tags","blobs_url":"https://api.github.com/repos/hail-is/hail/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/hail-is/hail/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/hail-is/hail/git/refs{/sha}","trees_url":"https://api.github.com/repos/hail-is/hail/git/trees{/sha}","statuses_url":"https://api.github.com/repos/hail-is/hail/statuses/{sha}","languages_url":"https://api.github.com/repos/hail-is/hail/languages","stargazers_url":"https://api.github.com/repos/hail-is/hail/stargazers","contributors_url":"https://api.github.com/repos/hail-is/hail/contributors","subscribers_url":"https://api.github.com/repos/hail-is/hail/subscribers","subscription_url":"https://api.github.com/repos/hail-is/hail/subscription","commits_url":"https://api.github.com/repos/hail-is/hail/commits{/sha}","git_commits_url":"https://api.github.com/repos/hail-is/hail/git/commits{/sha}","comments_url":"https://api.github.com/repos/hail-is/hail/comments{/number}","issue_comment_url":"https://api.github.com/repos/hail-is/hail/issues/comments{/number}","contents_url":"https://api.github.com/repos/hail-is/hail/contents/{+path}","compare_url":"https://api.github.com/repos/hail-is/hail/compare/{base}...{head}","merges_url":"https://api.github.com/repos/hail-is/hail/merges","archive_url":"https://api.github.com/repos/hail-is/hail/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/hail-is/hail/downloads","issues_url":"https://api.github.com/repos/hail-is/hail/issues{/number}","pulls_url":"https://api.github.com/repos/hail-is/hail/pulls{/number}","milestones_url":"https://api.github.com/repos/hail-is/hail/milestones{/number}","notifications_url":"https://api.github.com/repos/hail-is/hail/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/hail-is/hail/labels{/name}","releases_url":"https://api.github.com/repos/hail-is/hail/releases{/id}","deployments_url":"https://api.github.com/repos/hail-is/hail/deployments","created_at":"2015-10-27T20:55:42Z","updated_at":"2019-08-07T17:45:00Z","pushed_at":"2019-08-07T18:20:34Z","git_url":"git://github.com/hail-is/hail.git","ssh_url":"git@github.com:hail-is/hail.git","clone_url":"https://github.com/hail-is/hail.git","svn_url":"https://github.com/hail-is/hail","homepage":"https://hail.is","size":72034,"stargazers_count":543,"watchers_count":543,"language":"Scala","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":139,"mirror_url":null,"archived":false,"disabled":false,"open_issues_count":283,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit","node_id":"MDc6TGljZW5zZTEz"},"forks":139,"open_issues":283,"watchers":543,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/hail-is/hail/pulls/6822"},"html":{"href":"https://github.com/hail-is/hail/pull/6822"},"issue":{"href":"https://api.github.com/repos/hail-is/hail/issues/6822"},"comments":{"href":"https://api.github.com/repos/hail-is/hail/issues/6822/comments"},"review_comments":{"href":"https://api.github.com/repos/hail-is/hail/pulls/6822/comments"},"review_comment":{"href":"https://api.github.com/repos/hail-is/hail/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/hail-is/hail/pulls/6822/commits"},"statuses":{"href":"https://api.github.com/repos/hail-is/hail/statuses/b99800be6cf6025bed04b1aeef0bd7fb1db8ea62"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"iitalics"}}