{"action":"created","comment":{"url":"https://api.github.com/repos/hail-is/hail/pulls/comments/189435370","pull_request_review_id":121620001,"id":189435370,"diff_hunk":"@@ -133,42 +138,66 @@ case class GridPartitioner(blockSize: Int, nRows: Long, nCols: Long, maybeSparse\n   }\n   \n   // returns increasing array of all blocks intersecting the diagonal band consisting of\n-  //   all entries with -lowerBandwidth <= (colIndex - rowIndex) <= upperBandwidth\n-  def bandedBlocks(lowerBandwidth: Long, upperBandwidth: Long): Array[Int] = {\n-    require(lowerBandwidth >= 0 && upperBandwidth >= 0)\n+  //   all elements with lower <= jj - ii <= upper\n+  def bandBlocks(lower: Long, upper: Long): Array[Int] = {\n+    require(lower <= upper)\n     \n-    val lowerBlockBandwidth = indexBlockIndex(lowerBandwidth + blockSize - 1)\n-    val upperBlockBandwidth = indexBlockIndex(upperBandwidth + blockSize - 1)\n+    val lowerBlock = java.lang.Math.floorDiv(lower, blockSize).toInt\n+    val upperBlock = java.lang.Math.floorDiv(upper + blockSize - 1, blockSize).toInt\n \n     (for { j <- 0 until nBlockCols\n-           i <- ((j - upperBlockBandwidth) max 0) to\n-                ((j + lowerBlockBandwidth) min (nBlockRows - 1))\n+           i <- ((j - upperBlock) max 0) to\n+                ((j - lowerBlock) min (nBlockRows - 1))\n     } yield (j * nBlockRows) + i).toArray\n   }\n \n-  // returns increasing array of all blocks intersecting the rectangle [firstRow, lastRow] x [firstCol, lastCol]\n-  def rectangularBlocks(firstRow: Long, lastRow: Long, firstCol: Long, lastCol: Long): Array[Int] = {\n-    require(firstRow >= 0 && lastRow < nRows)\n-    require(firstCol >= 0 && lastCol < nCols)\n-    \n-    if (firstRow > lastRow || firstCol > lastCol)\n-      return Array.empty[Int]\n-    \n-    val firstBlockRow = indexBlockIndex(firstRow)\n-    val lastBlockRow = indexBlockIndex(lastRow)\n-    val firstBlockCol = indexBlockIndex(firstCol)\n-    val lastBlockCol = indexBlockIndex(lastCol)\n+  // returns increasing array of all blocks intersecting the rectangle\n+  // [r(0), r(1)) x [r(2), r(3)), i.e. [startRow, stopRow) x [startCol, stopCol)\n+  // rectangle checked in Python\n+  def rectangleBlocks(r: Array[Long]): Array[Int] = {\n+    val startBlockRow = indexBlockIndex(r(0))\n+    val stopBlockRow = java.lang.Math.floorDiv(r(1) - 1, blockSize).toInt + 1\n+    val startBlockCol = indexBlockIndex(r(2))\n+    val stopBlockCol = java.lang.Math.floorDiv(r(3) - 1, blockSize).toInt + 1\n     \n-    (for { j <- firstBlockCol to lastBlockCol\n-           i <- firstBlockRow to lastBlockRow\n+    (for { j <- startBlockCol until stopBlockCol\n+           i <- startBlockRow until stopBlockRow\n     } yield (j * nBlockRows) + i).toArray\n   }\n \n   // returns increasing array of all blocks intersecting the union of rectangles\n-  def rectangularBlocks(rectangles: Array[Array[Long]]): Array[Int] = {\n-    require(rectangles.forall(r => r.length == 4))\n-    val rects = rectangles.foldLeft(Set[Int]())((s, r) => s ++ rectangularBlocks(r(0), r(1), r(2), r(3))).toArray    \n-    scala.util.Sorting.quickSort(rects)\n-    rects\n+  // rectangles checked in Python\n+  def rectanglesBlocks(rectangles: Array[Array[Long]]): Array[Int] = {\n+    val blocks = rectangles.foldLeft(Set[Int]())((s, r) => s ++ rectangleBlocks(r)).toArray    \n+    scala.util.Sorting.quickSort(blocks)","path":"src/main/scala/is/hail/linalg/GridPartitioner.scala","position":77,"original_position":77,"commit_id":"7101a6cf1c74e2a386eb04695b42759fa1fbff4b","original_commit_id":"010763844312cee73226496f3a4f284e3f6fb25c","user":{"login":"jbloom22","id":3201642,"avatar_url":"https://avatars1.githubusercontent.com/u/3201642?v=4","gravatar_id":"","url":"https://api.github.com/users/jbloom22","html_url":"https://github.com/jbloom22","followers_url":"https://api.github.com/users/jbloom22/followers","following_url":"https://api.github.com/users/jbloom22/following{/other_user}","gists_url":"https://api.github.com/users/jbloom22/gists{/gist_id}","starred_url":"https://api.github.com/users/jbloom22/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jbloom22/subscriptions","organizations_url":"https://api.github.com/users/jbloom22/orgs","repos_url":"https://api.github.com/users/jbloom22/repos","events_url":"https://api.github.com/users/jbloom22/events{/privacy}","received_events_url":"https://api.github.com/users/jbloom22/received_events","type":"User","site_admin":false},"body":"Nm I see why. You’d rather I revert to earlier approach?","created_at":"2018-05-19T14:07:05Z","updated_at":"2018-05-19T14:07:05Z","html_url":"https://github.com/hail-is/hail/pull/3539#discussion_r189435370","pull_request_url":"https://api.github.com/repos/hail-is/hail/pulls/3539","author_association":"COLLABORATOR","_links":{"self":{"href":"https://api.github.com/repos/hail-is/hail/pulls/comments/189435370"},"html":{"href":"https://github.com/hail-is/hail/pull/3539#discussion_r189435370"},"pull_request":{"href":"https://api.github.com/repos/hail-is/hail/pulls/3539"}},"in_reply_to_id":189271496},"pull_request":{"url":"https://api.github.com/repos/hail-is/hail/pulls/3539","id":186791821,"html_url":"https://github.com/hail-is/hail/pull/3539","diff_url":"https://github.com/hail-is/hail/pull/3539.diff","patch_url":"https://github.com/hail-is/hail/pull/3539.patch","issue_url":"https://api.github.com/repos/hail-is/hail/issues/3539","number":3539,"state":"open","locked":false,"title":"sparsify_band/triangle/rectangles and export_rectangles","user":{"login":"jbloom22","id":3201642,"avatar_url":"https://avatars1.githubusercontent.com/u/3201642?v=4","gravatar_id":"","url":"https://api.github.com/users/jbloom22","html_url":"https://github.com/jbloom22","followers_url":"https://api.github.com/users/jbloom22/followers","following_url":"https://api.github.com/users/jbloom22/following{/other_user}","gists_url":"https://api.github.com/users/jbloom22/gists{/gist_id}","starred_url":"https://api.github.com/users/jbloom22/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jbloom22/subscriptions","organizations_url":"https://api.github.com/users/jbloom22/orgs","repos_url":"https://api.github.com/users/jbloom22/repos","events_url":"https://api.github.com/users/jbloom22/events{/privacy}","received_events_url":"https://api.github.com/users/jbloom22/received_events","type":"User","site_admin":false},"body":"Adds BlockMatrix sparsify functions for:\r\n- band matrix\r\n- upper/lower triangle (special case of band)\r\n- a collection of rectangles\r\n\r\nFor diagonal band, I switched GridPartitioner.filterBand to go from lower to upper diagonal index, rather than taking a lower and upper bandwidth. This is more general, e.g. the diagonal itself need not be in the band. Band and triangle zero out elements in partially overlapping blocks by default. Rectangles currently only supports dropping whole blocks.\r\n\r\nAlso adds `export_rectangles` for exporting rectangular regions to TSV in parallel.\r\nI use parameters `path_in` and `path_out`, and switched `BlockMatrix.export` to this convention as well from `input` and `output` to avoid using the reserved word `input`. I have not exposed export methods directly on BlockMatrix for now as it'd be very easy for users to needlessly write and read an already written BlockMatrix. I could add these in a later PR with a warning, or we can wait until we've moved to IR and can optimize read followed by export to export on the file. It'd also be good to add compression options (and float formatting options to `export` and `export_rectanges`).\r\n\r\nAlong the way I fixed NaN checking (due to Double.NaN != Double.NaN) on scalar and vector `/` sparse block ops and added NaN and Infinity checking to scalar and vector `*` sparse ops.\r\n\r\nTogether with `sparsify_row_intervals` in the first sparse matrix PR, this PR exposes all the BlockMatrix functionality needed for  big LD applications of Kate/Ran and Jacob/Masa.","created_at":"2018-05-09T00:48:51Z","updated_at":"2018-05-19T14:07:05Z","closed_at":null,"merged_at":null,"merge_commit_sha":"5f35836db39d6ed67ccb60ebccb5c862d4dc16b0","assignee":{"login":"tpoterba","id":10562794,"avatar_url":"https://avatars2.githubusercontent.com/u/10562794?v=4","gravatar_id":"","url":"https://api.github.com/users/tpoterba","html_url":"https://github.com/tpoterba","followers_url":"https://api.github.com/users/tpoterba/followers","following_url":"https://api.github.com/users/tpoterba/following{/other_user}","gists_url":"https://api.github.com/users/tpoterba/gists{/gist_id}","starred_url":"https://api.github.com/users/tpoterba/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tpoterba/subscriptions","organizations_url":"https://api.github.com/users/tpoterba/orgs","repos_url":"https://api.github.com/users/tpoterba/repos","events_url":"https://api.github.com/users/tpoterba/events{/privacy}","received_events_url":"https://api.github.com/users/tpoterba/received_events","type":"User","site_admin":false},"assignees":[{"login":"tpoterba","id":10562794,"avatar_url":"https://avatars2.githubusercontent.com/u/10562794?v=4","gravatar_id":"","url":"https://api.github.com/users/tpoterba","html_url":"https://github.com/tpoterba","followers_url":"https://api.github.com/users/tpoterba/followers","following_url":"https://api.github.com/users/tpoterba/following{/other_user}","gists_url":"https://api.github.com/users/tpoterba/gists{/gist_id}","starred_url":"https://api.github.com/users/tpoterba/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tpoterba/subscriptions","organizations_url":"https://api.github.com/users/tpoterba/orgs","repos_url":"https://api.github.com/users/tpoterba/repos","events_url":"https://api.github.com/users/tpoterba/events{/privacy}","received_events_url":"https://api.github.com/users/tpoterba/received_events","type":"User","site_admin":false}],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/hail-is/hail/pulls/3539/commits","review_comments_url":"https://api.github.com/repos/hail-is/hail/pulls/3539/comments","review_comment_url":"https://api.github.com/repos/hail-is/hail/pulls/comments{/number}","comments_url":"https://api.github.com/repos/hail-is/hail/issues/3539/comments","statuses_url":"https://api.github.com/repos/hail-is/hail/statuses/7101a6cf1c74e2a386eb04695b42759fa1fbff4b","head":{"label":"jbloom22:sparsify_band_tri_rects","ref":"sparsify_band_tri_rects","sha":"7101a6cf1c74e2a386eb04695b42759fa1fbff4b","user":{"login":"jbloom22","id":3201642,"avatar_url":"https://avatars1.githubusercontent.com/u/3201642?v=4","gravatar_id":"","url":"https://api.github.com/users/jbloom22","html_url":"https://github.com/jbloom22","followers_url":"https://api.github.com/users/jbloom22/followers","following_url":"https://api.github.com/users/jbloom22/following{/other_user}","gists_url":"https://api.github.com/users/jbloom22/gists{/gist_id}","starred_url":"https://api.github.com/users/jbloom22/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jbloom22/subscriptions","organizations_url":"https://api.github.com/users/jbloom22/orgs","repos_url":"https://api.github.com/users/jbloom22/repos","events_url":"https://api.github.com/users/jbloom22/events{/privacy}","received_events_url":"https://api.github.com/users/jbloom22/received_events","type":"User","site_admin":false},"repo":{"id":64696976,"name":"hail","full_name":"jbloom22/hail","owner":{"login":"jbloom22","id":3201642,"avatar_url":"https://avatars1.githubusercontent.com/u/3201642?v=4","gravatar_id":"","url":"https://api.github.com/users/jbloom22","html_url":"https://github.com/jbloom22","followers_url":"https://api.github.com/users/jbloom22/followers","following_url":"https://api.github.com/users/jbloom22/following{/other_user}","gists_url":"https://api.github.com/users/jbloom22/gists{/gist_id}","starred_url":"https://api.github.com/users/jbloom22/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jbloom22/subscriptions","organizations_url":"https://api.github.com/users/jbloom22/orgs","repos_url":"https://api.github.com/users/jbloom22/repos","events_url":"https://api.github.com/users/jbloom22/events{/privacy}","received_events_url":"https://api.github.com/users/jbloom22/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/jbloom22/hail","description":"Hail, a scalable framework for genomic data analysis","fork":true,"url":"https://api.github.com/repos/jbloom22/hail","forks_url":"https://api.github.com/repos/jbloom22/hail/forks","keys_url":"https://api.github.com/repos/jbloom22/hail/keys{/key_id}","collaborators_url":"https://api.github.com/repos/jbloom22/hail/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/jbloom22/hail/teams","hooks_url":"https://api.github.com/repos/jbloom22/hail/hooks","issue_events_url":"https://api.github.com/repos/jbloom22/hail/issues/events{/number}","events_url":"https://api.github.com/repos/jbloom22/hail/events","assignees_url":"https://api.github.com/repos/jbloom22/hail/assignees{/user}","branches_url":"https://api.github.com/repos/jbloom22/hail/branches{/branch}","tags_url":"https://api.github.com/repos/jbloom22/hail/tags","blobs_url":"https://api.github.com/repos/jbloom22/hail/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/jbloom22/hail/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/jbloom22/hail/git/refs{/sha}","trees_url":"https://api.github.com/repos/jbloom22/hail/git/trees{/sha}","statuses_url":"https://api.github.com/repos/jbloom22/hail/statuses/{sha}","languages_url":"https://api.github.com/repos/jbloom22/hail/languages","stargazers_url":"https://api.github.com/repos/jbloom22/hail/stargazers","contributors_url":"https://api.github.com/repos/jbloom22/hail/contributors","subscribers_url":"https://api.github.com/repos/jbloom22/hail/subscribers","subscription_url":"https://api.github.com/repos/jbloom22/hail/subscription","commits_url":"https://api.github.com/repos/jbloom22/hail/commits{/sha}","git_commits_url":"https://api.github.com/repos/jbloom22/hail/git/commits{/sha}","comments_url":"https://api.github.com/repos/jbloom22/hail/comments{/number}","issue_comment_url":"https://api.github.com/repos/jbloom22/hail/issues/comments{/number}","contents_url":"https://api.github.com/repos/jbloom22/hail/contents/{+path}","compare_url":"https://api.github.com/repos/jbloom22/hail/compare/{base}...{head}","merges_url":"https://api.github.com/repos/jbloom22/hail/merges","archive_url":"https://api.github.com/repos/jbloom22/hail/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/jbloom22/hail/downloads","issues_url":"https://api.github.com/repos/jbloom22/hail/issues{/number}","pulls_url":"https://api.github.com/repos/jbloom22/hail/pulls{/number}","milestones_url":"https://api.github.com/repos/jbloom22/hail/milestones{/number}","notifications_url":"https://api.github.com/repos/jbloom22/hail/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/jbloom22/hail/labels{/name}","releases_url":"https://api.github.com/repos/jbloom22/hail/releases{/id}","deployments_url":"https://api.github.com/repos/jbloom22/hail/deployments","created_at":"2016-08-01T19:51:47Z","updated_at":"2017-06-24T14:31:31Z","pushed_at":"2018-05-19T13:42:19Z","git_url":"git://github.com/jbloom22/hail.git","ssh_url":"git@github.com:jbloom22/hail.git","clone_url":"https://github.com/jbloom22/hail.git","svn_url":"https://github.com/jbloom22/hail","homepage":"","size":38928,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit"},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"hail-is:master","ref":"master","sha":"d4e8b05a999cf80f3cc7519a613e929b2f4ce465","user":{"login":"hail-is","id":20861792,"avatar_url":"https://avatars1.githubusercontent.com/u/20861792?v=4","gravatar_id":"","url":"https://api.github.com/users/hail-is","html_url":"https://github.com/hail-is","followers_url":"https://api.github.com/users/hail-is/followers","following_url":"https://api.github.com/users/hail-is/following{/other_user}","gists_url":"https://api.github.com/users/hail-is/gists{/gist_id}","starred_url":"https://api.github.com/users/hail-is/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hail-is/subscriptions","organizations_url":"https://api.github.com/users/hail-is/orgs","repos_url":"https://api.github.com/users/hail-is/repos","events_url":"https://api.github.com/users/hail-is/events{/privacy}","received_events_url":"https://api.github.com/users/hail-is/received_events","type":"Organization","site_admin":false},"repo":{"id":45069467,"name":"hail","full_name":"hail-is/hail","owner":{"login":"hail-is","id":20861792,"avatar_url":"https://avatars1.githubusercontent.com/u/20861792?v=4","gravatar_id":"","url":"https://api.github.com/users/hail-is","html_url":"https://github.com/hail-is","followers_url":"https://api.github.com/users/hail-is/followers","following_url":"https://api.github.com/users/hail-is/following{/other_user}","gists_url":"https://api.github.com/users/hail-is/gists{/gist_id}","starred_url":"https://api.github.com/users/hail-is/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hail-is/subscriptions","organizations_url":"https://api.github.com/users/hail-is/orgs","repos_url":"https://api.github.com/users/hail-is/repos","events_url":"https://api.github.com/users/hail-is/events{/privacy}","received_events_url":"https://api.github.com/users/hail-is/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/hail-is/hail","description":"Scalable genomic data analysis.","fork":false,"url":"https://api.github.com/repos/hail-is/hail","forks_url":"https://api.github.com/repos/hail-is/hail/forks","keys_url":"https://api.github.com/repos/hail-is/hail/keys{/key_id}","collaborators_url":"https://api.github.com/repos/hail-is/hail/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/hail-is/hail/teams","hooks_url":"https://api.github.com/repos/hail-is/hail/hooks","issue_events_url":"https://api.github.com/repos/hail-is/hail/issues/events{/number}","events_url":"https://api.github.com/repos/hail-is/hail/events","assignees_url":"https://api.github.com/repos/hail-is/hail/assignees{/user}","branches_url":"https://api.github.com/repos/hail-is/hail/branches{/branch}","tags_url":"https://api.github.com/repos/hail-is/hail/tags","blobs_url":"https://api.github.com/repos/hail-is/hail/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/hail-is/hail/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/hail-is/hail/git/refs{/sha}","trees_url":"https://api.github.com/repos/hail-is/hail/git/trees{/sha}","statuses_url":"https://api.github.com/repos/hail-is/hail/statuses/{sha}","languages_url":"https://api.github.com/repos/hail-is/hail/languages","stargazers_url":"https://api.github.com/repos/hail-is/hail/stargazers","contributors_url":"https://api.github.com/repos/hail-is/hail/contributors","subscribers_url":"https://api.github.com/repos/hail-is/hail/subscribers","subscription_url":"https://api.github.com/repos/hail-is/hail/subscription","commits_url":"https://api.github.com/repos/hail-is/hail/commits{/sha}","git_commits_url":"https://api.github.com/repos/hail-is/hail/git/commits{/sha}","comments_url":"https://api.github.com/repos/hail-is/hail/comments{/number}","issue_comment_url":"https://api.github.com/repos/hail-is/hail/issues/comments{/number}","contents_url":"https://api.github.com/repos/hail-is/hail/contents/{+path}","compare_url":"https://api.github.com/repos/hail-is/hail/compare/{base}...{head}","merges_url":"https://api.github.com/repos/hail-is/hail/merges","archive_url":"https://api.github.com/repos/hail-is/hail/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/hail-is/hail/downloads","issues_url":"https://api.github.com/repos/hail-is/hail/issues{/number}","pulls_url":"https://api.github.com/repos/hail-is/hail/pulls{/number}","milestones_url":"https://api.github.com/repos/hail-is/hail/milestones{/number}","notifications_url":"https://api.github.com/repos/hail-is/hail/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/hail-is/hail/labels{/name}","releases_url":"https://api.github.com/repos/hail-is/hail/releases{/id}","deployments_url":"https://api.github.com/repos/hail-is/hail/deployments","created_at":"2015-10-27T20:55:42Z","updated_at":"2018-05-18T20:46:07Z","pushed_at":"2018-05-19T13:42:21Z","git_url":"git://github.com/hail-is/hail.git","ssh_url":"git@github.com:hail-is/hail.git","clone_url":"https://github.com/hail-is/hail.git","svn_url":"https://github.com/hail-is/hail","homepage":"https://hail.is","size":35685,"stargazers_count":326,"watchers_count":326,"language":"Scala","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":98,"mirror_url":null,"archived":false,"open_issues_count":140,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit"},"forks":98,"open_issues":140,"watchers":326,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/hail-is/hail/pulls/3539"},"html":{"href":"https://github.com/hail-is/hail/pull/3539"},"issue":{"href":"https://api.github.com/repos/hail-is/hail/issues/3539"},"comments":{"href":"https://api.github.com/repos/hail-is/hail/issues/3539/comments"},"review_comments":{"href":"https://api.github.com/repos/hail-is/hail/pulls/3539/comments"},"review_comment":{"href":"https://api.github.com/repos/hail-is/hail/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/hail-is/hail/pulls/3539/commits"},"statuses":{"href":"https://api.github.com/repos/hail-is/hail/statuses/7101a6cf1c74e2a386eb04695b42759fa1fbff4b"}},"author_association":"COLLABORATOR"}}
{"actor":{"display_login":"jbloom22"}}