{"action":"created","comment":{"url":"https://api.github.com/repos/hail-is/hail/pulls/comments/199594977","pull_request_review_id":133745466,"id":199594977,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDE5OTU5NDk3Nw==","diff_hunk":"@@ -1360,26 +1362,34 @@ def test_ld_prune(self):\n                     (filtered_ds['GT'].n_alt_alleles() - filtered_ds['mean'])\n                     * filtered_ds['sd_reciprocal'] * (1 / hl.sqrt(n_samples)), 0))\n \n-        block_matrix = BlockMatrix.from_entry_expr(normalized_mean_imputed_genotype_expr)\n-        entries = ((block_matrix @ block_matrix.T) ** 2).entries()\n+        std_bm = BlockMatrix.from_entry_expr(normalized_mean_imputed_genotype_expr)\n+\n+        self.assertEqual(std_bm.n_rows, 14)\n+\n+        entries = ((std_bm @ std_bm.T) ** 2).entries()\n \n         index_table = filtered_ds.add_row_index().rows().key_by('row_idx').select('locus')\n         entries = entries.annotate(locus_i=index_table[entries.i].locus, locus_j=index_table[entries.j].locus)\n \n-        contig_filter = entries.locus_i.contig == entries.locus_j.contig\n-        window_filter = (hl.abs(entries.locus_i.position - entries.locus_j.position)) <= 1000000\n-        identical_filter = entries.i != entries.j\n+        bad_pair = (\n+            (entries.entry >= r2_threshold) &\n+            (entries.locus_i.contig == entries.locus_j.contig) &\n+            (hl.abs(entries.locus_j.position - entries.locus_i.position) <= window_size) &\n+            (entries.i != entries.j)\n+        )\n \n-        self.assertEqual(entries.filter(\n-            (entries['entry'] >= 0.2) & (contig_filter) & (window_filter) & (identical_filter)).count(), 0)\n+        self.assertEqual(entries.filter(bad_pair).count(), 0)\n \n     def test_ld_prune_inputs(self):\n-        ds = hl.split_multi_hts(hl.import_vcf(resource('sample.vcf')))\n-        self.assertRaises(ValueError, lambda: hl.ld_prune(ds.GT, r2=0.2, bp_window_size=1000000, memory_per_core=0))\n+        ds = hl.balding_nichols_model(n_populations=1, n_samples=1, n_variants=1)\n+        self.assertRaises(ValueError, lambda: hl.ld_prune(ds.GT, memory_per_core=0))\n+        self.assertRaises(ValueError, lambda: hl.ld_prune(ds.GT, bp_window_size=-1))\n+        self.assertRaises(ValueError, lambda: hl.ld_prune(ds.GT, r2=-1.0))\n+        self.assertRaises(ValueError, lambda: hl.ld_prune(ds.GT, r2=2.0))\n \n     def test_ld_prune_no_prune(self):\n-        ds = hl.balding_nichols_model(n_populations=1, n_samples=10, n_variants=100)\n-        pruned_table = hl.ld_prune(ds.GT, r2=0.1, bp_window_size=0)\n+        ds = hl.balding_nichols_model(n_populations=1, n_samples=10, n_variants=10)\n+        pruned_table = hl.ld_prune(ds.GT, r2=0.0, bp_window_size=0)","path":"python/hail/tests/test_methods.py","position":58,"original_position":58,"commit_id":"214e66876ce437ed9ba3a39870d85b5699863514","original_commit_id":"214e66876ce437ed9ba3a39870d85b5699863514","user":{"login":"jbloom22","id":3201642,"node_id":"MDQ6VXNlcjMyMDE2NDI=","avatar_url":"https://avatars1.githubusercontent.com/u/3201642?v=4","gravatar_id":"","url":"https://api.github.com/users/jbloom22","html_url":"https://github.com/jbloom22","followers_url":"https://api.github.com/users/jbloom22/followers","following_url":"https://api.github.com/users/jbloom22/following{/other_user}","gists_url":"https://api.github.com/users/jbloom22/gists{/gist_id}","starred_url":"https://api.github.com/users/jbloom22/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jbloom22/subscriptions","organizations_url":"https://api.github.com/users/jbloom22/orgs","repos_url":"https://api.github.com/users/jbloom22/repos","events_url":"https://api.github.com/users/jbloom22/events{/privacy}","received_events_url":"https://api.github.com/users/jbloom22/received_events","type":"User","site_admin":false},"body":"`r2=1.0` could still resulting in pruning if there were perfectly correlated variants.\r\n\r\nBy setting `r2=0.0` I'm being more explicit that no pruning results with `bp_window_size=0` on a matrix table with strictly increasing variant position.","created_at":"2018-07-02T19:14:30Z","updated_at":"2018-07-02T19:14:31Z","html_url":"https://github.com/hail-is/hail/pull/3873#discussion_r199594977","pull_request_url":"https://api.github.com/repos/hail-is/hail/pulls/3873","author_association":"COLLABORATOR","_links":{"self":{"href":"https://api.github.com/repos/hail-is/hail/pulls/comments/199594977"},"html":{"href":"https://github.com/hail-is/hail/pull/3873#discussion_r199594977"},"pull_request":{"href":"https://api.github.com/repos/hail-is/hail/pulls/3873"}},"in_reply_to_id":199564315},"pull_request":{"url":"https://api.github.com/repos/hail-is/hail/pulls/3873","id":198392208,"node_id":"MDExOlB1bGxSZXF1ZXN0MTk4MzkyMjA4","html_url":"https://github.com/hail-is/hail/pull/3873","diff_url":"https://github.com/hail-is/hail/pull/3873.diff","patch_url":"https://github.com/hail-is/hail/pull/3873.patch","issue_url":"https://api.github.com/repos/hail-is/hail/issues/3873","number":3873,"state":"open","locked":false,"title":"add array_windows / locus_windows, refactor ld_prune, remove filteredEntriesTable and UpperIndexBounds (v2)","user":{"login":"jbloom22","id":3201642,"node_id":"MDQ6VXNlcjMyMDE2NDI=","avatar_url":"https://avatars1.githubusercontent.com/u/3201642?v=4","gravatar_id":"","url":"https://api.github.com/users/jbloom22","html_url":"https://github.com/jbloom22","followers_url":"https://api.github.com/users/jbloom22/followers","following_url":"https://api.github.com/users/jbloom22/following{/other_user}","gists_url":"https://api.github.com/users/jbloom22/gists{/gist_id}","starred_url":"https://api.github.com/users/jbloom22/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jbloom22/subscriptions","organizations_url":"https://api.github.com/users/jbloom22/orgs","repos_url":"https://api.github.com/users/jbloom22/repos","events_url":"https://api.github.com/users/jbloom22/events{/privacy}","received_events_url":"https://api.github.com/users/jbloom22/received_events","type":"User","site_admin":false},"body":"This is a rebased update to the closed PR #3715. Updates:\r\n\r\n- `locus_table` is now `locus_expr`, an expression on a table or matrix table. This is more flexible, and avoids unnatural requirements on key (rather than order).\r\n\r\n- uses `global_position` and requires/checks ascending order, rather than reordering within the method. Re-ordering, if non-trivial, would silently invalidate the results with respect to the row-order of the source. This also avoids a potentially unnecessary shuffle, and addresses the issue that contig order may not be alphabetical in the reference.\r\n\r\n- keeps the size of data collected close to the minimal data necessary (in particular, no collection of contigs). When `coord_expr` is not set, its an array of int. When `coord_expr` is set, its an array of (int, float). A tuple of arrays would be better, but directly collecting two arrays in order would require two actions with the current infrastructure since agg.collect() does not preserve order.","created_at":"2018-06-29T18:31:21Z","updated_at":"2018-07-02T19:14:31Z","closed_at":null,"merged_at":null,"merge_commit_sha":"5aae271374d6c697ffa55e1aad96997bffbb0946","assignee":{"login":"jigold","id":1693348,"node_id":"MDQ6VXNlcjE2OTMzNDg=","avatar_url":"https://avatars2.githubusercontent.com/u/1693348?v=4","gravatar_id":"","url":"https://api.github.com/users/jigold","html_url":"https://github.com/jigold","followers_url":"https://api.github.com/users/jigold/followers","following_url":"https://api.github.com/users/jigold/following{/other_user}","gists_url":"https://api.github.com/users/jigold/gists{/gist_id}","starred_url":"https://api.github.com/users/jigold/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jigold/subscriptions","organizations_url":"https://api.github.com/users/jigold/orgs","repos_url":"https://api.github.com/users/jigold/repos","events_url":"https://api.github.com/users/jigold/events{/privacy}","received_events_url":"https://api.github.com/users/jigold/received_events","type":"User","site_admin":false},"assignees":[{"login":"jigold","id":1693348,"node_id":"MDQ6VXNlcjE2OTMzNDg=","avatar_url":"https://avatars2.githubusercontent.com/u/1693348?v=4","gravatar_id":"","url":"https://api.github.com/users/jigold","html_url":"https://github.com/jigold","followers_url":"https://api.github.com/users/jigold/followers","following_url":"https://api.github.com/users/jigold/following{/other_user}","gists_url":"https://api.github.com/users/jigold/gists{/gist_id}","starred_url":"https://api.github.com/users/jigold/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jigold/subscriptions","organizations_url":"https://api.github.com/users/jigold/orgs","repos_url":"https://api.github.com/users/jigold/repos","events_url":"https://api.github.com/users/jigold/events{/privacy}","received_events_url":"https://api.github.com/users/jigold/received_events","type":"User","site_admin":false}],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/hail-is/hail/pulls/3873/commits","review_comments_url":"https://api.github.com/repos/hail-is/hail/pulls/3873/comments","review_comment_url":"https://api.github.com/repos/hail-is/hail/pulls/comments{/number}","comments_url":"https://api.github.com/repos/hail-is/hail/issues/3873/comments","statuses_url":"https://api.github.com/repos/hail-is/hail/statuses/214e66876ce437ed9ba3a39870d85b5699863514","head":{"label":"jbloom22:ld_prune_refactor_expr","ref":"ld_prune_refactor_expr","sha":"214e66876ce437ed9ba3a39870d85b5699863514","user":{"login":"jbloom22","id":3201642,"node_id":"MDQ6VXNlcjMyMDE2NDI=","avatar_url":"https://avatars1.githubusercontent.com/u/3201642?v=4","gravatar_id":"","url":"https://api.github.com/users/jbloom22","html_url":"https://github.com/jbloom22","followers_url":"https://api.github.com/users/jbloom22/followers","following_url":"https://api.github.com/users/jbloom22/following{/other_user}","gists_url":"https://api.github.com/users/jbloom22/gists{/gist_id}","starred_url":"https://api.github.com/users/jbloom22/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jbloom22/subscriptions","organizations_url":"https://api.github.com/users/jbloom22/orgs","repos_url":"https://api.github.com/users/jbloom22/repos","events_url":"https://api.github.com/users/jbloom22/events{/privacy}","received_events_url":"https://api.github.com/users/jbloom22/received_events","type":"User","site_admin":false},"repo":{"id":64696976,"node_id":"MDEwOlJlcG9zaXRvcnk2NDY5Njk3Ng==","name":"hail","full_name":"jbloom22/hail","owner":{"login":"jbloom22","id":3201642,"node_id":"MDQ6VXNlcjMyMDE2NDI=","avatar_url":"https://avatars1.githubusercontent.com/u/3201642?v=4","gravatar_id":"","url":"https://api.github.com/users/jbloom22","html_url":"https://github.com/jbloom22","followers_url":"https://api.github.com/users/jbloom22/followers","following_url":"https://api.github.com/users/jbloom22/following{/other_user}","gists_url":"https://api.github.com/users/jbloom22/gists{/gist_id}","starred_url":"https://api.github.com/users/jbloom22/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jbloom22/subscriptions","organizations_url":"https://api.github.com/users/jbloom22/orgs","repos_url":"https://api.github.com/users/jbloom22/repos","events_url":"https://api.github.com/users/jbloom22/events{/privacy}","received_events_url":"https://api.github.com/users/jbloom22/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/jbloom22/hail","description":"Hail, a scalable framework for genomic data analysis","fork":true,"url":"https://api.github.com/repos/jbloom22/hail","forks_url":"https://api.github.com/repos/jbloom22/hail/forks","keys_url":"https://api.github.com/repos/jbloom22/hail/keys{/key_id}","collaborators_url":"https://api.github.com/repos/jbloom22/hail/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/jbloom22/hail/teams","hooks_url":"https://api.github.com/repos/jbloom22/hail/hooks","issue_events_url":"https://api.github.com/repos/jbloom22/hail/issues/events{/number}","events_url":"https://api.github.com/repos/jbloom22/hail/events","assignees_url":"https://api.github.com/repos/jbloom22/hail/assignees{/user}","branches_url":"https://api.github.com/repos/jbloom22/hail/branches{/branch}","tags_url":"https://api.github.com/repos/jbloom22/hail/tags","blobs_url":"https://api.github.com/repos/jbloom22/hail/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/jbloom22/hail/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/jbloom22/hail/git/refs{/sha}","trees_url":"https://api.github.com/repos/jbloom22/hail/git/trees{/sha}","statuses_url":"https://api.github.com/repos/jbloom22/hail/statuses/{sha}","languages_url":"https://api.github.com/repos/jbloom22/hail/languages","stargazers_url":"https://api.github.com/repos/jbloom22/hail/stargazers","contributors_url":"https://api.github.com/repos/jbloom22/hail/contributors","subscribers_url":"https://api.github.com/repos/jbloom22/hail/subscribers","subscription_url":"https://api.github.com/repos/jbloom22/hail/subscription","commits_url":"https://api.github.com/repos/jbloom22/hail/commits{/sha}","git_commits_url":"https://api.github.com/repos/jbloom22/hail/git/commits{/sha}","comments_url":"https://api.github.com/repos/jbloom22/hail/comments{/number}","issue_comment_url":"https://api.github.com/repos/jbloom22/hail/issues/comments{/number}","contents_url":"https://api.github.com/repos/jbloom22/hail/contents/{+path}","compare_url":"https://api.github.com/repos/jbloom22/hail/compare/{base}...{head}","merges_url":"https://api.github.com/repos/jbloom22/hail/merges","archive_url":"https://api.github.com/repos/jbloom22/hail/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/jbloom22/hail/downloads","issues_url":"https://api.github.com/repos/jbloom22/hail/issues{/number}","pulls_url":"https://api.github.com/repos/jbloom22/hail/pulls{/number}","milestones_url":"https://api.github.com/repos/jbloom22/hail/milestones{/number}","notifications_url":"https://api.github.com/repos/jbloom22/hail/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/jbloom22/hail/labels{/name}","releases_url":"https://api.github.com/repos/jbloom22/hail/releases{/id}","deployments_url":"https://api.github.com/repos/jbloom22/hail/deployments","created_at":"2016-08-01T19:51:47Z","updated_at":"2017-06-24T14:31:31Z","pushed_at":"2018-06-29T23:47:31Z","git_url":"git://github.com/jbloom22/hail.git","ssh_url":"git@github.com:jbloom22/hail.git","clone_url":"https://github.com/jbloom22/hail.git","svn_url":"https://github.com/jbloom22/hail","homepage":"","size":42184,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit","node_id":"MDc6TGljZW5zZTEz"},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"hail-is:master","ref":"master","sha":"957d94e8bd1804eec66ea3b493476afb3bf2c17e","user":{"login":"hail-is","id":20861792,"node_id":"MDEyOk9yZ2FuaXphdGlvbjIwODYxNzky","avatar_url":"https://avatars1.githubusercontent.com/u/20861792?v=4","gravatar_id":"","url":"https://api.github.com/users/hail-is","html_url":"https://github.com/hail-is","followers_url":"https://api.github.com/users/hail-is/followers","following_url":"https://api.github.com/users/hail-is/following{/other_user}","gists_url":"https://api.github.com/users/hail-is/gists{/gist_id}","starred_url":"https://api.github.com/users/hail-is/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hail-is/subscriptions","organizations_url":"https://api.github.com/users/hail-is/orgs","repos_url":"https://api.github.com/users/hail-is/repos","events_url":"https://api.github.com/users/hail-is/events{/privacy}","received_events_url":"https://api.github.com/users/hail-is/received_events","type":"Organization","site_admin":false},"repo":{"id":45069467,"node_id":"MDEwOlJlcG9zaXRvcnk0NTA2OTQ2Nw==","name":"hail","full_name":"hail-is/hail","owner":{"login":"hail-is","id":20861792,"node_id":"MDEyOk9yZ2FuaXphdGlvbjIwODYxNzky","avatar_url":"https://avatars1.githubusercontent.com/u/20861792?v=4","gravatar_id":"","url":"https://api.github.com/users/hail-is","html_url":"https://github.com/hail-is","followers_url":"https://api.github.com/users/hail-is/followers","following_url":"https://api.github.com/users/hail-is/following{/other_user}","gists_url":"https://api.github.com/users/hail-is/gists{/gist_id}","starred_url":"https://api.github.com/users/hail-is/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hail-is/subscriptions","organizations_url":"https://api.github.com/users/hail-is/orgs","repos_url":"https://api.github.com/users/hail-is/repos","events_url":"https://api.github.com/users/hail-is/events{/privacy}","received_events_url":"https://api.github.com/users/hail-is/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/hail-is/hail","description":"Scalable genomic data analysis.","fork":false,"url":"https://api.github.com/repos/hail-is/hail","forks_url":"https://api.github.com/repos/hail-is/hail/forks","keys_url":"https://api.github.com/repos/hail-is/hail/keys{/key_id}","collaborators_url":"https://api.github.com/repos/hail-is/hail/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/hail-is/hail/teams","hooks_url":"https://api.github.com/repos/hail-is/hail/hooks","issue_events_url":"https://api.github.com/repos/hail-is/hail/issues/events{/number}","events_url":"https://api.github.com/repos/hail-is/hail/events","assignees_url":"https://api.github.com/repos/hail-is/hail/assignees{/user}","branches_url":"https://api.github.com/repos/hail-is/hail/branches{/branch}","tags_url":"https://api.github.com/repos/hail-is/hail/tags","blobs_url":"https://api.github.com/repos/hail-is/hail/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/hail-is/hail/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/hail-is/hail/git/refs{/sha}","trees_url":"https://api.github.com/repos/hail-is/hail/git/trees{/sha}","statuses_url":"https://api.github.com/repos/hail-is/hail/statuses/{sha}","languages_url":"https://api.github.com/repos/hail-is/hail/languages","stargazers_url":"https://api.github.com/repos/hail-is/hail/stargazers","contributors_url":"https://api.github.com/repos/hail-is/hail/contributors","subscribers_url":"https://api.github.com/repos/hail-is/hail/subscribers","subscription_url":"https://api.github.com/repos/hail-is/hail/subscription","commits_url":"https://api.github.com/repos/hail-is/hail/commits{/sha}","git_commits_url":"https://api.github.com/repos/hail-is/hail/git/commits{/sha}","comments_url":"https://api.github.com/repos/hail-is/hail/comments{/number}","issue_comment_url":"https://api.github.com/repos/hail-is/hail/issues/comments{/number}","contents_url":"https://api.github.com/repos/hail-is/hail/contents/{+path}","compare_url":"https://api.github.com/repos/hail-is/hail/compare/{base}...{head}","merges_url":"https://api.github.com/repos/hail-is/hail/merges","archive_url":"https://api.github.com/repos/hail-is/hail/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/hail-is/hail/downloads","issues_url":"https://api.github.com/repos/hail-is/hail/issues{/number}","pulls_url":"https://api.github.com/repos/hail-is/hail/pulls{/number}","milestones_url":"https://api.github.com/repos/hail-is/hail/milestones{/number}","notifications_url":"https://api.github.com/repos/hail-is/hail/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/hail-is/hail/labels{/name}","releases_url":"https://api.github.com/repos/hail-is/hail/releases{/id}","deployments_url":"https://api.github.com/repos/hail-is/hail/deployments","created_at":"2015-10-27T20:55:42Z","updated_at":"2018-07-02T19:10:58Z","pushed_at":"2018-07-02T19:10:53Z","git_url":"git://github.com/hail-is/hail.git","ssh_url":"git@github.com:hail-is/hail.git","clone_url":"https://github.com/hail-is/hail.git","svn_url":"https://github.com/hail-is/hail","homepage":"https://hail.is","size":38615,"stargazers_count":340,"watchers_count":340,"language":"Scala","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":101,"mirror_url":null,"archived":false,"open_issues_count":146,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit","node_id":"MDc6TGljZW5zZTEz"},"forks":101,"open_issues":146,"watchers":340,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/hail-is/hail/pulls/3873"},"html":{"href":"https://github.com/hail-is/hail/pull/3873"},"issue":{"href":"https://api.github.com/repos/hail-is/hail/issues/3873"},"comments":{"href":"https://api.github.com/repos/hail-is/hail/issues/3873/comments"},"review_comments":{"href":"https://api.github.com/repos/hail-is/hail/pulls/3873/comments"},"review_comment":{"href":"https://api.github.com/repos/hail-is/hail/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/hail-is/hail/pulls/3873/commits"},"statuses":{"href":"https://api.github.com/repos/hail-is/hail/statuses/214e66876ce437ed9ba3a39870d85b5699863514"}},"author_association":"COLLABORATOR"}}
{"actor":{"display_login":"jbloom22"}}