{"action":"created","comment":{"url":"https://api.github.com/repos/hail-is/hail/pulls/comments/171878615","pull_request_review_id":100617607,"id":171878615,"diff_hunk":"@@ -37,6 +37,88 @@ def convert_struct_to_dict(x):\n     else:\n         return x\n \n+def create_all_values_datasets():\n+    all_values = hl.struct(\n+        f32=hl.float32(3.14),\n+        i64=hl.int64(-9),\n+        m=hl.null(hl.tfloat64),\n+        astruct=hl.struct(a=5, b=5.5),\n+        aset=hl.set(['foo', 'bar', 'baz']),\n+        d=hl.dict({hl.array(['a', 'b']): 0.5, hl.array(['x', 'y', 'z']): 0.3}),\n+        i=hl.interval(\n+            hl.locus('1', 999),\n+            hl.locus('1', 1001)),\n+        c=hl.call(0, 1),\n+        t=hl.tuple([hl.call(1, 2, phased=True), 'foo']))\n+\n+    def prefix(s, p):\n+        return hl.struct(**{p + k: s[k] for k in s})\n+\n+    all_values_table = (hl.utils.range_table(5, n_partitions=3)\n+                        .annotate_globals(**prefix(all_values, 'global_'))\n+                        .annotate(**all_values)\n+                        .cache())\n+\n+    all_values_matrix_table = (hl.utils.range_matrix_table(3, 2, n_partitions=2)\n+                               .annotate_globals(**prefix(all_values, 'global_'))\n+                               .annotate_rows(**prefix(all_values, 'row_'))\n+                               .annotate_cols(**prefix(all_values, 'col_'))\n+                               .annotate_entries(**prefix(all_values, 'entry_'))\n+                               .cache())\n+    return all_values_table, all_values_matrix_table\n+\n+def create_backward_compatability_test_files():\n+    import os\n+\n+    all_values_table, all_values_matrix_table = create_all_values_datasets()\n+\n+    file_version = Env.hail().variant.FileFormat.version().toString()\n+    supported_codecs = scala_object(Env.hail().io, 'CodecSpec').supportedCodecSpecs()\n+\n+    table_dir = 'backward_compatability/{}/table'.format(file_version)\n+    if not os.path.exists(table_dir):\n+        os.makedirs(table_dir)\n+\n+    matrix_table_dir = 'backward_compatability/{}/matrix_table'.format(file_version)\n+    if not os.path.exists(matrix_table_dir):\n+        os.makedirs(matrix_table_dir)\n+\n+    i = 0\n+    for codec in supported_codecs:\n+        all_values_table.write('{}/{}.ht'.format(table_dir, i), overwrite=True, _codec_spec=codec.toString())\n+        all_values_matrix_table.write('{}/{}.hmt'.format(matrix_table_dir, i), overwrite=True, _codec_spec=codec.toString())\n+        i = i + 1\n+\n+class FileFormatTests(unittest.TestCase):\n+    def test_backward_compatability(self):\n+        import os\n+        \n+        all_values_table, all_values_matrix_table = create_all_values_datasets()\n+\n+        table_dir = test_file('backward_compatability/1.0.0/table')\n+        matrix_table_dir = test_file('backward_compatability/1.0.0/matrix_table')\n+\n+        n = 0\n+        i = 0\n+        f = os.path.join(table_dir, '{}.ht'.format(i))\n+        while os.path.exists(f):\n+            ds = hl.read_table(f)\n+            self.assertTrue(ds._same(all_values_table))\n+            i = i + 1","path":"python/hail/tests.py","position":80,"original_position":80,"commit_id":"6ee7a7158bf72488957ba0db6c9e4640b4567185","original_commit_id":"6ee7a7158bf72488957ba0db6c9e4640b4567185","user":{"login":"jigold","id":1693348,"avatar_url":"https://avatars2.githubusercontent.com/u/1693348?v=4","gravatar_id":"","url":"https://api.github.com/users/jigold","html_url":"https://github.com/jigold","followers_url":"https://api.github.com/users/jigold/followers","following_url":"https://api.github.com/users/jigold/following{/other_user}","gists_url":"https://api.github.com/users/jigold/gists{/gist_id}","starred_url":"https://api.github.com/users/jigold/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jigold/subscriptions","organizations_url":"https://api.github.com/users/jigold/orgs","repos_url":"https://api.github.com/users/jigold/repos","events_url":"https://api.github.com/users/jigold/events{/privacy}","received_events_url":"https://api.github.com/users/jigold/received_events","type":"User","site_admin":false},"body":"+= here and below","created_at":"2018-03-02T15:34:59Z","updated_at":"2018-03-02T15:41:42Z","html_url":"https://github.com/hail-is/hail/pull/3043#discussion_r171878615","pull_request_url":"https://api.github.com/repos/hail-is/hail/pulls/3043","author_association":"COLLABORATOR","_links":{"self":{"href":"https://api.github.com/repos/hail-is/hail/pulls/comments/171878615"},"html":{"href":"https://github.com/hail-is/hail/pull/3043#discussion_r171878615"},"pull_request":{"href":"https://api.github.com/repos/hail-is/hail/pulls/3043"}}},"pull_request":{"url":"https://api.github.com/repos/hail-is/hail/pulls/3043","id":172322197,"html_url":"https://github.com/hail-is/hail/pull/3043","diff_url":"https://github.com/hail-is/hail/pull/3043.diff","patch_url":"https://github.com/hail-is/hail/pull/3043.patch","issue_url":"https://api.github.com/repos/hail-is/hail/issues/3043","number":3043,"state":"open","locked":false,"title":"added file format backward compatability tests","user":{"login":"cseed","id":1244990,"avatar_url":"https://avatars3.githubusercontent.com/u/1244990?v=4","gravatar_id":"","url":"https://api.github.com/users/cseed","html_url":"https://github.com/cseed","followers_url":"https://api.github.com/users/cseed/followers","following_url":"https://api.github.com/users/cseed/following{/other_user}","gists_url":"https://api.github.com/users/cseed/gists{/gist_id}","starred_url":"https://api.github.com/users/cseed/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cseed/subscriptions","organizations_url":"https://api.github.com/users/cseed/orgs","repos_url":"https://api.github.com/users/cseed/repos","events_url":"https://api.github.com/users/cseed/events{/privacy}","received_events_url":"https://api.github.com/users/cseed/received_events","type":"User","site_admin":false},"body":"This adds about 800KB total to the repo.\r\n\r\nI added a notion of supportedCodec, which includes all the codecs except DirectCodec (which currently depends on the in memory representation of region values which we don't want to promise).\r\n\r\nFYI @tpoterba I also added a micro optimization: if you get a field of a struct declaration, just grab out the AST for the field.  This fixed some massive expression size blowup in create_all_values_datasets.\r\n","created_at":"2018-03-01T18:53:05Z","updated_at":"2018-03-02T15:41:42Z","closed_at":null,"merged_at":null,"merge_commit_sha":"cb85eaf269c5ffda60927ae8b57442afd574e9cf","assignee":{"login":"jigold","id":1693348,"avatar_url":"https://avatars2.githubusercontent.com/u/1693348?v=4","gravatar_id":"","url":"https://api.github.com/users/jigold","html_url":"https://github.com/jigold","followers_url":"https://api.github.com/users/jigold/followers","following_url":"https://api.github.com/users/jigold/following{/other_user}","gists_url":"https://api.github.com/users/jigold/gists{/gist_id}","starred_url":"https://api.github.com/users/jigold/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jigold/subscriptions","organizations_url":"https://api.github.com/users/jigold/orgs","repos_url":"https://api.github.com/users/jigold/repos","events_url":"https://api.github.com/users/jigold/events{/privacy}","received_events_url":"https://api.github.com/users/jigold/received_events","type":"User","site_admin":false},"assignees":[{"login":"jigold","id":1693348,"avatar_url":"https://avatars2.githubusercontent.com/u/1693348?v=4","gravatar_id":"","url":"https://api.github.com/users/jigold","html_url":"https://github.com/jigold","followers_url":"https://api.github.com/users/jigold/followers","following_url":"https://api.github.com/users/jigold/following{/other_user}","gists_url":"https://api.github.com/users/jigold/gists{/gist_id}","starred_url":"https://api.github.com/users/jigold/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jigold/subscriptions","organizations_url":"https://api.github.com/users/jigold/orgs","repos_url":"https://api.github.com/users/jigold/repos","events_url":"https://api.github.com/users/jigold/events{/privacy}","received_events_url":"https://api.github.com/users/jigold/received_events","type":"User","site_admin":false}],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/hail-is/hail/pulls/3043/commits","review_comments_url":"https://api.github.com/repos/hail-is/hail/pulls/3043/comments","review_comment_url":"https://api.github.com/repos/hail-is/hail/pulls/comments{/number}","comments_url":"https://api.github.com/repos/hail-is/hail/issues/3043/comments","statuses_url":"https://api.github.com/repos/hail-is/hail/statuses/6ee7a7158bf72488957ba0db6c9e4640b4567185","head":{"label":"cseed:testfilebackwardcompat","ref":"testfilebackwardcompat","sha":"6ee7a7158bf72488957ba0db6c9e4640b4567185","user":{"login":"cseed","id":1244990,"avatar_url":"https://avatars3.githubusercontent.com/u/1244990?v=4","gravatar_id":"","url":"https://api.github.com/users/cseed","html_url":"https://github.com/cseed","followers_url":"https://api.github.com/users/cseed/followers","following_url":"https://api.github.com/users/cseed/following{/other_user}","gists_url":"https://api.github.com/users/cseed/gists{/gist_id}","starred_url":"https://api.github.com/users/cseed/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cseed/subscriptions","organizations_url":"https://api.github.com/users/cseed/orgs","repos_url":"https://api.github.com/users/cseed/repos","events_url":"https://api.github.com/users/cseed/events{/privacy}","received_events_url":"https://api.github.com/users/cseed/received_events","type":"User","site_admin":false},"repo":{"id":64697338,"name":"hail","full_name":"cseed/hail","owner":{"login":"cseed","id":1244990,"avatar_url":"https://avatars3.githubusercontent.com/u/1244990?v=4","gravatar_id":"","url":"https://api.github.com/users/cseed","html_url":"https://github.com/cseed","followers_url":"https://api.github.com/users/cseed/followers","following_url":"https://api.github.com/users/cseed/following{/other_user}","gists_url":"https://api.github.com/users/cseed/gists{/gist_id}","starred_url":"https://api.github.com/users/cseed/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cseed/subscriptions","organizations_url":"https://api.github.com/users/cseed/orgs","repos_url":"https://api.github.com/users/cseed/repos","events_url":"https://api.github.com/users/cseed/events{/privacy}","received_events_url":"https://api.github.com/users/cseed/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/cseed/hail","description":"Hail, a framework for scalable genetic data analysis","fork":true,"url":"https://api.github.com/repos/cseed/hail","forks_url":"https://api.github.com/repos/cseed/hail/forks","keys_url":"https://api.github.com/repos/cseed/hail/keys{/key_id}","collaborators_url":"https://api.github.com/repos/cseed/hail/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/cseed/hail/teams","hooks_url":"https://api.github.com/repos/cseed/hail/hooks","issue_events_url":"https://api.github.com/repos/cseed/hail/issues/events{/number}","events_url":"https://api.github.com/repos/cseed/hail/events","assignees_url":"https://api.github.com/repos/cseed/hail/assignees{/user}","branches_url":"https://api.github.com/repos/cseed/hail/branches{/branch}","tags_url":"https://api.github.com/repos/cseed/hail/tags","blobs_url":"https://api.github.com/repos/cseed/hail/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/cseed/hail/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/cseed/hail/git/refs{/sha}","trees_url":"https://api.github.com/repos/cseed/hail/git/trees{/sha}","statuses_url":"https://api.github.com/repos/cseed/hail/statuses/{sha}","languages_url":"https://api.github.com/repos/cseed/hail/languages","stargazers_url":"https://api.github.com/repos/cseed/hail/stargazers","contributors_url":"https://api.github.com/repos/cseed/hail/contributors","subscribers_url":"https://api.github.com/repos/cseed/hail/subscribers","subscription_url":"https://api.github.com/repos/cseed/hail/subscription","commits_url":"https://api.github.com/repos/cseed/hail/commits{/sha}","git_commits_url":"https://api.github.com/repos/cseed/hail/git/commits{/sha}","comments_url":"https://api.github.com/repos/cseed/hail/comments{/number}","issue_comment_url":"https://api.github.com/repos/cseed/hail/issues/comments{/number}","contents_url":"https://api.github.com/repos/cseed/hail/contents/{+path}","compare_url":"https://api.github.com/repos/cseed/hail/compare/{base}...{head}","merges_url":"https://api.github.com/repos/cseed/hail/merges","archive_url":"https://api.github.com/repos/cseed/hail/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/cseed/hail/downloads","issues_url":"https://api.github.com/repos/cseed/hail/issues{/number}","pulls_url":"https://api.github.com/repos/cseed/hail/pulls{/number}","milestones_url":"https://api.github.com/repos/cseed/hail/milestones{/number}","notifications_url":"https://api.github.com/repos/cseed/hail/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/cseed/hail/labels{/name}","releases_url":"https://api.github.com/repos/cseed/hail/releases{/id}","deployments_url":"https://api.github.com/repos/cseed/hail/deployments","created_at":"2016-08-01T19:57:20Z","updated_at":"2016-08-01T19:57:22Z","pushed_at":"2018-03-02T15:16:14Z","git_url":"git://github.com/cseed/hail.git","ssh_url":"git@github.com:cseed/hail.git","clone_url":"https://github.com/cseed/hail.git","svn_url":"https://github.com/cseed/hail","homepage":"","size":34497,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit"},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"hail-is:master","ref":"master","sha":"d42618f4e0c7c0398b187c3f1b5d9f994a472e95","user":{"login":"hail-is","id":20861792,"avatar_url":"https://avatars1.githubusercontent.com/u/20861792?v=4","gravatar_id":"","url":"https://api.github.com/users/hail-is","html_url":"https://github.com/hail-is","followers_url":"https://api.github.com/users/hail-is/followers","following_url":"https://api.github.com/users/hail-is/following{/other_user}","gists_url":"https://api.github.com/users/hail-is/gists{/gist_id}","starred_url":"https://api.github.com/users/hail-is/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hail-is/subscriptions","organizations_url":"https://api.github.com/users/hail-is/orgs","repos_url":"https://api.github.com/users/hail-is/repos","events_url":"https://api.github.com/users/hail-is/events{/privacy}","received_events_url":"https://api.github.com/users/hail-is/received_events","type":"Organization","site_admin":false},"repo":{"id":45069467,"name":"hail","full_name":"hail-is/hail","owner":{"login":"hail-is","id":20861792,"avatar_url":"https://avatars1.githubusercontent.com/u/20861792?v=4","gravatar_id":"","url":"https://api.github.com/users/hail-is","html_url":"https://github.com/hail-is","followers_url":"https://api.github.com/users/hail-is/followers","following_url":"https://api.github.com/users/hail-is/following{/other_user}","gists_url":"https://api.github.com/users/hail-is/gists{/gist_id}","starred_url":"https://api.github.com/users/hail-is/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hail-is/subscriptions","organizations_url":"https://api.github.com/users/hail-is/orgs","repos_url":"https://api.github.com/users/hail-is/repos","events_url":"https://api.github.com/users/hail-is/events{/privacy}","received_events_url":"https://api.github.com/users/hail-is/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/hail-is/hail","description":"Scalable genomic data analysis.","fork":false,"url":"https://api.github.com/repos/hail-is/hail","forks_url":"https://api.github.com/repos/hail-is/hail/forks","keys_url":"https://api.github.com/repos/hail-is/hail/keys{/key_id}","collaborators_url":"https://api.github.com/repos/hail-is/hail/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/hail-is/hail/teams","hooks_url":"https://api.github.com/repos/hail-is/hail/hooks","issue_events_url":"https://api.github.com/repos/hail-is/hail/issues/events{/number}","events_url":"https://api.github.com/repos/hail-is/hail/events","assignees_url":"https://api.github.com/repos/hail-is/hail/assignees{/user}","branches_url":"https://api.github.com/repos/hail-is/hail/branches{/branch}","tags_url":"https://api.github.com/repos/hail-is/hail/tags","blobs_url":"https://api.github.com/repos/hail-is/hail/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/hail-is/hail/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/hail-is/hail/git/refs{/sha}","trees_url":"https://api.github.com/repos/hail-is/hail/git/trees{/sha}","statuses_url":"https://api.github.com/repos/hail-is/hail/statuses/{sha}","languages_url":"https://api.github.com/repos/hail-is/hail/languages","stargazers_url":"https://api.github.com/repos/hail-is/hail/stargazers","contributors_url":"https://api.github.com/repos/hail-is/hail/contributors","subscribers_url":"https://api.github.com/repos/hail-is/hail/subscribers","subscription_url":"https://api.github.com/repos/hail-is/hail/subscription","commits_url":"https://api.github.com/repos/hail-is/hail/commits{/sha}","git_commits_url":"https://api.github.com/repos/hail-is/hail/git/commits{/sha}","comments_url":"https://api.github.com/repos/hail-is/hail/comments{/number}","issue_comment_url":"https://api.github.com/repos/hail-is/hail/issues/comments{/number}","contents_url":"https://api.github.com/repos/hail-is/hail/contents/{+path}","compare_url":"https://api.github.com/repos/hail-is/hail/compare/{base}...{head}","merges_url":"https://api.github.com/repos/hail-is/hail/merges","archive_url":"https://api.github.com/repos/hail-is/hail/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/hail-is/hail/downloads","issues_url":"https://api.github.com/repos/hail-is/hail/issues{/number}","pulls_url":"https://api.github.com/repos/hail-is/hail/pulls{/number}","milestones_url":"https://api.github.com/repos/hail-is/hail/milestones{/number}","notifications_url":"https://api.github.com/repos/hail-is/hail/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/hail-is/hail/labels{/name}","releases_url":"https://api.github.com/repos/hail-is/hail/releases{/id}","deployments_url":"https://api.github.com/repos/hail-is/hail/deployments","created_at":"2015-10-27T20:55:42Z","updated_at":"2018-02-28T14:07:07Z","pushed_at":"2018-03-02T15:16:15Z","git_url":"git://github.com/hail-is/hail.git","ssh_url":"git@github.com:hail-is/hail.git","clone_url":"https://github.com/hail-is/hail.git","svn_url":"https://github.com/hail-is/hail","homepage":"https://hail.is","size":31891,"stargazers_count":298,"watchers_count":298,"language":"Scala","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":94,"mirror_url":null,"archived":false,"open_issues_count":128,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit"},"forks":94,"open_issues":128,"watchers":298,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/hail-is/hail/pulls/3043"},"html":{"href":"https://github.com/hail-is/hail/pull/3043"},"issue":{"href":"https://api.github.com/repos/hail-is/hail/issues/3043"},"comments":{"href":"https://api.github.com/repos/hail-is/hail/issues/3043/comments"},"review_comments":{"href":"https://api.github.com/repos/hail-is/hail/pulls/3043/comments"},"review_comment":{"href":"https://api.github.com/repos/hail-is/hail/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/hail-is/hail/pulls/3043/commits"},"statuses":{"href":"https://api.github.com/repos/hail-is/hail/statuses/6ee7a7158bf72488957ba0db6c9e4640b4567185"}},"author_association":"COLLABORATOR"}}
{"actor":{"display_login":"jigold"}}