{"action":"created","comment":{"url":"https://api.github.com/repos/biocore/biom-format/pulls/comments/86674933","pull_request_review_id":7323179,"id":86674933,"diff_hunk":"@@ -97,6 +99,96 @@ def test_head_zero_or_neg(self):\n         with self.assertRaises(IndexError):\n             example_table.head(5, 0)\n \n+    def test_concat_samples(self):\n+        table2 = example_table.copy()\n+        table2.update_ids({'S1': 'S4', 'S2': 'S5', 'S3': 'S6'})\n+        \n+        exp = Table(np.array([[0, 1, 2, 0, 1, 2],\n+                              [3, 4, 5, 3, 4, 5]]), \n+                    ['O1', 'O2'],\n+                    ['S1', 'S2', 'S3', 'S4', 'S5', 'S6'],\n+                    example_table.metadata(axis='observation'),\n+                    list(example_table.metadata()) * 2)\n+        obs = example_table.concat([table2, ], axis='sample')\n+        self.assertEqual(obs, exp)\n+\n+    def test_concat_observations(self):\n+        table2 = example_table.copy()\n+        table2.update_ids({'O1': 'O3', 'O2': 'O4'}, axis='observation')\n+        \n+        exp = Table(np.array([[0, 1, 2],\n+                              [3, 4, 5],\n+                              [0, 1, 2],\n+                              [3, 4, 5]]), \n+                    ['O1', 'O2', 'O3', 'O4'],\n+                    ['S1', 'S2', 'S3'],\n+                    list(example_table.metadata(axis='observation')) * 2,\n+                    example_table.metadata())\n+        obs = example_table.concat([table2, ], axis='observation')\n+        self.assertEqual(obs, exp)\n+\n+    def test_concat_multiple(self):\n+        table2 = example_table.copy()\n+        table2.update_ids({'O1': 'O3', 'O2': 'O4'}, axis='observation')\n+        table3 = example_table.copy()\n+        table3.update_ids({'O1': 'O5', 'O2': 'O6'}, axis='observation')\n+        \n+        exp = Table(np.array([[0, 1, 2],\n+                              [3, 4, 5],\n+                              [0, 1, 2],\n+                              [3, 4, 5],\n+                              [0, 1, 2],\n+                              [3, 4, 5]]), \n+                    ['O1', 'O2', 'O3', 'O4', 'O5', 'O6'],\n+                    ['S1', 'S2', 'S3'],\n+                    list(example_table.metadata(axis='observation')) * 3,\n+                    example_table.metadata())\n+        obs = example_table.concat([table2, table3], axis='observation')\n+        self.assertEqual(obs, exp)\n+\n+    def test_concat_different_order(self):\n+        table2 = example_table.sort_order(['S3', 'S2', 'S1'])\n+        table2.update_ids({'S1': 'S4', 'S2': 'S5', 'S3': 'S6'})\n+        table2 = table2.sort_order(['O2', 'O1'], axis='observation')\n+        table3 = example_table.sort_order(['S2', 'S1', 'S3'])\n+        table3.update_ids({'S1': 'S7', 'S2': 'S8', 'S3': 'S9'})\n+        \n+        exp = Table(np.array([[0, 1, 2, 2, 1, 0, 1, 0, 2],\n+                              [3, 4, 5, 5, 4, 3, 4, 3, 5]]), \n+                    ['O1', 'O2'],\n+                    ['S1', 'S2', 'S3', 'S6', 'S5', 'S4', 'S8', 'S7', 'S9'],\n+                    example_table.metadata(axis='observation'),\n+                    list(example_table.metadata()) + \\\n+                            list(table2.metadata()) + \\\n+                            list(table3.metadata()))\n+        obs = example_table.concat([table2, table3], axis='sample')\n+        self.assertEqual(obs, exp)\n+\n+    def test_concat_pad_on_subset(self):\n+        table2 = example_table.copy()\n+        table2.update_ids({'O2': 'O3'}, axis='observation', strict=False)\n+        table2.update_ids({'S1': 'S4', 'S2': 'S5', 'S3': 'S6'})\n+        exp_obs_md = list(example_table.metadata(axis='observation'))\n+        exp_obs_md.append(example_table.metadata('O2', axis='observation'))\n+\n+        exp = Table(np.array([[0, 1, 2, 0, 1, 2],\n+                              [3, 4, 5, 0, 0, 0],\n+                              [0, 0, 0, 3, 4, 5]]),\n+                    ['O1', 'O2', 'O3'],\n+                    ['S1', 'S2', 'S3', 'S4', 'S5', 'S6'],\n+                    exp_obs_md,\n+                    list(example_table.metadata()) * 2)\n+\n+        obs = example_table.concat([table2, ], axis='sample')\n+        self.assertEqual(obs, exp)\n+\n+    def test_concat_raise_overlap(self):\n+        with self.assertRaises(DisjointIDError):\n+            example_table.concat([example_table])\n+\n+        with self.assertRaises(DisjointIDError):\n+            example_table.concat([example_table], axis='observation')\n+","path":"tests/test_table.py","position":106,"original_position":106,"commit_id":"fb824c87aabed6df132e2bebe81488a49bc5a63e","original_commit_id":"fb824c87aabed6df132e2bebe81488a49bc5a63e","user":{"login":"wasade","id":474290,"avatar_url":"https://avatars.githubusercontent.com/u/474290?v=3","gravatar_id":"","url":"https://api.github.com/users/wasade","html_url":"https://github.com/wasade","followers_url":"https://api.github.com/users/wasade/followers","following_url":"https://api.github.com/users/wasade/following{/other_user}","gists_url":"https://api.github.com/users/wasade/gists{/gist_id}","starred_url":"https://api.github.com/users/wasade/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/wasade/subscriptions","organizations_url":"https://api.github.com/users/wasade/orgs","repos_url":"https://api.github.com/users/wasade/repos","events_url":"https://api.github.com/users/wasade/events{/privacy}","received_events_url":"https://api.github.com/users/wasade/received_events","type":"User","site_admin":false},"body":"works as expected?\r\n\r\n```python\r\nIn [2]: biom.example_table.concat([biom.Table([], [], [])])\r\nOut[2]: 2 x 3 <class 'biom.table.Table'> with 5 nonzero entries (83% dense)\r\n```","created_at":"2016-11-05T22:57:28Z","updated_at":"2016-11-05T22:57:28Z","html_url":"https://github.com/biocore/biom-format/pull/720#discussion_r86674933","pull_request_url":"https://api.github.com/repos/biocore/biom-format/pulls/720","_links":{"self":{"href":"https://api.github.com/repos/biocore/biom-format/pulls/comments/86674933"},"html":{"href":"https://github.com/biocore/biom-format/pull/720#discussion_r86674933"},"pull_request":{"href":"https://api.github.com/repos/biocore/biom-format/pulls/720"}}},"pull_request":{"url":"https://api.github.com/repos/biocore/biom-format/pulls/720","id":92268806,"html_url":"https://github.com/biocore/biom-format/pull/720","diff_url":"https://github.com/biocore/biom-format/pull/720.diff","patch_url":"https://github.com/biocore/biom-format/pull/720.patch","issue_url":"https://api.github.com/repos/biocore/biom-format/issues/720","number":720,"state":"open","locked":false,"title":"Concat","user":{"login":"wasade","id":474290,"avatar_url":"https://avatars.githubusercontent.com/u/474290?v=3","gravatar_id":"","url":"https://api.github.com/users/wasade","html_url":"https://github.com/wasade","followers_url":"https://api.github.com/users/wasade/followers","following_url":"https://api.github.com/users/wasade/following{/other_user}","gists_url":"https://api.github.com/users/wasade/gists{/gist_id}","starred_url":"https://api.github.com/users/wasade/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/wasade/subscriptions","organizations_url":"https://api.github.com/users/wasade/orgs","repos_url":"https://api.github.com/users/wasade/repos","events_url":"https://api.github.com/users/wasade/events{/privacy}","received_events_url":"https://api.github.com/users/wasade/received_events","type":"User","site_admin":false},"body":"Resolves #716. This ended up growing in complexity to handle the general case in which one set of IDs are disjoint while the other axis doesn't need to be. I'd be curious if others see ways to reduce or simplify some of the code complexity here. \r\n\r\nNow, why is this method useful? Often, when merging tables, one axis is disjoint (e.g., the sample IDs in a meta-analysis). This method offers 80x or so improvement over the existing `merge` method for this case. In addition, it supports concatenating multiple tables while `merge` must operate only on pairs thus offering likely further performance gains in real world scenarios.\r\n\r\nThe first benchmark shows approximately a 20x improvement. When profiling, a hotspot was identified in `sort_order`, which was also rectified.\r\n\r\n```python\r\nIn [1]: import biom\r\n\r\nIn [2]: a = biom.load_table('twins.qiita-2014.biom')\r\n\r\nIn [3]: b = biom.load_table('ag.qiita-10317.biom')\r\n\r\nIn [4]: a\r\nOut[4]: 8735 x 1046 <class 'biom.table.Table'> with 344079 nonzero entries (3% dense)\r\n\r\nIn [5]: b\r\nOut[5]: 9128 x 9919 <class 'biom.table.Table'> with 1508178 nonzero entries (1% dense)\r\n\r\nIn [6]: %timeit c = a.concat([b])\r\n1 loop, best of 3: 2.25 s per loop\r\n\r\nIn [7]: %timeit c = a.merge(b)\r\n1 loop, best of 3: 43.9 s per loop\r\n```\r\n\r\nA quick profile suggested a hotspot in `sort_order` stemming from a) an implicit cast to dense (which also stood to bloat memory) and b) utilizing incremental indexing instead of fancy indexing. So I fixed that. New timing below:\r\n\r\n```python\r\nIn [1]: import biom\r\n\r\nIn [2]: a = biom.load_table('twins.qiita-2014.biom')\r\n\r\nIn [3]: b = biom.load_table('ag.qiita-10317.biom')\r\n\r\nIn [4]: %timeit c = a.concat([b])\r\n1 loop, best of 3: 582 ms per loop\r\n```","created_at":"2016-11-04T01:08:44Z","updated_at":"2016-11-05T22:57:28Z","closed_at":null,"merged_at":null,"merge_commit_sha":"467113119d4f4123dfa79ccf6aaecdaf6d4079a2","assignee":null,"assignees":[],"milestone":{"url":"https://api.github.com/repos/biocore/biom-format/milestones/11","html_url":"https://github.com/biocore/biom-format/milestones/2.1.6%20","labels_url":"https://api.github.com/repos/biocore/biom-format/milestones/11/labels","id":1365188,"number":11,"title":"2.1.6 ","description":"","creator":{"login":"gregcaporaso","id":192372,"avatar_url":"https://avatars.githubusercontent.com/u/192372?v=3","gravatar_id":"","url":"https://api.github.com/users/gregcaporaso","html_url":"https://github.com/gregcaporaso","followers_url":"https://api.github.com/users/gregcaporaso/followers","following_url":"https://api.github.com/users/gregcaporaso/following{/other_user}","gists_url":"https://api.github.com/users/gregcaporaso/gists{/gist_id}","starred_url":"https://api.github.com/users/gregcaporaso/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/gregcaporaso/subscriptions","organizations_url":"https://api.github.com/users/gregcaporaso/orgs","repos_url":"https://api.github.com/users/gregcaporaso/repos","events_url":"https://api.github.com/users/gregcaporaso/events{/privacy}","received_events_url":"https://api.github.com/users/gregcaporaso/received_events","type":"User","site_admin":false},"open_issues":27,"closed_issues":4,"state":"open","created_at":"2015-10-20T00:35:48Z","updated_at":"2016-11-05T22:15:47Z","due_on":null,"closed_at":null},"commits_url":"https://api.github.com/repos/biocore/biom-format/pulls/720/commits","review_comments_url":"https://api.github.com/repos/biocore/biom-format/pulls/720/comments","review_comment_url":"https://api.github.com/repos/biocore/biom-format/pulls/comments{/number}","comments_url":"https://api.github.com/repos/biocore/biom-format/issues/720/comments","statuses_url":"https://api.github.com/repos/biocore/biom-format/statuses/fb824c87aabed6df132e2bebe81488a49bc5a63e","head":{"label":"wasade:concat","ref":"concat","sha":"fb824c87aabed6df132e2bebe81488a49bc5a63e","user":{"login":"wasade","id":474290,"avatar_url":"https://avatars.githubusercontent.com/u/474290?v=3","gravatar_id":"","url":"https://api.github.com/users/wasade","html_url":"https://github.com/wasade","followers_url":"https://api.github.com/users/wasade/followers","following_url":"https://api.github.com/users/wasade/following{/other_user}","gists_url":"https://api.github.com/users/wasade/gists{/gist_id}","starred_url":"https://api.github.com/users/wasade/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/wasade/subscriptions","organizations_url":"https://api.github.com/users/wasade/orgs","repos_url":"https://api.github.com/users/wasade/repos","events_url":"https://api.github.com/users/wasade/events{/privacy}","received_events_url":"https://api.github.com/users/wasade/received_events","type":"User","site_admin":false},"repo":{"id":6601903,"name":"biom-format","full_name":"wasade/biom-format","owner":{"login":"wasade","id":474290,"avatar_url":"https://avatars.githubusercontent.com/u/474290?v=3","gravatar_id":"","url":"https://api.github.com/users/wasade","html_url":"https://github.com/wasade","followers_url":"https://api.github.com/users/wasade/followers","following_url":"https://api.github.com/users/wasade/following{/other_user}","gists_url":"https://api.github.com/users/wasade/gists{/gist_id}","starred_url":"https://api.github.com/users/wasade/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/wasade/subscriptions","organizations_url":"https://api.github.com/users/wasade/orgs","repos_url":"https://api.github.com/users/wasade/repos","events_url":"https://api.github.com/users/wasade/events{/privacy}","received_events_url":"https://api.github.com/users/wasade/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/wasade/biom-format","description":"The Biological Observation Matrix (BIOM) Format Project","fork":true,"url":"https://api.github.com/repos/wasade/biom-format","forks_url":"https://api.github.com/repos/wasade/biom-format/forks","keys_url":"https://api.github.com/repos/wasade/biom-format/keys{/key_id}","collaborators_url":"https://api.github.com/repos/wasade/biom-format/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/wasade/biom-format/teams","hooks_url":"https://api.github.com/repos/wasade/biom-format/hooks","issue_events_url":"https://api.github.com/repos/wasade/biom-format/issues/events{/number}","events_url":"https://api.github.com/repos/wasade/biom-format/events","assignees_url":"https://api.github.com/repos/wasade/biom-format/assignees{/user}","branches_url":"https://api.github.com/repos/wasade/biom-format/branches{/branch}","tags_url":"https://api.github.com/repos/wasade/biom-format/tags","blobs_url":"https://api.github.com/repos/wasade/biom-format/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/wasade/biom-format/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/wasade/biom-format/git/refs{/sha}","trees_url":"https://api.github.com/repos/wasade/biom-format/git/trees{/sha}","statuses_url":"https://api.github.com/repos/wasade/biom-format/statuses/{sha}","languages_url":"https://api.github.com/repos/wasade/biom-format/languages","stargazers_url":"https://api.github.com/repos/wasade/biom-format/stargazers","contributors_url":"https://api.github.com/repos/wasade/biom-format/contributors","subscribers_url":"https://api.github.com/repos/wasade/biom-format/subscribers","subscription_url":"https://api.github.com/repos/wasade/biom-format/subscription","commits_url":"https://api.github.com/repos/wasade/biom-format/commits{/sha}","git_commits_url":"https://api.github.com/repos/wasade/biom-format/git/commits{/sha}","comments_url":"https://api.github.com/repos/wasade/biom-format/comments{/number}","issue_comment_url":"https://api.github.com/repos/wasade/biom-format/issues/comments{/number}","contents_url":"https://api.github.com/repos/wasade/biom-format/contents/{+path}","compare_url":"https://api.github.com/repos/wasade/biom-format/compare/{base}...{head}","merges_url":"https://api.github.com/repos/wasade/biom-format/merges","archive_url":"https://api.github.com/repos/wasade/biom-format/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/wasade/biom-format/downloads","issues_url":"https://api.github.com/repos/wasade/biom-format/issues{/number}","pulls_url":"https://api.github.com/repos/wasade/biom-format/pulls{/number}","milestones_url":"https://api.github.com/repos/wasade/biom-format/milestones{/number}","notifications_url":"https://api.github.com/repos/wasade/biom-format/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/wasade/biom-format/labels{/name}","releases_url":"https://api.github.com/repos/wasade/biom-format/releases{/id}","deployments_url":"https://api.github.com/repos/wasade/biom-format/deployments","created_at":"2012-11-08T19:31:44Z","updated_at":"2015-03-27T19:40:13Z","pushed_at":"2016-11-05T21:16:44Z","git_url":"git://github.com/wasade/biom-format.git","ssh_url":"git@github.com:wasade/biom-format.git","clone_url":"https://github.com/wasade/biom-format.git","svn_url":"https://github.com/wasade/biom-format","homepage":"http://biom-format.org","size":20153,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":true,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"biocore:master","ref":"master","sha":"1ebd2d1781b3573dfedfb01235e66e99ee6e0b18","user":{"login":"biocore","id":6241857,"avatar_url":"https://avatars.githubusercontent.com/u/6241857?v=3","gravatar_id":"","url":"https://api.github.com/users/biocore","html_url":"https://github.com/biocore","followers_url":"https://api.github.com/users/biocore/followers","following_url":"https://api.github.com/users/biocore/following{/other_user}","gists_url":"https://api.github.com/users/biocore/gists{/gist_id}","starred_url":"https://api.github.com/users/biocore/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/biocore/subscriptions","organizations_url":"https://api.github.com/users/biocore/orgs","repos_url":"https://api.github.com/users/biocore/repos","events_url":"https://api.github.com/users/biocore/events{/privacy}","received_events_url":"https://api.github.com/users/biocore/received_events","type":"Organization","site_admin":false},"repo":{"id":3999367,"name":"biom-format","full_name":"biocore/biom-format","owner":{"login":"biocore","id":6241857,"avatar_url":"https://avatars.githubusercontent.com/u/6241857?v=3","gravatar_id":"","url":"https://api.github.com/users/biocore","html_url":"https://github.com/biocore","followers_url":"https://api.github.com/users/biocore/followers","following_url":"https://api.github.com/users/biocore/following{/other_user}","gists_url":"https://api.github.com/users/biocore/gists{/gist_id}","starred_url":"https://api.github.com/users/biocore/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/biocore/subscriptions","organizations_url":"https://api.github.com/users/biocore/orgs","repos_url":"https://api.github.com/users/biocore/repos","events_url":"https://api.github.com/users/biocore/events{/privacy}","received_events_url":"https://api.github.com/users/biocore/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/biocore/biom-format","description":"The Biological Observation Matrix (BIOM) Format Project","fork":false,"url":"https://api.github.com/repos/biocore/biom-format","forks_url":"https://api.github.com/repos/biocore/biom-format/forks","keys_url":"https://api.github.com/repos/biocore/biom-format/keys{/key_id}","collaborators_url":"https://api.github.com/repos/biocore/biom-format/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/biocore/biom-format/teams","hooks_url":"https://api.github.com/repos/biocore/biom-format/hooks","issue_events_url":"https://api.github.com/repos/biocore/biom-format/issues/events{/number}","events_url":"https://api.github.com/repos/biocore/biom-format/events","assignees_url":"https://api.github.com/repos/biocore/biom-format/assignees{/user}","branches_url":"https://api.github.com/repos/biocore/biom-format/branches{/branch}","tags_url":"https://api.github.com/repos/biocore/biom-format/tags","blobs_url":"https://api.github.com/repos/biocore/biom-format/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/biocore/biom-format/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/biocore/biom-format/git/refs{/sha}","trees_url":"https://api.github.com/repos/biocore/biom-format/git/trees{/sha}","statuses_url":"https://api.github.com/repos/biocore/biom-format/statuses/{sha}","languages_url":"https://api.github.com/repos/biocore/biom-format/languages","stargazers_url":"https://api.github.com/repos/biocore/biom-format/stargazers","contributors_url":"https://api.github.com/repos/biocore/biom-format/contributors","subscribers_url":"https://api.github.com/repos/biocore/biom-format/subscribers","subscription_url":"https://api.github.com/repos/biocore/biom-format/subscription","commits_url":"https://api.github.com/repos/biocore/biom-format/commits{/sha}","git_commits_url":"https://api.github.com/repos/biocore/biom-format/git/commits{/sha}","comments_url":"https://api.github.com/repos/biocore/biom-format/comments{/number}","issue_comment_url":"https://api.github.com/repos/biocore/biom-format/issues/comments{/number}","contents_url":"https://api.github.com/repos/biocore/biom-format/contents/{+path}","compare_url":"https://api.github.com/repos/biocore/biom-format/compare/{base}...{head}","merges_url":"https://api.github.com/repos/biocore/biom-format/merges","archive_url":"https://api.github.com/repos/biocore/biom-format/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/biocore/biom-format/downloads","issues_url":"https://api.github.com/repos/biocore/biom-format/issues{/number}","pulls_url":"https://api.github.com/repos/biocore/biom-format/pulls{/number}","milestones_url":"https://api.github.com/repos/biocore/biom-format/milestones{/number}","notifications_url":"https://api.github.com/repos/biocore/biom-format/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/biocore/biom-format/labels{/name}","releases_url":"https://api.github.com/repos/biocore/biom-format/releases{/id}","deployments_url":"https://api.github.com/repos/biocore/biom-format/deployments","created_at":"2012-04-11T23:10:28Z","updated_at":"2016-10-26T06:50:08Z","pushed_at":"2016-11-05T21:16:45Z","git_url":"git://github.com/biocore/biom-format.git","ssh_url":"git@github.com:biocore/biom-format.git","clone_url":"https://github.com/biocore/biom-format.git","svn_url":"https://github.com/biocore/biom-format","homepage":"http://biom-format.org","size":19731,"stargazers_count":28,"watchers_count":28,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":49,"mirror_url":null,"open_issues_count":49,"forks":49,"open_issues":49,"watchers":28,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/biocore/biom-format/pulls/720"},"html":{"href":"https://github.com/biocore/biom-format/pull/720"},"issue":{"href":"https://api.github.com/repos/biocore/biom-format/issues/720"},"comments":{"href":"https://api.github.com/repos/biocore/biom-format/issues/720/comments"},"review_comments":{"href":"https://api.github.com/repos/biocore/biom-format/pulls/720/comments"},"review_comment":{"href":"https://api.github.com/repos/biocore/biom-format/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/biocore/biom-format/pulls/720/commits"},"statuses":{"href":"https://api.github.com/repos/biocore/biom-format/statuses/fb824c87aabed6df132e2bebe81488a49bc5a63e"}}}}
{"actor":{"display_login":"wasade"}}