{"action":"created","comment":{"url":"https://api.github.com/repos/biocore/scikit-bio/pulls/comments/82466625","pull_request_review_id":3366718,"id":82466625,"diff_hunk":"@@ -0,0 +1,270 @@\n+# ----------------------------------------------------------------------------\n+# Copyright (c) 2013--, scikit-bio development team.\n+#\n+# Distributed under the terms of the Modified BSD License.\n+#\n+# The full license is in the file COPYING.txt, distributed with this software.\n+# ----------------------------------------------------------------------------\n+\n+import numpy as np\n+\n+from skbio.util._decorator import experimental\n+from skbio.diversity._driver import partial_beta_diversity\n+from skbio.stats.distance import DistanceMatrix\n+from skbio.diversity._util import _validate_counts_matrix\n+\n+\n+def _generate_id_blocks(ids=None, k=None, **kwargs):\n+    \"\"\"Generate blocks of IDs that map into a DistanceMatrix\n+\n+    Parameters\n+    ----------\n+    ids : Iterable\n+        An iterable of IDs of whatever type.\n+    k : int\n+        The size of a block to generate IDs for\n+\n+    Notes\n+    -----\n+\n+    This method is intended to facilitate partial beta diversity calculations.\n+    Blocks of IDs are generated from the upper triangle of the subsequent\n+    distance matrix. For instance, given the following distance matrix with\n+    IDs {A, B, C, D, E}:\n+\n+      A B C D E\n+    A 0 # # # #\n+    B # 0 # # #\n+    C # # 0 # #\n+    D # # # 0 #\n+    E # # # # 0\n+\n+    The goal of this method is to generate tuples of IDs of at most size k over\n+    the upper triangle which correspond to blocks of the matrix to compute. IDs\n+    are remapped as well into integers to facilitate downstream indexing.\n+    Given k=3, the following ID tuples would be generated:\n+\n+    ((0, 1, 2), (0, 1, 2))\n+    ((0, 1, 2), (3, 4))\n+    ((3, 4), (3, 4))\n+\n+    This method is not responsible for describing which specific pairs of IDs\n+    are to be computed, only the subset of the matrix of interest.\n+\n+    Returns\n+    -------\n+    tuple of 1D np.array\n+        Index 0 contains the row IDs, and index 1 contains the column IDs\n+    \"\"\"\n+    n = len(ids)\n+    ids_idx = np.arange(n)\n+\n+    for row_start in range(0, n, k):\n+        for col_start in range(row_start, n, k):\n+            row_ids = ids_idx[row_start:row_start + k]\n+            col_ids = ids_idx[col_start:col_start + k]\n+\n+            yield (row_ids, col_ids)\n+\n+\n+def _block_party(counts=None, row_ids=None, col_ids=None, **kwargs):\n+    \"\"\"Subset counts to relevant rows and columns\n+\n+    Parameters\n+    ----------\n+    counts : 2D array_like of ints or floats\n+        Matrix containing count/abundance data where each row contains counts\n+        of OTUs in a given sample.\n+    row_ids : 1D np.ndarray of int\n+        Block row IDs to keep in the counts matrix.\n+    col_ids : 1D np.ndarray of int\n+        Block column IDs to keep in the counts matrix. Note, these correspond\n+        to rows in the counts matrix, but columns in a subsequent distance\n+        matrix.\n+\n+    Returns\n+    -------\n+    dict\n+        kwargs that describe the block to compute. A filtered ``counts`` matrix\n+        is stored in kwargs. If applicable, a filtered ``tree`` and ``otu_ids``\n+        are also stored.\n+    \"\"\"\n+    # filter kwargs to valid keys for downstream block compute\n+    # explicit copies are not necessary as implicit copies or overwwrites are\n+    # performed below.\n+    valid_block_keys = {'counts', 'ids', 'tree', 'otu_ids', 'metric',\n+                        'id_pairs', 'validate'}\n+    block_kwargs = {k: v for k, v in kwargs.items() if k in valid_block_keys}\n+\n+    ids_to_keep = np.unique(np.hstack([row_ids, col_ids]))\n+\n+    # create a view of the relevant samples\n+    counts_block = counts[ids_to_keep]\n+\n+    # remove from the block any empty observations\n+    # NOTE: this will perform an implicit copy\n+    nonzero_cols = (counts_block != 0).any(axis=0)\n+    counts_block = counts_block[:, nonzero_cols]","path":"skbio/diversity/_block.py","position":107,"original_position":107,"commit_id":"a055c791a0a8e556c1f769a053c95712956a9d10","original_commit_id":"a055c791a0a8e556c1f769a053c95712956a9d10","user":{"login":"ElDeveloper","id":375307,"avatar_url":"https://avatars.githubusercontent.com/u/375307?v=3","gravatar_id":"","url":"https://api.github.com/users/ElDeveloper","html_url":"https://github.com/ElDeveloper","followers_url":"https://api.github.com/users/ElDeveloper/followers","following_url":"https://api.github.com/users/ElDeveloper/following{/other_user}","gists_url":"https://api.github.com/users/ElDeveloper/gists{/gist_id}","starred_url":"https://api.github.com/users/ElDeveloper/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ElDeveloper/subscriptions","organizations_url":"https://api.github.com/users/ElDeveloper/orgs","repos_url":"https://api.github.com/users/ElDeveloper/repos","events_url":"https://api.github.com/users/ElDeveloper/events{/privacy}","received_events_url":"https://api.github.com/users/ElDeveloper/received_events","type":"User","site_admin":false},"body":"This seems a bit odd you are slicing with a True value? This should return a warning or am I misunderstanding something?","created_at":"2016-10-07T20:44:48Z","updated_at":"2016-10-07T21:01:24Z","html_url":"https://github.com/biocore/scikit-bio/pull/1352#discussion_r82466625","pull_request_url":"https://api.github.com/repos/biocore/scikit-bio/pulls/1352","_links":{"self":{"href":"https://api.github.com/repos/biocore/scikit-bio/pulls/comments/82466625"},"html":{"href":"https://github.com/biocore/scikit-bio/pull/1352#discussion_r82466625"},"pull_request":{"href":"https://api.github.com/repos/biocore/scikit-bio/pulls/1352"}}},"pull_request":{"url":"https://api.github.com/repos/biocore/scikit-bio/pulls/1352","id":69714993,"html_url":"https://github.com/biocore/scikit-bio/pull/1352","diff_url":"https://github.com/biocore/scikit-bio/pull/1352.diff","patch_url":"https://github.com/biocore/scikit-bio/pull/1352.patch","issue_url":"https://api.github.com/repos/biocore/scikit-bio/issues/1352","number":1352,"state":"open","locked":false,"title":"beta diversity via block decomposition ","user":{"login":"wasade","id":474290,"avatar_url":"https://avatars.githubusercontent.com/u/474290?v=3","gravatar_id":"","url":"https://api.github.com/users/wasade","html_url":"https://github.com/wasade","followers_url":"https://api.github.com/users/wasade/followers","following_url":"https://api.github.com/users/wasade/following{/other_user}","gists_url":"https://api.github.com/users/wasade/gists{/gist_id}","starred_url":"https://api.github.com/users/wasade/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/wasade/subscriptions","organizations_url":"https://api.github.com/users/wasade/orgs","repos_url":"https://api.github.com/users/wasade/repos","events_url":"https://api.github.com/users/wasade/events{/privacy}","received_events_url":"https://api.github.com/users/wasade/received_events","type":"User","site_admin":false},"body":"This PR includes #1337, and assumes that `beta_diversity` can return a `PartialDistanceMatrix`. Right now, this is an unexercised possible direction to take with doing the block decomposition. \r\n\r\nThe design is such that it should allow for leveraging any parallel framework as the \"map\" and \"reduce\" functions can be arbitrary. The individual \"computable\" blocks are packed into `partial` functions so they can be executed by the `map_f` effectively deferring the heavy lifting (i.e., `_block_party` due to `shear` and `beta_diversity`) until the last possible moment.  \r\n\r\nNote: this is _unexercised_ code, so it is likely that there are problems. It may also make sense to revisit how the methods themselves are decomposed. I am primarily interested at this time in whether the approach to parallelism is one that makes sense in the bigger picture of scikit-bio. It is critical that this method is usable in a parallel context such that execution can be distributed to independent execution environments. \r\n\r\nA notebook highlighting the approach to the parallelism can be found [here](https://gist.github.com/wasade/c3b3e8baa7c16fe3699d4371c8fffb53). ","created_at":"2016-05-11T17:57:13Z","updated_at":"2016-10-07T21:01:24Z","closed_at":null,"merged_at":null,"merge_commit_sha":"9637bed09f30dd0299c144efd24994c030bea836","assignee":{"login":"ebolyen","id":3976804,"avatar_url":"https://avatars.githubusercontent.com/u/3976804?v=3","gravatar_id":"","url":"https://api.github.com/users/ebolyen","html_url":"https://github.com/ebolyen","followers_url":"https://api.github.com/users/ebolyen/followers","following_url":"https://api.github.com/users/ebolyen/following{/other_user}","gists_url":"https://api.github.com/users/ebolyen/gists{/gist_id}","starred_url":"https://api.github.com/users/ebolyen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ebolyen/subscriptions","organizations_url":"https://api.github.com/users/ebolyen/orgs","repos_url":"https://api.github.com/users/ebolyen/repos","events_url":"https://api.github.com/users/ebolyen/events{/privacy}","received_events_url":"https://api.github.com/users/ebolyen/received_events","type":"User","site_admin":false},"assignees":[{"login":"ebolyen","id":3976804,"avatar_url":"https://avatars.githubusercontent.com/u/3976804?v=3","gravatar_id":"","url":"https://api.github.com/users/ebolyen","html_url":"https://github.com/ebolyen","followers_url":"https://api.github.com/users/ebolyen/followers","following_url":"https://api.github.com/users/ebolyen/following{/other_user}","gists_url":"https://api.github.com/users/ebolyen/gists{/gist_id}","starred_url":"https://api.github.com/users/ebolyen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ebolyen/subscriptions","organizations_url":"https://api.github.com/users/ebolyen/orgs","repos_url":"https://api.github.com/users/ebolyen/repos","events_url":"https://api.github.com/users/ebolyen/events{/privacy}","received_events_url":"https://api.github.com/users/ebolyen/received_events","type":"User","site_admin":false}],"milestone":null,"commits_url":"https://api.github.com/repos/biocore/scikit-bio/pulls/1352/commits","review_comments_url":"https://api.github.com/repos/biocore/scikit-bio/pulls/1352/comments","review_comment_url":"https://api.github.com/repos/biocore/scikit-bio/pulls/comments{/number}","comments_url":"https://api.github.com/repos/biocore/scikit-bio/issues/1352/comments","statuses_url":"https://api.github.com/repos/biocore/scikit-bio/statuses/a055c791a0a8e556c1f769a053c95712956a9d10","head":{"label":"wasade:blockdecomp","ref":"blockdecomp","sha":"a055c791a0a8e556c1f769a053c95712956a9d10","user":{"login":"wasade","id":474290,"avatar_url":"https://avatars.githubusercontent.com/u/474290?v=3","gravatar_id":"","url":"https://api.github.com/users/wasade","html_url":"https://github.com/wasade","followers_url":"https://api.github.com/users/wasade/followers","following_url":"https://api.github.com/users/wasade/following{/other_user}","gists_url":"https://api.github.com/users/wasade/gists{/gist_id}","starred_url":"https://api.github.com/users/wasade/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/wasade/subscriptions","organizations_url":"https://api.github.com/users/wasade/orgs","repos_url":"https://api.github.com/users/wasade/repos","events_url":"https://api.github.com/users/wasade/events{/privacy}","received_events_url":"https://api.github.com/users/wasade/received_events","type":"User","site_admin":false},"repo":{"id":17808144,"name":"scikit-bio","full_name":"wasade/scikit-bio","owner":{"login":"wasade","id":474290,"avatar_url":"https://avatars.githubusercontent.com/u/474290?v=3","gravatar_id":"","url":"https://api.github.com/users/wasade","html_url":"https://github.com/wasade","followers_url":"https://api.github.com/users/wasade/followers","following_url":"https://api.github.com/users/wasade/following{/other_user}","gists_url":"https://api.github.com/users/wasade/gists{/gist_id}","starred_url":"https://api.github.com/users/wasade/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/wasade/subscriptions","organizations_url":"https://api.github.com/users/wasade/orgs","repos_url":"https://api.github.com/users/wasade/repos","events_url":"https://api.github.com/users/wasade/events{/privacy}","received_events_url":"https://api.github.com/users/wasade/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/wasade/scikit-bio","description":"Core objects, functions and statistics for working with biological data in Python.","fork":true,"url":"https://api.github.com/repos/wasade/scikit-bio","forks_url":"https://api.github.com/repos/wasade/scikit-bio/forks","keys_url":"https://api.github.com/repos/wasade/scikit-bio/keys{/key_id}","collaborators_url":"https://api.github.com/repos/wasade/scikit-bio/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/wasade/scikit-bio/teams","hooks_url":"https://api.github.com/repos/wasade/scikit-bio/hooks","issue_events_url":"https://api.github.com/repos/wasade/scikit-bio/issues/events{/number}","events_url":"https://api.github.com/repos/wasade/scikit-bio/events","assignees_url":"https://api.github.com/repos/wasade/scikit-bio/assignees{/user}","branches_url":"https://api.github.com/repos/wasade/scikit-bio/branches{/branch}","tags_url":"https://api.github.com/repos/wasade/scikit-bio/tags","blobs_url":"https://api.github.com/repos/wasade/scikit-bio/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/wasade/scikit-bio/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/wasade/scikit-bio/git/refs{/sha}","trees_url":"https://api.github.com/repos/wasade/scikit-bio/git/trees{/sha}","statuses_url":"https://api.github.com/repos/wasade/scikit-bio/statuses/{sha}","languages_url":"https://api.github.com/repos/wasade/scikit-bio/languages","stargazers_url":"https://api.github.com/repos/wasade/scikit-bio/stargazers","contributors_url":"https://api.github.com/repos/wasade/scikit-bio/contributors","subscribers_url":"https://api.github.com/repos/wasade/scikit-bio/subscribers","subscription_url":"https://api.github.com/repos/wasade/scikit-bio/subscription","commits_url":"https://api.github.com/repos/wasade/scikit-bio/commits{/sha}","git_commits_url":"https://api.github.com/repos/wasade/scikit-bio/git/commits{/sha}","comments_url":"https://api.github.com/repos/wasade/scikit-bio/comments{/number}","issue_comment_url":"https://api.github.com/repos/wasade/scikit-bio/issues/comments{/number}","contents_url":"https://api.github.com/repos/wasade/scikit-bio/contents/{+path}","compare_url":"https://api.github.com/repos/wasade/scikit-bio/compare/{base}...{head}","merges_url":"https://api.github.com/repos/wasade/scikit-bio/merges","archive_url":"https://api.github.com/repos/wasade/scikit-bio/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/wasade/scikit-bio/downloads","issues_url":"https://api.github.com/repos/wasade/scikit-bio/issues{/number}","pulls_url":"https://api.github.com/repos/wasade/scikit-bio/pulls{/number}","milestones_url":"https://api.github.com/repos/wasade/scikit-bio/milestones{/number}","notifications_url":"https://api.github.com/repos/wasade/scikit-bio/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/wasade/scikit-bio/labels{/name}","releases_url":"https://api.github.com/repos/wasade/scikit-bio/releases{/id}","deployments_url":"https://api.github.com/repos/wasade/scikit-bio/deployments","created_at":"2014-03-16T20:13:28Z","updated_at":"2014-06-25T14:53:19Z","pushed_at":"2016-10-07T15:43:09Z","git_url":"git://github.com/wasade/scikit-bio.git","ssh_url":"git@github.com:wasade/scikit-bio.git","clone_url":"https://github.com/wasade/scikit-bio.git","svn_url":"https://github.com/wasade/scikit-bio","homepage":"https://github.com/biocore/scikit-bio","size":22173,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"biocore:master","ref":"master","sha":"8c5d0d56c6c1a94584c98725cad70a02335d6c49","user":{"login":"biocore","id":6241857,"avatar_url":"https://avatars.githubusercontent.com/u/6241857?v=3","gravatar_id":"","url":"https://api.github.com/users/biocore","html_url":"https://github.com/biocore","followers_url":"https://api.github.com/users/biocore/followers","following_url":"https://api.github.com/users/biocore/following{/other_user}","gists_url":"https://api.github.com/users/biocore/gists{/gist_id}","starred_url":"https://api.github.com/users/biocore/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/biocore/subscriptions","organizations_url":"https://api.github.com/users/biocore/orgs","repos_url":"https://api.github.com/users/biocore/repos","events_url":"https://api.github.com/users/biocore/events{/privacy}","received_events_url":"https://api.github.com/users/biocore/received_events","type":"Organization","site_admin":false},"repo":{"id":15168264,"name":"scikit-bio","full_name":"biocore/scikit-bio","owner":{"login":"biocore","id":6241857,"avatar_url":"https://avatars.githubusercontent.com/u/6241857?v=3","gravatar_id":"","url":"https://api.github.com/users/biocore","html_url":"https://github.com/biocore","followers_url":"https://api.github.com/users/biocore/followers","following_url":"https://api.github.com/users/biocore/following{/other_user}","gists_url":"https://api.github.com/users/biocore/gists{/gist_id}","starred_url":"https://api.github.com/users/biocore/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/biocore/subscriptions","organizations_url":"https://api.github.com/users/biocore/orgs","repos_url":"https://api.github.com/users/biocore/repos","events_url":"https://api.github.com/users/biocore/events{/privacy}","received_events_url":"https://api.github.com/users/biocore/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/biocore/scikit-bio","description":"scikit-bio is an open-source, BSD-licensed, Python package providing data structures, algorithms, and educational resources for bioinformatics.","fork":false,"url":"https://api.github.com/repos/biocore/scikit-bio","forks_url":"https://api.github.com/repos/biocore/scikit-bio/forks","keys_url":"https://api.github.com/repos/biocore/scikit-bio/keys{/key_id}","collaborators_url":"https://api.github.com/repos/biocore/scikit-bio/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/biocore/scikit-bio/teams","hooks_url":"https://api.github.com/repos/biocore/scikit-bio/hooks","issue_events_url":"https://api.github.com/repos/biocore/scikit-bio/issues/events{/number}","events_url":"https://api.github.com/repos/biocore/scikit-bio/events","assignees_url":"https://api.github.com/repos/biocore/scikit-bio/assignees{/user}","branches_url":"https://api.github.com/repos/biocore/scikit-bio/branches{/branch}","tags_url":"https://api.github.com/repos/biocore/scikit-bio/tags","blobs_url":"https://api.github.com/repos/biocore/scikit-bio/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/biocore/scikit-bio/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/biocore/scikit-bio/git/refs{/sha}","trees_url":"https://api.github.com/repos/biocore/scikit-bio/git/trees{/sha}","statuses_url":"https://api.github.com/repos/biocore/scikit-bio/statuses/{sha}","languages_url":"https://api.github.com/repos/biocore/scikit-bio/languages","stargazers_url":"https://api.github.com/repos/biocore/scikit-bio/stargazers","contributors_url":"https://api.github.com/repos/biocore/scikit-bio/contributors","subscribers_url":"https://api.github.com/repos/biocore/scikit-bio/subscribers","subscription_url":"https://api.github.com/repos/biocore/scikit-bio/subscription","commits_url":"https://api.github.com/repos/biocore/scikit-bio/commits{/sha}","git_commits_url":"https://api.github.com/repos/biocore/scikit-bio/git/commits{/sha}","comments_url":"https://api.github.com/repos/biocore/scikit-bio/comments{/number}","issue_comment_url":"https://api.github.com/repos/biocore/scikit-bio/issues/comments{/number}","contents_url":"https://api.github.com/repos/biocore/scikit-bio/contents/{+path}","compare_url":"https://api.github.com/repos/biocore/scikit-bio/compare/{base}...{head}","merges_url":"https://api.github.com/repos/biocore/scikit-bio/merges","archive_url":"https://api.github.com/repos/biocore/scikit-bio/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/biocore/scikit-bio/downloads","issues_url":"https://api.github.com/repos/biocore/scikit-bio/issues{/number}","pulls_url":"https://api.github.com/repos/biocore/scikit-bio/pulls{/number}","milestones_url":"https://api.github.com/repos/biocore/scikit-bio/milestones{/number}","notifications_url":"https://api.github.com/repos/biocore/scikit-bio/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/biocore/scikit-bio/labels{/name}","releases_url":"https://api.github.com/repos/biocore/scikit-bio/releases{/id}","deployments_url":"https://api.github.com/repos/biocore/scikit-bio/deployments","created_at":"2013-12-13T16:24:41Z","updated_at":"2016-10-02T13:37:51Z","pushed_at":"2016-10-07T15:43:10Z","git_url":"git://github.com/biocore/scikit-bio.git","ssh_url":"git@github.com:biocore/scikit-bio.git","clone_url":"https://github.com/biocore/scikit-bio.git","svn_url":"https://github.com/biocore/scikit-bio","homepage":"http://scikit-bio.org","size":43272,"stargazers_count":233,"watchers_count":233,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":109,"mirror_url":null,"open_issues_count":164,"forks":109,"open_issues":164,"watchers":233,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/biocore/scikit-bio/pulls/1352"},"html":{"href":"https://github.com/biocore/scikit-bio/pull/1352"},"issue":{"href":"https://api.github.com/repos/biocore/scikit-bio/issues/1352"},"comments":{"href":"https://api.github.com/repos/biocore/scikit-bio/issues/1352/comments"},"review_comments":{"href":"https://api.github.com/repos/biocore/scikit-bio/pulls/1352/comments"},"review_comment":{"href":"https://api.github.com/repos/biocore/scikit-bio/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/biocore/scikit-bio/pulls/1352/commits"},"statuses":{"href":"https://api.github.com/repos/biocore/scikit-bio/statuses/a055c791a0a8e556c1f769a053c95712956a9d10"}}}}
{"actor":{"display_login":"ElDeveloper"}}