{"action":"created","comment":{"url":"https://api.github.com/repos/dask/dask/pulls/comments/55449753","id":55449753,"diff_hunk":"@@ -2152,6 +2181,99 @@ def quantile(df, q):\n     return return_type(dsk, name3, df.name, new_divisions)\n \n \n+def cov_corr(df, min_periods=None, corr=False, scalar=False):\n+    \"\"\"DataFrame covariance and pearson correlation.\n+\n+    Computes pairwise covariance or correlation of columns, excluding NA/null\n+    values.\n+\n+    Parameters\n+    ----------\n+    df : DataFrame\n+    min_periods : int, optional\n+        Minimum number of observations required per pair of columns\n+        to have a valid result.\n+    corr : bool, optional\n+        If True, compute the Pearson correlation. If False [default], compute\n+        the covariance.\n+    scalar : bool, optional\n+        If True, compute covariance between two variables as a scalar. Only\n+        valid if `df` has 2 columns.  If False [default], compute the entire\n+        covariance/correlation matrix.\n+    \"\"\"\n+    if min_periods is None:\n+        min_periods = 2\n+    elif min_periods < 2:\n+        raise ValueError(\"min_periods must be >= 2\")\n+    prefix = 'corr' if corr else 'cov'\n+    df = df._get_numeric_data()\n+    name = '{0}-agg-{1}'.format(prefix, tokenize(df, min_periods, scalar))\n+    if scalar and len(df.columns) != 2:\n+        raise ValueError(\"scalar only valid for 2 column dataframe\")\n+    k = '{0}-chunk-{1}'.format(prefix, df._name)\n+    dsk = dict(((k, i), (cov_corr_chunk, f, corr))\n+               for (i, f) in enumerate(df._keys()))\n+    dsk[(name, 0)] = (cov_corr_agg, list(dsk.keys()), df._pd, min_periods,\n+                      corr, scalar)\n+    dsk = merge(df.dask, dsk)\n+    if scalar:\n+        return Scalar(dsk, name)\n+    return DataFrame(dsk, name, df._pd, (df.columns[0], df.columns[-1]))\n+\n+\n+def cov_corr_chunk(df, corr=False):\n+    \"\"\"Chunk part of a covariance or correlation computation\"\"\"\n+    mat = df.values\n+    mask = np.isfinite(mat)\n+    keep = np.bitwise_and(mask[:, None, :], mask[:, :, None])\n+\n+    x = np.where(keep, mat[:, None, :], np.nan)\n+    sums = np.nansum(x, 0)\n+    counts = keep.astype('int').sum(0)\n+    cov = df.cov().values\n+    dtype = [('sum', sums.dtype), ('count', counts.dtype), ('cov', cov.dtype)]\n+    if corr:\n+        m = np.nansum((x - sums/np.where(counts, counts, np.nan))**2, 0)\n+        dtype.append(('m', m.dtype))\n+\n+    out = np.empty(counts.shape, dtype=dtype)\n+    out['sum'] = sums\n+    out['count'] = counts\n+    out['cov'] = cov * (counts - 1)\n+    if corr:\n+        out['m'] = m\n+    return out","path":"dask/dataframe/core.py","position":108,"original_position":108,"commit_id":"0698d9261b5a0fc1b91426a2227c9f5e27dc224e","original_commit_id":"0698d9261b5a0fc1b91426a2227c9f5e27dc224e","user":{"login":"jcrist","id":2783717,"avatar_url":"https://avatars.githubusercontent.com/u/2783717?v=3","gravatar_id":"","url":"https://api.github.com/users/jcrist","html_url":"https://github.com/jcrist","followers_url":"https://api.github.com/users/jcrist/followers","following_url":"https://api.github.com/users/jcrist/following{/other_user}","gists_url":"https://api.github.com/users/jcrist/gists{/gist_id}","starred_url":"https://api.github.com/users/jcrist/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jcrist/subscriptions","organizations_url":"https://api.github.com/users/jcrist/orgs","repos_url":"https://api.github.com/users/jcrist/repos","events_url":"https://api.github.com/users/jcrist/events{/privacy}","received_events_url":"https://api.github.com/users/jcrist/received_events","type":"User","site_admin":false},"body":"Intermediate result shape is `(len(df.columns), len(df.columns))`.","created_at":"2016-03-08T23:47:24Z","updated_at":"2016-03-08T23:47:24Z","html_url":"https://github.com/dask/dask/pull/1037#discussion_r55449753","pull_request_url":"https://api.github.com/repos/dask/dask/pulls/1037","_links":{"self":{"href":"https://api.github.com/repos/dask/dask/pulls/comments/55449753"},"html":{"href":"https://github.com/dask/dask/pull/1037#discussion_r55449753"},"pull_request":{"href":"https://api.github.com/repos/dask/dask/pulls/1037"}}},"pull_request":{"url":"https://api.github.com/repos/dask/dask/pulls/1037","id":62119663,"html_url":"https://github.com/dask/dask/pull/1037","diff_url":"https://github.com/dask/dask/pull/1037.diff","patch_url":"https://github.com/dask/dask/pull/1037.patch","issue_url":"https://api.github.com/repos/dask/dask/issues/1037","number":1037,"state":"open","locked":false,"title":"Add corr and cov for dask.dataframe","user":{"login":"jcrist","id":2783717,"avatar_url":"https://avatars.githubusercontent.com/u/2783717?v=3","gravatar_id":"","url":"https://api.github.com/users/jcrist","html_url":"https://github.com/jcrist","followers_url":"https://api.github.com/users/jcrist/followers","following_url":"https://api.github.com/users/jcrist/following{/other_user}","gists_url":"https://api.github.com/users/jcrist/gists{/gist_id}","starred_url":"https://api.github.com/users/jcrist/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jcrist/subscriptions","organizations_url":"https://api.github.com/users/jcrist/orgs","repos_url":"https://api.github.com/users/jcrist/repos","events_url":"https://api.github.com/users/jcrist/events{/privacy}","received_events_url":"https://api.github.com/users/jcrist/received_events","type":"User","site_admin":false},"body":"Implements `corr` and `cov` methods for both `DataFrame` and `Series` objects. Note that only Pearson correlation is implemented, the other methods require a sort. The computation is done in a single pass, and is numerically stable.\r\n\r\nPartially addresses #1000.","created_at":"2016-03-08T19:08:58Z","updated_at":"2016-03-08T23:47:24Z","closed_at":null,"merged_at":null,"merge_commit_sha":"5c46b9a99c2f1e1e8e377213dbb38439c30b4537","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/dask/dask/pulls/1037/commits","review_comments_url":"https://api.github.com/repos/dask/dask/pulls/1037/comments","review_comment_url":"https://api.github.com/repos/dask/dask/pulls/comments{/number}","comments_url":"https://api.github.com/repos/dask/dask/issues/1037/comments","statuses_url":"https://api.github.com/repos/dask/dask/statuses/0698d9261b5a0fc1b91426a2227c9f5e27dc224e","head":{"label":"jcrist:df_cov_corr","ref":"df_cov_corr","sha":"0698d9261b5a0fc1b91426a2227c9f5e27dc224e","user":{"login":"jcrist","id":2783717,"avatar_url":"https://avatars.githubusercontent.com/u/2783717?v=3","gravatar_id":"","url":"https://api.github.com/users/jcrist","html_url":"https://github.com/jcrist","followers_url":"https://api.github.com/users/jcrist/followers","following_url":"https://api.github.com/users/jcrist/following{/other_user}","gists_url":"https://api.github.com/users/jcrist/gists{/gist_id}","starred_url":"https://api.github.com/users/jcrist/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jcrist/subscriptions","organizations_url":"https://api.github.com/users/jcrist/orgs","repos_url":"https://api.github.com/users/jcrist/repos","events_url":"https://api.github.com/users/jcrist/events{/privacy}","received_events_url":"https://api.github.com/users/jcrist/received_events","type":"User","site_admin":false},"repo":{"id":32761805,"name":"dask","full_name":"jcrist/dask","owner":{"login":"jcrist","id":2783717,"avatar_url":"https://avatars.githubusercontent.com/u/2783717?v=3","gravatar_id":"","url":"https://api.github.com/users/jcrist","html_url":"https://github.com/jcrist","followers_url":"https://api.github.com/users/jcrist/followers","following_url":"https://api.github.com/users/jcrist/following{/other_user}","gists_url":"https://api.github.com/users/jcrist/gists{/gist_id}","starred_url":"https://api.github.com/users/jcrist/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jcrist/subscriptions","organizations_url":"https://api.github.com/users/jcrist/orgs","repos_url":"https://api.github.com/users/jcrist/repos","events_url":"https://api.github.com/users/jcrist/events{/privacy}","received_events_url":"https://api.github.com/users/jcrist/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/jcrist/dask","description":"Minimal task scheduling abstraction","fork":true,"url":"https://api.github.com/repos/jcrist/dask","forks_url":"https://api.github.com/repos/jcrist/dask/forks","keys_url":"https://api.github.com/repos/jcrist/dask/keys{/key_id}","collaborators_url":"https://api.github.com/repos/jcrist/dask/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/jcrist/dask/teams","hooks_url":"https://api.github.com/repos/jcrist/dask/hooks","issue_events_url":"https://api.github.com/repos/jcrist/dask/issues/events{/number}","events_url":"https://api.github.com/repos/jcrist/dask/events","assignees_url":"https://api.github.com/repos/jcrist/dask/assignees{/user}","branches_url":"https://api.github.com/repos/jcrist/dask/branches{/branch}","tags_url":"https://api.github.com/repos/jcrist/dask/tags","blobs_url":"https://api.github.com/repos/jcrist/dask/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/jcrist/dask/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/jcrist/dask/git/refs{/sha}","trees_url":"https://api.github.com/repos/jcrist/dask/git/trees{/sha}","statuses_url":"https://api.github.com/repos/jcrist/dask/statuses/{sha}","languages_url":"https://api.github.com/repos/jcrist/dask/languages","stargazers_url":"https://api.github.com/repos/jcrist/dask/stargazers","contributors_url":"https://api.github.com/repos/jcrist/dask/contributors","subscribers_url":"https://api.github.com/repos/jcrist/dask/subscribers","subscription_url":"https://api.github.com/repos/jcrist/dask/subscription","commits_url":"https://api.github.com/repos/jcrist/dask/commits{/sha}","git_commits_url":"https://api.github.com/repos/jcrist/dask/git/commits{/sha}","comments_url":"https://api.github.com/repos/jcrist/dask/comments{/number}","issue_comment_url":"https://api.github.com/repos/jcrist/dask/issues/comments{/number}","contents_url":"https://api.github.com/repos/jcrist/dask/contents/{+path}","compare_url":"https://api.github.com/repos/jcrist/dask/compare/{base}...{head}","merges_url":"https://api.github.com/repos/jcrist/dask/merges","archive_url":"https://api.github.com/repos/jcrist/dask/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/jcrist/dask/downloads","issues_url":"https://api.github.com/repos/jcrist/dask/issues{/number}","pulls_url":"https://api.github.com/repos/jcrist/dask/pulls{/number}","milestones_url":"https://api.github.com/repos/jcrist/dask/milestones{/number}","notifications_url":"https://api.github.com/repos/jcrist/dask/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/jcrist/dask/labels{/name}","releases_url":"https://api.github.com/repos/jcrist/dask/releases{/id}","deployments_url":"https://api.github.com/repos/jcrist/dask/deployments","created_at":"2015-03-23T22:11:35Z","updated_at":"2016-02-11T18:22:48Z","pushed_at":"2016-03-08T23:33:07Z","git_url":"git://github.com/jcrist/dask.git","ssh_url":"git@github.com:jcrist/dask.git","clone_url":"https://github.com/jcrist/dask.git","svn_url":"https://github.com/jcrist/dask","homepage":"http://dask.pydata.org","size":9295,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"dask:master","ref":"master","sha":"a0e297c3989e9416f8b035a210f385e96bb0ff4c","user":{"login":"dask","id":17131925,"avatar_url":"https://avatars.githubusercontent.com/u/17131925?v=3","gravatar_id":"","url":"https://api.github.com/users/dask","html_url":"https://github.com/dask","followers_url":"https://api.github.com/users/dask/followers","following_url":"https://api.github.com/users/dask/following{/other_user}","gists_url":"https://api.github.com/users/dask/gists{/gist_id}","starred_url":"https://api.github.com/users/dask/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dask/subscriptions","organizations_url":"https://api.github.com/users/dask/orgs","repos_url":"https://api.github.com/users/dask/repos","events_url":"https://api.github.com/users/dask/events{/privacy}","received_events_url":"https://api.github.com/users/dask/received_events","type":"Organization","site_admin":false},"repo":{"id":28782747,"name":"dask","full_name":"dask/dask","owner":{"login":"dask","id":17131925,"avatar_url":"https://avatars.githubusercontent.com/u/17131925?v=3","gravatar_id":"","url":"https://api.github.com/users/dask","html_url":"https://github.com/dask","followers_url":"https://api.github.com/users/dask/followers","following_url":"https://api.github.com/users/dask/following{/other_user}","gists_url":"https://api.github.com/users/dask/gists{/gist_id}","starred_url":"https://api.github.com/users/dask/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dask/subscriptions","organizations_url":"https://api.github.com/users/dask/orgs","repos_url":"https://api.github.com/users/dask/repos","events_url":"https://api.github.com/users/dask/events{/privacy}","received_events_url":"https://api.github.com/users/dask/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/dask/dask","description":"Task scheduling and blocked algorithms for parallel processing","fork":false,"url":"https://api.github.com/repos/dask/dask","forks_url":"https://api.github.com/repos/dask/dask/forks","keys_url":"https://api.github.com/repos/dask/dask/keys{/key_id}","collaborators_url":"https://api.github.com/repos/dask/dask/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/dask/dask/teams","hooks_url":"https://api.github.com/repos/dask/dask/hooks","issue_events_url":"https://api.github.com/repos/dask/dask/issues/events{/number}","events_url":"https://api.github.com/repos/dask/dask/events","assignees_url":"https://api.github.com/repos/dask/dask/assignees{/user}","branches_url":"https://api.github.com/repos/dask/dask/branches{/branch}","tags_url":"https://api.github.com/repos/dask/dask/tags","blobs_url":"https://api.github.com/repos/dask/dask/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/dask/dask/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/dask/dask/git/refs{/sha}","trees_url":"https://api.github.com/repos/dask/dask/git/trees{/sha}","statuses_url":"https://api.github.com/repos/dask/dask/statuses/{sha}","languages_url":"https://api.github.com/repos/dask/dask/languages","stargazers_url":"https://api.github.com/repos/dask/dask/stargazers","contributors_url":"https://api.github.com/repos/dask/dask/contributors","subscribers_url":"https://api.github.com/repos/dask/dask/subscribers","subscription_url":"https://api.github.com/repos/dask/dask/subscription","commits_url":"https://api.github.com/repos/dask/dask/commits{/sha}","git_commits_url":"https://api.github.com/repos/dask/dask/git/commits{/sha}","comments_url":"https://api.github.com/repos/dask/dask/comments{/number}","issue_comment_url":"https://api.github.com/repos/dask/dask/issues/comments{/number}","contents_url":"https://api.github.com/repos/dask/dask/contents/{+path}","compare_url":"https://api.github.com/repos/dask/dask/compare/{base}...{head}","merges_url":"https://api.github.com/repos/dask/dask/merges","archive_url":"https://api.github.com/repos/dask/dask/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/dask/dask/downloads","issues_url":"https://api.github.com/repos/dask/dask/issues{/number}","pulls_url":"https://api.github.com/repos/dask/dask/pulls{/number}","milestones_url":"https://api.github.com/repos/dask/dask/milestones{/number}","notifications_url":"https://api.github.com/repos/dask/dask/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/dask/dask/labels{/name}","releases_url":"https://api.github.com/repos/dask/dask/releases{/id}","deployments_url":"https://api.github.com/repos/dask/dask/deployments","created_at":"2015-01-04T18:50:00Z","updated_at":"2016-03-08T19:51:33Z","pushed_at":"2016-03-08T23:33:03Z","git_url":"git://github.com/dask/dask.git","ssh_url":"git@github.com:dask/dask.git","clone_url":"https://github.com/dask/dask.git","svn_url":"https://github.com/dask/dask","homepage":"http://dask.pydata.org","size":10307,"stargazers_count":607,"watchers_count":607,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":90,"mirror_url":null,"open_issues_count":91,"forks":90,"open_issues":91,"watchers":607,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/dask/dask/pulls/1037"},"html":{"href":"https://github.com/dask/dask/pull/1037"},"issue":{"href":"https://api.github.com/repos/dask/dask/issues/1037"},"comments":{"href":"https://api.github.com/repos/dask/dask/issues/1037/comments"},"review_comments":{"href":"https://api.github.com/repos/dask/dask/pulls/1037/comments"},"review_comment":{"href":"https://api.github.com/repos/dask/dask/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/dask/dask/pulls/1037/commits"},"statuses":{"href":"https://api.github.com/repos/dask/dask/statuses/0698d9261b5a0fc1b91426a2227c9f5e27dc224e"}}}}