{"action":"created","comment":{"url":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/comments/72359290","id":72359290,"diff_hunk":"@@ -0,0 +1,216 @@\n+# Author : Raghav RV <rvraghav93@gmail.com>\n+#\n+# Licence : BSD 3 clause\n+\n+import numpy as np\n+\n+from ..utils import check_X_y\n+from ..utils import check_random_state\n+\n+__all__ = [\"drop_values\"]\n+\n+\n+def drop_values(X, y=None, missing_mask=None,\n+                missing_values=\"NaN\",\n+                missing_fraction=0.1,\n+                label_correlation=1.0,\n+                n_labels=1,\n+                labels=None,\n+                missing_mask_only=False,\n+                return_labels=False,\n+                return_missing_mask=False,\n+                copy=False,\n+                random_state=None):\n+    \"\"\"Drop values based on a preset strategy.\n+\n+    Attributes\n+    ----------\n+\n+    X : ndarray like of shape (n_features, n_samples)\n+\n+        Data, in which the values must be dropped and set to\n+        missing_values.\n+\n+    y : ndarray like of shape (n_samples,), optional\n+        Target, in a supervised classification task.\n+\n+    missing_mask : bool ndarray shape (n_features, n_samples), optional\n+        This is used to either denote the missing values that already\n+        exist in the data or simply to specify a missing mask to modify\n+        inplace.\n+\n+    missing_values : {\"NaN\" (or np.nan) | int}, default \"NaN\"\n+        The missing value to use\n+\n+    label_correlation : float, default 1.0\n+        1 (MNAR) - Randomly choose n_targets (or take from given\n+        targets) and correlate the missing values with the occurence\n+        of those target labels.\n+\n+        0 (MCAR) - Randomly drop values without correlating the\n+        missingness with any target.\n+\n+        Any value inbetween would constitute a noisy MNAR missingness.\n+\n+    n_labels : int, optional, default 1\n+        The number of labels to pick at random and correlate with\n+\n+    labels : 1D list/ndarray, optional, default None\n+        The list of labels (must match with the labels in y) to\n+        correlate with.\n+\n+        If this is specified n_labels argument is ignored.\n+\n+    missing_mask_only : bool, default False\n+        Whether to modify/return only the missing mask without\n+        touching the actual data X.\n+\n+    return_missing_mask : bool, default False\n+        Whether to return the missing mask along with the data (X, y)\n+\n+    return_labels : bool, default False\n+        Whether to return the picked labels\n+\n+    copy : bool, default False\n+        Whether to copy the data (and missing_mask) or work inplace.\n+\n+    verbose : bool, default False\n+        Whether to print the number of correlated/non-correlated missing values\n+        that will be introduced.\n+\n+    random_state : int, optional\n+        The seed for the numpy's random number generator.\n+    Returns\n+    -------\n+\n+    X, y (, missing_mask, labels) : Tuple\n+        Returns missing_mask if return_missing_mask is set to True\n+        Returns labels if return_labels is set to True\n+\n+    \"\"\"\n+    X, y = check_X_y(X, y)\n+\n+    if missing_mask_only and not return_missing_mask:\n+        raise ValueError(\"Both missing_mask_only and return_missing_mask\"\n+                         \"cannot be True\")\n+\n+    if missing_fraction >= 1:\n+        raise ValueError(\"The missing_fraction cannot be greater than\"\n+                         \" or equal to 1.\")\n+\n+    if copy:\n+        X = X.copy()\n+        if missing_mask is not None:\n+            missing_mask = missing_mask.copy()\n+\n+    if (isinstance(missing_values, str) and\n+            missing_values.lower() == \"NaN\"):\n+        missing_values = np.nan\n+\n+    if missing_mask is None:\n+        if np.isnan(missing_values):\n+            missing_mask = np.isnan(X)\n+        else:\n+            missing_mask = X == missing_values\n+\n+    n_samples, n_features = X.shape\n+    n_elements = n_samples * n_features\n+\n+    current_n_missing = np.count_nonzero(missing_mask)\n+    required_n_missing = int(missing_fraction * n_elements)\n+\n+    if current_n_missing > required_n_missing:\n+        raise ValueError(\"There are currently %d missing values, \"\n+                         \"which is >= a fraction of %0.2f that is\"\n+                         \"expected to be missing.\"\n+                         % (current_n_missing, missing_fraction))\n+\n+    rng = check_random_state(random_state)\n+    n_more_missing = required_n_missing - current_n_missing\n+\n+    unique_labels = np.unique(y)\n+    n_unique_labels = len(unique_labels)\n+\n+    if labels is None:\n+        # Labels is an int specifying the no of labels to correlate\n+        # with\n+\n+        if n_labels > n_unique_labels:\n+            raise ValueError(\"The n_labels (%d) is greater than\"\n+                             \" no of unique labels in y (%d)\"\n+                             % (n_labels, n_unique_labels))\n+\n+        labels = rng.choice(n_unique_labels, n_labels, replace=False)\n+        # Reset the RNG as we don't want this operation to affect\n+        # the random selection\n+        rng = check_random_state(random_state)\n+\n+    # Filter based on labels\n+    n_correlated_missing = int(n_more_missing * label_correlation)\n+    n_non_correlated_missing = n_more_missing - n_correlated_missing\n+    if verbose:\n+        print(\"There are %d correlated and %d non-correlated new\"\n+              \"missing values introduced.\"\n+              % (n_correlated_missing, n_non_correlated_missing))\n+\n+    label_mask = np.zeros(missing_mask.shape, dtype=bool)\n+    if label_correlation != 0:\n+        for label in labels:\n+            label_mask[y==label] = True\n+\n+    # The logic of MCAR/MNAR is implemented here\n+    inv_missing_mask = ~missing_mask\n+    corr_available = inv_missing_mask & label_mask\n+    uncorr_available = inv_missing_mask & ~label_mask\n+\n+    n_corr_available = np.count_nonzero(corr_available)\n+    n_uncorr_available = np.count_nonzero(uncorr_available)\n+\n+    n_available = n_corr_available + n_uncorr_available\n+\n+    if n_available < n_more_missing:\n+        raise ValueError(\"There are only %d values available for \"\n+                         \"dropping. %d more are needed to reach\"\n+                         \" the missing_fraction of %0.2f\"\n+                         % (n_available, n_more_missing,\n+                            missing_fraction))\n+\n+    n_corr_chosen = int(n_more_missing * label_correlation)","path":"sklearn/preprocessing/value_dropper.py","position":178,"original_position":178,"commit_id":"7f9f866894c4a91d7df85ebeecaa6f434bf550dd","original_commit_id":"7f9f866894c4a91d7df85ebeecaa6f434bf550dd","user":{"login":"jnothman","id":78827,"avatar_url":"https://avatars.githubusercontent.com/u/78827?v=3","gravatar_id":"","url":"https://api.github.com/users/jnothman","html_url":"https://github.com/jnothman","followers_url":"https://api.github.com/users/jnothman/followers","following_url":"https://api.github.com/users/jnothman/following{/other_user}","gists_url":"https://api.github.com/users/jnothman/gists{/gist_id}","starred_url":"https://api.github.com/users/jnothman/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jnothman/subscriptions","organizations_url":"https://api.github.com/users/jnothman/orgs","repos_url":"https://api.github.com/users/jnothman/repos","events_url":"https://api.github.com/users/jnothman/events{/privacy}","received_events_url":"https://api.github.com/users/jnothman/received_events","type":"User","site_admin":false},"body":"Is it possible for this to be greater than `n_corr_available`?","created_at":"2016-07-26T23:51:34Z","updated_at":"2016-07-26T23:51:34Z","html_url":"https://github.com/scikit-learn/scikit-learn/pull/7084#discussion_r72359290","pull_request_url":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/7084","_links":{"self":{"href":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/comments/72359290"},"html":{"href":"https://github.com/scikit-learn/scikit-learn/pull/7084#discussion_r72359290"},"pull_request":{"href":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/7084"}}},"pull_request":{"url":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/7084","id":78865546,"html_url":"https://github.com/scikit-learn/scikit-learn/pull/7084","diff_url":"https://github.com/scikit-learn/scikit-learn/pull/7084.diff","patch_url":"https://github.com/scikit-learn/scikit-learn/pull/7084.patch","issue_url":"https://api.github.com/repos/scikit-learn/scikit-learn/issues/7084","number":7084,"state":"open","locked":false,"title":"[WIP] ENH add a value_dropper to artificially add missing_values to dataset","user":{"login":"raghavrv","id":9487348,"avatar_url":"https://avatars.githubusercontent.com/u/9487348?v=3","gravatar_id":"","url":"https://api.github.com/users/raghavrv","html_url":"https://github.com/raghavrv","followers_url":"https://api.github.com/users/raghavrv/followers","following_url":"https://api.github.com/users/raghavrv/following{/other_user}","gists_url":"https://api.github.com/users/raghavrv/gists{/gist_id}","starred_url":"https://api.github.com/users/raghavrv/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/raghavrv/subscriptions","organizations_url":"https://api.github.com/users/raghavrv/orgs","repos_url":"https://api.github.com/users/raghavrv/repos","events_url":"https://api.github.com/users/raghavrv/events{/privacy}","received_events_url":"https://api.github.com/users/raghavrv/received_events","type":"User","site_admin":false},"body":"Fixes #6284 \r\n\r\n`value_dropper` function has a `label_correlation` attribute which will control the extent to which missingness will correlate with the (given) `labels`.\r\n\r\n- [x] `value_dropper` to introduce missing values.\r\n- [ ] Tests\r\n- [ ] Narrative doc\r\n\r\nI'd add an example for #5974 which will use this function to compare the missing value handling strategy for MCAR/MNAR missingness. \r\n\r\n@agramfort @GaelVaroquaux @amueller @jnothman @glouppe @MechCoder ","created_at":"2016-07-26T14:49:38Z","updated_at":"2016-07-26T23:51:34Z","closed_at":null,"merged_at":null,"merge_commit_sha":"4e451bef13c00c5242fb5f986302d00217287c98","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/7084/commits","review_comments_url":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/7084/comments","review_comment_url":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/comments{/number}","comments_url":"https://api.github.com/repos/scikit-learn/scikit-learn/issues/7084/comments","statuses_url":"https://api.github.com/repos/scikit-learn/scikit-learn/statuses/7f9f866894c4a91d7df85ebeecaa6f434bf550dd","head":{"label":"raghavrv:value_dropper","ref":"value_dropper","sha":"7f9f866894c4a91d7df85ebeecaa6f434bf550dd","user":{"login":"raghavrv","id":9487348,"avatar_url":"https://avatars.githubusercontent.com/u/9487348?v=3","gravatar_id":"","url":"https://api.github.com/users/raghavrv","html_url":"https://github.com/raghavrv","followers_url":"https://api.github.com/users/raghavrv/followers","following_url":"https://api.github.com/users/raghavrv/following{/other_user}","gists_url":"https://api.github.com/users/raghavrv/gists{/gist_id}","starred_url":"https://api.github.com/users/raghavrv/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/raghavrv/subscriptions","organizations_url":"https://api.github.com/users/raghavrv/orgs","repos_url":"https://api.github.com/users/raghavrv/repos","events_url":"https://api.github.com/users/raghavrv/events{/privacy}","received_events_url":"https://api.github.com/users/raghavrv/received_events","type":"User","site_admin":false},"repo":{"id":26038110,"name":"scikit-learn","full_name":"raghavrv/scikit-learn","owner":{"login":"raghavrv","id":9487348,"avatar_url":"https://avatars.githubusercontent.com/u/9487348?v=3","gravatar_id":"","url":"https://api.github.com/users/raghavrv","html_url":"https://github.com/raghavrv","followers_url":"https://api.github.com/users/raghavrv/followers","following_url":"https://api.github.com/users/raghavrv/following{/other_user}","gists_url":"https://api.github.com/users/raghavrv/gists{/gist_id}","starred_url":"https://api.github.com/users/raghavrv/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/raghavrv/subscriptions","organizations_url":"https://api.github.com/users/raghavrv/orgs","repos_url":"https://api.github.com/users/raghavrv/repos","events_url":"https://api.github.com/users/raghavrv/events{/privacy}","received_events_url":"https://api.github.com/users/raghavrv/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/raghavrv/scikit-learn","description":"scikit-learn: machine learning in Python","fork":true,"url":"https://api.github.com/repos/raghavrv/scikit-learn","forks_url":"https://api.github.com/repos/raghavrv/scikit-learn/forks","keys_url":"https://api.github.com/repos/raghavrv/scikit-learn/keys{/key_id}","collaborators_url":"https://api.github.com/repos/raghavrv/scikit-learn/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/raghavrv/scikit-learn/teams","hooks_url":"https://api.github.com/repos/raghavrv/scikit-learn/hooks","issue_events_url":"https://api.github.com/repos/raghavrv/scikit-learn/issues/events{/number}","events_url":"https://api.github.com/repos/raghavrv/scikit-learn/events","assignees_url":"https://api.github.com/repos/raghavrv/scikit-learn/assignees{/user}","branches_url":"https://api.github.com/repos/raghavrv/scikit-learn/branches{/branch}","tags_url":"https://api.github.com/repos/raghavrv/scikit-learn/tags","blobs_url":"https://api.github.com/repos/raghavrv/scikit-learn/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/raghavrv/scikit-learn/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/raghavrv/scikit-learn/git/refs{/sha}","trees_url":"https://api.github.com/repos/raghavrv/scikit-learn/git/trees{/sha}","statuses_url":"https://api.github.com/repos/raghavrv/scikit-learn/statuses/{sha}","languages_url":"https://api.github.com/repos/raghavrv/scikit-learn/languages","stargazers_url":"https://api.github.com/repos/raghavrv/scikit-learn/stargazers","contributors_url":"https://api.github.com/repos/raghavrv/scikit-learn/contributors","subscribers_url":"https://api.github.com/repos/raghavrv/scikit-learn/subscribers","subscription_url":"https://api.github.com/repos/raghavrv/scikit-learn/subscription","commits_url":"https://api.github.com/repos/raghavrv/scikit-learn/commits{/sha}","git_commits_url":"https://api.github.com/repos/raghavrv/scikit-learn/git/commits{/sha}","comments_url":"https://api.github.com/repos/raghavrv/scikit-learn/comments{/number}","issue_comment_url":"https://api.github.com/repos/raghavrv/scikit-learn/issues/comments{/number}","contents_url":"https://api.github.com/repos/raghavrv/scikit-learn/contents/{+path}","compare_url":"https://api.github.com/repos/raghavrv/scikit-learn/compare/{base}...{head}","merges_url":"https://api.github.com/repos/raghavrv/scikit-learn/merges","archive_url":"https://api.github.com/repos/raghavrv/scikit-learn/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/raghavrv/scikit-learn/downloads","issues_url":"https://api.github.com/repos/raghavrv/scikit-learn/issues{/number}","pulls_url":"https://api.github.com/repos/raghavrv/scikit-learn/pulls{/number}","milestones_url":"https://api.github.com/repos/raghavrv/scikit-learn/milestones{/number}","notifications_url":"https://api.github.com/repos/raghavrv/scikit-learn/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/raghavrv/scikit-learn/labels{/name}","releases_url":"https://api.github.com/repos/raghavrv/scikit-learn/releases{/id}","deployments_url":"https://api.github.com/repos/raghavrv/scikit-learn/deployments","created_at":"2014-11-01T00:56:31Z","updated_at":"2016-04-27T18:29:41Z","pushed_at":"2016-07-26T14:43:21Z","git_url":"git://github.com/raghavrv/scikit-learn.git","ssh_url":"git@github.com:raghavrv/scikit-learn.git","clone_url":"https://github.com/raghavrv/scikit-learn.git","svn_url":"https://github.com/raghavrv/scikit-learn","homepage":"http://scikit-learn.org","size":91394,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":1,"mirror_url":null,"open_issues_count":1,"forks":1,"open_issues":1,"watchers":0,"default_branch":"master"}},"base":{"label":"scikit-learn:master","ref":"master","sha":"ab6dd8957e537205f153b847240d262374b034f6","user":{"login":"scikit-learn","id":365630,"avatar_url":"https://avatars.githubusercontent.com/u/365630?v=3","gravatar_id":"","url":"https://api.github.com/users/scikit-learn","html_url":"https://github.com/scikit-learn","followers_url":"https://api.github.com/users/scikit-learn/followers","following_url":"https://api.github.com/users/scikit-learn/following{/other_user}","gists_url":"https://api.github.com/users/scikit-learn/gists{/gist_id}","starred_url":"https://api.github.com/users/scikit-learn/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/scikit-learn/subscriptions","organizations_url":"https://api.github.com/users/scikit-learn/orgs","repos_url":"https://api.github.com/users/scikit-learn/repos","events_url":"https://api.github.com/users/scikit-learn/events{/privacy}","received_events_url":"https://api.github.com/users/scikit-learn/received_events","type":"Organization","site_admin":false},"repo":{"id":843222,"name":"scikit-learn","full_name":"scikit-learn/scikit-learn","owner":{"login":"scikit-learn","id":365630,"avatar_url":"https://avatars.githubusercontent.com/u/365630?v=3","gravatar_id":"","url":"https://api.github.com/users/scikit-learn","html_url":"https://github.com/scikit-learn","followers_url":"https://api.github.com/users/scikit-learn/followers","following_url":"https://api.github.com/users/scikit-learn/following{/other_user}","gists_url":"https://api.github.com/users/scikit-learn/gists{/gist_id}","starred_url":"https://api.github.com/users/scikit-learn/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/scikit-learn/subscriptions","organizations_url":"https://api.github.com/users/scikit-learn/orgs","repos_url":"https://api.github.com/users/scikit-learn/repos","events_url":"https://api.github.com/users/scikit-learn/events{/privacy}","received_events_url":"https://api.github.com/users/scikit-learn/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/scikit-learn/scikit-learn","description":"scikit-learn: machine learning in Python","fork":false,"url":"https://api.github.com/repos/scikit-learn/scikit-learn","forks_url":"https://api.github.com/repos/scikit-learn/scikit-learn/forks","keys_url":"https://api.github.com/repos/scikit-learn/scikit-learn/keys{/key_id}","collaborators_url":"https://api.github.com/repos/scikit-learn/scikit-learn/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/scikit-learn/scikit-learn/teams","hooks_url":"https://api.github.com/repos/scikit-learn/scikit-learn/hooks","issue_events_url":"https://api.github.com/repos/scikit-learn/scikit-learn/issues/events{/number}","events_url":"https://api.github.com/repos/scikit-learn/scikit-learn/events","assignees_url":"https://api.github.com/repos/scikit-learn/scikit-learn/assignees{/user}","branches_url":"https://api.github.com/repos/scikit-learn/scikit-learn/branches{/branch}","tags_url":"https://api.github.com/repos/scikit-learn/scikit-learn/tags","blobs_url":"https://api.github.com/repos/scikit-learn/scikit-learn/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/scikit-learn/scikit-learn/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/scikit-learn/scikit-learn/git/refs{/sha}","trees_url":"https://api.github.com/repos/scikit-learn/scikit-learn/git/trees{/sha}","statuses_url":"https://api.github.com/repos/scikit-learn/scikit-learn/statuses/{sha}","languages_url":"https://api.github.com/repos/scikit-learn/scikit-learn/languages","stargazers_url":"https://api.github.com/repos/scikit-learn/scikit-learn/stargazers","contributors_url":"https://api.github.com/repos/scikit-learn/scikit-learn/contributors","subscribers_url":"https://api.github.com/repos/scikit-learn/scikit-learn/subscribers","subscription_url":"https://api.github.com/repos/scikit-learn/scikit-learn/subscription","commits_url":"https://api.github.com/repos/scikit-learn/scikit-learn/commits{/sha}","git_commits_url":"https://api.github.com/repos/scikit-learn/scikit-learn/git/commits{/sha}","comments_url":"https://api.github.com/repos/scikit-learn/scikit-learn/comments{/number}","issue_comment_url":"https://api.github.com/repos/scikit-learn/scikit-learn/issues/comments{/number}","contents_url":"https://api.github.com/repos/scikit-learn/scikit-learn/contents/{+path}","compare_url":"https://api.github.com/repos/scikit-learn/scikit-learn/compare/{base}...{head}","merges_url":"https://api.github.com/repos/scikit-learn/scikit-learn/merges","archive_url":"https://api.github.com/repos/scikit-learn/scikit-learn/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/scikit-learn/scikit-learn/downloads","issues_url":"https://api.github.com/repos/scikit-learn/scikit-learn/issues{/number}","pulls_url":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls{/number}","milestones_url":"https://api.github.com/repos/scikit-learn/scikit-learn/milestones{/number}","notifications_url":"https://api.github.com/repos/scikit-learn/scikit-learn/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/scikit-learn/scikit-learn/labels{/name}","releases_url":"https://api.github.com/repos/scikit-learn/scikit-learn/releases{/id}","deployments_url":"https://api.github.com/repos/scikit-learn/scikit-learn/deployments","created_at":"2010-08-17T09:43:38Z","updated_at":"2016-07-26T23:06:41Z","pushed_at":"2016-07-26T19:58:34Z","git_url":"git://github.com/scikit-learn/scikit-learn.git","ssh_url":"git@github.com:scikit-learn/scikit-learn.git","clone_url":"https://github.com/scikit-learn/scikit-learn.git","svn_url":"https://github.com/scikit-learn/scikit-learn","homepage":"http://scikit-learn.org","size":121806,"stargazers_count":12546,"watchers_count":12546,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":7264,"mirror_url":null,"open_issues_count":1245,"forks":7264,"open_issues":1245,"watchers":12546,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/7084"},"html":{"href":"https://github.com/scikit-learn/scikit-learn/pull/7084"},"issue":{"href":"https://api.github.com/repos/scikit-learn/scikit-learn/issues/7084"},"comments":{"href":"https://api.github.com/repos/scikit-learn/scikit-learn/issues/7084/comments"},"review_comments":{"href":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/7084/comments"},"review_comment":{"href":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/7084/commits"},"statuses":{"href":"https://api.github.com/repos/scikit-learn/scikit-learn/statuses/7f9f866894c4a91d7df85ebeecaa6f434bf550dd"}}}}
{"actor":{"display_login":"jnothman"}}