{"action":"created","comment":{"url":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/comments/307165980","pull_request_review_id":266457254,"id":307165980,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDMwNzE2NTk4MA==","diff_hunk":"@@ -1254,6 +1256,119 @@ def test_ranking_loss_ties_handling():\n     assert_almost_equal(label_ranking_loss([[1, 1, 0]], [[0.25, 0.5, 0.5]]), 1)\n \n \n+def test_dcg_score():\n+    _, y_true = make_multilabel_classification(random_state=0, n_classes=10)\n+    y_score = - y_true + 1\n+    _test_dcg_score_for(y_true, y_score)\n+    y_true, y_score = np.random.RandomState(0).random_sample((2, 100, 10))\n+    _test_dcg_score_for(y_true, y_score)\n+\n+\n+def _test_dcg_score_for(y_true, y_score):\n+    discount = np.log2(np.arange(y_true.shape[1]) + 2)\n+    ideal = _dcg_sample_scores(y_true, y_true)\n+    score = _dcg_sample_scores(y_true, y_score)\n+    assert (score <= ideal).all()\n+    assert (_dcg_sample_scores(y_true, y_true, k=5) <= ideal).all()\n+    assert ideal.shape == (y_true.shape[0], )\n+    assert score.shape == (y_true.shape[0], )\n+    assert ideal == pytest.approx(\n+        (np.sort(y_true)[:, ::-1] / discount).sum(axis=1))\n+\n+\n+def test_dcg_ties():\n+    y_true = np.asarray([np.arange(5)])\n+    y_score = np.zeros(y_true.shape)\n+    dcg = _dcg_sample_scores(y_true, y_score)\n+    dcg_ignore_ties = _dcg_sample_scores(y_true, y_score, ignore_ties=True)\n+    discounts = 1 / np.log2(np.arange(2, 7))\n+    assert dcg == pytest.approx([discounts.sum() * y_true.mean()])\n+    assert dcg_ignore_ties == pytest.approx(\n+        [(discounts * y_true[:, ::-1]).sum()])\n+    y_score[0, 3:] = 1\n+    dcg = _dcg_sample_scores(y_true, y_score)\n+    dcg_ignore_ties = _dcg_sample_scores(y_true, y_score, ignore_ties=True)\n+    assert dcg_ignore_ties == pytest.approx(\n+        [(discounts * y_true[:, ::-1]).sum()])\n+    assert dcg == pytest.approx([\n+        discounts[:2].sum() * y_true[0, 3:].mean() +\n+        discounts[2:].sum() * y_true[0, :3].mean()\n+    ])\n+\n+\n+def test_ndcg_ignore_ties_with_k():\n+    a = np.arange(12).reshape((2, 6))\n+    ndcg_score(a, a, k=3, ignore_ties=True)\n+\n+\n+def test_ndcg_invariant():\n+    y_true = np.arange(70).reshape(7, 10)\n+    y_score = y_true + np.random.RandomState(0).uniform(\n+        -.2, .2, size=y_true.shape)\n+    ndcg = ndcg_score(y_true, y_score)\n+    ndcg_no_ties = ndcg_score(y_true, y_score, ignore_ties=True)\n+    assert ndcg == pytest.approx(ndcg_no_ties)\n+    assert ndcg == pytest.approx(1.)\n+    y_score += 1000\n+    assert ndcg_score(y_true, y_score) == pytest.approx(1.)\n+\n+\n+def test_ndcg_toy_examples():\n+    y_true = 3 * np.eye(7)[:5]\n+    y_score = np.tile(np.arange(6, -1, -1), (5, 1))\n+    y_score_noisy = y_score + np.random.RandomState(0).uniform(\n+        -.2, .2, size=y_score.shape)\n+    assert _dcg_sample_scores(y_true, y_score) == pytest.approx(\n+        3 / np.log2(np.arange(2, 7)))\n+    assert _dcg_sample_scores(y_true, y_score) == pytest.approx(","path":"sklearn/metrics/tests/test_ranking.py","position":77,"original_position":77,"commit_id":"67f4dc31a4f69ab4dfe6dcfd8cb6562608a120b8","original_commit_id":"67f4dc31a4f69ab4dfe6dcfd8cb6562608a120b8","user":{"login":"jnothman","id":78827,"node_id":"MDQ6VXNlcjc4ODI3","avatar_url":"https://avatars2.githubusercontent.com/u/78827?v=4","gravatar_id":"","url":"https://api.github.com/users/jnothman","html_url":"https://github.com/jnothman","followers_url":"https://api.github.com/users/jnothman/followers","following_url":"https://api.github.com/users/jnothman/following{/other_user}","gists_url":"https://api.github.com/users/jnothman/gists{/gist_id}","starred_url":"https://api.github.com/users/jnothman/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jnothman/subscriptions","organizations_url":"https://api.github.com/users/jnothman/orgs","repos_url":"https://api.github.com/users/jnothman/repos","events_url":"https://api.github.com/users/jnothman/events{/privacy}","received_events_url":"https://api.github.com/users/jnothman/received_events","type":"User","site_admin":false},"body":"Can we use pytest.mark.parameterize to test the ignore_ties equivalence?","created_at":"2019-07-25T08:09:25Z","updated_at":"2019-07-25T08:14:14Z","html_url":"https://github.com/scikit-learn/scikit-learn/pull/9951#discussion_r307165980","pull_request_url":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/9951","author_association":"MEMBER","_links":{"self":{"href":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/comments/307165980"},"html":{"href":"https://github.com/scikit-learn/scikit-learn/pull/9951#discussion_r307165980"},"pull_request":{"href":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/9951"}}},"pull_request":{"url":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/9951","id":147388210,"node_id":"MDExOlB1bGxSZXF1ZXN0MTQ3Mzg4MjEw","html_url":"https://github.com/scikit-learn/scikit-learn/pull/9951","diff_url":"https://github.com/scikit-learn/scikit-learn/pull/9951.diff","patch_url":"https://github.com/scikit-learn/scikit-learn/pull/9951.patch","issue_url":"https://api.github.com/repos/scikit-learn/scikit-learn/issues/9951","number":9951,"state":"open","locked":false,"title":"[MRG+1] Add Normalized Discounted Cumulative Gain","user":{"login":"jeromedockes","id":9196501,"node_id":"MDQ6VXNlcjkxOTY1MDE=","avatar_url":"https://avatars3.githubusercontent.com/u/9196501?v=4","gravatar_id":"","url":"https://api.github.com/users/jeromedockes","html_url":"https://github.com/jeromedockes","followers_url":"https://api.github.com/users/jeromedockes/followers","following_url":"https://api.github.com/users/jeromedockes/following{/other_user}","gists_url":"https://api.github.com/users/jeromedockes/gists{/gist_id}","starred_url":"https://api.github.com/users/jeromedockes/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jeromedockes/subscriptions","organizations_url":"https://api.github.com/users/jeromedockes/orgs","repos_url":"https://api.github.com/users/jeromedockes/repos","events_url":"https://api.github.com/users/jeromedockes/events{/privacy}","received_events_url":"https://api.github.com/users/jeromedockes/received_events","type":"User","site_admin":false},"body":"After #9921, it was decided that the old implementation of NDCG would be removed (#9932), but that a new one might be useful.\r\n\r\nDiscounted Cumulative Gain and Normalized Discounted Cumulative Gain are popular ranking metrics (https://en.wikipedia.org/wiki/Discounted_cumulative_gain)\r\n\r\nTODO:\r\n\r\n- [x] handle ties correctly\r\n- [x] test on toy examples\r\n- [x] test boundary cases (all scores equal for some samples, perfect score)\r\n- [x] test invariants due to perturbing perfect y_score\r\n- [x] add narrative docs.\r\n- [x] examples section in the docstring\r\n- [x] add public functions to tests","created_at":"2017-10-18T19:22:30Z","updated_at":"2019-07-25T08:14:13Z","closed_at":null,"merged_at":null,"merge_commit_sha":"7cd6a70468d441ea38833d341e62e8a777a0af00","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[{"id":12995756,"node_id":"MDU6TGFiZWwxMjk5NTc1Ng==","url":"https://api.github.com/repos/scikit-learn/scikit-learn/labels/New%20Feature","name":"New Feature","color":"02e10c","default":false}],"milestone":null,"commits_url":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/9951/commits","review_comments_url":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/9951/comments","review_comment_url":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/comments{/number}","comments_url":"https://api.github.com/repos/scikit-learn/scikit-learn/issues/9951/comments","statuses_url":"https://api.github.com/repos/scikit-learn/scikit-learn/statuses/67f4dc31a4f69ab4dfe6dcfd8cb6562608a120b8","head":{"label":"jeromedockes:add_ndcg","ref":"add_ndcg","sha":"67f4dc31a4f69ab4dfe6dcfd8cb6562608a120b8","user":{"login":"jeromedockes","id":9196501,"node_id":"MDQ6VXNlcjkxOTY1MDE=","avatar_url":"https://avatars3.githubusercontent.com/u/9196501?v=4","gravatar_id":"","url":"https://api.github.com/users/jeromedockes","html_url":"https://github.com/jeromedockes","followers_url":"https://api.github.com/users/jeromedockes/followers","following_url":"https://api.github.com/users/jeromedockes/following{/other_user}","gists_url":"https://api.github.com/users/jeromedockes/gists{/gist_id}","starred_url":"https://api.github.com/users/jeromedockes/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jeromedockes/subscriptions","organizations_url":"https://api.github.com/users/jeromedockes/orgs","repos_url":"https://api.github.com/users/jeromedockes/repos","events_url":"https://api.github.com/users/jeromedockes/events{/privacy}","received_events_url":"https://api.github.com/users/jeromedockes/received_events","type":"User","site_admin":false},"repo":{"id":107433235,"node_id":"MDEwOlJlcG9zaXRvcnkxMDc0MzMyMzU=","name":"scikit-learn","full_name":"jeromedockes/scikit-learn","private":false,"owner":{"login":"jeromedockes","id":9196501,"node_id":"MDQ6VXNlcjkxOTY1MDE=","avatar_url":"https://avatars3.githubusercontent.com/u/9196501?v=4","gravatar_id":"","url":"https://api.github.com/users/jeromedockes","html_url":"https://github.com/jeromedockes","followers_url":"https://api.github.com/users/jeromedockes/followers","following_url":"https://api.github.com/users/jeromedockes/following{/other_user}","gists_url":"https://api.github.com/users/jeromedockes/gists{/gist_id}","starred_url":"https://api.github.com/users/jeromedockes/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jeromedockes/subscriptions","organizations_url":"https://api.github.com/users/jeromedockes/orgs","repos_url":"https://api.github.com/users/jeromedockes/repos","events_url":"https://api.github.com/users/jeromedockes/events{/privacy}","received_events_url":"https://api.github.com/users/jeromedockes/received_events","type":"User","site_admin":false},"html_url":"https://github.com/jeromedockes/scikit-learn","description":"scikit-learn: machine learning in Python","fork":true,"url":"https://api.github.com/repos/jeromedockes/scikit-learn","forks_url":"https://api.github.com/repos/jeromedockes/scikit-learn/forks","keys_url":"https://api.github.com/repos/jeromedockes/scikit-learn/keys{/key_id}","collaborators_url":"https://api.github.com/repos/jeromedockes/scikit-learn/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/jeromedockes/scikit-learn/teams","hooks_url":"https://api.github.com/repos/jeromedockes/scikit-learn/hooks","issue_events_url":"https://api.github.com/repos/jeromedockes/scikit-learn/issues/events{/number}","events_url":"https://api.github.com/repos/jeromedockes/scikit-learn/events","assignees_url":"https://api.github.com/repos/jeromedockes/scikit-learn/assignees{/user}","branches_url":"https://api.github.com/repos/jeromedockes/scikit-learn/branches{/branch}","tags_url":"https://api.github.com/repos/jeromedockes/scikit-learn/tags","blobs_url":"https://api.github.com/repos/jeromedockes/scikit-learn/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/jeromedockes/scikit-learn/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/jeromedockes/scikit-learn/git/refs{/sha}","trees_url":"https://api.github.com/repos/jeromedockes/scikit-learn/git/trees{/sha}","statuses_url":"https://api.github.com/repos/jeromedockes/scikit-learn/statuses/{sha}","languages_url":"https://api.github.com/repos/jeromedockes/scikit-learn/languages","stargazers_url":"https://api.github.com/repos/jeromedockes/scikit-learn/stargazers","contributors_url":"https://api.github.com/repos/jeromedockes/scikit-learn/contributors","subscribers_url":"https://api.github.com/repos/jeromedockes/scikit-learn/subscribers","subscription_url":"https://api.github.com/repos/jeromedockes/scikit-learn/subscription","commits_url":"https://api.github.com/repos/jeromedockes/scikit-learn/commits{/sha}","git_commits_url":"https://api.github.com/repos/jeromedockes/scikit-learn/git/commits{/sha}","comments_url":"https://api.github.com/repos/jeromedockes/scikit-learn/comments{/number}","issue_comment_url":"https://api.github.com/repos/jeromedockes/scikit-learn/issues/comments{/number}","contents_url":"https://api.github.com/repos/jeromedockes/scikit-learn/contents/{+path}","compare_url":"https://api.github.com/repos/jeromedockes/scikit-learn/compare/{base}...{head}","merges_url":"https://api.github.com/repos/jeromedockes/scikit-learn/merges","archive_url":"https://api.github.com/repos/jeromedockes/scikit-learn/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/jeromedockes/scikit-learn/downloads","issues_url":"https://api.github.com/repos/jeromedockes/scikit-learn/issues{/number}","pulls_url":"https://api.github.com/repos/jeromedockes/scikit-learn/pulls{/number}","milestones_url":"https://api.github.com/repos/jeromedockes/scikit-learn/milestones{/number}","notifications_url":"https://api.github.com/repos/jeromedockes/scikit-learn/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/jeromedockes/scikit-learn/labels{/name}","releases_url":"https://api.github.com/repos/jeromedockes/scikit-learn/releases{/id}","deployments_url":"https://api.github.com/repos/jeromedockes/scikit-learn/deployments","created_at":"2017-10-18T16:19:29Z","updated_at":"2019-07-19T09:54:19Z","pushed_at":"2019-07-22T08:35:43Z","git_url":"git://github.com/jeromedockes/scikit-learn.git","ssh_url":"git@github.com:jeromedockes/scikit-learn.git","clone_url":"https://github.com/jeromedockes/scikit-learn.git","svn_url":"https://github.com/jeromedockes/scikit-learn","homepage":"http://scikit-learn.org","size":104355,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"disabled":false,"open_issues_count":0,"license":{"key":"other","name":"Other","spdx_id":"NOASSERTION","url":null,"node_id":"MDc6TGljZW5zZTA="},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"scikit-learn:master","ref":"master","sha":"f82d966d7b6286aa34a11374304c0bca515f4331","user":{"login":"scikit-learn","id":365630,"node_id":"MDEyOk9yZ2FuaXphdGlvbjM2NTYzMA==","avatar_url":"https://avatars2.githubusercontent.com/u/365630?v=4","gravatar_id":"","url":"https://api.github.com/users/scikit-learn","html_url":"https://github.com/scikit-learn","followers_url":"https://api.github.com/users/scikit-learn/followers","following_url":"https://api.github.com/users/scikit-learn/following{/other_user}","gists_url":"https://api.github.com/users/scikit-learn/gists{/gist_id}","starred_url":"https://api.github.com/users/scikit-learn/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/scikit-learn/subscriptions","organizations_url":"https://api.github.com/users/scikit-learn/orgs","repos_url":"https://api.github.com/users/scikit-learn/repos","events_url":"https://api.github.com/users/scikit-learn/events{/privacy}","received_events_url":"https://api.github.com/users/scikit-learn/received_events","type":"Organization","site_admin":false},"repo":{"id":843222,"node_id":"MDEwOlJlcG9zaXRvcnk4NDMyMjI=","name":"scikit-learn","full_name":"scikit-learn/scikit-learn","private":false,"owner":{"login":"scikit-learn","id":365630,"node_id":"MDEyOk9yZ2FuaXphdGlvbjM2NTYzMA==","avatar_url":"https://avatars2.githubusercontent.com/u/365630?v=4","gravatar_id":"","url":"https://api.github.com/users/scikit-learn","html_url":"https://github.com/scikit-learn","followers_url":"https://api.github.com/users/scikit-learn/followers","following_url":"https://api.github.com/users/scikit-learn/following{/other_user}","gists_url":"https://api.github.com/users/scikit-learn/gists{/gist_id}","starred_url":"https://api.github.com/users/scikit-learn/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/scikit-learn/subscriptions","organizations_url":"https://api.github.com/users/scikit-learn/orgs","repos_url":"https://api.github.com/users/scikit-learn/repos","events_url":"https://api.github.com/users/scikit-learn/events{/privacy}","received_events_url":"https://api.github.com/users/scikit-learn/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/scikit-learn/scikit-learn","description":"scikit-learn: machine learning in Python","fork":false,"url":"https://api.github.com/repos/scikit-learn/scikit-learn","forks_url":"https://api.github.com/repos/scikit-learn/scikit-learn/forks","keys_url":"https://api.github.com/repos/scikit-learn/scikit-learn/keys{/key_id}","collaborators_url":"https://api.github.com/repos/scikit-learn/scikit-learn/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/scikit-learn/scikit-learn/teams","hooks_url":"https://api.github.com/repos/scikit-learn/scikit-learn/hooks","issue_events_url":"https://api.github.com/repos/scikit-learn/scikit-learn/issues/events{/number}","events_url":"https://api.github.com/repos/scikit-learn/scikit-learn/events","assignees_url":"https://api.github.com/repos/scikit-learn/scikit-learn/assignees{/user}","branches_url":"https://api.github.com/repos/scikit-learn/scikit-learn/branches{/branch}","tags_url":"https://api.github.com/repos/scikit-learn/scikit-learn/tags","blobs_url":"https://api.github.com/repos/scikit-learn/scikit-learn/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/scikit-learn/scikit-learn/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/scikit-learn/scikit-learn/git/refs{/sha}","trees_url":"https://api.github.com/repos/scikit-learn/scikit-learn/git/trees{/sha}","statuses_url":"https://api.github.com/repos/scikit-learn/scikit-learn/statuses/{sha}","languages_url":"https://api.github.com/repos/scikit-learn/scikit-learn/languages","stargazers_url":"https://api.github.com/repos/scikit-learn/scikit-learn/stargazers","contributors_url":"https://api.github.com/repos/scikit-learn/scikit-learn/contributors","subscribers_url":"https://api.github.com/repos/scikit-learn/scikit-learn/subscribers","subscription_url":"https://api.github.com/repos/scikit-learn/scikit-learn/subscription","commits_url":"https://api.github.com/repos/scikit-learn/scikit-learn/commits{/sha}","git_commits_url":"https://api.github.com/repos/scikit-learn/scikit-learn/git/commits{/sha}","comments_url":"https://api.github.com/repos/scikit-learn/scikit-learn/comments{/number}","issue_comment_url":"https://api.github.com/repos/scikit-learn/scikit-learn/issues/comments{/number}","contents_url":"https://api.github.com/repos/scikit-learn/scikit-learn/contents/{+path}","compare_url":"https://api.github.com/repos/scikit-learn/scikit-learn/compare/{base}...{head}","merges_url":"https://api.github.com/repos/scikit-learn/scikit-learn/merges","archive_url":"https://api.github.com/repos/scikit-learn/scikit-learn/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/scikit-learn/scikit-learn/downloads","issues_url":"https://api.github.com/repos/scikit-learn/scikit-learn/issues{/number}","pulls_url":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls{/number}","milestones_url":"https://api.github.com/repos/scikit-learn/scikit-learn/milestones{/number}","notifications_url":"https://api.github.com/repos/scikit-learn/scikit-learn/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/scikit-learn/scikit-learn/labels{/name}","releases_url":"https://api.github.com/repos/scikit-learn/scikit-learn/releases{/id}","deployments_url":"https://api.github.com/repos/scikit-learn/scikit-learn/deployments","created_at":"2010-08-17T09:43:38Z","updated_at":"2019-07-25T07:50:13Z","pushed_at":"2019-07-25T07:47:22Z","git_url":"git://github.com/scikit-learn/scikit-learn.git","ssh_url":"git@github.com:scikit-learn/scikit-learn.git","clone_url":"https://github.com/scikit-learn/scikit-learn.git","svn_url":"https://github.com/scikit-learn/scikit-learn","homepage":"https://scikit-learn.org","size":106019,"stargazers_count":36273,"watchers_count":36273,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":17787,"mirror_url":null,"archived":false,"disabled":false,"open_issues_count":1957,"license":{"key":"other","name":"Other","spdx_id":"NOASSERTION","url":null,"node_id":"MDc6TGljZW5zZTA="},"forks":17787,"open_issues":1957,"watchers":36273,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/9951"},"html":{"href":"https://github.com/scikit-learn/scikit-learn/pull/9951"},"issue":{"href":"https://api.github.com/repos/scikit-learn/scikit-learn/issues/9951"},"comments":{"href":"https://api.github.com/repos/scikit-learn/scikit-learn/issues/9951/comments"},"review_comments":{"href":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/9951/comments"},"review_comment":{"href":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/9951/commits"},"statuses":{"href":"https://api.github.com/repos/scikit-learn/scikit-learn/statuses/67f4dc31a4f69ab4dfe6dcfd8cb6562608a120b8"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"jnothman"}}