{"action":"created","comment":{"url":"https://api.github.com/repos/xapian/xapian/pulls/comments/194233097","pull_request_review_id":127372492,"id":194233097,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDE5NDIzMzA5Nw==","diff_hunk":"@@ -0,0 +1,218 @@\n+/** @file diversify.cc\n+ *  @brief Diversification API\n+ */\n+/* Copyright (C) 2018 Uppinder Chugh\n+ *\n+ * This program is free software; you can redistribute it and/or\n+ * modify it under the terms of the GNU General Public License as\n+ * published by the Free Software Foundation; either version 2 of the\n+ * License, or (at your option) any later version.\n+ *\n+ * This program is distributed in the hope that it will be useful,\n+ * but WITHOUT ANY WARRANTY; without even the implied warranty of\n+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n+ * GNU General Public License for more details.\n+ *\n+ * You should have received a copy of the GNU General Public License\n+ * along with this program; if not, write to the Free Software\n+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301\n+ * USA\n+ */\n+\n+#include <config.h>\n+\n+#include \"xapian/diversify.h\"\n+#include \"xapian/error.h\"\n+\n+#include \"debuglog.h\"\n+\n+#include <algorithm>\n+#include <cmath>\n+#include <limits>\n+#include <vector>\n+\n+using namespace Xapian;\n+using namespace std;\n+\n+Diversify::Diversify(unsigned int k_,\n+\t\t     double lambda_,\n+\t\t     double b_,\n+\t\t     double sigma_sqr_)\n+    : k(k_), lambda(lambda_), b(b_), sigma_sqr(sigma_sqr_)\n+{\n+    LOGCALL_CTOR(API, \"Diversify\", k_ | lambda_ | b_ | sigma_sqr_);\n+}\n+\n+string\n+Diversify::get_description() const\n+{\n+    return \"Diversify()\";\n+}\n+\n+void\n+Diversify::initialise_points(const MSet &source)\n+{\n+    LOGCALL_VOID(API, \"Diversify::initialise_points\", source);\n+    unsigned int count = 0;\n+    TermListGroup tlg(source);\n+    for (MSetIterator it = source.begin(); it != source.end(); ++it) {\n+\tpoints.emplace(*it, Xapian::Point(tlg, it.get_document()));\n+\tscores[*it] = it.get_weight();\n+\t// Initial top-k diversified documents\n+\tif (count < k) {\n+\t    main_dmset.push_back(*it);\n+\t    ++count;\n+\t}\n+    }\n+}\n+\n+pair<Xapian::docid, Xapian::docid>\n+Diversify::get_key(const Xapian::docid &docid_a, const Xapian::docid &docid_b)\n+{\n+    pair<Xapian::docid, Xapian::docid> key;\n+    if (docid_a <= docid_b) {\n+\tkey = make_pair(docid_a, docid_b);\n+    } else {\n+\tkey = make_pair(docid_b, docid_a);\n+    }\n+\n+    return key;\n+}\n+\n+void\n+Diversify::compute_similarities()\n+{\n+    LOGCALL_VOID(API, \"Diversify::compute_similarities\", NO_ARGS);\n+    Xapian::CosineDistance d;\n+    for (auto p_a : points) {\n+\tXapian::docid pointid_a = p_a.first;\n+\tXapian::Point point_a = p_a.second;\n+\tfor (auto p_b : points) {\n+\t    Xapian::docid pointid_b = p_b.first;\n+\t    Xapian::Point point_b = p_b.second;\n+\n+\t    if (pointid_a > pointid_b) {\n+\t\tcontinue;\n+\t    }\n+\n+\t    double sim = d.similarity(point_a, point_b);\n+\t    auto key = get_key(pointid_a, pointid_b);\n+\t    pairwise_sim[key] = sim;\n+\t}\n+    }\n+}\n+\n+vector<Xapian::docid>\n+Diversify::compute_diff_dmset(const vector<Xapian::docid> &dmset)\n+{\n+    LOGCALL(API, vector<Xapian::docid>, \"Diversify::compute_diff_dmset\", dmset);\n+    vector<Xapian::docid> diff_dmset;\n+    for (auto point : points) {\n+\tXapian::docid point_id = point.first;\n+\tbool found_point = false;\n+\tfor (auto doc_id : dmset) {\n+\t    if (point_id == doc_id) {\n+\t\tfound_point = true;\n+\t\tbreak;\n+\t    }\n+\t}\n+\n+\tif (!found_point) {\n+\t    diff_dmset.push_back(point_id);\n+\t}\n+    }\n+\n+    return diff_dmset;\n+}\n+\n+double\n+Diversify::evaluate_dmset(const vector<Xapian::docid> &dmset)\n+{\n+    LOGCALL(API, double, \"Diversify::evaluate_dmset\", dmset);\n+    double score_1 = 0, score_2 = 0;\n+\n+    for (auto doc_id : dmset)\n+\tscore_1 += scores[doc_id];\n+\n+    vector<Xapian::docid> diff_dmset = compute_diff_dmset(dmset);\n+\n+    for (auto point_id : diff_dmset) {\n+\tdouble min_dist = numeric_limits<double>::max();\n+\tunsigned int pos = 1;\n+\tfor (auto doc_id : dmset) {\n+\t    auto key = get_key(point_id, doc_id);\n+\t    double sim = pairwise_sim[key];\n+\t    double weight = 2 * b * sigma_sqr * (1 / log(1 + pos)) * (1 - sim);","path":"xapian-core/diversify/diversify.cc","position":null,"original_position":145,"commit_id":"ebcc1075d905c2918df0d4d31ec3709f516bac85","original_commit_id":"4648702a9bc72f551d1303d7cff67b08275b657a","user":{"login":"uppinder","id":12787473,"node_id":"MDQ6VXNlcjEyNzg3NDcz","avatar_url":"https://avatars1.githubusercontent.com/u/12787473?v=4","gravatar_id":"","url":"https://api.github.com/users/uppinder","html_url":"https://github.com/uppinder","followers_url":"https://api.github.com/users/uppinder/followers","following_url":"https://api.github.com/users/uppinder/following{/other_user}","gists_url":"https://api.github.com/users/uppinder/gists{/gist_id}","starred_url":"https://api.github.com/users/uppinder/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/uppinder/subscriptions","organizations_url":"https://api.github.com/users/uppinder/orgs","repos_url":"https://api.github.com/users/uppinder/repos","events_url":"https://api.github.com/users/uppinder/events{/privacy}","received_events_url":"https://api.github.com/users/uppinder/received_events","type":"User","site_admin":false},"body":"Ah, I see. In hindsight, there was never any emphasis on such micro-level optimisations in my university courses. Thanks for pointing this out, definitely something I didn't know earlier.","created_at":"2018-06-09T16:42:20Z","updated_at":"2018-06-09T16:42:21Z","html_url":"https://github.com/xapian/xapian/pull/198#discussion_r194233097","pull_request_url":"https://api.github.com/repos/xapian/xapian/pulls/198","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/xapian/xapian/pulls/comments/194233097"},"html":{"href":"https://github.com/xapian/xapian/pull/198#discussion_r194233097"},"pull_request":{"href":"https://api.github.com/repos/xapian/xapian/pulls/198"}},"in_reply_to_id":193292570},"pull_request":{"url":"https://api.github.com/repos/xapian/xapian/pulls/198","id":181690091,"node_id":"MDExOlB1bGxSZXF1ZXN0MTgxNjkwMDkx","html_url":"https://github.com/xapian/xapian/pull/198","diff_url":"https://github.com/xapian/xapian/pull/198.diff","patch_url":"https://github.com/xapian/xapian/pull/198.patch","issue_url":"https://api.github.com/repos/xapian/xapian/issues/198","number":198,"state":"open","locked":false,"title":"Diversification Module: GLS-MPT","user":{"login":"uppinder","id":12787473,"node_id":"MDQ6VXNlcjEyNzg3NDcz","avatar_url":"https://avatars1.githubusercontent.com/u/12787473?v=4","gravatar_id":"","url":"https://api.github.com/users/uppinder","html_url":"https://github.com/uppinder","followers_url":"https://api.github.com/users/uppinder/followers","following_url":"https://api.github.com/users/uppinder/following{/other_user}","gists_url":"https://api.github.com/users/uppinder/gists{/gist_id}","starred_url":"https://api.github.com/users/uppinder/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/uppinder/subscriptions","organizations_url":"https://api.github.com/users/uppinder/orgs","repos_url":"https://api.github.com/users/uppinder/repos","events_url":"https://api.github.com/users/uppinder/events{/privacy}","received_events_url":"https://api.github.com/users/uppinder/received_events","type":"User","site_admin":false},"body":"Contains implementation of GLS with MPT as the evaluation function for diversification.\r\n\r\nPending tasks:\r\n- [x] Pre-compute pairwise cosine similarities.\r\n- [x] Add tests: Working and performance test for diversification.","created_at":"2018-04-15T09:44:59Z","updated_at":"2018-06-09T16:42:21Z","closed_at":null,"merged_at":null,"merge_commit_sha":"ee6c53317e39edb481b3a99a25cdbe806cb0f4cd","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/xapian/xapian/pulls/198/commits","review_comments_url":"https://api.github.com/repos/xapian/xapian/pulls/198/comments","review_comment_url":"https://api.github.com/repos/xapian/xapian/pulls/comments{/number}","comments_url":"https://api.github.com/repos/xapian/xapian/issues/198/comments","statuses_url":"https://api.github.com/repos/xapian/xapian/statuses/ebcc1075d905c2918df0d4d31ec3709f516bac85","head":{"label":"uppinder:search-diversification","ref":"search-diversification","sha":"ebcc1075d905c2918df0d4d31ec3709f516bac85","user":{"login":"uppinder","id":12787473,"node_id":"MDQ6VXNlcjEyNzg3NDcz","avatar_url":"https://avatars1.githubusercontent.com/u/12787473?v=4","gravatar_id":"","url":"https://api.github.com/users/uppinder","html_url":"https://github.com/uppinder","followers_url":"https://api.github.com/users/uppinder/followers","following_url":"https://api.github.com/users/uppinder/following{/other_user}","gists_url":"https://api.github.com/users/uppinder/gists{/gist_id}","starred_url":"https://api.github.com/users/uppinder/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/uppinder/subscriptions","organizations_url":"https://api.github.com/users/uppinder/orgs","repos_url":"https://api.github.com/users/uppinder/repos","events_url":"https://api.github.com/users/uppinder/events{/privacy}","received_events_url":"https://api.github.com/users/uppinder/received_events","type":"User","site_admin":false},"repo":{"id":121769343,"node_id":"MDEwOlJlcG9zaXRvcnkxMjE3NjkzNDM=","name":"xapian","full_name":"uppinder/xapian","owner":{"login":"uppinder","id":12787473,"node_id":"MDQ6VXNlcjEyNzg3NDcz","avatar_url":"https://avatars1.githubusercontent.com/u/12787473?v=4","gravatar_id":"","url":"https://api.github.com/users/uppinder","html_url":"https://github.com/uppinder","followers_url":"https://api.github.com/users/uppinder/followers","following_url":"https://api.github.com/users/uppinder/following{/other_user}","gists_url":"https://api.github.com/users/uppinder/gists{/gist_id}","starred_url":"https://api.github.com/users/uppinder/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/uppinder/subscriptions","organizations_url":"https://api.github.com/users/uppinder/orgs","repos_url":"https://api.github.com/users/uppinder/repos","events_url":"https://api.github.com/users/uppinder/events{/privacy}","received_events_url":"https://api.github.com/users/uppinder/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/uppinder/xapian","description":"Mirror of the Xapian repository.  You're welcome to open pull requests on github (they'll just get merged indirectly).","fork":true,"url":"https://api.github.com/repos/uppinder/xapian","forks_url":"https://api.github.com/repos/uppinder/xapian/forks","keys_url":"https://api.github.com/repos/uppinder/xapian/keys{/key_id}","collaborators_url":"https://api.github.com/repos/uppinder/xapian/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/uppinder/xapian/teams","hooks_url":"https://api.github.com/repos/uppinder/xapian/hooks","issue_events_url":"https://api.github.com/repos/uppinder/xapian/issues/events{/number}","events_url":"https://api.github.com/repos/uppinder/xapian/events","assignees_url":"https://api.github.com/repos/uppinder/xapian/assignees{/user}","branches_url":"https://api.github.com/repos/uppinder/xapian/branches{/branch}","tags_url":"https://api.github.com/repos/uppinder/xapian/tags","blobs_url":"https://api.github.com/repos/uppinder/xapian/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/uppinder/xapian/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/uppinder/xapian/git/refs{/sha}","trees_url":"https://api.github.com/repos/uppinder/xapian/git/trees{/sha}","statuses_url":"https://api.github.com/repos/uppinder/xapian/statuses/{sha}","languages_url":"https://api.github.com/repos/uppinder/xapian/languages","stargazers_url":"https://api.github.com/repos/uppinder/xapian/stargazers","contributors_url":"https://api.github.com/repos/uppinder/xapian/contributors","subscribers_url":"https://api.github.com/repos/uppinder/xapian/subscribers","subscription_url":"https://api.github.com/repos/uppinder/xapian/subscription","commits_url":"https://api.github.com/repos/uppinder/xapian/commits{/sha}","git_commits_url":"https://api.github.com/repos/uppinder/xapian/git/commits{/sha}","comments_url":"https://api.github.com/repos/uppinder/xapian/comments{/number}","issue_comment_url":"https://api.github.com/repos/uppinder/xapian/issues/comments{/number}","contents_url":"https://api.github.com/repos/uppinder/xapian/contents/{+path}","compare_url":"https://api.github.com/repos/uppinder/xapian/compare/{base}...{head}","merges_url":"https://api.github.com/repos/uppinder/xapian/merges","archive_url":"https://api.github.com/repos/uppinder/xapian/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/uppinder/xapian/downloads","issues_url":"https://api.github.com/repos/uppinder/xapian/issues{/number}","pulls_url":"https://api.github.com/repos/uppinder/xapian/pulls{/number}","milestones_url":"https://api.github.com/repos/uppinder/xapian/milestones{/number}","notifications_url":"https://api.github.com/repos/uppinder/xapian/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/uppinder/xapian/labels{/name}","releases_url":"https://api.github.com/repos/uppinder/xapian/releases{/id}","deployments_url":"https://api.github.com/repos/uppinder/xapian/deployments","created_at":"2018-02-16T15:55:57Z","updated_at":"2018-06-05T17:19:22Z","pushed_at":"2018-06-09T16:29:19Z","git_url":"git://github.com/uppinder/xapian.git","ssh_url":"git@github.com:uppinder/xapian.git","clone_url":"https://github.com/uppinder/xapian.git","svn_url":"https://github.com/uppinder/xapian","homepage":"https://xapian.org/","size":81733,"stargazers_count":0,"watchers_count":0,"language":"C++","has_issues":false,"has_projects":true,"has_downloads":false,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":null,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"xapian:master","ref":"master","sha":"361324198141f2c9b44dfb65dfcbd96f93c005c3","user":{"login":"xapian","id":312861,"node_id":"MDEyOk9yZ2FuaXphdGlvbjMxMjg2MQ==","avatar_url":"https://avatars0.githubusercontent.com/u/312861?v=4","gravatar_id":"","url":"https://api.github.com/users/xapian","html_url":"https://github.com/xapian","followers_url":"https://api.github.com/users/xapian/followers","following_url":"https://api.github.com/users/xapian/following{/other_user}","gists_url":"https://api.github.com/users/xapian/gists{/gist_id}","starred_url":"https://api.github.com/users/xapian/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/xapian/subscriptions","organizations_url":"https://api.github.com/users/xapian/orgs","repos_url":"https://api.github.com/users/xapian/repos","events_url":"https://api.github.com/users/xapian/events{/privacy}","received_events_url":"https://api.github.com/users/xapian/received_events","type":"Organization","site_admin":false},"repo":{"id":735981,"node_id":"MDEwOlJlcG9zaXRvcnk3MzU5ODE=","name":"xapian","full_name":"xapian/xapian","owner":{"login":"xapian","id":312861,"node_id":"MDEyOk9yZ2FuaXphdGlvbjMxMjg2MQ==","avatar_url":"https://avatars0.githubusercontent.com/u/312861?v=4","gravatar_id":"","url":"https://api.github.com/users/xapian","html_url":"https://github.com/xapian","followers_url":"https://api.github.com/users/xapian/followers","following_url":"https://api.github.com/users/xapian/following{/other_user}","gists_url":"https://api.github.com/users/xapian/gists{/gist_id}","starred_url":"https://api.github.com/users/xapian/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/xapian/subscriptions","organizations_url":"https://api.github.com/users/xapian/orgs","repos_url":"https://api.github.com/users/xapian/repos","events_url":"https://api.github.com/users/xapian/events{/privacy}","received_events_url":"https://api.github.com/users/xapian/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/xapian/xapian","description":"Mirror of the Xapian repository.  You're welcome to open pull requests on github (they'll just get merged indirectly).","fork":false,"url":"https://api.github.com/repos/xapian/xapian","forks_url":"https://api.github.com/repos/xapian/xapian/forks","keys_url":"https://api.github.com/repos/xapian/xapian/keys{/key_id}","collaborators_url":"https://api.github.com/repos/xapian/xapian/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/xapian/xapian/teams","hooks_url":"https://api.github.com/repos/xapian/xapian/hooks","issue_events_url":"https://api.github.com/repos/xapian/xapian/issues/events{/number}","events_url":"https://api.github.com/repos/xapian/xapian/events","assignees_url":"https://api.github.com/repos/xapian/xapian/assignees{/user}","branches_url":"https://api.github.com/repos/xapian/xapian/branches{/branch}","tags_url":"https://api.github.com/repos/xapian/xapian/tags","blobs_url":"https://api.github.com/repos/xapian/xapian/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/xapian/xapian/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/xapian/xapian/git/refs{/sha}","trees_url":"https://api.github.com/repos/xapian/xapian/git/trees{/sha}","statuses_url":"https://api.github.com/repos/xapian/xapian/statuses/{sha}","languages_url":"https://api.github.com/repos/xapian/xapian/languages","stargazers_url":"https://api.github.com/repos/xapian/xapian/stargazers","contributors_url":"https://api.github.com/repos/xapian/xapian/contributors","subscribers_url":"https://api.github.com/repos/xapian/xapian/subscribers","subscription_url":"https://api.github.com/repos/xapian/xapian/subscription","commits_url":"https://api.github.com/repos/xapian/xapian/commits{/sha}","git_commits_url":"https://api.github.com/repos/xapian/xapian/git/commits{/sha}","comments_url":"https://api.github.com/repos/xapian/xapian/comments{/number}","issue_comment_url":"https://api.github.com/repos/xapian/xapian/issues/comments{/number}","contents_url":"https://api.github.com/repos/xapian/xapian/contents/{+path}","compare_url":"https://api.github.com/repos/xapian/xapian/compare/{base}...{head}","merges_url":"https://api.github.com/repos/xapian/xapian/merges","archive_url":"https://api.github.com/repos/xapian/xapian/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/xapian/xapian/downloads","issues_url":"https://api.github.com/repos/xapian/xapian/issues{/number}","pulls_url":"https://api.github.com/repos/xapian/xapian/pulls{/number}","milestones_url":"https://api.github.com/repos/xapian/xapian/milestones{/number}","notifications_url":"https://api.github.com/repos/xapian/xapian/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/xapian/xapian/labels{/name}","releases_url":"https://api.github.com/repos/xapian/xapian/releases{/id}","deployments_url":"https://api.github.com/repos/xapian/xapian/deployments","created_at":"2010-06-23T15:51:03Z","updated_at":"2018-06-08T19:41:27Z","pushed_at":"2018-06-09T16:29:20Z","git_url":"git://github.com/xapian/xapian.git","ssh_url":"git@github.com:xapian/xapian.git","clone_url":"https://github.com/xapian/xapian.git","svn_url":"https://github.com/xapian/xapian","homepage":"https://xapian.org/","size":81909,"stargazers_count":416,"watchers_count":416,"language":"C++","has_issues":false,"has_projects":false,"has_downloads":false,"has_wiki":false,"has_pages":false,"forks_count":201,"mirror_url":null,"archived":false,"open_issues_count":10,"license":null,"forks":201,"open_issues":10,"watchers":416,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/xapian/xapian/pulls/198"},"html":{"href":"https://github.com/xapian/xapian/pull/198"},"issue":{"href":"https://api.github.com/repos/xapian/xapian/issues/198"},"comments":{"href":"https://api.github.com/repos/xapian/xapian/issues/198/comments"},"review_comments":{"href":"https://api.github.com/repos/xapian/xapian/pulls/198/comments"},"review_comment":{"href":"https://api.github.com/repos/xapian/xapian/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/xapian/xapian/pulls/198/commits"},"statuses":{"href":"https://api.github.com/repos/xapian/xapian/statuses/ebcc1075d905c2918df0d4d31ec3709f516bac85"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"uppinder"}}