{"action":"created","comment":{"url":"https://api.github.com/repos/piskvorky/gensim/pulls/comments/58667276","id":58667276,"diff_hunk":"@@ -382,6 +383,93 @@ def cossim(vec1, vec2):\n     return result\n \n \n+def kullback_leibler(vec1,vec2,num_of_features=None):\n+    \"\"\"\n+    A similarity metric between two probability distributions.\n+    Returns a similarity in range <0,1> where values closer to 0 mean a higher similarity.\n+    Uses the scipy.stats.entropy method to identify kullback_leibler convergence value.\n+    If the distribution draws from a certain number of docs, that value must be passed.\n+    \"\"\"\n+    if scipy.sparse.issparse(vec1) and scipy.sparse.issparse(vec2):\n+        vec1 = vec1.todense()\n+        vec2 = vec2.todense()\n+        if vec1.shape[1] == 2 and vec2.shape[1] == 2:\n+            vec1 = vec1.tolist()\n+            vec2 = vec2.tolist()\n+            max_len = max(len(vec1),len(vec2),num_of_features)\n+            dense1 = sparse2full(vec1,max_len)\n+            dense2 = sparse2full(vec2,max_len)     \n+            return scipy.stats.entropy(dense1,dense2)\n+        else:\n+            return scipy.stats.entropy(vec1,vec2)\n+    elif isinstance(vec1,numpy.ndarray) and isinstance(vec2,numpy.ndarray):\n+        if vec1.shape[1] == 2 and vec2.shape[1] == 2:\n+            vec1 = vec1.tolist()\n+            vec2 = vec2.tolist()\n+            max_len = max(len(vec1),len(vec2),num_of_features)\n+            dense1 = sparse2full(vec1,max_len)\n+            dense2 = sparse2full(vec2,max_len)\n+            return scipy.stats.entropy(dense1,dense2)\n+        else:\n+            return scipy.stats.entropy(vec1,vec2)\n+    elif type(vec1) is list and type(vec2) is list:\n+        max_len = max(len(vec1),len(vec2),num_of_features)\n+        dense1 = sparse2full(vec1,max_len)\n+        dense2 = sparse2full(vec2,max_len)\n+        return scipy.stats.entropy(dense1,dense2)\n+\n+\n+def hellinger(vec1, vec2):\n+    \"\"\"\n+    Hellinger distance is a distance metric to quanitfy the similarity between two probability distributions.\n+    Similarity between distributions will be a number between <0,1>, where 0 is maximum similarity and 1 is minimum similarity.\n+    \"\"\"\n+    if scipy.sparse.issparse(vec1) and scipy.sparse.issparse(vec2):\n+        dense1 = vec1.todense()\n+        dense2 = vec2.todense()\n+        # if it is a sparse matrix in bag of words format, convert to list and go ahead.\n+        if dense1.shape[1] == 2 and dense2.shape[1] == 2:\n+            dense1,dense2 = dict(dense1.tolist()),dict(dense2.tolist())\n+            if len(dense2) < len(dense1):\n+                dense1, dense2 = dense1, dense2 # swap references so that we iterate over the shorter vector\n+            sim = numpy.sqrt(0.5*sum((numpy.sqrt(value) - numpy.sqrt(dense2.get(index, 0.0)))**2 for index, value in iteritems(dense1)))\n+            return sim\n+        # else if it is in a different format, directly go about it\n+        sim = numpy.sqrt(0.5 * ((numpy.sqrt(dense1) - numpy.sqrt(dense2))**2).sum())\n+        return sim\n+    elif isinstance(vec1,numpy.ndarray) and isinstance(vec2,numpy.ndarray):\n+        # if it is in a bag of words format, go ahead with list method\n+        if vec1.shape[1] == 2 and vec2.shape[1] == 2:\n+            vec1, vec2 = dict(vec1.tolist()), dict(vec2.tolist())\n+            if len(vec2) < len(vec1):\n+                vec1, vec2 = vec2, vec1\n+            sim = numpy.sqrt(0.5*sum((numpy.sqrt(value) - numpy.sqrt(vec2.get(index, 0.0)))**2 for index, value in iteritems(vec1)))\n+            return sim\n+        # else if it is in a different format, directly go about it\n+        sim = numpy.sqrt(0.5 * ((numpy.sqrt(vec1) - numpy.sqrt(vec2))**2).sum())\n+        return sim\n+    elif type(vec1) is list and type(vec2) is list:\n+        vec1, vec2 = dict(vec1), dict(vec2)\n+        if len(vec2) < len(vec1):\n+            vec1, vec2 = vec2, vec1\n+        sim = numpy.sqrt(0.5*sum((numpy.sqrt(value) - numpy.sqrt(vec2.get(index, 0.0)))**2 for index, value in iteritems(vec1)))\n+        return sim\n+\n+\n+def jaccard(vec1,vec2):\n+    \"\"\"\n+    A similarity metric between bags of words representation.\n+    Return the intersection divided by union, where union is the sum of the size of the two bags.\n+    Highest value jaccard similarity of two bags is 1/2, which indicates the highest similarity.\n+    \"\"\"\n+    intersection = 0\n+    union = sum(row[1] for row in vec1) + sum(row[1] for row in vec2)","path":"gensim/matutils.py","position":92,"original_position":92,"commit_id":"fa2949f56dedde6a2f3acd0e3c489e095f4907da","original_commit_id":"fa2949f56dedde6a2f3acd0e3c489e095f4907da","user":{"login":"bhargavvader","id":8374816,"avatar_url":"https://avatars.githubusercontent.com/u/8374816?v=3","gravatar_id":"","url":"https://api.github.com/users/bhargavvader","html_url":"https://github.com/bhargavvader","followers_url":"https://api.github.com/users/bhargavvader/followers","following_url":"https://api.github.com/users/bhargavvader/following{/other_user}","gists_url":"https://api.github.com/users/bhargavvader/gists{/gist_id}","starred_url":"https://api.github.com/users/bhargavvader/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bhargavvader/subscriptions","organizations_url":"https://api.github.com/users/bhargavvader/orgs","repos_url":"https://api.github.com/users/bhargavvader/repos","events_url":"https://api.github.com/users/bhargavvader/events{/privacy}","received_events_url":"https://api.github.com/users/bhargavvader/received_events","type":"User","site_admin":false},"body":"`row` is probably a poor choice of word, what I mean to do is to get the value of the `weight` of the corresponding `id` in a bag of words representation which Jaccard makes sense for. What should I change this variable to to make more sense?\r\n\r\nThe union is meant to be the sum of the sizes of the two bags you are comparing. This is the definition of jaccard similarity according to [this](http://infolab.stanford.edu/~ullman/mmds/ch3.pdf) text. For e.g, The bag-similarity of bags `{a, a, a, b}` and `{a, a, b, b, c}` is `1/3`. The size of the bags is `9` and the size of intersection is `3`. I'd expect these bags to be represented as `[(0,3),(1,1)]`  and `[(0,2),(1,2),(2,1)]`. \r\nThough now that I think about this more, I think I have to re-write my jaccard method to accept other kinds of inputs and be more generic... Will work on it.","created_at":"2016-04-06T08:17:18Z","updated_at":"2016-04-06T08:17:18Z","html_url":"https://github.com/piskvorky/gensim/pull/656#discussion_r58667276","pull_request_url":"https://api.github.com/repos/piskvorky/gensim/pulls/656","_links":{"self":{"href":"https://api.github.com/repos/piskvorky/gensim/pulls/comments/58667276"},"html":{"href":"https://github.com/piskvorky/gensim/pull/656#discussion_r58667276"},"pull_request":{"href":"https://api.github.com/repos/piskvorky/gensim/pulls/656"}}},"pull_request":{"url":"https://api.github.com/repos/piskvorky/gensim/pulls/656","id":65179907,"html_url":"https://github.com/piskvorky/gensim/pull/656","diff_url":"https://github.com/piskvorky/gensim/pull/656.diff","patch_url":"https://github.com/piskvorky/gensim/pull/656.patch","issue_url":"https://api.github.com/repos/piskvorky/gensim/issues/656","number":656,"state":"open","locked":false,"title":"[WIP] Adding similarity/distance metrics to matutils.","user":{"login":"bhargavvader","id":8374816,"avatar_url":"https://avatars.githubusercontent.com/u/8374816?v=3","gravatar_id":"","url":"https://api.github.com/users/bhargavvader","html_url":"https://github.com/bhargavvader","followers_url":"https://api.github.com/users/bhargavvader/followers","following_url":"https://api.github.com/users/bhargavvader/following{/other_user}","gists_url":"https://api.github.com/users/bhargavvader/gists{/gist_id}","starred_url":"https://api.github.com/users/bhargavvader/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bhargavvader/subscriptions","organizations_url":"https://api.github.com/users/bhargavvader/orgs","repos_url":"https://api.github.com/users/bhargavvader/repos","events_url":"https://api.github.com/users/bhargavvader/events{/privacy}","received_events_url":"https://api.github.com/users/bhargavvader/received_events","type":"User","site_admin":false},"body":"@tmylk , @piskvorky , I've created 3 different metric functions: `Hellinger` and `Kullback-Leibler` for probability distributions, and `Jaccard` Similarity for Bag of Words representations. \r\n\r\nI have a few questions about possible changes, particularly:\r\n\r\n1) The Hellinger and Kullback-Leibler methods right now only takes **2-tuple** representations (in numpy array, scipy sparse, or gensim vector representations). Is that ok? Most probability distributions passed should look like that. I can improve it to include dense numpy arrays on input as well as well, if it is expected that people will pass that into the function.\r\n\r\n2) The Kullback-Leibler uses the [scipy.sparse.entropy ](http://scipy.github.io/devdocs/generated/scipy.stats.entropy.html) method, which fails if the probability distributions of any of the tuples is 0. For most cases we deal with, this should not be a problem, but if we want to allow 0s to be present in the input, I will have to rewrite the kullback-leibler calculation myself.\r\n\r\nJaccard similarity is primarily only used for bag of words, so I've not bothered to include other kinds of formats on input.\r\n\r\nIs this approach for adding similarity functions ok? If it is, I can make the changes you'd suggest, add test cases, and add more of these functions if needed.\r\n\r\nThis is related to issue #64 .","created_at":"2016-04-04T17:10:48Z","updated_at":"2016-04-06T08:17:18Z","closed_at":null,"merged_at":null,"merge_commit_sha":"6719a01b4ea9169bf9f79c01886625466a395f3e","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/piskvorky/gensim/pulls/656/commits","review_comments_url":"https://api.github.com/repos/piskvorky/gensim/pulls/656/comments","review_comment_url":"https://api.github.com/repos/piskvorky/gensim/pulls/comments{/number}","comments_url":"https://api.github.com/repos/piskvorky/gensim/issues/656/comments","statuses_url":"https://api.github.com/repos/piskvorky/gensim/statuses/fa2949f56dedde6a2f3acd0e3c489e095f4907da","head":{"label":"bhargavvader:Similarity","ref":"Similarity","sha":"fa2949f56dedde6a2f3acd0e3c489e095f4907da","user":{"login":"bhargavvader","id":8374816,"avatar_url":"https://avatars.githubusercontent.com/u/8374816?v=3","gravatar_id":"","url":"https://api.github.com/users/bhargavvader","html_url":"https://github.com/bhargavvader","followers_url":"https://api.github.com/users/bhargavvader/followers","following_url":"https://api.github.com/users/bhargavvader/following{/other_user}","gists_url":"https://api.github.com/users/bhargavvader/gists{/gist_id}","starred_url":"https://api.github.com/users/bhargavvader/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bhargavvader/subscriptions","organizations_url":"https://api.github.com/users/bhargavvader/orgs","repos_url":"https://api.github.com/users/bhargavvader/repos","events_url":"https://api.github.com/users/bhargavvader/events{/privacy}","received_events_url":"https://api.github.com/users/bhargavvader/received_events","type":"User","site_admin":false},"repo":{"id":54395982,"name":"gensim","full_name":"bhargavvader/gensim","owner":{"login":"bhargavvader","id":8374816,"avatar_url":"https://avatars.githubusercontent.com/u/8374816?v=3","gravatar_id":"","url":"https://api.github.com/users/bhargavvader","html_url":"https://github.com/bhargavvader","followers_url":"https://api.github.com/users/bhargavvader/followers","following_url":"https://api.github.com/users/bhargavvader/following{/other_user}","gists_url":"https://api.github.com/users/bhargavvader/gists{/gist_id}","starred_url":"https://api.github.com/users/bhargavvader/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bhargavvader/subscriptions","organizations_url":"https://api.github.com/users/bhargavvader/orgs","repos_url":"https://api.github.com/users/bhargavvader/repos","events_url":"https://api.github.com/users/bhargavvader/events{/privacy}","received_events_url":"https://api.github.com/users/bhargavvader/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/bhargavvader/gensim","description":"Topic Modelling for Humans","fork":true,"url":"https://api.github.com/repos/bhargavvader/gensim","forks_url":"https://api.github.com/repos/bhargavvader/gensim/forks","keys_url":"https://api.github.com/repos/bhargavvader/gensim/keys{/key_id}","collaborators_url":"https://api.github.com/repos/bhargavvader/gensim/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/bhargavvader/gensim/teams","hooks_url":"https://api.github.com/repos/bhargavvader/gensim/hooks","issue_events_url":"https://api.github.com/repos/bhargavvader/gensim/issues/events{/number}","events_url":"https://api.github.com/repos/bhargavvader/gensim/events","assignees_url":"https://api.github.com/repos/bhargavvader/gensim/assignees{/user}","branches_url":"https://api.github.com/repos/bhargavvader/gensim/branches{/branch}","tags_url":"https://api.github.com/repos/bhargavvader/gensim/tags","blobs_url":"https://api.github.com/repos/bhargavvader/gensim/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/bhargavvader/gensim/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/bhargavvader/gensim/git/refs{/sha}","trees_url":"https://api.github.com/repos/bhargavvader/gensim/git/trees{/sha}","statuses_url":"https://api.github.com/repos/bhargavvader/gensim/statuses/{sha}","languages_url":"https://api.github.com/repos/bhargavvader/gensim/languages","stargazers_url":"https://api.github.com/repos/bhargavvader/gensim/stargazers","contributors_url":"https://api.github.com/repos/bhargavvader/gensim/contributors","subscribers_url":"https://api.github.com/repos/bhargavvader/gensim/subscribers","subscription_url":"https://api.github.com/repos/bhargavvader/gensim/subscription","commits_url":"https://api.github.com/repos/bhargavvader/gensim/commits{/sha}","git_commits_url":"https://api.github.com/repos/bhargavvader/gensim/git/commits{/sha}","comments_url":"https://api.github.com/repos/bhargavvader/gensim/comments{/number}","issue_comment_url":"https://api.github.com/repos/bhargavvader/gensim/issues/comments{/number}","contents_url":"https://api.github.com/repos/bhargavvader/gensim/contents/{+path}","compare_url":"https://api.github.com/repos/bhargavvader/gensim/compare/{base}...{head}","merges_url":"https://api.github.com/repos/bhargavvader/gensim/merges","archive_url":"https://api.github.com/repos/bhargavvader/gensim/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/bhargavvader/gensim/downloads","issues_url":"https://api.github.com/repos/bhargavvader/gensim/issues{/number}","pulls_url":"https://api.github.com/repos/bhargavvader/gensim/pulls{/number}","milestones_url":"https://api.github.com/repos/bhargavvader/gensim/milestones{/number}","notifications_url":"https://api.github.com/repos/bhargavvader/gensim/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/bhargavvader/gensim/labels{/name}","releases_url":"https://api.github.com/repos/bhargavvader/gensim/releases{/id}","deployments_url":"https://api.github.com/repos/bhargavvader/gensim/deployments","created_at":"2016-03-21T14:33:07Z","updated_at":"2016-03-21T14:33:08Z","pushed_at":"2016-04-05T17:45:20Z","git_url":"git://github.com/bhargavvader/gensim.git","ssh_url":"git@github.com:bhargavvader/gensim.git","clone_url":"https://github.com/bhargavvader/gensim.git","svn_url":"https://github.com/bhargavvader/gensim","homepage":"http://radimrehurek.com/gensim/","size":14952,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"develop"}},"base":{"label":"piskvorky:develop","ref":"develop","sha":"89bee2eadf011230737412fcdc8047ce500a8bc9","user":{"login":"piskvorky","id":610412,"avatar_url":"https://avatars.githubusercontent.com/u/610412?v=3","gravatar_id":"","url":"https://api.github.com/users/piskvorky","html_url":"https://github.com/piskvorky","followers_url":"https://api.github.com/users/piskvorky/followers","following_url":"https://api.github.com/users/piskvorky/following{/other_user}","gists_url":"https://api.github.com/users/piskvorky/gists{/gist_id}","starred_url":"https://api.github.com/users/piskvorky/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/piskvorky/subscriptions","organizations_url":"https://api.github.com/users/piskvorky/orgs","repos_url":"https://api.github.com/users/piskvorky/repos","events_url":"https://api.github.com/users/piskvorky/events{/privacy}","received_events_url":"https://api.github.com/users/piskvorky/received_events","type":"User","site_admin":false},"repo":{"id":1349775,"name":"gensim","full_name":"piskvorky/gensim","owner":{"login":"piskvorky","id":610412,"avatar_url":"https://avatars.githubusercontent.com/u/610412?v=3","gravatar_id":"","url":"https://api.github.com/users/piskvorky","html_url":"https://github.com/piskvorky","followers_url":"https://api.github.com/users/piskvorky/followers","following_url":"https://api.github.com/users/piskvorky/following{/other_user}","gists_url":"https://api.github.com/users/piskvorky/gists{/gist_id}","starred_url":"https://api.github.com/users/piskvorky/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/piskvorky/subscriptions","organizations_url":"https://api.github.com/users/piskvorky/orgs","repos_url":"https://api.github.com/users/piskvorky/repos","events_url":"https://api.github.com/users/piskvorky/events{/privacy}","received_events_url":"https://api.github.com/users/piskvorky/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/piskvorky/gensim","description":"Topic Modelling for Humans","fork":false,"url":"https://api.github.com/repos/piskvorky/gensim","forks_url":"https://api.github.com/repos/piskvorky/gensim/forks","keys_url":"https://api.github.com/repos/piskvorky/gensim/keys{/key_id}","collaborators_url":"https://api.github.com/repos/piskvorky/gensim/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/piskvorky/gensim/teams","hooks_url":"https://api.github.com/repos/piskvorky/gensim/hooks","issue_events_url":"https://api.github.com/repos/piskvorky/gensim/issues/events{/number}","events_url":"https://api.github.com/repos/piskvorky/gensim/events","assignees_url":"https://api.github.com/repos/piskvorky/gensim/assignees{/user}","branches_url":"https://api.github.com/repos/piskvorky/gensim/branches{/branch}","tags_url":"https://api.github.com/repos/piskvorky/gensim/tags","blobs_url":"https://api.github.com/repos/piskvorky/gensim/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/piskvorky/gensim/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/piskvorky/gensim/git/refs{/sha}","trees_url":"https://api.github.com/repos/piskvorky/gensim/git/trees{/sha}","statuses_url":"https://api.github.com/repos/piskvorky/gensim/statuses/{sha}","languages_url":"https://api.github.com/repos/piskvorky/gensim/languages","stargazers_url":"https://api.github.com/repos/piskvorky/gensim/stargazers","contributors_url":"https://api.github.com/repos/piskvorky/gensim/contributors","subscribers_url":"https://api.github.com/repos/piskvorky/gensim/subscribers","subscription_url":"https://api.github.com/repos/piskvorky/gensim/subscription","commits_url":"https://api.github.com/repos/piskvorky/gensim/commits{/sha}","git_commits_url":"https://api.github.com/repos/piskvorky/gensim/git/commits{/sha}","comments_url":"https://api.github.com/repos/piskvorky/gensim/comments{/number}","issue_comment_url":"https://api.github.com/repos/piskvorky/gensim/issues/comments{/number}","contents_url":"https://api.github.com/repos/piskvorky/gensim/contents/{+path}","compare_url":"https://api.github.com/repos/piskvorky/gensim/compare/{base}...{head}","merges_url":"https://api.github.com/repos/piskvorky/gensim/merges","archive_url":"https://api.github.com/repos/piskvorky/gensim/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/piskvorky/gensim/downloads","issues_url":"https://api.github.com/repos/piskvorky/gensim/issues{/number}","pulls_url":"https://api.github.com/repos/piskvorky/gensim/pulls{/number}","milestones_url":"https://api.github.com/repos/piskvorky/gensim/milestones{/number}","notifications_url":"https://api.github.com/repos/piskvorky/gensim/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/piskvorky/gensim/labels{/name}","releases_url":"https://api.github.com/repos/piskvorky/gensim/releases{/id}","deployments_url":"https://api.github.com/repos/piskvorky/gensim/deployments","created_at":"2011-02-10T07:43:04Z","updated_at":"2016-04-05T12:34:27Z","pushed_at":"2016-04-05T17:45:21Z","git_url":"git://github.com/piskvorky/gensim.git","ssh_url":"git@github.com:piskvorky/gensim.git","clone_url":"https://github.com/piskvorky/gensim.git","svn_url":"https://github.com/piskvorky/gensim","homepage":"http://radimrehurek.com/gensim/","size":19064,"stargazers_count":2392,"watchers_count":2392,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":846,"mirror_url":null,"open_issues_count":115,"forks":846,"open_issues":115,"watchers":2392,"default_branch":"develop"}},"_links":{"self":{"href":"https://api.github.com/repos/piskvorky/gensim/pulls/656"},"html":{"href":"https://github.com/piskvorky/gensim/pull/656"},"issue":{"href":"https://api.github.com/repos/piskvorky/gensim/issues/656"},"comments":{"href":"https://api.github.com/repos/piskvorky/gensim/issues/656/comments"},"review_comments":{"href":"https://api.github.com/repos/piskvorky/gensim/pulls/656/comments"},"review_comment":{"href":"https://api.github.com/repos/piskvorky/gensim/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/piskvorky/gensim/pulls/656/commits"},"statuses":{"href":"https://api.github.com/repos/piskvorky/gensim/statuses/fa2949f56dedde6a2f3acd0e3c489e095f4907da"}}}}