{"action":"created","comment":{"url":"https://api.github.com/repos/src-d/code2vec/pulls/comments/203306631","pull_request_review_id":138167012,"id":203306631,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDIwMzMwNjYzMQ==","diff_hunk":"@@ -0,0 +1,81 @@\n+import operator\n+\n+from pyspark import RDD, Row\n+from models.code2vec_features import Code2VecFeatures\n+\n+from ast import literal_eval as make_tuple\n+from sourced.ml.transformers import Transformer\n+\n+\n+class Vocabulary2Id(Transformer):\n+    def __init__(self, keep_freq: bool, output: str, **kwargs):\n+        super().__init__(**kwargs)\n+        self.keep_freq = keep_freq\n+        self.output = output\n+\n+    def __call__(self, rows: RDD):\n+        self.value2index, self.path2index = self.build_vocabularies(rows)\n+\n+        doc2path_contexts = self.build_doc2pc(rows)\n+\n+        doc2path_contexts = doc2path_contexts.collect()\n+\n+        Code2VecFeatures().construct(value2index=self.value2index, path2index=self.path2index,\n+                                     path_contexts=doc2path_contexts).save(\n+            self.output)\n+\n+    @staticmethod\n+    def _unstringify_path_context(row):\n+        \"\"\"\n+        Takes a row containing ((pc, doc), freq) and returns a tuple (u, path, v)\n+        (removes namespace prefix v.)\n+        \"\"\"\n+        return make_tuple(row[0][0][2:])\n+\n+    def _get_path(self, row: Row):\n+        # 2: removes the namespace v. from the string to parse it as tuple\n+        k = Vocabulary2Id._unstringify_path_context(row)\n+        return [(k[0], 1), (k[1], 1), (k[2], 1)]\n+\n+    def build_vocabularies(self, rows: RDD):\n+        \"\"\"\n+        Process rows to gather values and paths with/without their frequencies.\n+        \"\"\"\n+\n+        rows = rows \\\n+            .flatMap(self._get_path) \\\n+            .reduceByKey(operator.add) \\\n+            .persist()\n+\n+        get_elem = lambda x: x[0]\n+        if not self.keep_freq:\n+            rows = rows.map(lambda x: x[0])\n+            get_elem = lambda x: x\n+\n+        values = rows.filter(lambda x: type(get_elem(x)) == str).collect()\n+        paths = rows.filter(lambda x: type(get_elem(x)) == tuple).collect()\n+\n+        value2index = {w: i for i, w in enumerate(values)}","path":"src/transformers/vocabulary2id.py","position":58,"original_position":58,"commit_id":"b406d188386664d6cd33b7051dfe3325937754a8","original_commit_id":"b406d188386664d6cd33b7051dfe3325937754a8","user":{"login":"r0mainK","id":32878976,"node_id":"MDQ6VXNlcjMyODc4OTc2","avatar_url":"https://avatars2.githubusercontent.com/u/32878976?v=4","gravatar_id":"","url":"https://api.github.com/users/r0mainK","html_url":"https://github.com/r0mainK","followers_url":"https://api.github.com/users/r0mainK/followers","following_url":"https://api.github.com/users/r0mainK/following{/other_user}","gists_url":"https://api.github.com/users/r0mainK/gists{/gist_id}","starred_url":"https://api.github.com/users/r0mainK/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/r0mainK/subscriptions","organizations_url":"https://api.github.com/users/r0mainK/orgs","repos_url":"https://api.github.com/users/r0mainK/repos","events_url":"https://api.github.com/users/r0mainK/events{/privacy}","received_events_url":"https://api.github.com/users/r0mainK/received_events","type":"User","site_admin":false},"body":"this is a problem if `keep_freq is True`, in one case you get the dictionnaries but in the other you get `(key, value): index`. I would move the index creation to the call, and use `self.docfreq` to potentially create the frequency dict, and pass it as an optional dict parameter for the Model that default to None.","created_at":"2018-07-18T09:14:54Z","updated_at":"2018-07-18T09:14:54Z","html_url":"https://github.com/src-d/code2vec/pull/5#discussion_r203306631","pull_request_url":"https://api.github.com/repos/src-d/code2vec/pulls/5","author_association":"COLLABORATOR","_links":{"self":{"href":"https://api.github.com/repos/src-d/code2vec/pulls/comments/203306631"},"html":{"href":"https://github.com/src-d/code2vec/pull/5#discussion_r203306631"},"pull_request":{"href":"https://api.github.com/repos/src-d/code2vec/pulls/5"}}},"pull_request":{"url":"https://api.github.com/repos/src-d/code2vec/pulls/5","id":201562264,"node_id":"MDExOlB1bGxSZXF1ZXN0MjAxNTYyMjY0","html_url":"https://github.com/src-d/code2vec/pull/5","diff_url":"https://github.com/src-d/code2vec/pull/5.diff","patch_url":"https://github.com/src-d/code2vec/pull/5.patch","issue_url":"https://api.github.com/repos/src-d/code2vec/issues/5","number":5,"state":"open","locked":false,"title":"WIP: Added Vocabulary builder and model","user":{"login":"kafkasl","id":2961095,"node_id":"MDQ6VXNlcjI5NjEwOTU=","avatar_url":"https://avatars0.githubusercontent.com/u/2961095?v=4","gravatar_id":"","url":"https://api.github.com/users/kafkasl","html_url":"https://github.com/kafkasl","followers_url":"https://api.github.com/users/kafkasl/followers","following_url":"https://api.github.com/users/kafkasl/following{/other_user}","gists_url":"https://api.github.com/users/kafkasl/gists{/gist_id}","starred_url":"https://api.github.com/users/kafkasl/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kafkasl/subscriptions","organizations_url":"https://api.github.com/users/kafkasl/orgs","repos_url":"https://api.github.com/users/kafkasl/repos","events_url":"https://api.github.com/users/kafkasl/events{/privacy}","received_events_url":"https://api.github.com/users/kafkasl/received_events","type":"User","site_admin":false},"body":"This PR is not finished but the code2vec model and Vocabulary2id are pretty much done, so I thought you could start reviewing it in case something is really off.\r\n\r\nCurrently I'm getting an error when trying to save the model and I'm not sure why. Any help would be appreciated:\r\n\r\n```\r\n  File \"/home/hydra/projects/code2vec/src/../src/code2vec.py\", line 23, in code2vec\r\n    .link(Vocabulary2Id(args.keep_freq, args.output)) \\\r\n  File \"/usr/local/lib/python3.5/dist-packages/sourced/ml/transformers/transformer.py\", line 95, in execute\r\n    head = node(head)\r\n  File \"/home/hydra/projects/code2vec/src/transformers/vocabulary2id.py\", line 28, in __call__\r\n    \r\n  File \"/usr/local/lib/python3.5/dist-packages/modelforge/model.py\", line 269, in save\r\n    self._write_tree(tree, output)\r\n  File \"/usr/local/lib/python3.5/dist-packages/modelforge/model.py\", line 285, in _write_tree\r\n    asdf.AsdfFile(final_tree).write_to(output, all_array_compression=ARRAY_COMPRESSION)\r\n  File \"/usr/local/lib/python3.5/dist-packages/asdf/asdf.py\", line 899, in write_to\r\n    self._post_write(fd)\r\n  File \"/usr/local/lib/python3.5/dist-packages/asdf/generic_io.py\", line 313, in __exit__\r\n    self._fd.__exit__(type, value, traceback)\r\n  File \"/usr/local/lib/python3.5/dist-packages/asdf/extern/atomicfile.py\", line 113, in __exit__\r\n    self.close()\r\n  File \"/usr/local/lib/python3.5/dist-packages/asdf/extern/atomicfile.py\", line 109, in close\r\n    atomic_rename(self._tmp_filename, self._filename)\r\nNotADirectoryError: [Errno 20] Not a directory: '/home/hydra/projects/code2vec/results/.___atomic_writeg1zpv8zx' -> '/home/hydra/projects/code2vec/src/../results/'\r\n\r\n```\r\n","created_at":"2018-07-16T07:53:21Z","updated_at":"2018-07-18T09:14:54Z","closed_at":null,"merged_at":null,"merge_commit_sha":"a6f007679655731d3ae50db7cda979ee1f82ef64","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/src-d/code2vec/pulls/5/commits","review_comments_url":"https://api.github.com/repos/src-d/code2vec/pulls/5/comments","review_comment_url":"https://api.github.com/repos/src-d/code2vec/pulls/comments{/number}","comments_url":"https://api.github.com/repos/src-d/code2vec/issues/5/comments","statuses_url":"https://api.github.com/repos/src-d/code2vec/statuses/b406d188386664d6cd33b7051dfe3325937754a8","head":{"label":"kafkasl:master","ref":"master","sha":"b406d188386664d6cd33b7051dfe3325937754a8","user":{"login":"kafkasl","id":2961095,"node_id":"MDQ6VXNlcjI5NjEwOTU=","avatar_url":"https://avatars0.githubusercontent.com/u/2961095?v=4","gravatar_id":"","url":"https://api.github.com/users/kafkasl","html_url":"https://github.com/kafkasl","followers_url":"https://api.github.com/users/kafkasl/followers","following_url":"https://api.github.com/users/kafkasl/following{/other_user}","gists_url":"https://api.github.com/users/kafkasl/gists{/gist_id}","starred_url":"https://api.github.com/users/kafkasl/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kafkasl/subscriptions","organizations_url":"https://api.github.com/users/kafkasl/orgs","repos_url":"https://api.github.com/users/kafkasl/repos","events_url":"https://api.github.com/users/kafkasl/events{/privacy}","received_events_url":"https://api.github.com/users/kafkasl/received_events","type":"User","site_admin":false},"repo":{"id":140399466,"node_id":"MDEwOlJlcG9zaXRvcnkxNDAzOTk0NjY=","name":"code2vec","full_name":"kafkasl/code2vec","owner":{"login":"kafkasl","id":2961095,"node_id":"MDQ6VXNlcjI5NjEwOTU=","avatar_url":"https://avatars0.githubusercontent.com/u/2961095?v=4","gravatar_id":"","url":"https://api.github.com/users/kafkasl","html_url":"https://github.com/kafkasl","followers_url":"https://api.github.com/users/kafkasl/followers","following_url":"https://api.github.com/users/kafkasl/following{/other_user}","gists_url":"https://api.github.com/users/kafkasl/gists{/gist_id}","starred_url":"https://api.github.com/users/kafkasl/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/kafkasl/subscriptions","organizations_url":"https://api.github.com/users/kafkasl/orgs","repos_url":"https://api.github.com/users/kafkasl/repos","events_url":"https://api.github.com/users/kafkasl/events{/privacy}","received_events_url":"https://api.github.com/users/kafkasl/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/kafkasl/code2vec","description":"MLonCode community effort to implement Learning Distributed Representations of Code (https://arxiv.org/pdf/1803.09473.pdf)","fork":true,"url":"https://api.github.com/repos/kafkasl/code2vec","forks_url":"https://api.github.com/repos/kafkasl/code2vec/forks","keys_url":"https://api.github.com/repos/kafkasl/code2vec/keys{/key_id}","collaborators_url":"https://api.github.com/repos/kafkasl/code2vec/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/kafkasl/code2vec/teams","hooks_url":"https://api.github.com/repos/kafkasl/code2vec/hooks","issue_events_url":"https://api.github.com/repos/kafkasl/code2vec/issues/events{/number}","events_url":"https://api.github.com/repos/kafkasl/code2vec/events","assignees_url":"https://api.github.com/repos/kafkasl/code2vec/assignees{/user}","branches_url":"https://api.github.com/repos/kafkasl/code2vec/branches{/branch}","tags_url":"https://api.github.com/repos/kafkasl/code2vec/tags","blobs_url":"https://api.github.com/repos/kafkasl/code2vec/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/kafkasl/code2vec/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/kafkasl/code2vec/git/refs{/sha}","trees_url":"https://api.github.com/repos/kafkasl/code2vec/git/trees{/sha}","statuses_url":"https://api.github.com/repos/kafkasl/code2vec/statuses/{sha}","languages_url":"https://api.github.com/repos/kafkasl/code2vec/languages","stargazers_url":"https://api.github.com/repos/kafkasl/code2vec/stargazers","contributors_url":"https://api.github.com/repos/kafkasl/code2vec/contributors","subscribers_url":"https://api.github.com/repos/kafkasl/code2vec/subscribers","subscription_url":"https://api.github.com/repos/kafkasl/code2vec/subscription","commits_url":"https://api.github.com/repos/kafkasl/code2vec/commits{/sha}","git_commits_url":"https://api.github.com/repos/kafkasl/code2vec/git/commits{/sha}","comments_url":"https://api.github.com/repos/kafkasl/code2vec/comments{/number}","issue_comment_url":"https://api.github.com/repos/kafkasl/code2vec/issues/comments{/number}","contents_url":"https://api.github.com/repos/kafkasl/code2vec/contents/{+path}","compare_url":"https://api.github.com/repos/kafkasl/code2vec/compare/{base}...{head}","merges_url":"https://api.github.com/repos/kafkasl/code2vec/merges","archive_url":"https://api.github.com/repos/kafkasl/code2vec/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/kafkasl/code2vec/downloads","issues_url":"https://api.github.com/repos/kafkasl/code2vec/issues{/number}","pulls_url":"https://api.github.com/repos/kafkasl/code2vec/pulls{/number}","milestones_url":"https://api.github.com/repos/kafkasl/code2vec/milestones{/number}","notifications_url":"https://api.github.com/repos/kafkasl/code2vec/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/kafkasl/code2vec/labels{/name}","releases_url":"https://api.github.com/repos/kafkasl/code2vec/releases{/id}","deployments_url":"https://api.github.com/repos/kafkasl/code2vec/deployments","created_at":"2018-07-10T08:12:59Z","updated_at":"2018-07-17T16:42:34Z","pushed_at":"2018-07-17T16:42:32Z","git_url":"git://github.com/kafkasl/code2vec.git","ssh_url":"git@github.com:kafkasl/code2vec.git","clone_url":"https://github.com/kafkasl/code2vec.git","svn_url":"https://github.com/kafkasl/code2vec","homepage":null,"size":720,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"src-d:master","ref":"master","sha":"3284c3055870d7fec293e09cadad013418a6837b","user":{"login":"src-d","id":15128793,"node_id":"MDEyOk9yZ2FuaXphdGlvbjE1MTI4Nzkz","avatar_url":"https://avatars2.githubusercontent.com/u/15128793?v=4","gravatar_id":"","url":"https://api.github.com/users/src-d","html_url":"https://github.com/src-d","followers_url":"https://api.github.com/users/src-d/followers","following_url":"https://api.github.com/users/src-d/following{/other_user}","gists_url":"https://api.github.com/users/src-d/gists{/gist_id}","starred_url":"https://api.github.com/users/src-d/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/src-d/subscriptions","organizations_url":"https://api.github.com/users/src-d/orgs","repos_url":"https://api.github.com/users/src-d/repos","events_url":"https://api.github.com/users/src-d/events{/privacy}","received_events_url":"https://api.github.com/users/src-d/received_events","type":"Organization","site_admin":false},"repo":{"id":139014384,"node_id":"MDEwOlJlcG9zaXRvcnkxMzkwMTQzODQ=","name":"code2vec","full_name":"src-d/code2vec","owner":{"login":"src-d","id":15128793,"node_id":"MDEyOk9yZ2FuaXphdGlvbjE1MTI4Nzkz","avatar_url":"https://avatars2.githubusercontent.com/u/15128793?v=4","gravatar_id":"","url":"https://api.github.com/users/src-d","html_url":"https://github.com/src-d","followers_url":"https://api.github.com/users/src-d/followers","following_url":"https://api.github.com/users/src-d/following{/other_user}","gists_url":"https://api.github.com/users/src-d/gists{/gist_id}","starred_url":"https://api.github.com/users/src-d/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/src-d/subscriptions","organizations_url":"https://api.github.com/users/src-d/orgs","repos_url":"https://api.github.com/users/src-d/repos","events_url":"https://api.github.com/users/src-d/events{/privacy}","received_events_url":"https://api.github.com/users/src-d/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/src-d/code2vec","description":"MLonCode community effort to implement Learning Distributed Representations of Code (https://arxiv.org/pdf/1803.09473.pdf)","fork":false,"url":"https://api.github.com/repos/src-d/code2vec","forks_url":"https://api.github.com/repos/src-d/code2vec/forks","keys_url":"https://api.github.com/repos/src-d/code2vec/keys{/key_id}","collaborators_url":"https://api.github.com/repos/src-d/code2vec/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/src-d/code2vec/teams","hooks_url":"https://api.github.com/repos/src-d/code2vec/hooks","issue_events_url":"https://api.github.com/repos/src-d/code2vec/issues/events{/number}","events_url":"https://api.github.com/repos/src-d/code2vec/events","assignees_url":"https://api.github.com/repos/src-d/code2vec/assignees{/user}","branches_url":"https://api.github.com/repos/src-d/code2vec/branches{/branch}","tags_url":"https://api.github.com/repos/src-d/code2vec/tags","blobs_url":"https://api.github.com/repos/src-d/code2vec/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/src-d/code2vec/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/src-d/code2vec/git/refs{/sha}","trees_url":"https://api.github.com/repos/src-d/code2vec/git/trees{/sha}","statuses_url":"https://api.github.com/repos/src-d/code2vec/statuses/{sha}","languages_url":"https://api.github.com/repos/src-d/code2vec/languages","stargazers_url":"https://api.github.com/repos/src-d/code2vec/stargazers","contributors_url":"https://api.github.com/repos/src-d/code2vec/contributors","subscribers_url":"https://api.github.com/repos/src-d/code2vec/subscribers","subscription_url":"https://api.github.com/repos/src-d/code2vec/subscription","commits_url":"https://api.github.com/repos/src-d/code2vec/commits{/sha}","git_commits_url":"https://api.github.com/repos/src-d/code2vec/git/commits{/sha}","comments_url":"https://api.github.com/repos/src-d/code2vec/comments{/number}","issue_comment_url":"https://api.github.com/repos/src-d/code2vec/issues/comments{/number}","contents_url":"https://api.github.com/repos/src-d/code2vec/contents/{+path}","compare_url":"https://api.github.com/repos/src-d/code2vec/compare/{base}...{head}","merges_url":"https://api.github.com/repos/src-d/code2vec/merges","archive_url":"https://api.github.com/repos/src-d/code2vec/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/src-d/code2vec/downloads","issues_url":"https://api.github.com/repos/src-d/code2vec/issues{/number}","pulls_url":"https://api.github.com/repos/src-d/code2vec/pulls{/number}","milestones_url":"https://api.github.com/repos/src-d/code2vec/milestones{/number}","notifications_url":"https://api.github.com/repos/src-d/code2vec/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/src-d/code2vec/labels{/name}","releases_url":"https://api.github.com/repos/src-d/code2vec/releases{/id}","deployments_url":"https://api.github.com/repos/src-d/code2vec/deployments","created_at":"2018-06-28T12:21:22Z","updated_at":"2018-07-14T13:06:58Z","pushed_at":"2018-07-17T16:42:34Z","git_url":"git://github.com/src-d/code2vec.git","ssh_url":"git@github.com:src-d/code2vec.git","clone_url":"https://github.com/src-d/code2vec.git","svn_url":"https://github.com/src-d/code2vec","homepage":null,"size":716,"stargazers_count":4,"watchers_count":4,"language":"Python","has_issues":true,"has_projects":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":6,"mirror_url":null,"archived":false,"open_issues_count":1,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":6,"open_issues":1,"watchers":4,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/src-d/code2vec/pulls/5"},"html":{"href":"https://github.com/src-d/code2vec/pull/5"},"issue":{"href":"https://api.github.com/repos/src-d/code2vec/issues/5"},"comments":{"href":"https://api.github.com/repos/src-d/code2vec/issues/5/comments"},"review_comments":{"href":"https://api.github.com/repos/src-d/code2vec/pulls/5/comments"},"review_comment":{"href":"https://api.github.com/repos/src-d/code2vec/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/src-d/code2vec/pulls/5/commits"},"statuses":{"href":"https://api.github.com/repos/src-d/code2vec/statuses/b406d188386664d6cd33b7051dfe3325937754a8"}},"author_association":"COLLABORATOR"}}
{"actor":{"display_login":"r0mainK"}}