{"action":"created","comment":{"url":"https://api.github.com/repos/Copper-Head/the-three-stooges/pulls/comments/68990303","id":68990303,"diff_hunk":"@@ -0,0 +1,103 @@\n+import argparse\n+from collections import Counter\n+from itertools import chain\n+from numpy import save, array\n+from os import listdir\n+from os import path\n+import re\n+import sys\n+\n+import dataproc as dp\n+\n+parser = argparse.ArgumentParser()\n+parser.add_argument('-d', '--dir', type=str, help='Source directory to conll-data.')\n+parser.add_argument('-n', '--name', type=str, help='Specifies the corpus name.')\n+parser.add_argument('-u', '--maxlength', type=int, default=sys.maxsize, help='Max sentence length to avoid memory errors.')\n+parser.add_argument('-l', '--minlength', type=int, default=0, help='Minimal sentence length, default is 0.')\n+parser.add_argument('-r', '--removeunknown', type=int, default=0, help='Determines if sentences with rare words should be dropped. Any number != 0 says no-drop.')\n+parser.add_argument('-w', '--wordfreq', type=int, default=1, help='Minimum frequence of words, words with lower'\n+                                                                  +' frequence will be replaced by <UNKNOWN>. Default=5')\n+parser.add_argument('-c', '--case', type=int, default=1, help='Determines, if the vocabulary should be case sensitive. It is on per default, 0 means non-case sensitive.')\n+parser.add_argument('-p', '--punctuation', type=int, default=1, help='0 - delete punctuation; any other number - keep punctuation')\n+\n+args = parser.parse_args()\n+\n+target_file = './data/'+args.name+'_data.hdf5'","path":"conll2hdf5.py","position":25,"original_position":25,"commit_id":"4b0c436cc1b745f49aaeeec66c6d77cab7c7a678","original_commit_id":"4b0c436cc1b745f49aaeeec66c6d77cab7c7a678","user":{"login":"Copper-Head","id":899591,"avatar_url":"https://avatars.githubusercontent.com/u/899591?v=3","gravatar_id":"","url":"https://api.github.com/users/Copper-Head","html_url":"https://github.com/Copper-Head","followers_url":"https://api.github.com/users/Copper-Head/followers","following_url":"https://api.github.com/users/Copper-Head/following{/other_user}","gists_url":"https://api.github.com/users/Copper-Head/gists{/gist_id}","starred_url":"https://api.github.com/users/Copper-Head/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Copper-Head/subscriptions","organizations_url":"https://api.github.com/users/Copper-Head/orgs","repos_url":"https://api.github.com/users/Copper-Head/repos","events_url":"https://api.github.com/users/Copper-Head/events{/privacy}","received_events_url":"https://api.github.com/users/Copper-Head/received_events","type":"User","site_admin":false},"body":"It'd be nice to have a space around all operators :)","created_at":"2016-06-29T17:26:16Z","updated_at":"2016-06-29T17:26:16Z","html_url":"https://github.com/Copper-Head/the-three-stooges/pull/1#discussion_r68990303","pull_request_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/pulls/1","_links":{"self":{"href":"https://api.github.com/repos/Copper-Head/the-three-stooges/pulls/comments/68990303"},"html":{"href":"https://github.com/Copper-Head/the-three-stooges/pull/1#discussion_r68990303"},"pull_request":{"href":"https://api.github.com/repos/Copper-Head/the-three-stooges/pulls/1"}}},"pull_request":{"url":"https://api.github.com/repos/Copper-Head/the-three-stooges/pulls/1","id":75389177,"html_url":"https://github.com/Copper-Head/the-three-stooges/pull/1","diff_url":"https://github.com/Copper-Head/the-three-stooges/pull/1.diff","patch_url":"https://github.com/Copper-Head/the-three-stooges/pull/1.patch","issue_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/issues/1","number":1,"state":"open","locked":false,"title":"generalizing changes on files; new files included","user":{"login":"MartinKl","id":2787060,"avatar_url":"https://avatars.githubusercontent.com/u/2787060?v=3","gravatar_id":"","url":"https://api.github.com/users/MartinKl","html_url":"https://github.com/MartinKl","followers_url":"https://api.github.com/users/MartinKl/followers","following_url":"https://api.github.com/users/MartinKl/following{/other_user}","gists_url":"https://api.github.com/users/MartinKl/gists{/gist_id}","starred_url":"https://api.github.com/users/MartinKl/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/MartinKl/subscriptions","organizations_url":"https://api.github.com/users/MartinKl/orgs","repos_url":"https://api.github.com/users/MartinKl/repos","events_url":"https://api.github.com/users/MartinKl/events{/privacy}","received_events_url":"https://api.github.com/users/MartinKl/received_events","type":"User","site_admin":false},"body":"I introduced some changes implying an \"api\"-break:\r\n\r\n+ `Network` has now 2 positional arguments: *network_type* and *input_dim*; *input_dim_file* is no longer available, compute the dimensions from the file beforehand instead\r\n+ `train.py` is generalized, which requires several command line arguments: __-b__ is batch size, if set to 0, all examples will be put in one batch; __-f__ has to be followed by the path of the training data-hdf5; __-a__ by the alphabet file (for the bible data this should be compatible with *char_to_ind.pkl*; __-m__ is 0 or any number, where 0 implies read data from disk and any number means \"load to memory\", default is non-zero;\r\n+ there is now `conll2hdf5.py` which can compile any conll-file to hdf5 (sequences); for command line arguments please see source; the alphabet file will be generated as well\r\n+ the same holds for `prepare_lk.py`, which prepares the linux kernel data in a similar way","created_at":"2016-06-27T22:15:47Z","updated_at":"2016-06-29T17:26:16Z","closed_at":null,"merged_at":null,"merge_commit_sha":"b7831d9406e5f0c297040fa27058b81d73a6ea4e","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/pulls/1/commits","review_comments_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/pulls/1/comments","review_comment_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/pulls/comments{/number}","comments_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/issues/1/comments","statuses_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/statuses/4b0c436cc1b745f49aaeeec66c6d77cab7c7a678","head":{"label":"Copper-Head:martin","ref":"martin","sha":"4b0c436cc1b745f49aaeeec66c6d77cab7c7a678","user":{"login":"Copper-Head","id":899591,"avatar_url":"https://avatars.githubusercontent.com/u/899591?v=3","gravatar_id":"","url":"https://api.github.com/users/Copper-Head","html_url":"https://github.com/Copper-Head","followers_url":"https://api.github.com/users/Copper-Head/followers","following_url":"https://api.github.com/users/Copper-Head/following{/other_user}","gists_url":"https://api.github.com/users/Copper-Head/gists{/gist_id}","starred_url":"https://api.github.com/users/Copper-Head/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Copper-Head/subscriptions","organizations_url":"https://api.github.com/users/Copper-Head/orgs","repos_url":"https://api.github.com/users/Copper-Head/repos","events_url":"https://api.github.com/users/Copper-Head/events{/privacy}","received_events_url":"https://api.github.com/users/Copper-Head/received_events","type":"User","site_admin":false},"repo":{"id":60348511,"name":"the-three-stooges","full_name":"Copper-Head/the-three-stooges","owner":{"login":"Copper-Head","id":899591,"avatar_url":"https://avatars.githubusercontent.com/u/899591?v=3","gravatar_id":"","url":"https://api.github.com/users/Copper-Head","html_url":"https://github.com/Copper-Head","followers_url":"https://api.github.com/users/Copper-Head/followers","following_url":"https://api.github.com/users/Copper-Head/following{/other_user}","gists_url":"https://api.github.com/users/Copper-Head/gists{/gist_id}","starred_url":"https://api.github.com/users/Copper-Head/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Copper-Head/subscriptions","organizations_url":"https://api.github.com/users/Copper-Head/orgs","repos_url":"https://api.github.com/users/Copper-Head/repos","events_url":"https://api.github.com/users/Copper-Head/events{/privacy}","received_events_url":"https://api.github.com/users/Copper-Head/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/Copper-Head/the-three-stooges","description":"Final project for Deep Learning at Uni Potsdam","fork":false,"url":"https://api.github.com/repos/Copper-Head/the-three-stooges","forks_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/forks","keys_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/keys{/key_id}","collaborators_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/teams","hooks_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/hooks","issue_events_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/issues/events{/number}","events_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/events","assignees_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/assignees{/user}","branches_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/branches{/branch}","tags_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/tags","blobs_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/git/refs{/sha}","trees_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/git/trees{/sha}","statuses_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/statuses/{sha}","languages_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/languages","stargazers_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/stargazers","contributors_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/contributors","subscribers_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/subscribers","subscription_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/subscription","commits_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/commits{/sha}","git_commits_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/git/commits{/sha}","comments_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/comments{/number}","issue_comment_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/issues/comments{/number}","contents_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/contents/{+path}","compare_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/compare/{base}...{head}","merges_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/merges","archive_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/downloads","issues_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/issues{/number}","pulls_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/pulls{/number}","milestones_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/milestones{/number}","notifications_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/labels{/name}","releases_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/releases{/id}","deployments_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/deployments","created_at":"2016-06-03T13:09:30Z","updated_at":"2016-06-15T17:57:34Z","pushed_at":"2016-06-29T12:24:31Z","git_url":"git://github.com/Copper-Head/the-three-stooges.git","ssh_url":"git@github.com:Copper-Head/the-three-stooges.git","clone_url":"https://github.com/Copper-Head/the-three-stooges.git","svn_url":"https://github.com/Copper-Head/the-three-stooges","homepage":"","size":1278,"stargazers_count":0,"watchers_count":0,"language":"Jupyter Notebook","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":1,"forks":0,"open_issues":1,"watchers":0,"default_branch":"master"}},"base":{"label":"Copper-Head:master","ref":"master","sha":"0dd52c6da5784ce281e257e51637db09d7bf5421","user":{"login":"Copper-Head","id":899591,"avatar_url":"https://avatars.githubusercontent.com/u/899591?v=3","gravatar_id":"","url":"https://api.github.com/users/Copper-Head","html_url":"https://github.com/Copper-Head","followers_url":"https://api.github.com/users/Copper-Head/followers","following_url":"https://api.github.com/users/Copper-Head/following{/other_user}","gists_url":"https://api.github.com/users/Copper-Head/gists{/gist_id}","starred_url":"https://api.github.com/users/Copper-Head/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Copper-Head/subscriptions","organizations_url":"https://api.github.com/users/Copper-Head/orgs","repos_url":"https://api.github.com/users/Copper-Head/repos","events_url":"https://api.github.com/users/Copper-Head/events{/privacy}","received_events_url":"https://api.github.com/users/Copper-Head/received_events","type":"User","site_admin":false},"repo":{"id":60348511,"name":"the-three-stooges","full_name":"Copper-Head/the-three-stooges","owner":{"login":"Copper-Head","id":899591,"avatar_url":"https://avatars.githubusercontent.com/u/899591?v=3","gravatar_id":"","url":"https://api.github.com/users/Copper-Head","html_url":"https://github.com/Copper-Head","followers_url":"https://api.github.com/users/Copper-Head/followers","following_url":"https://api.github.com/users/Copper-Head/following{/other_user}","gists_url":"https://api.github.com/users/Copper-Head/gists{/gist_id}","starred_url":"https://api.github.com/users/Copper-Head/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Copper-Head/subscriptions","organizations_url":"https://api.github.com/users/Copper-Head/orgs","repos_url":"https://api.github.com/users/Copper-Head/repos","events_url":"https://api.github.com/users/Copper-Head/events{/privacy}","received_events_url":"https://api.github.com/users/Copper-Head/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/Copper-Head/the-three-stooges","description":"Final project for Deep Learning at Uni Potsdam","fork":false,"url":"https://api.github.com/repos/Copper-Head/the-three-stooges","forks_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/forks","keys_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/keys{/key_id}","collaborators_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/teams","hooks_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/hooks","issue_events_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/issues/events{/number}","events_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/events","assignees_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/assignees{/user}","branches_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/branches{/branch}","tags_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/tags","blobs_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/git/refs{/sha}","trees_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/git/trees{/sha}","statuses_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/statuses/{sha}","languages_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/languages","stargazers_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/stargazers","contributors_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/contributors","subscribers_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/subscribers","subscription_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/subscription","commits_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/commits{/sha}","git_commits_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/git/commits{/sha}","comments_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/comments{/number}","issue_comment_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/issues/comments{/number}","contents_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/contents/{+path}","compare_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/compare/{base}...{head}","merges_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/merges","archive_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/downloads","issues_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/issues{/number}","pulls_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/pulls{/number}","milestones_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/milestones{/number}","notifications_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/labels{/name}","releases_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/releases{/id}","deployments_url":"https://api.github.com/repos/Copper-Head/the-three-stooges/deployments","created_at":"2016-06-03T13:09:30Z","updated_at":"2016-06-15T17:57:34Z","pushed_at":"2016-06-29T12:24:31Z","git_url":"git://github.com/Copper-Head/the-three-stooges.git","ssh_url":"git@github.com:Copper-Head/the-three-stooges.git","clone_url":"https://github.com/Copper-Head/the-three-stooges.git","svn_url":"https://github.com/Copper-Head/the-three-stooges","homepage":"","size":1278,"stargazers_count":0,"watchers_count":0,"language":"Jupyter Notebook","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":1,"forks":0,"open_issues":1,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/Copper-Head/the-three-stooges/pulls/1"},"html":{"href":"https://github.com/Copper-Head/the-three-stooges/pull/1"},"issue":{"href":"https://api.github.com/repos/Copper-Head/the-three-stooges/issues/1"},"comments":{"href":"https://api.github.com/repos/Copper-Head/the-three-stooges/issues/1/comments"},"review_comments":{"href":"https://api.github.com/repos/Copper-Head/the-three-stooges/pulls/1/comments"},"review_comment":{"href":"https://api.github.com/repos/Copper-Head/the-three-stooges/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/Copper-Head/the-three-stooges/pulls/1/commits"},"statuses":{"href":"https://api.github.com/repos/Copper-Head/the-three-stooges/statuses/4b0c436cc1b745f49aaeeec66c6d77cab7c7a678"}}}}
{"actor":{"display_login":"Copper-Head"}}