{"action":"opened","number":20,"pull_request":{"url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/pulls/20","id":70509910,"html_url":"https://github.com/Unbabel/unbabel-text-utils/pull/20","diff_url":"https://github.com/Unbabel/unbabel-text-utils/pull/20.diff","patch_url":"https://github.com/Unbabel/unbabel-text-utils/pull/20.patch","issue_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/issues/20","number":20,"state":"open","locked":false,"title":"Tokenizer","user":{"login":"andre-martins","id":1390019,"avatar_url":"https://avatars.githubusercontent.com/u/1390019?v=3","gravatar_id":"","url":"https://api.github.com/users/andre-martins","html_url":"https://github.com/andre-martins","followers_url":"https://api.github.com/users/andre-martins/followers","following_url":"https://api.github.com/users/andre-martins/following{/other_user}","gists_url":"https://api.github.com/users/andre-martins/gists{/gist_id}","starred_url":"https://api.github.com/users/andre-martins/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/andre-martins/subscriptions","organizations_url":"https://api.github.com/users/andre-martins/orgs","repos_url":"https://api.github.com/users/andre-martins/repos","events_url":"https://api.github.com/users/andre-martins/events{/privacy}","received_events_url":"https://api.github.com/users/andre-martins/received_events","type":"User","site_admin":false},"body":"Implements an Unbabel tokenizer in Python. It reproduces the Moses tokenizer but keeps track of the token positions in the buffer. It's about 15x slower than the Moses tokenizer but its speed (about 1000 sentences per second) is probably still acceptable. A class AlignedString is also provided that can be useful to other things.","created_at":"2016-05-18T11:18:43Z","updated_at":"2016-05-18T11:18:43Z","closed_at":null,"merged_at":null,"merge_commit_sha":"962d8c2f461ca2ad77d60e362cb5a21b09d9e812","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/pulls/20/commits","review_comments_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/pulls/20/comments","review_comment_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/pulls/comments{/number}","comments_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/issues/20/comments","statuses_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/statuses/f6cda9b278edd2108ea6a780d3c00621caeb7fd0","head":{"label":"Unbabel:tokenizer","ref":"tokenizer","sha":"f6cda9b278edd2108ea6a780d3c00621caeb7fd0","user":{"login":"Unbabel","id":6173964,"avatar_url":"https://avatars.githubusercontent.com/u/6173964?v=3","gravatar_id":"","url":"https://api.github.com/users/Unbabel","html_url":"https://github.com/Unbabel","followers_url":"https://api.github.com/users/Unbabel/followers","following_url":"https://api.github.com/users/Unbabel/following{/other_user}","gists_url":"https://api.github.com/users/Unbabel/gists{/gist_id}","starred_url":"https://api.github.com/users/Unbabel/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Unbabel/subscriptions","organizations_url":"https://api.github.com/users/Unbabel/orgs","repos_url":"https://api.github.com/users/Unbabel/repos","events_url":"https://api.github.com/users/Unbabel/events{/privacy}","received_events_url":"https://api.github.com/users/Unbabel/received_events","type":"Organization","site_admin":false},"repo":{"id":45260496,"name":"unbabel-text-utils","full_name":"Unbabel/unbabel-text-utils","owner":{"login":"Unbabel","id":6173964,"avatar_url":"https://avatars.githubusercontent.com/u/6173964?v=3","gravatar_id":"","url":"https://api.github.com/users/Unbabel","html_url":"https://github.com/Unbabel","followers_url":"https://api.github.com/users/Unbabel/followers","following_url":"https://api.github.com/users/Unbabel/following{/other_user}","gists_url":"https://api.github.com/users/Unbabel/gists{/gist_id}","starred_url":"https://api.github.com/users/Unbabel/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Unbabel/subscriptions","organizations_url":"https://api.github.com/users/Unbabel/orgs","repos_url":"https://api.github.com/users/Unbabel/repos","events_url":"https://api.github.com/users/Unbabel/events{/privacy}","received_events_url":"https://api.github.com/users/Unbabel/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/Unbabel/unbabel-text-utils","description":"Basic Text Utilities for Unbabel.","fork":false,"url":"https://api.github.com/repos/Unbabel/unbabel-text-utils","forks_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/forks","keys_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/keys{/key_id}","collaborators_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/teams","hooks_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/hooks","issue_events_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/issues/events{/number}","events_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/events","assignees_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/assignees{/user}","branches_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/branches{/branch}","tags_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/tags","blobs_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/git/refs{/sha}","trees_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/git/trees{/sha}","statuses_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/statuses/{sha}","languages_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/languages","stargazers_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/stargazers","contributors_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/contributors","subscribers_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/subscribers","subscription_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/subscription","commits_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/commits{/sha}","git_commits_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/git/commits{/sha}","comments_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/comments{/number}","issue_comment_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/issues/comments{/number}","contents_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/contents/{+path}","compare_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/compare/{base}...{head}","merges_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/merges","archive_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/downloads","issues_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/issues{/number}","pulls_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/pulls{/number}","milestones_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/milestones{/number}","notifications_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/labels{/name}","releases_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/releases{/id}","deployments_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/deployments","created_at":"2015-10-30T15:46:48Z","updated_at":"2016-01-15T17:39:29Z","pushed_at":"2016-05-18T11:18:43Z","git_url":"git://github.com/Unbabel/unbabel-text-utils.git","ssh_url":"git@github.com:Unbabel/unbabel-text-utils.git","clone_url":"https://github.com/Unbabel/unbabel-text-utils.git","svn_url":"https://github.com/Unbabel/unbabel-text-utils","homepage":null,"size":28433,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":4,"forks":0,"open_issues":4,"watchers":0,"default_branch":"master"}},"base":{"label":"Unbabel:master","ref":"master","sha":"c0dc84d3c9079c0f29e25b9f0743f197cf3037b9","user":{"login":"Unbabel","id":6173964,"avatar_url":"https://avatars.githubusercontent.com/u/6173964?v=3","gravatar_id":"","url":"https://api.github.com/users/Unbabel","html_url":"https://github.com/Unbabel","followers_url":"https://api.github.com/users/Unbabel/followers","following_url":"https://api.github.com/users/Unbabel/following{/other_user}","gists_url":"https://api.github.com/users/Unbabel/gists{/gist_id}","starred_url":"https://api.github.com/users/Unbabel/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Unbabel/subscriptions","organizations_url":"https://api.github.com/users/Unbabel/orgs","repos_url":"https://api.github.com/users/Unbabel/repos","events_url":"https://api.github.com/users/Unbabel/events{/privacy}","received_events_url":"https://api.github.com/users/Unbabel/received_events","type":"Organization","site_admin":false},"repo":{"id":45260496,"name":"unbabel-text-utils","full_name":"Unbabel/unbabel-text-utils","owner":{"login":"Unbabel","id":6173964,"avatar_url":"https://avatars.githubusercontent.com/u/6173964?v=3","gravatar_id":"","url":"https://api.github.com/users/Unbabel","html_url":"https://github.com/Unbabel","followers_url":"https://api.github.com/users/Unbabel/followers","following_url":"https://api.github.com/users/Unbabel/following{/other_user}","gists_url":"https://api.github.com/users/Unbabel/gists{/gist_id}","starred_url":"https://api.github.com/users/Unbabel/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Unbabel/subscriptions","organizations_url":"https://api.github.com/users/Unbabel/orgs","repos_url":"https://api.github.com/users/Unbabel/repos","events_url":"https://api.github.com/users/Unbabel/events{/privacy}","received_events_url":"https://api.github.com/users/Unbabel/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/Unbabel/unbabel-text-utils","description":"Basic Text Utilities for Unbabel.","fork":false,"url":"https://api.github.com/repos/Unbabel/unbabel-text-utils","forks_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/forks","keys_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/keys{/key_id}","collaborators_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/teams","hooks_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/hooks","issue_events_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/issues/events{/number}","events_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/events","assignees_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/assignees{/user}","branches_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/branches{/branch}","tags_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/tags","blobs_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/git/refs{/sha}","trees_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/git/trees{/sha}","statuses_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/statuses/{sha}","languages_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/languages","stargazers_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/stargazers","contributors_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/contributors","subscribers_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/subscribers","subscription_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/subscription","commits_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/commits{/sha}","git_commits_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/git/commits{/sha}","comments_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/comments{/number}","issue_comment_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/issues/comments{/number}","contents_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/contents/{+path}","compare_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/compare/{base}...{head}","merges_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/merges","archive_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/downloads","issues_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/issues{/number}","pulls_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/pulls{/number}","milestones_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/milestones{/number}","notifications_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/labels{/name}","releases_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/releases{/id}","deployments_url":"https://api.github.com/repos/Unbabel/unbabel-text-utils/deployments","created_at":"2015-10-30T15:46:48Z","updated_at":"2016-01-15T17:39:29Z","pushed_at":"2016-05-18T11:18:43Z","git_url":"git://github.com/Unbabel/unbabel-text-utils.git","ssh_url":"git@github.com:Unbabel/unbabel-text-utils.git","clone_url":"https://github.com/Unbabel/unbabel-text-utils.git","svn_url":"https://github.com/Unbabel/unbabel-text-utils","homepage":null,"size":28433,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":4,"forks":0,"open_issues":4,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/Unbabel/unbabel-text-utils/pulls/20"},"html":{"href":"https://github.com/Unbabel/unbabel-text-utils/pull/20"},"issue":{"href":"https://api.github.com/repos/Unbabel/unbabel-text-utils/issues/20"},"comments":{"href":"https://api.github.com/repos/Unbabel/unbabel-text-utils/issues/20/comments"},"review_comments":{"href":"https://api.github.com/repos/Unbabel/unbabel-text-utils/pulls/20/comments"},"review_comment":{"href":"https://api.github.com/repos/Unbabel/unbabel-text-utils/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/Unbabel/unbabel-text-utils/pulls/20/commits"},"statuses":{"href":"https://api.github.com/repos/Unbabel/unbabel-text-utils/statuses/f6cda9b278edd2108ea6a780d3c00621caeb7fd0"}},"merged":false,"mergeable":true,"mergeable_state":"clean","merged_by":null,"comments":0,"review_comments":0,"commits":11,"additions":636,"deletions":1,"changed_files":7}}