{"action":"created","comment":{"url":"https://api.github.com/repos/biolab/orange3-text/pulls/comments/52012823","id":52012823,"diff_hunk":"@@ -1,94 +1,298 @@\n-from nltk.stem.porter import PorterStemmer\n+from nltk.corpus import stopwords\n from nltk.stem import WordNetLemmatizer\n-from sklearn.feature_extraction.text import CountVectorizer\n+from nltk.stem.porter import PorterStemmer\n+from nltk.stem.snowball import SnowballStemmer\n+from nltk.tokenize import word_tokenize\n+\n+from orangecontrib.text.corpus import Corpus\n \n \n class Preprocessor:\n+    \"\"\"A pre-processing class for the Orange text mining extension.\n+\n+    The class is capable of tokenizing, lowercasing, stemming and lemmatizing\n+    the input. Returns a list of preprocessed tokens, sorted by order of\n+    appearance in text.\n     \"\"\"\n-        Holds pre-processing flags and other information, about stop word\n-        removal, lowercasing, text morphing etc.(the options are set via\n-        the Preprocess widget).\n-    \"\"\"\n-    def __init__(self, incl_punct=False, lowercase=True, stop_words='english', trans=None, min_df=1):\n+    def __init__(self, lowercase=True, stop_words=None,\n+                 min_df=None, max_df=None, transformation=None,\n+                 use_twitter_tokenizer=False):\n         \"\"\"\n-        :param incl_punct: Determines whether the tokenizer should include punctuation in the tokens.\n-        :type incl_punct: boolean\n-        :param lowercase: If set, transform the tokens to lower case, before returning them.\n-        :type lowercase: boolean\n-        :param stop_words: Determines whether stop words should(\"english\"), or should not(None) be removed.\n-            If this is list, it should contain stopwords.\n-        :type stop_words: 'english' or list or None\n-        :param trans: An optional pre-processor object to perform the morphological\n-            transformation on the tokens before returning them.\n-        :type trans: :class: `orangecontrib.text.preprocess.Lemmatizer`\n-            or :class: `orangecontrib.text.preprocess.Stemmer`\n-        :return: :class: `orangecontrib.text.preprocess.Preprocessor`\n+        Args:\n+            lowercase (Optional[bool]): If set, transform the tokens to lower\n+                case, before returning them. Defaults to False.","path":"orangecontrib/text/preprocess.py","position":42,"original_position":42,"commit_id":"d06b8aa739503e0ae192dd8122d17e97e7587554","original_commit_id":"d06b8aa739503e0ae192dd8122d17e97e7587554","user":{"login":"nikicc","id":713026,"avatar_url":"https://avatars.githubusercontent.com/u/713026?v=3","gravatar_id":"","url":"https://api.github.com/users/nikicc","html_url":"https://github.com/nikicc","followers_url":"https://api.github.com/users/nikicc/followers","following_url":"https://api.github.com/users/nikicc/following{/other_user}","gists_url":"https://api.github.com/users/nikicc/gists{/gist_id}","starred_url":"https://api.github.com/users/nikicc/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nikicc/subscriptions","organizations_url":"https://api.github.com/users/nikicc/orgs","repos_url":"https://api.github.com/users/nikicc/repos","events_url":"https://api.github.com/users/nikicc/events{/privacy}","received_events_url":"https://api.github.com/users/nikicc/received_events","type":"User","site_admin":false},"body":"Default is set to `True`, not `False`.","created_at":"2016-02-05T13:03:40Z","updated_at":"2016-02-05T13:03:40Z","html_url":"https://github.com/biolab/orange3-text/pull/21#discussion_r52012823","pull_request_url":"https://api.github.com/repos/biolab/orange3-text/pulls/21","_links":{"self":{"href":"https://api.github.com/repos/biolab/orange3-text/pulls/comments/52012823"},"html":{"href":"https://github.com/biolab/orange3-text/pull/21#discussion_r52012823"},"pull_request":{"href":"https://api.github.com/repos/biolab/orange3-text/pulls/21"}}},"pull_request":{"url":"https://api.github.com/repos/biolab/orange3-text/pulls/21","id":53530385,"html_url":"https://github.com/biolab/orange3-text/pull/21","diff_url":"https://github.com/biolab/orange3-text/pull/21.diff","patch_url":"https://github.com/biolab/orange3-text/pull/21.patch","issue_url":"https://api.github.com/repos/biolab/orange3-text/issues/21","number":21,"state":"open","locked":false,"title":"Preprocess V2","user":{"login":"david-novak","id":8188862,"avatar_url":"https://avatars.githubusercontent.com/u/8188862?v=3","gravatar_id":"","url":"https://api.github.com/users/david-novak","html_url":"https://github.com/david-novak","followers_url":"https://api.github.com/users/david-novak/followers","following_url":"https://api.github.com/users/david-novak/following{/other_user}","gists_url":"https://api.github.com/users/david-novak/gists{/gist_id}","starred_url":"https://api.github.com/users/david-novak/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/david-novak/subscriptions","organizations_url":"https://api.github.com/users/david-novak/orgs","repos_url":"https://api.github.com/users/david-novak/repos","events_url":"https://api.github.com/users/david-novak/events{/privacy}","received_events_url":"https://api.github.com/users/david-novak/received_events","type":"User","site_admin":false},"body":"Reworked preprocessing widget.","created_at":"2015-12-14T06:57:31Z","updated_at":"2016-02-05T13:03:40Z","closed_at":null,"merged_at":null,"merge_commit_sha":"6531b22575e6af265121e5bcee9c5dbd5d2bbd95","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/biolab/orange3-text/pulls/21/commits","review_comments_url":"https://api.github.com/repos/biolab/orange3-text/pulls/21/comments","review_comment_url":"https://api.github.com/repos/biolab/orange3-text/pulls/comments{/number}","comments_url":"https://api.github.com/repos/biolab/orange3-text/issues/21/comments","statuses_url":"https://api.github.com/repos/biolab/orange3-text/statuses/d06b8aa739503e0ae192dd8122d17e97e7587554","head":{"label":"david-novak:preprocessv2","ref":"preprocessv2","sha":"d06b8aa739503e0ae192dd8122d17e97e7587554","user":{"login":"david-novak","id":8188862,"avatar_url":"https://avatars.githubusercontent.com/u/8188862?v=3","gravatar_id":"","url":"https://api.github.com/users/david-novak","html_url":"https://github.com/david-novak","followers_url":"https://api.github.com/users/david-novak/followers","following_url":"https://api.github.com/users/david-novak/following{/other_user}","gists_url":"https://api.github.com/users/david-novak/gists{/gist_id}","starred_url":"https://api.github.com/users/david-novak/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/david-novak/subscriptions","organizations_url":"https://api.github.com/users/david-novak/orgs","repos_url":"https://api.github.com/users/david-novak/repos","events_url":"https://api.github.com/users/david-novak/events{/privacy}","received_events_url":"https://api.github.com/users/david-novak/received_events","type":"User","site_admin":false},"repo":{"id":38234383,"name":"orange3-text","full_name":"david-novak/orange3-text","owner":{"login":"david-novak","id":8188862,"avatar_url":"https://avatars.githubusercontent.com/u/8188862?v=3","gravatar_id":"","url":"https://api.github.com/users/david-novak","html_url":"https://github.com/david-novak","followers_url":"https://api.github.com/users/david-novak/followers","following_url":"https://api.github.com/users/david-novak/following{/other_user}","gists_url":"https://api.github.com/users/david-novak/gists{/gist_id}","starred_url":"https://api.github.com/users/david-novak/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/david-novak/subscriptions","organizations_url":"https://api.github.com/users/david-novak/orgs","repos_url":"https://api.github.com/users/david-novak/repos","events_url":"https://api.github.com/users/david-novak/events{/privacy}","received_events_url":"https://api.github.com/users/david-novak/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/david-novak/orange3-text","description":"Text Mining Addon for Orange3","fork":true,"url":"https://api.github.com/repos/david-novak/orange3-text","forks_url":"https://api.github.com/repos/david-novak/orange3-text/forks","keys_url":"https://api.github.com/repos/david-novak/orange3-text/keys{/key_id}","collaborators_url":"https://api.github.com/repos/david-novak/orange3-text/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/david-novak/orange3-text/teams","hooks_url":"https://api.github.com/repos/david-novak/orange3-text/hooks","issue_events_url":"https://api.github.com/repos/david-novak/orange3-text/issues/events{/number}","events_url":"https://api.github.com/repos/david-novak/orange3-text/events","assignees_url":"https://api.github.com/repos/david-novak/orange3-text/assignees{/user}","branches_url":"https://api.github.com/repos/david-novak/orange3-text/branches{/branch}","tags_url":"https://api.github.com/repos/david-novak/orange3-text/tags","blobs_url":"https://api.github.com/repos/david-novak/orange3-text/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/david-novak/orange3-text/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/david-novak/orange3-text/git/refs{/sha}","trees_url":"https://api.github.com/repos/david-novak/orange3-text/git/trees{/sha}","statuses_url":"https://api.github.com/repos/david-novak/orange3-text/statuses/{sha}","languages_url":"https://api.github.com/repos/david-novak/orange3-text/languages","stargazers_url":"https://api.github.com/repos/david-novak/orange3-text/stargazers","contributors_url":"https://api.github.com/repos/david-novak/orange3-text/contributors","subscribers_url":"https://api.github.com/repos/david-novak/orange3-text/subscribers","subscription_url":"https://api.github.com/repos/david-novak/orange3-text/subscription","commits_url":"https://api.github.com/repos/david-novak/orange3-text/commits{/sha}","git_commits_url":"https://api.github.com/repos/david-novak/orange3-text/git/commits{/sha}","comments_url":"https://api.github.com/repos/david-novak/orange3-text/comments{/number}","issue_comment_url":"https://api.github.com/repos/david-novak/orange3-text/issues/comments{/number}","contents_url":"https://api.github.com/repos/david-novak/orange3-text/contents/{+path}","compare_url":"https://api.github.com/repos/david-novak/orange3-text/compare/{base}...{head}","merges_url":"https://api.github.com/repos/david-novak/orange3-text/merges","archive_url":"https://api.github.com/repos/david-novak/orange3-text/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/david-novak/orange3-text/downloads","issues_url":"https://api.github.com/repos/david-novak/orange3-text/issues{/number}","pulls_url":"https://api.github.com/repos/david-novak/orange3-text/pulls{/number}","milestones_url":"https://api.github.com/repos/david-novak/orange3-text/milestones{/number}","notifications_url":"https://api.github.com/repos/david-novak/orange3-text/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/david-novak/orange3-text/labels{/name}","releases_url":"https://api.github.com/repos/david-novak/orange3-text/releases{/id}","deployments_url":"https://api.github.com/repos/david-novak/orange3-text/deployments","created_at":"2015-06-29T07:44:21Z","updated_at":"2015-06-29T07:44:22Z","pushed_at":"2016-02-01T18:34:21Z","git_url":"git://github.com/david-novak/orange3-text.git","ssh_url":"git@github.com:david-novak/orange3-text.git","clone_url":"https://github.com/david-novak/orange3-text.git","svn_url":"https://github.com/david-novak/orange3-text","homepage":null,"size":22884,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"biolab:master","ref":"master","sha":"f957532a1eeebd24606006a28213735b032f4934","user":{"login":"biolab","id":555189,"avatar_url":"https://avatars.githubusercontent.com/u/555189?v=3","gravatar_id":"","url":"https://api.github.com/users/biolab","html_url":"https://github.com/biolab","followers_url":"https://api.github.com/users/biolab/followers","following_url":"https://api.github.com/users/biolab/following{/other_user}","gists_url":"https://api.github.com/users/biolab/gists{/gist_id}","starred_url":"https://api.github.com/users/biolab/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/biolab/subscriptions","organizations_url":"https://api.github.com/users/biolab/orgs","repos_url":"https://api.github.com/users/biolab/repos","events_url":"https://api.github.com/users/biolab/events{/privacy}","received_events_url":"https://api.github.com/users/biolab/received_events","type":"Organization","site_admin":false},"repo":{"id":38108816,"name":"orange3-text","full_name":"biolab/orange3-text","owner":{"login":"biolab","id":555189,"avatar_url":"https://avatars.githubusercontent.com/u/555189?v=3","gravatar_id":"","url":"https://api.github.com/users/biolab","html_url":"https://github.com/biolab","followers_url":"https://api.github.com/users/biolab/followers","following_url":"https://api.github.com/users/biolab/following{/other_user}","gists_url":"https://api.github.com/users/biolab/gists{/gist_id}","starred_url":"https://api.github.com/users/biolab/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/biolab/subscriptions","organizations_url":"https://api.github.com/users/biolab/orgs","repos_url":"https://api.github.com/users/biolab/repos","events_url":"https://api.github.com/users/biolab/events{/privacy}","received_events_url":"https://api.github.com/users/biolab/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/biolab/orange3-text","description":"Text Mining add-on for Orange3","fork":false,"url":"https://api.github.com/repos/biolab/orange3-text","forks_url":"https://api.github.com/repos/biolab/orange3-text/forks","keys_url":"https://api.github.com/repos/biolab/orange3-text/keys{/key_id}","collaborators_url":"https://api.github.com/repos/biolab/orange3-text/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/biolab/orange3-text/teams","hooks_url":"https://api.github.com/repos/biolab/orange3-text/hooks","issue_events_url":"https://api.github.com/repos/biolab/orange3-text/issues/events{/number}","events_url":"https://api.github.com/repos/biolab/orange3-text/events","assignees_url":"https://api.github.com/repos/biolab/orange3-text/assignees{/user}","branches_url":"https://api.github.com/repos/biolab/orange3-text/branches{/branch}","tags_url":"https://api.github.com/repos/biolab/orange3-text/tags","blobs_url":"https://api.github.com/repos/biolab/orange3-text/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/biolab/orange3-text/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/biolab/orange3-text/git/refs{/sha}","trees_url":"https://api.github.com/repos/biolab/orange3-text/git/trees{/sha}","statuses_url":"https://api.github.com/repos/biolab/orange3-text/statuses/{sha}","languages_url":"https://api.github.com/repos/biolab/orange3-text/languages","stargazers_url":"https://api.github.com/repos/biolab/orange3-text/stargazers","contributors_url":"https://api.github.com/repos/biolab/orange3-text/contributors","subscribers_url":"https://api.github.com/repos/biolab/orange3-text/subscribers","subscription_url":"https://api.github.com/repos/biolab/orange3-text/subscription","commits_url":"https://api.github.com/repos/biolab/orange3-text/commits{/sha}","git_commits_url":"https://api.github.com/repos/biolab/orange3-text/git/commits{/sha}","comments_url":"https://api.github.com/repos/biolab/orange3-text/comments{/number}","issue_comment_url":"https://api.github.com/repos/biolab/orange3-text/issues/comments{/number}","contents_url":"https://api.github.com/repos/biolab/orange3-text/contents/{+path}","compare_url":"https://api.github.com/repos/biolab/orange3-text/compare/{base}...{head}","merges_url":"https://api.github.com/repos/biolab/orange3-text/merges","archive_url":"https://api.github.com/repos/biolab/orange3-text/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/biolab/orange3-text/downloads","issues_url":"https://api.github.com/repos/biolab/orange3-text/issues{/number}","pulls_url":"https://api.github.com/repos/biolab/orange3-text/pulls{/number}","milestones_url":"https://api.github.com/repos/biolab/orange3-text/milestones{/number}","notifications_url":"https://api.github.com/repos/biolab/orange3-text/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/biolab/orange3-text/labels{/name}","releases_url":"https://api.github.com/repos/biolab/orange3-text/releases{/id}","deployments_url":"https://api.github.com/repos/biolab/orange3-text/deployments","created_at":"2015-06-26T12:02:06Z","updated_at":"2015-12-06T00:09:41Z","pushed_at":"2016-02-01T18:34:21Z","git_url":"git://github.com/biolab/orange3-text.git","ssh_url":"git@github.com:biolab/orange3-text.git","clone_url":"https://github.com/biolab/orange3-text.git","svn_url":"https://github.com/biolab/orange3-text","homepage":"","size":22906,"stargazers_count":2,"watchers_count":2,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":7,"mirror_url":null,"open_issues_count":3,"forks":7,"open_issues":3,"watchers":2,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/biolab/orange3-text/pulls/21"},"html":{"href":"https://github.com/biolab/orange3-text/pull/21"},"issue":{"href":"https://api.github.com/repos/biolab/orange3-text/issues/21"},"comments":{"href":"https://api.github.com/repos/biolab/orange3-text/issues/21/comments"},"review_comments":{"href":"https://api.github.com/repos/biolab/orange3-text/pulls/21/comments"},"review_comment":{"href":"https://api.github.com/repos/biolab/orange3-text/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/biolab/orange3-text/pulls/21/commits"},"statuses":{"href":"https://api.github.com/repos/biolab/orange3-text/statuses/d06b8aa739503e0ae192dd8122d17e97e7587554"}}}}