{"action":"created","comment":{"url":"https://api.github.com/repos/chartbeat-labs/textacy/pulls/comments/54199255","id":54199255,"diff_hunk":"@@ -0,0 +1,187 @@\n+\"\"\"\n+Module with functions for reading content from disk in common formats.\n+\"\"\"\n+import bz2\n+from functools import partial\n+import gzip\n+import io\n+import json\n+import os\n+\n+import ijson\n+from spacy.tokens.doc import Doc\n+\n+\n+JSON_DECODER = json.JSONDecoder()\n+\n+\n+def read_json(filename, mode='rt', encoding=None, prefix=''):\n+    \"\"\"\n+    Iterate over JSON objects matching the field given by ``prefix``.\n+    Useful for reading a large JSON array one item (with ``prefix='item'``)\n+    or sub-item (``prefix='item.fieldname'``) at a time.\n+\n+    Args:\n+        filename (str): /path/to/file on disk from which json items will be streamed,\n+            such as items in a JSON array; for example::\n+\n+                [\n+                    {\"title\": \"Harrison Bergeron\", \"text\": \"The year was 2081, and everybody was finally equal.\"},\n+                    {\"title\": \"2BR02B\", \"text\": \"Everything was perfectly swell.\"}\n+                ]\n+\n+        mode (str, optional)\n+        encoding (str, optional)\n+        prefix (str, optional): if '', the entire JSON object will be read in at once;\n+            if 'item', each item in a top-level array will be read in successively;\n+            if 'item.text', each array item's 'text' value will be read in successively\n+\n+    Yields:\n+        next matching JSON object; could be a dict, list, int, float, str,\n+            depending on the value of ``prefix``\n+\n+    Notes:\n+        Refer to ``ijson`` at https://pypi.python.org/pypi/ijson/ for usage details.\n+    \"\"\"\n+    with io.open(filename, mode=mode, encoding=encoding) as f:\n+        if prefix == '':\n+            yield json.load(f)\n+        else:\n+            for item in ijson.items(f, prefix):\n+                yield item\n+\n+\n+def read_json_lines(filename, mode='rt', encoding=None):\n+    \"\"\"\n+    Iterate over a stream of JSON objects, where each line of file ``filename``\n+    is a valid JSON object but no JSON object (e.g. array) exists at the top level.\n+\n+    Args:\n+        filename (str): /path/to/file on disk from which json objects will be streamed,\n+            where each line in the file must be its own json object; for example::\n+\n+                {\"title\": \"Harrison Bergeron\", \"text\": \"The year was 2081, and everybody was finally equal.\"}\\n\n+                {\"title\": \"2BR02B\", \"text\": \"Everything was perfectly swell.\"}\n+\n+        mode (str, optional)\n+        encoding (str, optional)\n+\n+    Yields:\n+        dict: next valid JSON object, converted to native Python equivalent\n+    \"\"\"\n+    with io.open(filename, mode=mode, encoding=encoding) as f:\n+        for line in f:\n+            yield json.loads(line)\n+\n+\n+def read_json_mash(filename, mode='rt', encoding=None, buffersize=2048):\n+    \"\"\"\n+    Iterate over a stream of JSON objects, all of them mashed together, end-to-end,\n+    on a single line of a file. Bad form, but still manageable.\n+\n+    Args:\n+        filename (str): /path/to/file on disk from which json objects will be streamed,\n+            where all json objects are mashed together, end-to-end, on a single line,;\n+            for example::\n+\n+                {\"title\": \"Harrison Bergeron\", \"text\": \"The year was 2081, and everybody was finally equal.\"}{\"title\": \"2BR02B\", \"text\": \"Everything was perfectly swell.\"}\n+\n+        mode (str, optional)\n+        encoding (str, optional)\n+        buffersize (int, optional): number of bytes to read in as a chunk\n+\n+    Yields:\n+        dict: next valid JSON object, converted to native Python equivalent\n+    \"\"\"\n+    with io.open(filename, mode=mode, encoding=encoding) as f:\n+        buffer = ''\n+        for chunk in iter(partial(f.read, buffersize), ''):\n+            buffer += chunk\n+            while buffer:\n+                try:\n+                    result, index = JSON_DECODER.raw_decode(buffer)\n+                    yield result\n+                    buffer = buffer[index:]\n+                # not enough data to decode => read another chunk\n+                except ValueError:\n+                    break\n+\n+\n+def read_file(filename, mode='rt', encoding=None):\n+    \"\"\"\n+    Read the full contents of a file. Files compressed with gzip or bz2 are handled\n+    automatically.\n+    \"\"\"\n+    _open = gzip.open if filename.endswith('.gz') \\\n+        else bz2.open if filename.endswith('.bz2') \\\n+        else io.open\n+    with _open(filename, mode=mode, encoding=encoding) as f:\n+        return f.read()\n+\n+\n+def read_file_lines(filename, mode='rt', encoding=None):\n+    \"\"\"\n+    Read the contents of a file, line by line. Files compressed with gzip or bz2\n+    are handled automatically.\n+    \"\"\"\n+    _open = gzip.open if filename.endswith('.gz') \\\n+        else bz2.open if filename.endswith('.bz2') \\\n+        else io.open\n+    with _open(filename, mode=mode, encoding=encoding) as f:\n+        for line in f:\n+            yield f\n+\n+\n+def read_spacy_docs(spacy_vocab, filename):\n+    \"\"\"\n+    Stream ``spacy.Doc`` s from disk at ``filename`` where they were serialized\n+    using Spacy's ``spacy.Doc.to_bytes()`` functionality.\n+\n+    Args:\n+        spacy_vocab (``spacy.Vocab``): the spacy vocab object used to serialize\n+            the docs in ``filename``\n+        filename (str): /path/to/file on disk from which spacy docs will be streamed\n+\n+    Yields:\n+        the next deserialized ``spacy.Doc``\n+    \"\"\"\n+    with io.open(filename, mode='rb') as f:\n+        for bytes_string in Doc.read_bytes(f):\n+            yield Doc(spacy_vocab).from_bytes(bytes_string)\n+\n+\n+def get_filenames_in_dir(dirname, mode='rt', encoding='utf8',\n+                         file_type=None, subdirs=False):\n+    \"\"\"\n+    Yield the full names of files under directory ``dirname``, optionally\n+    filtering for a certain file type and crawling all subdirectories.\n+\n+    Args:\n+        dirname (str): /path/to/dir on disk where files to read are saved\n+        mode (str, optional): mode with which to open files; default is probably correct\n+        encoding (str, optional): encoding with which to open files","path":"textacy/fileio/read.py","position":162,"original_position":162,"commit_id":"d4d7f138288502e9649ec4815cbb4149b20b2f25","original_commit_id":"d4d7f138288502e9649ec4815cbb4149b20b2f25","user":{"login":"danvalente","id":3912229,"avatar_url":"https://avatars.githubusercontent.com/u/3912229?v=3","gravatar_id":"","url":"https://api.github.com/users/danvalente","html_url":"https://github.com/danvalente","followers_url":"https://api.github.com/users/danvalente/followers","following_url":"https://api.github.com/users/danvalente/following{/other_user}","gists_url":"https://api.github.com/users/danvalente/gists{/gist_id}","starred_url":"https://api.github.com/users/danvalente/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/danvalente/subscriptions","organizations_url":"https://api.github.com/users/danvalente/orgs","repos_url":"https://api.github.com/users/danvalente/repos","events_url":"https://api.github.com/users/danvalente/events{/privacy}","received_events_url":"https://api.github.com/users/danvalente/received_events","type":"User","site_admin":false},"body":"`mode` and `encoding` are not used in this function.","created_at":"2016-02-26T02:46:05Z","updated_at":"2016-02-26T02:46:05Z","html_url":"https://github.com/chartbeat-labs/textacy/pull/4#discussion_r54199255","pull_request_url":"https://api.github.com/repos/chartbeat-labs/textacy/pulls/4","_links":{"self":{"href":"https://api.github.com/repos/chartbeat-labs/textacy/pulls/comments/54199255"},"html":{"href":"https://github.com/chartbeat-labs/textacy/pull/4#discussion_r54199255"},"pull_request":{"href":"https://api.github.com/repos/chartbeat-labs/textacy/pulls/4"}}},"pull_request":{"url":"https://api.github.com/repos/chartbeat-labs/textacy/pulls/4","id":60450376,"html_url":"https://github.com/chartbeat-labs/textacy/pull/4","diff_url":"https://github.com/chartbeat-labs/textacy/pull/4.diff","patch_url":"https://github.com/chartbeat-labs/textacy/pull/4.patch","issue_url":"https://api.github.com/repos/chartbeat-labs/textacy/issues/4","number":4,"state":"open","locked":false,"title":"added fileio subpackage for reading/writing content from/to disk","user":{"login":"bdewilde","id":2514535,"avatar_url":"https://avatars.githubusercontent.com/u/2514535?v=3","gravatar_id":"","url":"https://api.github.com/users/bdewilde","html_url":"https://github.com/bdewilde","followers_url":"https://api.github.com/users/bdewilde/followers","following_url":"https://api.github.com/users/bdewilde/following{/other_user}","gists_url":"https://api.github.com/users/bdewilde/gists{/gist_id}","starred_url":"https://api.github.com/users/bdewilde/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bdewilde/subscriptions","organizations_url":"https://api.github.com/users/bdewilde/orgs","repos_url":"https://api.github.com/users/bdewilde/repos","events_url":"https://api.github.com/users/bdewilde/events{/privacy}","received_events_url":"https://api.github.com/users/bdewilde/received_events","type":"User","site_admin":false},"body":"You know the drill, @danvalente. :bow:\r\n\r\nThree things I could use some help with:\r\n\r\n1. Have I implemented the existing functions correctly and efficiently?\r\n1. Have I omitted any common formats for storing text(s) on disk? Which?\r\n1. How should I go about testing these functions?","created_at":"2016-02-24T03:23:18Z","updated_at":"2016-02-26T02:46:05Z","closed_at":null,"merged_at":null,"merge_commit_sha":"c077301eec09f9f870a4cad306f2c53a0279dee0","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/chartbeat-labs/textacy/pulls/4/commits","review_comments_url":"https://api.github.com/repos/chartbeat-labs/textacy/pulls/4/comments","review_comment_url":"https://api.github.com/repos/chartbeat-labs/textacy/pulls/comments{/number}","comments_url":"https://api.github.com/repos/chartbeat-labs/textacy/issues/4/comments","statuses_url":"https://api.github.com/repos/chartbeat-labs/textacy/statuses/d4d7f138288502e9649ec4815cbb4149b20b2f25","head":{"label":"chartbeat-labs:fileio","ref":"fileio","sha":"d4d7f138288502e9649ec4815cbb4149b20b2f25","user":{"login":"chartbeat-labs","id":5552574,"avatar_url":"https://avatars.githubusercontent.com/u/5552574?v=3","gravatar_id":"","url":"https://api.github.com/users/chartbeat-labs","html_url":"https://github.com/chartbeat-labs","followers_url":"https://api.github.com/users/chartbeat-labs/followers","following_url":"https://api.github.com/users/chartbeat-labs/following{/other_user}","gists_url":"https://api.github.com/users/chartbeat-labs/gists{/gist_id}","starred_url":"https://api.github.com/users/chartbeat-labs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/chartbeat-labs/subscriptions","organizations_url":"https://api.github.com/users/chartbeat-labs/orgs","repos_url":"https://api.github.com/users/chartbeat-labs/repos","events_url":"https://api.github.com/users/chartbeat-labs/events{/privacy}","received_events_url":"https://api.github.com/users/chartbeat-labs/received_events","type":"Organization","site_admin":false},"repo":{"id":51014761,"name":"textacy","full_name":"chartbeat-labs/textacy","owner":{"login":"chartbeat-labs","id":5552574,"avatar_url":"https://avatars.githubusercontent.com/u/5552574?v=3","gravatar_id":"","url":"https://api.github.com/users/chartbeat-labs","html_url":"https://github.com/chartbeat-labs","followers_url":"https://api.github.com/users/chartbeat-labs/followers","following_url":"https://api.github.com/users/chartbeat-labs/following{/other_user}","gists_url":"https://api.github.com/users/chartbeat-labs/gists{/gist_id}","starred_url":"https://api.github.com/users/chartbeat-labs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/chartbeat-labs/subscriptions","organizations_url":"https://api.github.com/users/chartbeat-labs/orgs","repos_url":"https://api.github.com/users/chartbeat-labs/repos","events_url":"https://api.github.com/users/chartbeat-labs/events{/privacy}","received_events_url":"https://api.github.com/users/chartbeat-labs/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/chartbeat-labs/textacy","description":"higher-level NLP built on Spacy","fork":false,"url":"https://api.github.com/repos/chartbeat-labs/textacy","forks_url":"https://api.github.com/repos/chartbeat-labs/textacy/forks","keys_url":"https://api.github.com/repos/chartbeat-labs/textacy/keys{/key_id}","collaborators_url":"https://api.github.com/repos/chartbeat-labs/textacy/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/chartbeat-labs/textacy/teams","hooks_url":"https://api.github.com/repos/chartbeat-labs/textacy/hooks","issue_events_url":"https://api.github.com/repos/chartbeat-labs/textacy/issues/events{/number}","events_url":"https://api.github.com/repos/chartbeat-labs/textacy/events","assignees_url":"https://api.github.com/repos/chartbeat-labs/textacy/assignees{/user}","branches_url":"https://api.github.com/repos/chartbeat-labs/textacy/branches{/branch}","tags_url":"https://api.github.com/repos/chartbeat-labs/textacy/tags","blobs_url":"https://api.github.com/repos/chartbeat-labs/textacy/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/chartbeat-labs/textacy/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/chartbeat-labs/textacy/git/refs{/sha}","trees_url":"https://api.github.com/repos/chartbeat-labs/textacy/git/trees{/sha}","statuses_url":"https://api.github.com/repos/chartbeat-labs/textacy/statuses/{sha}","languages_url":"https://api.github.com/repos/chartbeat-labs/textacy/languages","stargazers_url":"https://api.github.com/repos/chartbeat-labs/textacy/stargazers","contributors_url":"https://api.github.com/repos/chartbeat-labs/textacy/contributors","subscribers_url":"https://api.github.com/repos/chartbeat-labs/textacy/subscribers","subscription_url":"https://api.github.com/repos/chartbeat-labs/textacy/subscription","commits_url":"https://api.github.com/repos/chartbeat-labs/textacy/commits{/sha}","git_commits_url":"https://api.github.com/repos/chartbeat-labs/textacy/git/commits{/sha}","comments_url":"https://api.github.com/repos/chartbeat-labs/textacy/comments{/number}","issue_comment_url":"https://api.github.com/repos/chartbeat-labs/textacy/issues/comments{/number}","contents_url":"https://api.github.com/repos/chartbeat-labs/textacy/contents/{+path}","compare_url":"https://api.github.com/repos/chartbeat-labs/textacy/compare/{base}...{head}","merges_url":"https://api.github.com/repos/chartbeat-labs/textacy/merges","archive_url":"https://api.github.com/repos/chartbeat-labs/textacy/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/chartbeat-labs/textacy/downloads","issues_url":"https://api.github.com/repos/chartbeat-labs/textacy/issues{/number}","pulls_url":"https://api.github.com/repos/chartbeat-labs/textacy/pulls{/number}","milestones_url":"https://api.github.com/repos/chartbeat-labs/textacy/milestones{/number}","notifications_url":"https://api.github.com/repos/chartbeat-labs/textacy/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/chartbeat-labs/textacy/labels{/name}","releases_url":"https://api.github.com/repos/chartbeat-labs/textacy/releases{/id}","deployments_url":"https://api.github.com/repos/chartbeat-labs/textacy/deployments","created_at":"2016-02-03T16:52:45Z","updated_at":"2016-02-19T16:56:44Z","pushed_at":"2016-02-24T20:00:44Z","git_url":"git://github.com/chartbeat-labs/textacy.git","ssh_url":"git@github.com:chartbeat-labs/textacy.git","clone_url":"https://github.com/chartbeat-labs/textacy.git","svn_url":"https://github.com/chartbeat-labs/textacy","homepage":"http://textacy.readthedocs.org/","size":4364,"stargazers_count":6,"watchers_count":6,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":1,"forks":0,"open_issues":1,"watchers":6,"default_branch":"master"}},"base":{"label":"chartbeat-labs:master","ref":"master","sha":"f0ffa7d7b96c2ddd33f21d1e7611f8bf78aae65d","user":{"login":"chartbeat-labs","id":5552574,"avatar_url":"https://avatars.githubusercontent.com/u/5552574?v=3","gravatar_id":"","url":"https://api.github.com/users/chartbeat-labs","html_url":"https://github.com/chartbeat-labs","followers_url":"https://api.github.com/users/chartbeat-labs/followers","following_url":"https://api.github.com/users/chartbeat-labs/following{/other_user}","gists_url":"https://api.github.com/users/chartbeat-labs/gists{/gist_id}","starred_url":"https://api.github.com/users/chartbeat-labs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/chartbeat-labs/subscriptions","organizations_url":"https://api.github.com/users/chartbeat-labs/orgs","repos_url":"https://api.github.com/users/chartbeat-labs/repos","events_url":"https://api.github.com/users/chartbeat-labs/events{/privacy}","received_events_url":"https://api.github.com/users/chartbeat-labs/received_events","type":"Organization","site_admin":false},"repo":{"id":51014761,"name":"textacy","full_name":"chartbeat-labs/textacy","owner":{"login":"chartbeat-labs","id":5552574,"avatar_url":"https://avatars.githubusercontent.com/u/5552574?v=3","gravatar_id":"","url":"https://api.github.com/users/chartbeat-labs","html_url":"https://github.com/chartbeat-labs","followers_url":"https://api.github.com/users/chartbeat-labs/followers","following_url":"https://api.github.com/users/chartbeat-labs/following{/other_user}","gists_url":"https://api.github.com/users/chartbeat-labs/gists{/gist_id}","starred_url":"https://api.github.com/users/chartbeat-labs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/chartbeat-labs/subscriptions","organizations_url":"https://api.github.com/users/chartbeat-labs/orgs","repos_url":"https://api.github.com/users/chartbeat-labs/repos","events_url":"https://api.github.com/users/chartbeat-labs/events{/privacy}","received_events_url":"https://api.github.com/users/chartbeat-labs/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/chartbeat-labs/textacy","description":"higher-level NLP built on Spacy","fork":false,"url":"https://api.github.com/repos/chartbeat-labs/textacy","forks_url":"https://api.github.com/repos/chartbeat-labs/textacy/forks","keys_url":"https://api.github.com/repos/chartbeat-labs/textacy/keys{/key_id}","collaborators_url":"https://api.github.com/repos/chartbeat-labs/textacy/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/chartbeat-labs/textacy/teams","hooks_url":"https://api.github.com/repos/chartbeat-labs/textacy/hooks","issue_events_url":"https://api.github.com/repos/chartbeat-labs/textacy/issues/events{/number}","events_url":"https://api.github.com/repos/chartbeat-labs/textacy/events","assignees_url":"https://api.github.com/repos/chartbeat-labs/textacy/assignees{/user}","branches_url":"https://api.github.com/repos/chartbeat-labs/textacy/branches{/branch}","tags_url":"https://api.github.com/repos/chartbeat-labs/textacy/tags","blobs_url":"https://api.github.com/repos/chartbeat-labs/textacy/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/chartbeat-labs/textacy/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/chartbeat-labs/textacy/git/refs{/sha}","trees_url":"https://api.github.com/repos/chartbeat-labs/textacy/git/trees{/sha}","statuses_url":"https://api.github.com/repos/chartbeat-labs/textacy/statuses/{sha}","languages_url":"https://api.github.com/repos/chartbeat-labs/textacy/languages","stargazers_url":"https://api.github.com/repos/chartbeat-labs/textacy/stargazers","contributors_url":"https://api.github.com/repos/chartbeat-labs/textacy/contributors","subscribers_url":"https://api.github.com/repos/chartbeat-labs/textacy/subscribers","subscription_url":"https://api.github.com/repos/chartbeat-labs/textacy/subscription","commits_url":"https://api.github.com/repos/chartbeat-labs/textacy/commits{/sha}","git_commits_url":"https://api.github.com/repos/chartbeat-labs/textacy/git/commits{/sha}","comments_url":"https://api.github.com/repos/chartbeat-labs/textacy/comments{/number}","issue_comment_url":"https://api.github.com/repos/chartbeat-labs/textacy/issues/comments{/number}","contents_url":"https://api.github.com/repos/chartbeat-labs/textacy/contents/{+path}","compare_url":"https://api.github.com/repos/chartbeat-labs/textacy/compare/{base}...{head}","merges_url":"https://api.github.com/repos/chartbeat-labs/textacy/merges","archive_url":"https://api.github.com/repos/chartbeat-labs/textacy/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/chartbeat-labs/textacy/downloads","issues_url":"https://api.github.com/repos/chartbeat-labs/textacy/issues{/number}","pulls_url":"https://api.github.com/repos/chartbeat-labs/textacy/pulls{/number}","milestones_url":"https://api.github.com/repos/chartbeat-labs/textacy/milestones{/number}","notifications_url":"https://api.github.com/repos/chartbeat-labs/textacy/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/chartbeat-labs/textacy/labels{/name}","releases_url":"https://api.github.com/repos/chartbeat-labs/textacy/releases{/id}","deployments_url":"https://api.github.com/repos/chartbeat-labs/textacy/deployments","created_at":"2016-02-03T16:52:45Z","updated_at":"2016-02-19T16:56:44Z","pushed_at":"2016-02-24T20:00:44Z","git_url":"git://github.com/chartbeat-labs/textacy.git","ssh_url":"git@github.com:chartbeat-labs/textacy.git","clone_url":"https://github.com/chartbeat-labs/textacy.git","svn_url":"https://github.com/chartbeat-labs/textacy","homepage":"http://textacy.readthedocs.org/","size":4364,"stargazers_count":6,"watchers_count":6,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":1,"forks":0,"open_issues":1,"watchers":6,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/chartbeat-labs/textacy/pulls/4"},"html":{"href":"https://github.com/chartbeat-labs/textacy/pull/4"},"issue":{"href":"https://api.github.com/repos/chartbeat-labs/textacy/issues/4"},"comments":{"href":"https://api.github.com/repos/chartbeat-labs/textacy/issues/4/comments"},"review_comments":{"href":"https://api.github.com/repos/chartbeat-labs/textacy/pulls/4/comments"},"review_comment":{"href":"https://api.github.com/repos/chartbeat-labs/textacy/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/chartbeat-labs/textacy/pulls/4/commits"},"statuses":{"href":"https://api.github.com/repos/chartbeat-labs/textacy/statuses/d4d7f138288502e9649ec4815cbb4149b20b2f25"}}}}