{"action":"created","comment":{"url":"https://api.github.com/repos/devin-petersohn/ray/pulls/comments/170430559","pull_request_review_id":99121474,"id":170430559,"diff_hunk":"@@ -0,0 +1,137 @@\n+from __future__ import absolute_import\n+from __future__ import division\n+from __future__ import print_function\n+\n+from itertools import chain\n+from io import BytesIO\n+import os\n+\n+from pyarrow.parquet import ParquetFile\n+import pandas as pd\n+\n+from .dataframe import from_pandas, ray, DataFrame\n+from . import get_nprtitions\n+\n+\n+def read_parquet(path, engine='auto', columns=None, **kwargs):\n+    \"\"\"Load a parquet object from the file path, returning a DataFrame.\n+    Ray DataFrame only supports pyarrow engine for now.\n+\n+    Args:\n+        path: The filepath of the parquet file.\n+              We only support local files for now.\n+        engine: Ray only support pyarrow reader. This argument doesn't do anything for now. \n+        kwargs: Pass into parquet's read_row_group function.\n+    \"\"\"\n+    pf = ParquetFile(path)\n+    n_row_groups = pf.metadata.num_row_groups\n+    partition_per_row_group = get_nprtitions() // n_row_groups\n+    ray_row_groups = [\n+        _read_parquet_row_group.remote(path, columns, i,\n+                                       partition_per_row_group, kwargs)\n+        for i in range(n_row_groups)\n+    ]\n+    return _vertical_concat(ray.get(ray_row_groups))\n+\n+\n+@ray.remote\n+def _read_parquet_row_group(path, columns, row_group, npartitions, kwargs={}):\n+    \"\"\"Read a parquet row_group given file_path. This function returns\n+    a Ray DataFrame\n+    \"\"\"\n+    pf = ParquetFile(path)\n+    df = pf.read_row_group(row_group, columns=columns, **kwargs).to_pandas()\n+    return from_pandas(df, npartitions=npartitions)\n+\n+\n+def _compute_offset(fn, npartitions):\n+    \"\"\"\n+    Calculate the currect bytes offsets for a csv file.\n+    Return a list of (start, end) tuple where the end == \\n or EOF.\n+    \"\"\"\n+    total_bytes = os.path.getsize(fn)\n+    chunksize = total_bytes // npartitions\n+    if chunksize == 0:\n+        chunksize = 1\n+\n+    bio = open(fn, 'rb')\n+\n+    offsets = []\n+    start = 0\n+    while start <= total_bytes:\n+        bio.seek(chunksize, 1)  # Move forward {chunksize} bytes\n+        extend_line = bio.readline()  # Move after the next \\n\n+        total_offset = chunksize + len(extend_line)\n+        # The position of the \\n we just crossed.\n+        new_line_cursor = start + total_offset - 1\n+        offsets.append((start, new_line_cursor))\n+        start = new_line_cursor + 1\n+\n+    bio.close()\n+    return offsets\n+\n+\n+def _get_first_line(fn):\n+    bio = open(fn, 'rb')\n+    first = bio.readline()\n+    bio.close()\n+    return first\n+\n+\n+@ray.remote\n+def _read_csv_with_offset(fn, start, end, header=b'', kwargs={}):\n+    bio = open(fn, 'rb')\n+    bio.seek(start)\n+    to_read = header + bio.read(end - start)\n+    bio.close()\n+    return from_pandas(pd.read_csv(BytesIO(to_read), **kwargs), npartitions=1)\n+\n+\n+def read_csv(filepath, **kwargs):","path":"python/ray/dataframe/io.py","position":90,"original_position":90,"commit_id":"32dbd9e4e484aa7e6dca3f764eb47cca00eb3169","original_commit_id":"32dbd9e4e484aa7e6dca3f764eb47cca00eb3169","user":{"login":"simon-mo","id":21118851,"avatar_url":"https://avatars3.githubusercontent.com/u/21118851?v=4","gravatar_id":"","url":"https://api.github.com/users/simon-mo","html_url":"https://github.com/simon-mo","followers_url":"https://api.github.com/users/simon-mo/followers","following_url":"https://api.github.com/users/simon-mo/following{/other_user}","gists_url":"https://api.github.com/users/simon-mo/gists{/gist_id}","starred_url":"https://api.github.com/users/simon-mo/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/simon-mo/subscriptions","organizations_url":"https://api.github.com/users/simon-mo/orgs","repos_url":"https://api.github.com/users/simon-mo/repos","events_url":"https://api.github.com/users/simon-mo/events{/privacy}","received_events_url":"https://api.github.com/users/simon-mo/received_events","type":"User","site_admin":false},"body":"They don't work if they are not specified by key word. Ok I'm putting all the args to our code","created_at":"2018-02-24T19:46:20Z","updated_at":"2018-02-24T19:46:20Z","html_url":"https://github.com/devin-petersohn/ray/pull/1#discussion_r170430559","pull_request_url":"https://api.github.com/repos/devin-petersohn/ray/pulls/1","author_association":"NONE","_links":{"self":{"href":"https://api.github.com/repos/devin-petersohn/ray/pulls/comments/170430559"},"html":{"href":"https://github.com/devin-petersohn/ray/pull/1#discussion_r170430559"},"pull_request":{"href":"https://api.github.com/repos/devin-petersohn/ray/pulls/1"}},"in_reply_to_id":170415583},"pull_request":{"url":"https://api.github.com/repos/devin-petersohn/ray/pulls/1","id":171136990,"html_url":"https://github.com/devin-petersohn/ray/pull/1","diff_url":"https://github.com/devin-petersohn/ray/pull/1.diff","patch_url":"https://github.com/devin-petersohn/ray/pull/1.patch","issue_url":"https://api.github.com/repos/devin-petersohn/ray/issues/1","number":1,"state":"open","locked":false,"title":"DataFrame IO","user":{"login":"simon-mo","id":21118851,"avatar_url":"https://avatars3.githubusercontent.com/u/21118851?v=4","gravatar_id":"","url":"https://api.github.com/users/simon-mo","html_url":"https://github.com/simon-mo","followers_url":"https://api.github.com/users/simon-mo/followers","following_url":"https://api.github.com/users/simon-mo/following{/other_user}","gists_url":"https://api.github.com/users/simon-mo/gists{/gist_id}","starred_url":"https://api.github.com/users/simon-mo/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/simon-mo/subscriptions","organizations_url":"https://api.github.com/users/simon-mo/orgs","repos_url":"https://api.github.com/users/simon-mo/repos","events_url":"https://api.github.com/users/simon-mo/events{/privacy}","received_events_url":"https://api.github.com/users/simon-mo/received_events","type":"User","site_admin":false},"body":"- Add `read_csv` and `read_parquet` with tests. \r\n- Modified to_pandas that put back `df.index`\r\n","created_at":"2018-02-23T23:01:58Z","updated_at":"2018-02-24T19:46:20Z","closed_at":null,"merged_at":null,"merge_commit_sha":"381a807ab4cd04847069ce9b0b78c06f5217b7bd","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/devin-petersohn/ray/pulls/1/commits","review_comments_url":"https://api.github.com/repos/devin-petersohn/ray/pulls/1/comments","review_comment_url":"https://api.github.com/repos/devin-petersohn/ray/pulls/comments{/number}","comments_url":"https://api.github.com/repos/devin-petersohn/ray/issues/1/comments","statuses_url":"https://api.github.com/repos/devin-petersohn/ray/statuses/32dbd9e4e484aa7e6dca3f764eb47cca00eb3169","head":{"label":"simon-mo:df_io","ref":"df_io","sha":"32dbd9e4e484aa7e6dca3f764eb47cca00eb3169","user":{"login":"simon-mo","id":21118851,"avatar_url":"https://avatars3.githubusercontent.com/u/21118851?v=4","gravatar_id":"","url":"https://api.github.com/users/simon-mo","html_url":"https://github.com/simon-mo","followers_url":"https://api.github.com/users/simon-mo/followers","following_url":"https://api.github.com/users/simon-mo/following{/other_user}","gists_url":"https://api.github.com/users/simon-mo/gists{/gist_id}","starred_url":"https://api.github.com/users/simon-mo/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/simon-mo/subscriptions","organizations_url":"https://api.github.com/users/simon-mo/orgs","repos_url":"https://api.github.com/users/simon-mo/repos","events_url":"https://api.github.com/users/simon-mo/events{/privacy}","received_events_url":"https://api.github.com/users/simon-mo/received_events","type":"User","site_admin":false},"repo":{"id":122156396,"name":"ray","full_name":"simon-mo/ray","owner":{"login":"simon-mo","id":21118851,"avatar_url":"https://avatars3.githubusercontent.com/u/21118851?v=4","gravatar_id":"","url":"https://api.github.com/users/simon-mo","html_url":"https://github.com/simon-mo","followers_url":"https://api.github.com/users/simon-mo/followers","following_url":"https://api.github.com/users/simon-mo/following{/other_user}","gists_url":"https://api.github.com/users/simon-mo/gists{/gist_id}","starred_url":"https://api.github.com/users/simon-mo/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/simon-mo/subscriptions","organizations_url":"https://api.github.com/users/simon-mo/orgs","repos_url":"https://api.github.com/users/simon-mo/repos","events_url":"https://api.github.com/users/simon-mo/events{/privacy}","received_events_url":"https://api.github.com/users/simon-mo/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/simon-mo/ray","description":"A high-performance distributed execution engine","fork":true,"url":"https://api.github.com/repos/simon-mo/ray","forks_url":"https://api.github.com/repos/simon-mo/ray/forks","keys_url":"https://api.github.com/repos/simon-mo/ray/keys{/key_id}","collaborators_url":"https://api.github.com/repos/simon-mo/ray/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/simon-mo/ray/teams","hooks_url":"https://api.github.com/repos/simon-mo/ray/hooks","issue_events_url":"https://api.github.com/repos/simon-mo/ray/issues/events{/number}","events_url":"https://api.github.com/repos/simon-mo/ray/events","assignees_url":"https://api.github.com/repos/simon-mo/ray/assignees{/user}","branches_url":"https://api.github.com/repos/simon-mo/ray/branches{/branch}","tags_url":"https://api.github.com/repos/simon-mo/ray/tags","blobs_url":"https://api.github.com/repos/simon-mo/ray/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/simon-mo/ray/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/simon-mo/ray/git/refs{/sha}","trees_url":"https://api.github.com/repos/simon-mo/ray/git/trees{/sha}","statuses_url":"https://api.github.com/repos/simon-mo/ray/statuses/{sha}","languages_url":"https://api.github.com/repos/simon-mo/ray/languages","stargazers_url":"https://api.github.com/repos/simon-mo/ray/stargazers","contributors_url":"https://api.github.com/repos/simon-mo/ray/contributors","subscribers_url":"https://api.github.com/repos/simon-mo/ray/subscribers","subscription_url":"https://api.github.com/repos/simon-mo/ray/subscription","commits_url":"https://api.github.com/repos/simon-mo/ray/commits{/sha}","git_commits_url":"https://api.github.com/repos/simon-mo/ray/git/commits{/sha}","comments_url":"https://api.github.com/repos/simon-mo/ray/comments{/number}","issue_comment_url":"https://api.github.com/repos/simon-mo/ray/issues/comments{/number}","contents_url":"https://api.github.com/repos/simon-mo/ray/contents/{+path}","compare_url":"https://api.github.com/repos/simon-mo/ray/compare/{base}...{head}","merges_url":"https://api.github.com/repos/simon-mo/ray/merges","archive_url":"https://api.github.com/repos/simon-mo/ray/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/simon-mo/ray/downloads","issues_url":"https://api.github.com/repos/simon-mo/ray/issues{/number}","pulls_url":"https://api.github.com/repos/simon-mo/ray/pulls{/number}","milestones_url":"https://api.github.com/repos/simon-mo/ray/milestones{/number}","notifications_url":"https://api.github.com/repos/simon-mo/ray/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/simon-mo/ray/labels{/name}","releases_url":"https://api.github.com/repos/simon-mo/ray/releases{/id}","deployments_url":"https://api.github.com/repos/simon-mo/ray/deployments","created_at":"2018-02-20T04:47:06Z","updated_at":"2018-02-20T04:47:09Z","pushed_at":"2018-02-23T22:58:00Z","git_url":"git://github.com/simon-mo/ray.git","ssh_url":"git@github.com:simon-mo/ray.git","clone_url":"https://github.com/simon-mo/ray.git","svn_url":"https://github.com/simon-mo/ray","homepage":"","size":9109,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":1,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0"},"forks":0,"open_issues":1,"watchers":0,"default_branch":"master"}},"base":{"label":"devin-petersohn:df_patch03","ref":"df_patch03","sha":"7efea344ece51354e78ae4da6b059fd3e823f08a","user":{"login":"devin-petersohn","id":10732128,"avatar_url":"https://avatars0.githubusercontent.com/u/10732128?v=4","gravatar_id":"","url":"https://api.github.com/users/devin-petersohn","html_url":"https://github.com/devin-petersohn","followers_url":"https://api.github.com/users/devin-petersohn/followers","following_url":"https://api.github.com/users/devin-petersohn/following{/other_user}","gists_url":"https://api.github.com/users/devin-petersohn/gists{/gist_id}","starred_url":"https://api.github.com/users/devin-petersohn/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/devin-petersohn/subscriptions","organizations_url":"https://api.github.com/users/devin-petersohn/orgs","repos_url":"https://api.github.com/users/devin-petersohn/repos","events_url":"https://api.github.com/users/devin-petersohn/events{/privacy}","received_events_url":"https://api.github.com/users/devin-petersohn/received_events","type":"User","site_admin":false},"repo":{"id":113401323,"name":"ray","full_name":"devin-petersohn/ray","owner":{"login":"devin-petersohn","id":10732128,"avatar_url":"https://avatars0.githubusercontent.com/u/10732128?v=4","gravatar_id":"","url":"https://api.github.com/users/devin-petersohn","html_url":"https://github.com/devin-petersohn","followers_url":"https://api.github.com/users/devin-petersohn/followers","following_url":"https://api.github.com/users/devin-petersohn/following{/other_user}","gists_url":"https://api.github.com/users/devin-petersohn/gists{/gist_id}","starred_url":"https://api.github.com/users/devin-petersohn/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/devin-petersohn/subscriptions","organizations_url":"https://api.github.com/users/devin-petersohn/orgs","repos_url":"https://api.github.com/users/devin-petersohn/repos","events_url":"https://api.github.com/users/devin-petersohn/events{/privacy}","received_events_url":"https://api.github.com/users/devin-petersohn/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/devin-petersohn/ray","description":"A high-performance distributed execution engine","fork":true,"url":"https://api.github.com/repos/devin-petersohn/ray","forks_url":"https://api.github.com/repos/devin-petersohn/ray/forks","keys_url":"https://api.github.com/repos/devin-petersohn/ray/keys{/key_id}","collaborators_url":"https://api.github.com/repos/devin-petersohn/ray/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/devin-petersohn/ray/teams","hooks_url":"https://api.github.com/repos/devin-petersohn/ray/hooks","issue_events_url":"https://api.github.com/repos/devin-petersohn/ray/issues/events{/number}","events_url":"https://api.github.com/repos/devin-petersohn/ray/events","assignees_url":"https://api.github.com/repos/devin-petersohn/ray/assignees{/user}","branches_url":"https://api.github.com/repos/devin-petersohn/ray/branches{/branch}","tags_url":"https://api.github.com/repos/devin-petersohn/ray/tags","blobs_url":"https://api.github.com/repos/devin-petersohn/ray/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/devin-petersohn/ray/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/devin-petersohn/ray/git/refs{/sha}","trees_url":"https://api.github.com/repos/devin-petersohn/ray/git/trees{/sha}","statuses_url":"https://api.github.com/repos/devin-petersohn/ray/statuses/{sha}","languages_url":"https://api.github.com/repos/devin-petersohn/ray/languages","stargazers_url":"https://api.github.com/repos/devin-petersohn/ray/stargazers","contributors_url":"https://api.github.com/repos/devin-petersohn/ray/contributors","subscribers_url":"https://api.github.com/repos/devin-petersohn/ray/subscribers","subscription_url":"https://api.github.com/repos/devin-petersohn/ray/subscription","commits_url":"https://api.github.com/repos/devin-petersohn/ray/commits{/sha}","git_commits_url":"https://api.github.com/repos/devin-petersohn/ray/git/commits{/sha}","comments_url":"https://api.github.com/repos/devin-petersohn/ray/comments{/number}","issue_comment_url":"https://api.github.com/repos/devin-petersohn/ray/issues/comments{/number}","contents_url":"https://api.github.com/repos/devin-petersohn/ray/contents/{+path}","compare_url":"https://api.github.com/repos/devin-petersohn/ray/compare/{base}...{head}","merges_url":"https://api.github.com/repos/devin-petersohn/ray/merges","archive_url":"https://api.github.com/repos/devin-petersohn/ray/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/devin-petersohn/ray/downloads","issues_url":"https://api.github.com/repos/devin-petersohn/ray/issues{/number}","pulls_url":"https://api.github.com/repos/devin-petersohn/ray/pulls{/number}","milestones_url":"https://api.github.com/repos/devin-petersohn/ray/milestones{/number}","notifications_url":"https://api.github.com/repos/devin-petersohn/ray/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/devin-petersohn/ray/labels{/name}","releases_url":"https://api.github.com/repos/devin-petersohn/ray/releases{/id}","deployments_url":"https://api.github.com/repos/devin-petersohn/ray/deployments","created_at":"2017-12-07T04:06:28Z","updated_at":"2017-12-15T19:36:55Z","pushed_at":"2018-02-24T19:40:22Z","git_url":"git://github.com/devin-petersohn/ray.git","ssh_url":"git@github.com:devin-petersohn/ray.git","clone_url":"https://github.com/devin-petersohn/ray.git","svn_url":"https://github.com/devin-petersohn/ray","homepage":"","size":9133,"stargazers_count":1,"watchers_count":1,"language":"Python","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":1,"license":null,"forks":0,"open_issues":1,"watchers":1,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/devin-petersohn/ray/pulls/1"},"html":{"href":"https://github.com/devin-petersohn/ray/pull/1"},"issue":{"href":"https://api.github.com/repos/devin-petersohn/ray/issues/1"},"comments":{"href":"https://api.github.com/repos/devin-petersohn/ray/issues/1/comments"},"review_comments":{"href":"https://api.github.com/repos/devin-petersohn/ray/pulls/1/comments"},"review_comment":{"href":"https://api.github.com/repos/devin-petersohn/ray/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/devin-petersohn/ray/pulls/1/commits"},"statuses":{"href":"https://api.github.com/repos/devin-petersohn/ray/statuses/32dbd9e4e484aa7e6dca3f764eb47cca00eb3169"}},"author_association":"NONE"}}
{"actor":{"display_login":"simon-mo"}}