{"action":"created","comment":{"url":"https://api.github.com/repos/dask/dask/pulls/comments/68492021","id":68492021,"diff_hunk":"@@ -115,7 +120,22 @@ def read_csv_from_bytes(block_lists, header, head, kwargs, collection=True,\n         return dfs\n \n \n-def read_csv(urlpath, blocksize=2**25, chunkbytes=None,\n+def auto_blocksize(total_memory, cpu_count):\n+    memory_factor = 10\n+    blocksize = int(total_memory // cpu_count / memory_factor)\n+    return min(blocksize, int(64e6))\n+\n+\n+# guess blocksize if psutil is installed or use acceptable default one if not\n+if psutil is not None:\n+    TOTAL_MEM = psutil.virtual_memory().total\n+    CPU_COUNT = psutil.cpu_count()\n+    AUTO_BLOCKSIZE = auto_blocksize(TOTAL_MEM, CPU_COUNT)\n+else:\n+    AUTO_BLOCKSIZE = 2**25\n+\n+\n+def read_csv(urlpath, blocksize='auto', chunkbytes=None,","path":"dask/dataframe/csv.py","position":32,"original_position":32,"commit_id":"85f81095be3b90a9dd6453971f94dfc03d939198","original_commit_id":"85f81095be3b90a9dd6453971f94dfc03d939198","user":{"login":"mrocklin","id":306380,"avatar_url":"https://avatars.githubusercontent.com/u/306380?v=3","gravatar_id":"","url":"https://api.github.com/users/mrocklin","html_url":"https://github.com/mrocklin","followers_url":"https://api.github.com/users/mrocklin/followers","following_url":"https://api.github.com/users/mrocklin/following{/other_user}","gists_url":"https://api.github.com/users/mrocklin/gists{/gist_id}","starred_url":"https://api.github.com/users/mrocklin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mrocklin/subscriptions","organizations_url":"https://api.github.com/users/mrocklin/orgs","repos_url":"https://api.github.com/users/mrocklin/repos","events_url":"https://api.github.com/users/mrocklin/events{/privacy}","received_events_url":"https://api.github.com/users/mrocklin/received_events","type":"User","site_admin":false},"body":"Perhaps just put `AUTO_BLOCKSIZE` in here directly?","created_at":"2016-06-25T18:27:35Z","updated_at":"2016-06-25T18:27:35Z","html_url":"https://github.com/dask/dask/pull/1328#discussion_r68492021","pull_request_url":"https://api.github.com/repos/dask/dask/pulls/1328","_links":{"self":{"href":"https://api.github.com/repos/dask/dask/pulls/comments/68492021"},"html":{"href":"https://github.com/dask/dask/pull/1328#discussion_r68492021"},"pull_request":{"href":"https://api.github.com/repos/dask/dask/pulls/1328"}}},"pull_request":{"url":"https://api.github.com/repos/dask/dask/pulls/1328","id":75156475,"html_url":"https://github.com/dask/dask/pull/1328","diff_url":"https://github.com/dask/dask/pull/1328.diff","patch_url":"https://github.com/dask/dask/pull/1328.patch","issue_url":"https://api.github.com/repos/dask/dask/issues/1328","number":1328,"state":"open","locked":false,"title":"Automatic blocksize for read_csv based on available memory and number of cores","user":{"login":"dukebody","id":41953,"avatar_url":"https://avatars.githubusercontent.com/u/41953?v=3","gravatar_id":"","url":"https://api.github.com/users/dukebody","html_url":"https://github.com/dukebody","followers_url":"https://api.github.com/users/dukebody/followers","following_url":"https://api.github.com/users/dukebody/following{/other_user}","gists_url":"https://api.github.com/users/dukebody/gists{/gist_id}","starred_url":"https://api.github.com/users/dukebody/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dukebody/subscriptions","organizations_url":"https://api.github.com/users/dukebody/orgs","repos_url":"https://api.github.com/users/dukebody/repos","events_url":"https://api.github.com/users/dukebody/events{/privacy}","received_events_url":"https://api.github.com/users/dukebody/received_events","type":"User","site_admin":false},"body":"See #1147 . ","created_at":"2016-06-24T18:45:40Z","updated_at":"2016-06-25T18:27:35Z","closed_at":null,"merged_at":null,"merge_commit_sha":"ec41085d6a74088dd15182b6f6e8b3d5bc1d8151","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/dask/dask/pulls/1328/commits","review_comments_url":"https://api.github.com/repos/dask/dask/pulls/1328/comments","review_comment_url":"https://api.github.com/repos/dask/dask/pulls/comments{/number}","comments_url":"https://api.github.com/repos/dask/dask/issues/1328/comments","statuses_url":"https://api.github.com/repos/dask/dask/statuses/85f81095be3b90a9dd6453971f94dfc03d939198","head":{"label":"dukebody:automatic_blocksize_read_csv","ref":"automatic_blocksize_read_csv","sha":"85f81095be3b90a9dd6453971f94dfc03d939198","user":{"login":"dukebody","id":41953,"avatar_url":"https://avatars.githubusercontent.com/u/41953?v=3","gravatar_id":"","url":"https://api.github.com/users/dukebody","html_url":"https://github.com/dukebody","followers_url":"https://api.github.com/users/dukebody/followers","following_url":"https://api.github.com/users/dukebody/following{/other_user}","gists_url":"https://api.github.com/users/dukebody/gists{/gist_id}","starred_url":"https://api.github.com/users/dukebody/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dukebody/subscriptions","organizations_url":"https://api.github.com/users/dukebody/orgs","repos_url":"https://api.github.com/users/dukebody/repos","events_url":"https://api.github.com/users/dukebody/events{/privacy}","received_events_url":"https://api.github.com/users/dukebody/received_events","type":"User","site_admin":false},"repo":{"id":61496524,"name":"dask","full_name":"dukebody/dask","owner":{"login":"dukebody","id":41953,"avatar_url":"https://avatars.githubusercontent.com/u/41953?v=3","gravatar_id":"","url":"https://api.github.com/users/dukebody","html_url":"https://github.com/dukebody","followers_url":"https://api.github.com/users/dukebody/followers","following_url":"https://api.github.com/users/dukebody/following{/other_user}","gists_url":"https://api.github.com/users/dukebody/gists{/gist_id}","starred_url":"https://api.github.com/users/dukebody/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dukebody/subscriptions","organizations_url":"https://api.github.com/users/dukebody/orgs","repos_url":"https://api.github.com/users/dukebody/repos","events_url":"https://api.github.com/users/dukebody/events{/privacy}","received_events_url":"https://api.github.com/users/dukebody/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/dukebody/dask","description":"Versatile parallel programming with task scheduling","fork":true,"url":"https://api.github.com/repos/dukebody/dask","forks_url":"https://api.github.com/repos/dukebody/dask/forks","keys_url":"https://api.github.com/repos/dukebody/dask/keys{/key_id}","collaborators_url":"https://api.github.com/repos/dukebody/dask/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/dukebody/dask/teams","hooks_url":"https://api.github.com/repos/dukebody/dask/hooks","issue_events_url":"https://api.github.com/repos/dukebody/dask/issues/events{/number}","events_url":"https://api.github.com/repos/dukebody/dask/events","assignees_url":"https://api.github.com/repos/dukebody/dask/assignees{/user}","branches_url":"https://api.github.com/repos/dukebody/dask/branches{/branch}","tags_url":"https://api.github.com/repos/dukebody/dask/tags","blobs_url":"https://api.github.com/repos/dukebody/dask/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/dukebody/dask/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/dukebody/dask/git/refs{/sha}","trees_url":"https://api.github.com/repos/dukebody/dask/git/trees{/sha}","statuses_url":"https://api.github.com/repos/dukebody/dask/statuses/{sha}","languages_url":"https://api.github.com/repos/dukebody/dask/languages","stargazers_url":"https://api.github.com/repos/dukebody/dask/stargazers","contributors_url":"https://api.github.com/repos/dukebody/dask/contributors","subscribers_url":"https://api.github.com/repos/dukebody/dask/subscribers","subscription_url":"https://api.github.com/repos/dukebody/dask/subscription","commits_url":"https://api.github.com/repos/dukebody/dask/commits{/sha}","git_commits_url":"https://api.github.com/repos/dukebody/dask/git/commits{/sha}","comments_url":"https://api.github.com/repos/dukebody/dask/comments{/number}","issue_comment_url":"https://api.github.com/repos/dukebody/dask/issues/comments{/number}","contents_url":"https://api.github.com/repos/dukebody/dask/contents/{+path}","compare_url":"https://api.github.com/repos/dukebody/dask/compare/{base}...{head}","merges_url":"https://api.github.com/repos/dukebody/dask/merges","archive_url":"https://api.github.com/repos/dukebody/dask/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/dukebody/dask/downloads","issues_url":"https://api.github.com/repos/dukebody/dask/issues{/number}","pulls_url":"https://api.github.com/repos/dukebody/dask/pulls{/number}","milestones_url":"https://api.github.com/repos/dukebody/dask/milestones{/number}","notifications_url":"https://api.github.com/repos/dukebody/dask/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/dukebody/dask/labels{/name}","releases_url":"https://api.github.com/repos/dukebody/dask/releases{/id}","deployments_url":"https://api.github.com/repos/dukebody/dask/deployments","created_at":"2016-06-19T19:26:49Z","updated_at":"2016-06-19T19:26:50Z","pushed_at":"2016-06-25T18:11:31Z","git_url":"git://github.com/dukebody/dask.git","ssh_url":"git@github.com:dukebody/dask.git","clone_url":"https://github.com/dukebody/dask.git","svn_url":"https://github.com/dukebody/dask","homepage":"http://dask.pydata.org","size":10648,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"dask:master","ref":"master","sha":"f460b5d762384f3360b8303e9e4a938e1aca4120","user":{"login":"dask","id":17131925,"avatar_url":"https://avatars.githubusercontent.com/u/17131925?v=3","gravatar_id":"","url":"https://api.github.com/users/dask","html_url":"https://github.com/dask","followers_url":"https://api.github.com/users/dask/followers","following_url":"https://api.github.com/users/dask/following{/other_user}","gists_url":"https://api.github.com/users/dask/gists{/gist_id}","starred_url":"https://api.github.com/users/dask/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dask/subscriptions","organizations_url":"https://api.github.com/users/dask/orgs","repos_url":"https://api.github.com/users/dask/repos","events_url":"https://api.github.com/users/dask/events{/privacy}","received_events_url":"https://api.github.com/users/dask/received_events","type":"Organization","site_admin":false},"repo":{"id":28782747,"name":"dask","full_name":"dask/dask","owner":{"login":"dask","id":17131925,"avatar_url":"https://avatars.githubusercontent.com/u/17131925?v=3","gravatar_id":"","url":"https://api.github.com/users/dask","html_url":"https://github.com/dask","followers_url":"https://api.github.com/users/dask/followers","following_url":"https://api.github.com/users/dask/following{/other_user}","gists_url":"https://api.github.com/users/dask/gists{/gist_id}","starred_url":"https://api.github.com/users/dask/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dask/subscriptions","organizations_url":"https://api.github.com/users/dask/orgs","repos_url":"https://api.github.com/users/dask/repos","events_url":"https://api.github.com/users/dask/events{/privacy}","received_events_url":"https://api.github.com/users/dask/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/dask/dask","description":"Versatile parallel programming with task scheduling","fork":false,"url":"https://api.github.com/repos/dask/dask","forks_url":"https://api.github.com/repos/dask/dask/forks","keys_url":"https://api.github.com/repos/dask/dask/keys{/key_id}","collaborators_url":"https://api.github.com/repos/dask/dask/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/dask/dask/teams","hooks_url":"https://api.github.com/repos/dask/dask/hooks","issue_events_url":"https://api.github.com/repos/dask/dask/issues/events{/number}","events_url":"https://api.github.com/repos/dask/dask/events","assignees_url":"https://api.github.com/repos/dask/dask/assignees{/user}","branches_url":"https://api.github.com/repos/dask/dask/branches{/branch}","tags_url":"https://api.github.com/repos/dask/dask/tags","blobs_url":"https://api.github.com/repos/dask/dask/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/dask/dask/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/dask/dask/git/refs{/sha}","trees_url":"https://api.github.com/repos/dask/dask/git/trees{/sha}","statuses_url":"https://api.github.com/repos/dask/dask/statuses/{sha}","languages_url":"https://api.github.com/repos/dask/dask/languages","stargazers_url":"https://api.github.com/repos/dask/dask/stargazers","contributors_url":"https://api.github.com/repos/dask/dask/contributors","subscribers_url":"https://api.github.com/repos/dask/dask/subscribers","subscription_url":"https://api.github.com/repos/dask/dask/subscription","commits_url":"https://api.github.com/repos/dask/dask/commits{/sha}","git_commits_url":"https://api.github.com/repos/dask/dask/git/commits{/sha}","comments_url":"https://api.github.com/repos/dask/dask/comments{/number}","issue_comment_url":"https://api.github.com/repos/dask/dask/issues/comments{/number}","contents_url":"https://api.github.com/repos/dask/dask/contents/{+path}","compare_url":"https://api.github.com/repos/dask/dask/compare/{base}...{head}","merges_url":"https://api.github.com/repos/dask/dask/merges","archive_url":"https://api.github.com/repos/dask/dask/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/dask/dask/downloads","issues_url":"https://api.github.com/repos/dask/dask/issues{/number}","pulls_url":"https://api.github.com/repos/dask/dask/pulls{/number}","milestones_url":"https://api.github.com/repos/dask/dask/milestones{/number}","notifications_url":"https://api.github.com/repos/dask/dask/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/dask/dask/labels{/name}","releases_url":"https://api.github.com/repos/dask/dask/releases{/id}","deployments_url":"https://api.github.com/repos/dask/dask/deployments","created_at":"2015-01-04T18:50:00Z","updated_at":"2016-06-25T14:17:43Z","pushed_at":"2016-06-25T18:13:29Z","git_url":"git://github.com/dask/dask.git","ssh_url":"git@github.com:dask/dask.git","clone_url":"https://github.com/dask/dask.git","svn_url":"https://github.com/dask/dask","homepage":"http://dask.pydata.org","size":11914,"stargazers_count":763,"watchers_count":763,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":132,"mirror_url":null,"open_issues_count":85,"forks":132,"open_issues":85,"watchers":763,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/dask/dask/pulls/1328"},"html":{"href":"https://github.com/dask/dask/pull/1328"},"issue":{"href":"https://api.github.com/repos/dask/dask/issues/1328"},"comments":{"href":"https://api.github.com/repos/dask/dask/issues/1328/comments"},"review_comments":{"href":"https://api.github.com/repos/dask/dask/pulls/1328/comments"},"review_comment":{"href":"https://api.github.com/repos/dask/dask/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/dask/dask/pulls/1328/commits"},"statuses":{"href":"https://api.github.com/repos/dask/dask/statuses/85f81095be3b90a9dd6453971f94dfc03d939198"}}}}
{"actor":{"display_login":"mrocklin"}}