{"action":"created","comment":{"url":"https://api.github.com/repos/dask/dask/pulls/comments/60482030","id":60482030,"diff_hunk":"@@ -724,10 +736,57 @@ def one_path_one_key(path, key, start, stop, columns, chunksize, lock):\n     if (start != 0 or stop is not None) and len(keys) > 1:\n         raise NotImplementedError(read_hdf_error_msg)\n     from .multi import concat\n-    return concat([one_path_one_key(path, k, start, s, columns, chunksize, lock)\n+    return concat([one_path_one_key(path, k, start, s, columns, chunksize,\n+                                    lock, division_column)\n                    for k, s in zip(keys, stops)])\n \n \n+def _find_divisions(division_column, start, stop, chunksize, path, key):\n+    divisions = list()\n+    starts = [start]\n+    stops = list()\n+    row = start\n+    stop = pd.HDFStore(path).get_storer(key).nrows if stop is None else stop\n+    data = _read_hdf_column(path, key, division_column, row, 1)\n+    divisions.append(data[0])\n+    for row in range(start + chunksize, stop, chunksize):\n+        data = _read_hdf_column(path, key, division_column, row - 1, 100)\n+        split = _find_split(data)\n+        while not split and row < stop:\n+            row = row + 100\n+            data = _read_hdf_column(path, key, division_column, row - 1, 100)\n+            if len(data) == 0:\n+                split = False\n+            else:\n+                split = _find_split(data)\n+        if split > 0:\n+            starts.append(row)\n+            stops.append(row)\n+            divisions.append(data[split])\n+    stops.append(stop)\n+    data = _read_hdf_column(path, key, division_column, stop - 1, 1)\n+    divisions.append(data[0])\n+    return divisions, starts, stops\n+\n+\n+def _find_split(data):\n+    i = 0\n+    while data[i] == data[i + 1]:\n+        i += 1\n+        if i == len(data) - 1:\n+            return False\n+    return i + 1\n+\n+\n+def _read_hdf_column(path, key, column, start, rows):\n+    if isinstance(column, string_types):\n+        return pd.HDFStore(path).select(key, start=start, stop=start + rows,\n+                                        columns=[column])[column].values","path":"dask/dataframe/io.py","position":108,"original_position":108,"commit_id":"93893b4db36f3575191987b0fad2c9af3891f49d","original_commit_id":"93893b4db36f3575191987b0fad2c9af3891f49d","user":{"login":"jreback","id":953992,"avatar_url":"https://avatars.githubusercontent.com/u/953992?v=3","gravatar_id":"","url":"https://api.github.com/users/jreback","html_url":"https://github.com/jreback","followers_url":"https://api.github.com/users/jreback/followers","following_url":"https://api.github.com/users/jreback/following{/other_user}","gists_url":"https://api.github.com/users/jreback/gists{/gist_id}","starred_url":"https://api.github.com/users/jreback/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jreback/subscriptions","organizations_url":"https://api.github.com/users/jreback/orgs","repos_url":"https://api.github.com/users/jreback/repos","events_url":"https://api.github.com/users/jreback/events{/privacy}","received_events_url":"https://api.github.com/users/jreback/received_events","type":"User","site_admin":false},"body":"if you are selecting a single columns, use ``.select_column``, you cannot use a where clauses, but ``start/stop`` will work. This will be MUCH faster as the ``columns`` kw selects ALL columns then does a reindex (its how PyTables works).","created_at":"2016-04-20T20:32:46Z","updated_at":"2016-04-20T20:32:46Z","html_url":"https://github.com/dask/dask/pull/1058#discussion_r60482030","pull_request_url":"https://api.github.com/repos/dask/dask/pulls/1058","_links":{"self":{"href":"https://api.github.com/repos/dask/dask/pulls/comments/60482030"},"html":{"href":"https://github.com/dask/dask/pull/1058#discussion_r60482030"},"pull_request":{"href":"https://api.github.com/repos/dask/dask/pulls/1058"}}},"pull_request":{"url":"https://api.github.com/repos/dask/dask/pulls/1058","id":63503761,"html_url":"https://github.com/dask/dask/pull/1058","diff_url":"https://github.com/dask/dask/pull/1058.diff","patch_url":"https://github.com/dask/dask/pull/1058.patch","issue_url":"https://api.github.com/repos/dask/dask/issues/1058","number":1058,"state":"open","locked":false,"title":"ENH: Adding divisions based on index for HDF5","user":{"login":"thequackdaddy","id":8186432,"avatar_url":"https://avatars.githubusercontent.com/u/8186432?v=3","gravatar_id":"","url":"https://api.github.com/users/thequackdaddy","html_url":"https://github.com/thequackdaddy","followers_url":"https://api.github.com/users/thequackdaddy/followers","following_url":"https://api.github.com/users/thequackdaddy/following{/other_user}","gists_url":"https://api.github.com/users/thequackdaddy/gists{/gist_id}","starred_url":"https://api.github.com/users/thequackdaddy/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/thequackdaddy/subscriptions","organizations_url":"https://api.github.com/users/thequackdaddy/orgs","repos_url":"https://api.github.com/users/thequackdaddy/repos","events_url":"https://api.github.com/users/thequackdaddy/events{/privacy}","received_events_url":"https://api.github.com/users/thequackdaddy/received_events","type":"User","site_admin":false},"body":"Hello, \r\n\r\n#1057 \r\n\r\nSo I don't think divisions are broken for HDF5, but it would be nice to have divisions with the accurate index. This enables that. \r\n\r\nThanks for your consideration. ","created_at":"2016-03-20T18:50:25Z","updated_at":"2016-04-20T20:32:46Z","closed_at":null,"merged_at":null,"merge_commit_sha":"585653051214c1f8ba9968985f8d137f10c835b3","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/dask/dask/pulls/1058/commits","review_comments_url":"https://api.github.com/repos/dask/dask/pulls/1058/comments","review_comment_url":"https://api.github.com/repos/dask/dask/pulls/comments{/number}","comments_url":"https://api.github.com/repos/dask/dask/issues/1058/comments","statuses_url":"https://api.github.com/repos/dask/dask/statuses/93893b4db36f3575191987b0fad2c9af3891f49d","head":{"label":"thequackdaddy:hdf_index","ref":"hdf_index","sha":"93893b4db36f3575191987b0fad2c9af3891f49d","user":{"login":"thequackdaddy","id":8186432,"avatar_url":"https://avatars.githubusercontent.com/u/8186432?v=3","gravatar_id":"","url":"https://api.github.com/users/thequackdaddy","html_url":"https://github.com/thequackdaddy","followers_url":"https://api.github.com/users/thequackdaddy/followers","following_url":"https://api.github.com/users/thequackdaddy/following{/other_user}","gists_url":"https://api.github.com/users/thequackdaddy/gists{/gist_id}","starred_url":"https://api.github.com/users/thequackdaddy/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/thequackdaddy/subscriptions","organizations_url":"https://api.github.com/users/thequackdaddy/orgs","repos_url":"https://api.github.com/users/thequackdaddy/repos","events_url":"https://api.github.com/users/thequackdaddy/events{/privacy}","received_events_url":"https://api.github.com/users/thequackdaddy/received_events","type":"User","site_admin":false},"repo":{"id":53228881,"name":"dask","full_name":"thequackdaddy/dask","owner":{"login":"thequackdaddy","id":8186432,"avatar_url":"https://avatars.githubusercontent.com/u/8186432?v=3","gravatar_id":"","url":"https://api.github.com/users/thequackdaddy","html_url":"https://github.com/thequackdaddy","followers_url":"https://api.github.com/users/thequackdaddy/followers","following_url":"https://api.github.com/users/thequackdaddy/following{/other_user}","gists_url":"https://api.github.com/users/thequackdaddy/gists{/gist_id}","starred_url":"https://api.github.com/users/thequackdaddy/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/thequackdaddy/subscriptions","organizations_url":"https://api.github.com/users/thequackdaddy/orgs","repos_url":"https://api.github.com/users/thequackdaddy/repos","events_url":"https://api.github.com/users/thequackdaddy/events{/privacy}","received_events_url":"https://api.github.com/users/thequackdaddy/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/thequackdaddy/dask","description":"Task scheduling and blocked algorithms for parallel processing","fork":true,"url":"https://api.github.com/repos/thequackdaddy/dask","forks_url":"https://api.github.com/repos/thequackdaddy/dask/forks","keys_url":"https://api.github.com/repos/thequackdaddy/dask/keys{/key_id}","collaborators_url":"https://api.github.com/repos/thequackdaddy/dask/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/thequackdaddy/dask/teams","hooks_url":"https://api.github.com/repos/thequackdaddy/dask/hooks","issue_events_url":"https://api.github.com/repos/thequackdaddy/dask/issues/events{/number}","events_url":"https://api.github.com/repos/thequackdaddy/dask/events","assignees_url":"https://api.github.com/repos/thequackdaddy/dask/assignees{/user}","branches_url":"https://api.github.com/repos/thequackdaddy/dask/branches{/branch}","tags_url":"https://api.github.com/repos/thequackdaddy/dask/tags","blobs_url":"https://api.github.com/repos/thequackdaddy/dask/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/thequackdaddy/dask/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/thequackdaddy/dask/git/refs{/sha}","trees_url":"https://api.github.com/repos/thequackdaddy/dask/git/trees{/sha}","statuses_url":"https://api.github.com/repos/thequackdaddy/dask/statuses/{sha}","languages_url":"https://api.github.com/repos/thequackdaddy/dask/languages","stargazers_url":"https://api.github.com/repos/thequackdaddy/dask/stargazers","contributors_url":"https://api.github.com/repos/thequackdaddy/dask/contributors","subscribers_url":"https://api.github.com/repos/thequackdaddy/dask/subscribers","subscription_url":"https://api.github.com/repos/thequackdaddy/dask/subscription","commits_url":"https://api.github.com/repos/thequackdaddy/dask/commits{/sha}","git_commits_url":"https://api.github.com/repos/thequackdaddy/dask/git/commits{/sha}","comments_url":"https://api.github.com/repos/thequackdaddy/dask/comments{/number}","issue_comment_url":"https://api.github.com/repos/thequackdaddy/dask/issues/comments{/number}","contents_url":"https://api.github.com/repos/thequackdaddy/dask/contents/{+path}","compare_url":"https://api.github.com/repos/thequackdaddy/dask/compare/{base}...{head}","merges_url":"https://api.github.com/repos/thequackdaddy/dask/merges","archive_url":"https://api.github.com/repos/thequackdaddy/dask/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/thequackdaddy/dask/downloads","issues_url":"https://api.github.com/repos/thequackdaddy/dask/issues{/number}","pulls_url":"https://api.github.com/repos/thequackdaddy/dask/pulls{/number}","milestones_url":"https://api.github.com/repos/thequackdaddy/dask/milestones{/number}","notifications_url":"https://api.github.com/repos/thequackdaddy/dask/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/thequackdaddy/dask/labels{/name}","releases_url":"https://api.github.com/repos/thequackdaddy/dask/releases{/id}","deployments_url":"https://api.github.com/repos/thequackdaddy/dask/deployments","created_at":"2016-03-05T23:47:30Z","updated_at":"2016-03-05T23:47:31Z","pushed_at":"2016-04-15T16:10:26Z","git_url":"git://github.com/thequackdaddy/dask.git","ssh_url":"git@github.com:thequackdaddy/dask.git","clone_url":"https://github.com/thequackdaddy/dask.git","svn_url":"https://github.com/thequackdaddy/dask","homepage":"http://dask.pydata.org","size":9447,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"dask:master","ref":"master","sha":"e49a21b20c864a7113cbee4ad4104cbb021495b1","user":{"login":"dask","id":17131925,"avatar_url":"https://avatars.githubusercontent.com/u/17131925?v=3","gravatar_id":"","url":"https://api.github.com/users/dask","html_url":"https://github.com/dask","followers_url":"https://api.github.com/users/dask/followers","following_url":"https://api.github.com/users/dask/following{/other_user}","gists_url":"https://api.github.com/users/dask/gists{/gist_id}","starred_url":"https://api.github.com/users/dask/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dask/subscriptions","organizations_url":"https://api.github.com/users/dask/orgs","repos_url":"https://api.github.com/users/dask/repos","events_url":"https://api.github.com/users/dask/events{/privacy}","received_events_url":"https://api.github.com/users/dask/received_events","type":"Organization","site_admin":false},"repo":{"id":28782747,"name":"dask","full_name":"dask/dask","owner":{"login":"dask","id":17131925,"avatar_url":"https://avatars.githubusercontent.com/u/17131925?v=3","gravatar_id":"","url":"https://api.github.com/users/dask","html_url":"https://github.com/dask","followers_url":"https://api.github.com/users/dask/followers","following_url":"https://api.github.com/users/dask/following{/other_user}","gists_url":"https://api.github.com/users/dask/gists{/gist_id}","starred_url":"https://api.github.com/users/dask/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dask/subscriptions","organizations_url":"https://api.github.com/users/dask/orgs","repos_url":"https://api.github.com/users/dask/repos","events_url":"https://api.github.com/users/dask/events{/privacy}","received_events_url":"https://api.github.com/users/dask/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/dask/dask","description":"Task scheduling and blocked algorithms for parallel processing","fork":false,"url":"https://api.github.com/repos/dask/dask","forks_url":"https://api.github.com/repos/dask/dask/forks","keys_url":"https://api.github.com/repos/dask/dask/keys{/key_id}","collaborators_url":"https://api.github.com/repos/dask/dask/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/dask/dask/teams","hooks_url":"https://api.github.com/repos/dask/dask/hooks","issue_events_url":"https://api.github.com/repos/dask/dask/issues/events{/number}","events_url":"https://api.github.com/repos/dask/dask/events","assignees_url":"https://api.github.com/repos/dask/dask/assignees{/user}","branches_url":"https://api.github.com/repos/dask/dask/branches{/branch}","tags_url":"https://api.github.com/repos/dask/dask/tags","blobs_url":"https://api.github.com/repos/dask/dask/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/dask/dask/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/dask/dask/git/refs{/sha}","trees_url":"https://api.github.com/repos/dask/dask/git/trees{/sha}","statuses_url":"https://api.github.com/repos/dask/dask/statuses/{sha}","languages_url":"https://api.github.com/repos/dask/dask/languages","stargazers_url":"https://api.github.com/repos/dask/dask/stargazers","contributors_url":"https://api.github.com/repos/dask/dask/contributors","subscribers_url":"https://api.github.com/repos/dask/dask/subscribers","subscription_url":"https://api.github.com/repos/dask/dask/subscription","commits_url":"https://api.github.com/repos/dask/dask/commits{/sha}","git_commits_url":"https://api.github.com/repos/dask/dask/git/commits{/sha}","comments_url":"https://api.github.com/repos/dask/dask/comments{/number}","issue_comment_url":"https://api.github.com/repos/dask/dask/issues/comments{/number}","contents_url":"https://api.github.com/repos/dask/dask/contents/{+path}","compare_url":"https://api.github.com/repos/dask/dask/compare/{base}...{head}","merges_url":"https://api.github.com/repos/dask/dask/merges","archive_url":"https://api.github.com/repos/dask/dask/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/dask/dask/downloads","issues_url":"https://api.github.com/repos/dask/dask/issues{/number}","pulls_url":"https://api.github.com/repos/dask/dask/pulls{/number}","milestones_url":"https://api.github.com/repos/dask/dask/milestones{/number}","notifications_url":"https://api.github.com/repos/dask/dask/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/dask/dask/labels{/name}","releases_url":"https://api.github.com/repos/dask/dask/releases{/id}","deployments_url":"https://api.github.com/repos/dask/dask/deployments","created_at":"2015-01-04T18:50:00Z","updated_at":"2016-04-20T18:12:58Z","pushed_at":"2016-04-20T20:32:02Z","git_url":"git://github.com/dask/dask.git","ssh_url":"git@github.com:dask/dask.git","clone_url":"https://github.com/dask/dask.git","svn_url":"https://github.com/dask/dask","homepage":"http://dask.pydata.org","size":10813,"stargazers_count":666,"watchers_count":666,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":102,"mirror_url":null,"open_issues_count":93,"forks":102,"open_issues":93,"watchers":666,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/dask/dask/pulls/1058"},"html":{"href":"https://github.com/dask/dask/pull/1058"},"issue":{"href":"https://api.github.com/repos/dask/dask/issues/1058"},"comments":{"href":"https://api.github.com/repos/dask/dask/issues/1058/comments"},"review_comments":{"href":"https://api.github.com/repos/dask/dask/pulls/1058/comments"},"review_comment":{"href":"https://api.github.com/repos/dask/dask/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/dask/dask/pulls/1058/commits"},"statuses":{"href":"https://api.github.com/repos/dask/dask/statuses/93893b4db36f3575191987b0fad2c9af3891f49d"}}}}