{"action":"created","comment":{"url":"https://api.github.com/repos/dask/dask/pulls/comments/190421175","pull_request_review_id":122793217,"id":190421175,"diff_hunk":"@@ -1883,34 +1893,79 @@ def ensure_int(f):\n     return i\n \n \n-def normalize_chunks(chunks, shape=None):\n+def normalize_chunks(chunks, shape=None, limit=None, dtype=None,\n+                     previous_chunks=None):\n     \"\"\" Normalize chunks to tuple of tuples\n \n+    This takes in a variety of input types and information and produces a full\n+    tuple-of-tuples result for chunks, sutiable to be passed to Array or\n+    rechunk or any other operation that creates a Dask array.\n+\n+    Parameters\n+    ----------\n+    chunks: tuple, int, or string\n+        The chunks to be normalized.  See example below for more details\n+    shape: Tuple[int]\n+        The shape of the array\n+    limit: int (optional)\n+        The maximum block size to target in bytes,\n+        if freedom is given to choose\n+    itemsize: int (optional)\n+        The size in bytes of the dtype to be used if this determines chunk sizes\n+    previous_chunks: Tuple[Tuple[int]] optional\n+        Chunks from a previous array that we should use for inspiration when\n+        rechunking auto dimensions.  If not provided but auto-chunking exists\n+        then auto-dimensions will prefer square-like chunk shapes.\n+\n+    Examples\n+    --------\n+    Specify uniform chunk sizes\n+\n     >>> normalize_chunks((2, 2), shape=(5, 6))\n     ((2, 2, 1), (2, 2, 2))\n \n-    >>> normalize_chunks(((2, 2, 1), (2, 2, 2)), shape=(5, 6))  # Idempotent\n+    Also passes through fully explicit tuple-of-tuples\n+\n+    >>> normalize_chunks(((2, 2, 1), (2, 2, 2)), shape=(5, 6))\n     ((2, 2, 1), (2, 2, 2))\n \n-    >>> normalize_chunks([[2, 2], [3, 3]])  # Cleans up lists to tuples\n+    Cleans up lists to tuples\n+\n+    >>> normalize_chunks([[2, 2], [3, 3]])\n     ((2, 2), (3, 3))\n \n-    >>> normalize_chunks(10, shape=(30, 5))  # Supports integer inputs\n+    Expands integer inputs 10 -> (10, 10)\n+\n+    >>> normalize_chunks(10, shape=(30, 5))\n     ((10, 10, 10), (5,))\n \n-    >>> normalize_chunks((-1,), shape=(10,))  # -1 gets mapped to full size\n+    The value -1 gets mapped to full size\n+\n+    >>> normalize_chunks((-1,), shape=(10,))\n     ((10,),)\n \n-    >>> normalize_chunks((), shape=(0, 0))  #  respects null dimensions\n+    Use the value \"auto\" to automatically determine chunk sizes along certain\n+    dimensions.  This uses the ``limit=`` and ``itemsize=`` keywords to\n+    determine how large to make the chunks\n+\n+    >>> normalize_chunks((\"auto\",), shape=(20,), limit=5, dtype='uint8')","path":"dask/array/core.py","position":126,"original_position":126,"commit_id":"8d982b8dfc9bc94b433b7e1faf5f38865b0c91b8","original_commit_id":"8d982b8dfc9bc94b433b7e1faf5f38865b0c91b8","user":{"login":"mrocklin","id":306380,"avatar_url":"https://avatars3.githubusercontent.com/u/306380?v=4","gravatar_id":"","url":"https://api.github.com/users/mrocklin","html_url":"https://github.com/mrocklin","followers_url":"https://api.github.com/users/mrocklin/followers","following_url":"https://api.github.com/users/mrocklin/following{/other_user}","gists_url":"https://api.github.com/users/mrocklin/gists{/gist_id}","starred_url":"https://api.github.com/users/mrocklin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mrocklin/subscriptions","organizations_url":"https://api.github.com/users/mrocklin/orgs","repos_url":"https://api.github.com/users/mrocklin/repos","events_url":"https://api.github.com/users/mrocklin/events{/privacy}","received_events_url":"https://api.github.com/users/mrocklin/received_events","type":"User","site_admin":false},"body":"Added this to the docstring","created_at":"2018-05-23T22:41:52Z","updated_at":"2018-05-23T22:41:52Z","html_url":"https://github.com/dask/dask/pull/3507#discussion_r190421175","pull_request_url":"https://api.github.com/repos/dask/dask/pulls/3507","author_association":"MEMBER","_links":{"self":{"href":"https://api.github.com/repos/dask/dask/pulls/comments/190421175"},"html":{"href":"https://github.com/dask/dask/pull/3507#discussion_r190421175"},"pull_request":{"href":"https://api.github.com/repos/dask/dask/pulls/3507"}},"in_reply_to_id":190374928},"pull_request":{"url":"https://api.github.com/repos/dask/dask/pulls/3507","id":188527454,"html_url":"https://github.com/dask/dask/pull/3507","diff_url":"https://github.com/dask/dask/pull/3507.diff","patch_url":"https://github.com/dask/dask/pull/3507.patch","issue_url":"https://api.github.com/repos/dask/dask/issues/3507","number":3507,"state":"open","locked":false,"title":"Introduce \"auto\" value for normalize_chunks","user":{"login":"mrocklin","id":306380,"avatar_url":"https://avatars3.githubusercontent.com/u/306380?v=4","gravatar_id":"","url":"https://api.github.com/users/mrocklin","html_url":"https://github.com/mrocklin","followers_url":"https://api.github.com/users/mrocklin/followers","following_url":"https://api.github.com/users/mrocklin/following{/other_user}","gists_url":"https://api.github.com/users/mrocklin/gists{/gist_id}","starred_url":"https://api.github.com/users/mrocklin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mrocklin/subscriptions","organizations_url":"https://api.github.com/users/mrocklin/orgs","repos_url":"https://api.github.com/users/mrocklin/repos","events_url":"https://api.github.com/users/mrocklin/events{/privacy}","received_events_url":"https://api.github.com/users/mrocklin/received_events","type":"User","site_admin":false},"body":"This allows for automatic decisions when rechunking in certain dimensions.\r\n\r\n```python\r\nIn [1]: import dask.array as da\r\n\r\nIn [2]: x = da.ones((20, 20), chunks=(2, 2))\r\n\r\nIn [3]: x.rechunk({0: -1, 1: \"auto\"}, block_size_limit=20*8).chunks\r\nOut[3]: ((20,), (1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1))\r\n\r\nIn [4]: x.rechunk({0: -1, 1: \"auto\"}, block_size_limit=100*8).chunks\r\nOut[4]: ((20,), (5, 5, 5, 5))\r\n```\r\n\r\nThis also centralizes some logic for dict and -1 handling from rechunk into `normalize_chunks`.  This introduces options for this logic to also be used at construction time.\r\n\r\n```python\r\nIn [1]: import numpy as np\r\n\r\nIn [2]: import dask.array as da\r\n\r\nIn [3]: x = np.random.random((1000, 1000, 1000))\r\n\r\nIn [4]: y = da.from_array(x, chunks={0: 100, 1: -1, 2: \"auto\"})\r\n\r\nIn [5]: y\r\nOut[5]: dask.array<array, shape=(1000, 1000, 1000), dtype=float64, chunksize=(100, 1000, 125)>\r\n```\r\n\r\n- [x] Tests added / passed\r\n- [x] Passes `flake8 dask`\r\n","created_at":"2018-05-16T18:53:56Z","updated_at":"2018-05-23T22:41:52Z","closed_at":null,"merged_at":null,"merge_commit_sha":"48857dac34d0e508112db54de4ce785e65d92ff4","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/dask/dask/pulls/3507/commits","review_comments_url":"https://api.github.com/repos/dask/dask/pulls/3507/comments","review_comment_url":"https://api.github.com/repos/dask/dask/pulls/comments{/number}","comments_url":"https://api.github.com/repos/dask/dask/issues/3507/comments","statuses_url":"https://api.github.com/repos/dask/dask/statuses/8d982b8dfc9bc94b433b7e1faf5f38865b0c91b8","head":{"label":"mrocklin:rechunk-auto","ref":"rechunk-auto","sha":"8d982b8dfc9bc94b433b7e1faf5f38865b0c91b8","user":{"login":"mrocklin","id":306380,"avatar_url":"https://avatars3.githubusercontent.com/u/306380?v=4","gravatar_id":"","url":"https://api.github.com/users/mrocklin","html_url":"https://github.com/mrocklin","followers_url":"https://api.github.com/users/mrocklin/followers","following_url":"https://api.github.com/users/mrocklin/following{/other_user}","gists_url":"https://api.github.com/users/mrocklin/gists{/gist_id}","starred_url":"https://api.github.com/users/mrocklin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mrocklin/subscriptions","organizations_url":"https://api.github.com/users/mrocklin/orgs","repos_url":"https://api.github.com/users/mrocklin/repos","events_url":"https://api.github.com/users/mrocklin/events{/privacy}","received_events_url":"https://api.github.com/users/mrocklin/received_events","type":"User","site_admin":false},"repo":{"id":28313768,"name":"dask","full_name":"mrocklin/dask","owner":{"login":"mrocklin","id":306380,"avatar_url":"https://avatars3.githubusercontent.com/u/306380?v=4","gravatar_id":"","url":"https://api.github.com/users/mrocklin","html_url":"https://github.com/mrocklin","followers_url":"https://api.github.com/users/mrocklin/followers","following_url":"https://api.github.com/users/mrocklin/following{/other_user}","gists_url":"https://api.github.com/users/mrocklin/gists{/gist_id}","starred_url":"https://api.github.com/users/mrocklin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mrocklin/subscriptions","organizations_url":"https://api.github.com/users/mrocklin/orgs","repos_url":"https://api.github.com/users/mrocklin/repos","events_url":"https://api.github.com/users/mrocklin/events{/privacy}","received_events_url":"https://api.github.com/users/mrocklin/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/mrocklin/dask","description":"Task scheduling and blocked algorithms for parallel processing","fork":true,"url":"https://api.github.com/repos/mrocklin/dask","forks_url":"https://api.github.com/repos/mrocklin/dask/forks","keys_url":"https://api.github.com/repos/mrocklin/dask/keys{/key_id}","collaborators_url":"https://api.github.com/repos/mrocklin/dask/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/mrocklin/dask/teams","hooks_url":"https://api.github.com/repos/mrocklin/dask/hooks","issue_events_url":"https://api.github.com/repos/mrocklin/dask/issues/events{/number}","events_url":"https://api.github.com/repos/mrocklin/dask/events","assignees_url":"https://api.github.com/repos/mrocklin/dask/assignees{/user}","branches_url":"https://api.github.com/repos/mrocklin/dask/branches{/branch}","tags_url":"https://api.github.com/repos/mrocklin/dask/tags","blobs_url":"https://api.github.com/repos/mrocklin/dask/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/mrocklin/dask/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/mrocklin/dask/git/refs{/sha}","trees_url":"https://api.github.com/repos/mrocklin/dask/git/trees{/sha}","statuses_url":"https://api.github.com/repos/mrocklin/dask/statuses/{sha}","languages_url":"https://api.github.com/repos/mrocklin/dask/languages","stargazers_url":"https://api.github.com/repos/mrocklin/dask/stargazers","contributors_url":"https://api.github.com/repos/mrocklin/dask/contributors","subscribers_url":"https://api.github.com/repos/mrocklin/dask/subscribers","subscription_url":"https://api.github.com/repos/mrocklin/dask/subscription","commits_url":"https://api.github.com/repos/mrocklin/dask/commits{/sha}","git_commits_url":"https://api.github.com/repos/mrocklin/dask/git/commits{/sha}","comments_url":"https://api.github.com/repos/mrocklin/dask/comments{/number}","issue_comment_url":"https://api.github.com/repos/mrocklin/dask/issues/comments{/number}","contents_url":"https://api.github.com/repos/mrocklin/dask/contents/{+path}","compare_url":"https://api.github.com/repos/mrocklin/dask/compare/{base}...{head}","merges_url":"https://api.github.com/repos/mrocklin/dask/merges","archive_url":"https://api.github.com/repos/mrocklin/dask/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/mrocklin/dask/downloads","issues_url":"https://api.github.com/repos/mrocklin/dask/issues{/number}","pulls_url":"https://api.github.com/repos/mrocklin/dask/pulls{/number}","milestones_url":"https://api.github.com/repos/mrocklin/dask/milestones{/number}","notifications_url":"https://api.github.com/repos/mrocklin/dask/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/mrocklin/dask/labels{/name}","releases_url":"https://api.github.com/repos/mrocklin/dask/releases{/id}","deployments_url":"https://api.github.com/repos/mrocklin/dask/deployments","created_at":"2014-12-21T21:26:56Z","updated_at":"2018-05-05T23:24:48Z","pushed_at":"2018-05-23T11:16:18Z","git_url":"git://github.com/mrocklin/dask.git","ssh_url":"git@github.com:mrocklin/dask.git","clone_url":"https://github.com/mrocklin/dask.git","svn_url":"https://github.com/mrocklin/dask","homepage":"","size":17149,"stargazers_count":15,"watchers_count":15,"language":"Python","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"other","name":"Other","spdx_id":null,"url":null},"forks":0,"open_issues":0,"watchers":15,"default_branch":"master"}},"base":{"label":"dask:master","ref":"master","sha":"6f9bf19a0f1bf2f410fef08b3cdaf8f78f024737","user":{"login":"dask","id":17131925,"avatar_url":"https://avatars1.githubusercontent.com/u/17131925?v=4","gravatar_id":"","url":"https://api.github.com/users/dask","html_url":"https://github.com/dask","followers_url":"https://api.github.com/users/dask/followers","following_url":"https://api.github.com/users/dask/following{/other_user}","gists_url":"https://api.github.com/users/dask/gists{/gist_id}","starred_url":"https://api.github.com/users/dask/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dask/subscriptions","organizations_url":"https://api.github.com/users/dask/orgs","repos_url":"https://api.github.com/users/dask/repos","events_url":"https://api.github.com/users/dask/events{/privacy}","received_events_url":"https://api.github.com/users/dask/received_events","type":"Organization","site_admin":false},"repo":{"id":28782747,"name":"dask","full_name":"dask/dask","owner":{"login":"dask","id":17131925,"avatar_url":"https://avatars1.githubusercontent.com/u/17131925?v=4","gravatar_id":"","url":"https://api.github.com/users/dask","html_url":"https://github.com/dask","followers_url":"https://api.github.com/users/dask/followers","following_url":"https://api.github.com/users/dask/following{/other_user}","gists_url":"https://api.github.com/users/dask/gists{/gist_id}","starred_url":"https://api.github.com/users/dask/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dask/subscriptions","organizations_url":"https://api.github.com/users/dask/orgs","repos_url":"https://api.github.com/users/dask/repos","events_url":"https://api.github.com/users/dask/events{/privacy}","received_events_url":"https://api.github.com/users/dask/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/dask/dask","description":"Parallel computing with task scheduling","fork":false,"url":"https://api.github.com/repos/dask/dask","forks_url":"https://api.github.com/repos/dask/dask/forks","keys_url":"https://api.github.com/repos/dask/dask/keys{/key_id}","collaborators_url":"https://api.github.com/repos/dask/dask/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/dask/dask/teams","hooks_url":"https://api.github.com/repos/dask/dask/hooks","issue_events_url":"https://api.github.com/repos/dask/dask/issues/events{/number}","events_url":"https://api.github.com/repos/dask/dask/events","assignees_url":"https://api.github.com/repos/dask/dask/assignees{/user}","branches_url":"https://api.github.com/repos/dask/dask/branches{/branch}","tags_url":"https://api.github.com/repos/dask/dask/tags","blobs_url":"https://api.github.com/repos/dask/dask/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/dask/dask/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/dask/dask/git/refs{/sha}","trees_url":"https://api.github.com/repos/dask/dask/git/trees{/sha}","statuses_url":"https://api.github.com/repos/dask/dask/statuses/{sha}","languages_url":"https://api.github.com/repos/dask/dask/languages","stargazers_url":"https://api.github.com/repos/dask/dask/stargazers","contributors_url":"https://api.github.com/repos/dask/dask/contributors","subscribers_url":"https://api.github.com/repos/dask/dask/subscribers","subscription_url":"https://api.github.com/repos/dask/dask/subscription","commits_url":"https://api.github.com/repos/dask/dask/commits{/sha}","git_commits_url":"https://api.github.com/repos/dask/dask/git/commits{/sha}","comments_url":"https://api.github.com/repos/dask/dask/comments{/number}","issue_comment_url":"https://api.github.com/repos/dask/dask/issues/comments{/number}","contents_url":"https://api.github.com/repos/dask/dask/contents/{+path}","compare_url":"https://api.github.com/repos/dask/dask/compare/{base}...{head}","merges_url":"https://api.github.com/repos/dask/dask/merges","archive_url":"https://api.github.com/repos/dask/dask/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/dask/dask/downloads","issues_url":"https://api.github.com/repos/dask/dask/issues{/number}","pulls_url":"https://api.github.com/repos/dask/dask/pulls{/number}","milestones_url":"https://api.github.com/repos/dask/dask/milestones{/number}","notifications_url":"https://api.github.com/repos/dask/dask/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/dask/dask/labels{/name}","releases_url":"https://api.github.com/repos/dask/dask/releases{/id}","deployments_url":"https://api.github.com/repos/dask/dask/deployments","created_at":"2015-01-04T18:50:00Z","updated_at":"2018-05-23T15:29:04Z","pushed_at":"2018-05-23T22:06:22Z","git_url":"git://github.com/dask/dask.git","ssh_url":"git@github.com:dask/dask.git","clone_url":"https://github.com/dask/dask.git","svn_url":"https://github.com/dask/dask","homepage":"http://dask.pydata.org","size":16861,"stargazers_count":2883,"watchers_count":2883,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":487,"mirror_url":null,"archived":false,"open_issues_count":431,"license":{"key":"other","name":"Other","spdx_id":null,"url":null},"forks":487,"open_issues":431,"watchers":2883,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/dask/dask/pulls/3507"},"html":{"href":"https://github.com/dask/dask/pull/3507"},"issue":{"href":"https://api.github.com/repos/dask/dask/issues/3507"},"comments":{"href":"https://api.github.com/repos/dask/dask/issues/3507/comments"},"review_comments":{"href":"https://api.github.com/repos/dask/dask/pulls/3507/comments"},"review_comment":{"href":"https://api.github.com/repos/dask/dask/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/dask/dask/pulls/3507/commits"},"statuses":{"href":"https://api.github.com/repos/dask/dask/statuses/8d982b8dfc9bc94b433b7e1faf5f38865b0c91b8"}},"author_association":"MEMBER"}}
{"actor":{"display_login":"mrocklin"}}