{"action":"created","comment":{"url":"https://api.github.com/repos/xhochy/fletcher/pulls/comments/201090239","pull_request_review_id":135524002,"id":201090239,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDIwMTA5MDIzOQ==","diff_hunk":"@@ -262,47 +256,57 @@ def __setitem__(self, key, value):\n         None\n         \"\"\"\n         # Convert all possible input key types to an array of integers\n-        if is_bool_dtype(key) or isinstance(key, slice):\n-            key = np.array(range(len(self)))[key]\n+        if is_bool_dtype(key):\n+            key = np.argwhere(key).flatten()\n+        elif isinstance(key, slice):\n+            key = np.array(range(len(self))[key])\n         elif is_integer(key):\n             key = np.array([key])\n+\n         if pd.api.types.is_scalar(value):\n-            value = np.full(len(key), value)\n+            value = np.broadcast_to(value, len(key))\n         else:\n             value = np.asarray(value)\n \n-        affected_chunks = self._get_chunk_indexer(key)\n-        chunks = []\n-        offset = 0\n-        for ix, chunk in enumerate(self.data.iterchunks()):\n-            if ix in affected_chunks:\n-                key_chunk_indices = np.argwhere(affected_chunks == ix).flatten()\n-                array_chunk_indices = key[key_chunk_indices] - offset\n-                if pa.types.is_date64(self.dtype.arrow_dtype):\n-                    # ARROW-2741: pa.array from np.datetime[D] and type=pa.date64 produces invalid results\n-                    arr = np.array(chunk.to_pylist())\n-                    arr[array_chunk_indices] = np.array(value)[key_chunk_indices]\n-                    chunks.append(pa.array(arr, self.dtype.arrow_dtype))\n-                else:\n-                    arr = chunk.to_pandas()\n-                    # In the case where we zero-copy Arrow to Pandas conversion, the\n-                    # the resulting arrays are read-only.\n-                    if not arr.flags.writeable:\n-                        arr = arr.copy()\n-                    arr[array_chunk_indices] = np.array(value)[key_chunk_indices]\n-                    # ARROW-2806: Inconsistent handling of np.nan requires adding a mask\n-                    if pa.types.is_integer(\n-                        self.dtype.arrow_dtype\n-                    ) or pa.types.is_floating(self.dtype.arrow_dtype):\n-                        mask = pd.isna(arr)\n-                    else:\n-                        mask = None\n-                    chunks.append(pa.array(arr, self.dtype.arrow_dtype, mask=mask))\n+        affected_chunks_index = self._get_chunk_indexer(key)\n+        affected_chunks_unique = np.unique(affected_chunks_index)\n+\n+        all_chunks = list(self.data.iterchunks())\n+\n+        for ix, offset in zip(\n+            affected_chunks_unique, self.offsets[affected_chunks_unique]\n+        ):\n+            chunk = all_chunks[ix]\n+\n+            # Translate the array-wide indices to indices of the chunk\n+            key_chunk_indices = np.argwhere(affected_chunks_index == ix).flatten()\n+            array_chunk_indices = key[key_chunk_indices] - offset\n+\n+            if pa.types.is_date64(self.dtype.arrow_dtype):\n+                # ARROW-2741: pa.array from np.datetime[D] and type=pa.date64 produces invalid results\n+                arr = np.array(chunk.to_pylist())\n+                arr[array_chunk_indices] = np.array(value)[key_chunk_indices]\n+                pa_arr = pa.array(arr, self.dtype.arrow_dtype)\n             else:\n-                chunks.append(chunk)\n-            offset += len(chunk)\n+                arr = chunk.to_pandas()\n+                # In the case where we zero-copy Arrow to Pandas conversion, the\n+                # the resulting arrays are read-only.\n+                if not arr.flags.writeable:\n+                    arr = arr.copy()\n+\n+                arr[array_chunk_indices] = value[key_chunk_indices]\n+\n+                # ARROW-2806: Inconsistent handling of np.nan requires adding a mask\n+                if pa.types.is_integer(self.dtype.arrow_dtype) or pa.types.is_floating(\n+                    self.dtype.arrow_dtype\n+                ):\n+                    mask = pd.isna(arr)\n+                else:\n+                    mask = None\n+                pa_arr = pa.array(arr, self.dtype.arrow_dtype, mask=mask)\n+            all_chunks[ix] = pa_arr\n \n-        self.data = pa.chunked_array(chunks)\n+        self.data = pa.chunked_array(all_chunks)","path":"fletcher/base.py","position":129,"original_position":129,"commit_id":"ee020c0968b5b96a92f4214e9a85377d4ab581d8","original_commit_id":"ee020c0968b5b96a92f4214e9a85377d4ab581d8","user":{"login":"fjetter","id":8629629,"node_id":"MDQ6VXNlcjg2Mjk2Mjk=","avatar_url":"https://avatars0.githubusercontent.com/u/8629629?v=4","gravatar_id":"","url":"https://api.github.com/users/fjetter","html_url":"https://github.com/fjetter","followers_url":"https://api.github.com/users/fjetter/followers","following_url":"https://api.github.com/users/fjetter/following{/other_user}","gists_url":"https://api.github.com/users/fjetter/gists{/gist_id}","starred_url":"https://api.github.com/users/fjetter/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fjetter/subscriptions","organizations_url":"https://api.github.com/users/fjetter/orgs","repos_url":"https://api.github.com/users/fjetter/repos","events_url":"https://api.github.com/users/fjetter/events{/privacy}","received_events_url":"https://api.github.com/users/fjetter/received_events","type":"User","site_admin":false},"body":"setitem doesn't change the size of the chunks. the only operation we currently support which changes the offsets is concat which calls the constructor. ","created_at":"2018-07-09T17:46:35Z","updated_at":"2018-07-09T17:46:35Z","html_url":"https://github.com/xhochy/fletcher/pull/55#discussion_r201090239","pull_request_url":"https://api.github.com/repos/xhochy/fletcher/pulls/55","author_association":"COLLABORATOR","_links":{"self":{"href":"https://api.github.com/repos/xhochy/fletcher/pulls/comments/201090239"},"html":{"href":"https://github.com/xhochy/fletcher/pull/55#discussion_r201090239"},"pull_request":{"href":"https://api.github.com/repos/xhochy/fletcher/pulls/55"}},"in_reply_to_id":200899252},"pull_request":{"url":"https://api.github.com/repos/xhochy/fletcher/pulls/55","id":199959250,"node_id":"MDExOlB1bGxSZXF1ZXN0MTk5OTU5MjUw","html_url":"https://github.com/xhochy/fletcher/pull/55","diff_url":"https://github.com/xhochy/fletcher/pull/55.diff","patch_url":"https://github.com/xhochy/fletcher/pull/55.patch","issue_url":"https://api.github.com/repos/xhochy/fletcher/issues/55","number":55,"state":"open","locked":false,"title":"Get and setitem performance","user":{"login":"fjetter","id":8629629,"node_id":"MDQ6VXNlcjg2Mjk2Mjk=","avatar_url":"https://avatars0.githubusercontent.com/u/8629629?v=4","gravatar_id":"","url":"https://api.github.com/users/fjetter","html_url":"https://github.com/fjetter","followers_url":"https://api.github.com/users/fjetter/followers","following_url":"https://api.github.com/users/fjetter/following{/other_user}","gists_url":"https://api.github.com/users/fjetter/gists{/gist_id}","starred_url":"https://api.github.com/users/fjetter/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fjetter/subscriptions","organizations_url":"https://api.github.com/users/fjetter/orgs","repos_url":"https://api.github.com/users/fjetter/repos","events_url":"https://api.github.com/users/fjetter/events{/privacy}","received_events_url":"https://api.github.com/users/fjetter/received_events","type":"User","site_admin":false},"body":"This adds benchmarks for get/seitem and improves performance of setitem by upto a factor of 5. We're still *far* away from where we should be but with these changes we're basically limited in how fast `arrow->pandas->arrow` conversion is done\r\n\r\nThe _complexity_ of the benchmark setup is there to receive better output since the parameters are strings instead of arrays","created_at":"2018-07-08T20:35:36Z","updated_at":"2018-07-09T17:46:35Z","closed_at":null,"merged_at":null,"merge_commit_sha":"00759c766df3d95e8e7a3fdc5580bedd41a48bfc","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/xhochy/fletcher/pulls/55/commits","review_comments_url":"https://api.github.com/repos/xhochy/fletcher/pulls/55/comments","review_comment_url":"https://api.github.com/repos/xhochy/fletcher/pulls/comments{/number}","comments_url":"https://api.github.com/repos/xhochy/fletcher/issues/55/comments","statuses_url":"https://api.github.com/repos/xhochy/fletcher/statuses/ee020c0968b5b96a92f4214e9a85377d4ab581d8","head":{"label":"fjetter:get_setitem_perf","ref":"get_setitem_perf","sha":"ee020c0968b5b96a92f4214e9a85377d4ab581d8","user":{"login":"fjetter","id":8629629,"node_id":"MDQ6VXNlcjg2Mjk2Mjk=","avatar_url":"https://avatars0.githubusercontent.com/u/8629629?v=4","gravatar_id":"","url":"https://api.github.com/users/fjetter","html_url":"https://github.com/fjetter","followers_url":"https://api.github.com/users/fjetter/followers","following_url":"https://api.github.com/users/fjetter/following{/other_user}","gists_url":"https://api.github.com/users/fjetter/gists{/gist_id}","starred_url":"https://api.github.com/users/fjetter/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fjetter/subscriptions","organizations_url":"https://api.github.com/users/fjetter/orgs","repos_url":"https://api.github.com/users/fjetter/repos","events_url":"https://api.github.com/users/fjetter/events{/privacy}","received_events_url":"https://api.github.com/users/fjetter/received_events","type":"User","site_admin":false},"repo":{"id":136715441,"node_id":"MDEwOlJlcG9zaXRvcnkxMzY3MTU0NDE=","name":"fletcher","full_name":"fjetter/fletcher","owner":{"login":"fjetter","id":8629629,"node_id":"MDQ6VXNlcjg2Mjk2Mjk=","avatar_url":"https://avatars0.githubusercontent.com/u/8629629?v=4","gravatar_id":"","url":"https://api.github.com/users/fjetter","html_url":"https://github.com/fjetter","followers_url":"https://api.github.com/users/fjetter/followers","following_url":"https://api.github.com/users/fjetter/following{/other_user}","gists_url":"https://api.github.com/users/fjetter/gists{/gist_id}","starred_url":"https://api.github.com/users/fjetter/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fjetter/subscriptions","organizations_url":"https://api.github.com/users/fjetter/orgs","repos_url":"https://api.github.com/users/fjetter/repos","events_url":"https://api.github.com/users/fjetter/events{/privacy}","received_events_url":"https://api.github.com/users/fjetter/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/fjetter/fletcher","description":"Pandas ExtensionDType/Array backed by Apache Arrow","fork":true,"url":"https://api.github.com/repos/fjetter/fletcher","forks_url":"https://api.github.com/repos/fjetter/fletcher/forks","keys_url":"https://api.github.com/repos/fjetter/fletcher/keys{/key_id}","collaborators_url":"https://api.github.com/repos/fjetter/fletcher/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/fjetter/fletcher/teams","hooks_url":"https://api.github.com/repos/fjetter/fletcher/hooks","issue_events_url":"https://api.github.com/repos/fjetter/fletcher/issues/events{/number}","events_url":"https://api.github.com/repos/fjetter/fletcher/events","assignees_url":"https://api.github.com/repos/fjetter/fletcher/assignees{/user}","branches_url":"https://api.github.com/repos/fjetter/fletcher/branches{/branch}","tags_url":"https://api.github.com/repos/fjetter/fletcher/tags","blobs_url":"https://api.github.com/repos/fjetter/fletcher/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/fjetter/fletcher/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/fjetter/fletcher/git/refs{/sha}","trees_url":"https://api.github.com/repos/fjetter/fletcher/git/trees{/sha}","statuses_url":"https://api.github.com/repos/fjetter/fletcher/statuses/{sha}","languages_url":"https://api.github.com/repos/fjetter/fletcher/languages","stargazers_url":"https://api.github.com/repos/fjetter/fletcher/stargazers","contributors_url":"https://api.github.com/repos/fjetter/fletcher/contributors","subscribers_url":"https://api.github.com/repos/fjetter/fletcher/subscribers","subscription_url":"https://api.github.com/repos/fjetter/fletcher/subscription","commits_url":"https://api.github.com/repos/fjetter/fletcher/commits{/sha}","git_commits_url":"https://api.github.com/repos/fjetter/fletcher/git/commits{/sha}","comments_url":"https://api.github.com/repos/fjetter/fletcher/comments{/number}","issue_comment_url":"https://api.github.com/repos/fjetter/fletcher/issues/comments{/number}","contents_url":"https://api.github.com/repos/fjetter/fletcher/contents/{+path}","compare_url":"https://api.github.com/repos/fjetter/fletcher/compare/{base}...{head}","merges_url":"https://api.github.com/repos/fjetter/fletcher/merges","archive_url":"https://api.github.com/repos/fjetter/fletcher/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/fjetter/fletcher/downloads","issues_url":"https://api.github.com/repos/fjetter/fletcher/issues{/number}","pulls_url":"https://api.github.com/repos/fjetter/fletcher/pulls{/number}","milestones_url":"https://api.github.com/repos/fjetter/fletcher/milestones{/number}","notifications_url":"https://api.github.com/repos/fjetter/fletcher/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/fjetter/fletcher/labels{/name}","releases_url":"https://api.github.com/repos/fjetter/fletcher/releases{/id}","deployments_url":"https://api.github.com/repos/fjetter/fletcher/deployments","created_at":"2018-06-09T10:58:59Z","updated_at":"2018-06-09T10:59:00Z","pushed_at":"2018-07-08T20:36:44Z","git_url":"git://github.com/fjetter/fletcher.git","ssh_url":"git@github.com:fjetter/fletcher.git","clone_url":"https://github.com/fjetter/fletcher.git","svn_url":"https://github.com/fjetter/fletcher","homepage":"","size":127,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit","node_id":"MDc6TGljZW5zZTEz"},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"xhochy:master","ref":"master","sha":"384211f4a03284ca4c6b4c7303b428e448e629d3","user":{"login":"xhochy","id":70274,"node_id":"MDQ6VXNlcjcwMjc0","avatar_url":"https://avatars2.githubusercontent.com/u/70274?v=4","gravatar_id":"","url":"https://api.github.com/users/xhochy","html_url":"https://github.com/xhochy","followers_url":"https://api.github.com/users/xhochy/followers","following_url":"https://api.github.com/users/xhochy/following{/other_user}","gists_url":"https://api.github.com/users/xhochy/gists{/gist_id}","starred_url":"https://api.github.com/users/xhochy/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/xhochy/subscriptions","organizations_url":"https://api.github.com/users/xhochy/orgs","repos_url":"https://api.github.com/users/xhochy/repos","events_url":"https://api.github.com/users/xhochy/events{/privacy}","received_events_url":"https://api.github.com/users/xhochy/received_events","type":"User","site_admin":false},"repo":{"id":123808274,"node_id":"MDEwOlJlcG9zaXRvcnkxMjM4MDgyNzQ=","name":"fletcher","full_name":"xhochy/fletcher","owner":{"login":"xhochy","id":70274,"node_id":"MDQ6VXNlcjcwMjc0","avatar_url":"https://avatars2.githubusercontent.com/u/70274?v=4","gravatar_id":"","url":"https://api.github.com/users/xhochy","html_url":"https://github.com/xhochy","followers_url":"https://api.github.com/users/xhochy/followers","following_url":"https://api.github.com/users/xhochy/following{/other_user}","gists_url":"https://api.github.com/users/xhochy/gists{/gist_id}","starred_url":"https://api.github.com/users/xhochy/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/xhochy/subscriptions","organizations_url":"https://api.github.com/users/xhochy/orgs","repos_url":"https://api.github.com/users/xhochy/repos","events_url":"https://api.github.com/users/xhochy/events{/privacy}","received_events_url":"https://api.github.com/users/xhochy/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/xhochy/fletcher","description":"Pandas ExtensionDType/Array backed by Apache Arrow","fork":false,"url":"https://api.github.com/repos/xhochy/fletcher","forks_url":"https://api.github.com/repos/xhochy/fletcher/forks","keys_url":"https://api.github.com/repos/xhochy/fletcher/keys{/key_id}","collaborators_url":"https://api.github.com/repos/xhochy/fletcher/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/xhochy/fletcher/teams","hooks_url":"https://api.github.com/repos/xhochy/fletcher/hooks","issue_events_url":"https://api.github.com/repos/xhochy/fletcher/issues/events{/number}","events_url":"https://api.github.com/repos/xhochy/fletcher/events","assignees_url":"https://api.github.com/repos/xhochy/fletcher/assignees{/user}","branches_url":"https://api.github.com/repos/xhochy/fletcher/branches{/branch}","tags_url":"https://api.github.com/repos/xhochy/fletcher/tags","blobs_url":"https://api.github.com/repos/xhochy/fletcher/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/xhochy/fletcher/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/xhochy/fletcher/git/refs{/sha}","trees_url":"https://api.github.com/repos/xhochy/fletcher/git/trees{/sha}","statuses_url":"https://api.github.com/repos/xhochy/fletcher/statuses/{sha}","languages_url":"https://api.github.com/repos/xhochy/fletcher/languages","stargazers_url":"https://api.github.com/repos/xhochy/fletcher/stargazers","contributors_url":"https://api.github.com/repos/xhochy/fletcher/contributors","subscribers_url":"https://api.github.com/repos/xhochy/fletcher/subscribers","subscription_url":"https://api.github.com/repos/xhochy/fletcher/subscription","commits_url":"https://api.github.com/repos/xhochy/fletcher/commits{/sha}","git_commits_url":"https://api.github.com/repos/xhochy/fletcher/git/commits{/sha}","comments_url":"https://api.github.com/repos/xhochy/fletcher/comments{/number}","issue_comment_url":"https://api.github.com/repos/xhochy/fletcher/issues/comments{/number}","contents_url":"https://api.github.com/repos/xhochy/fletcher/contents/{+path}","compare_url":"https://api.github.com/repos/xhochy/fletcher/compare/{base}...{head}","merges_url":"https://api.github.com/repos/xhochy/fletcher/merges","archive_url":"https://api.github.com/repos/xhochy/fletcher/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/xhochy/fletcher/downloads","issues_url":"https://api.github.com/repos/xhochy/fletcher/issues{/number}","pulls_url":"https://api.github.com/repos/xhochy/fletcher/pulls{/number}","milestones_url":"https://api.github.com/repos/xhochy/fletcher/milestones{/number}","notifications_url":"https://api.github.com/repos/xhochy/fletcher/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/xhochy/fletcher/labels{/name}","releases_url":"https://api.github.com/repos/xhochy/fletcher/releases{/id}","deployments_url":"https://api.github.com/repos/xhochy/fletcher/deployments","created_at":"2018-03-04T16:44:22Z","updated_at":"2018-07-09T16:56:58Z","pushed_at":"2018-07-09T07:17:43Z","git_url":"git://github.com/xhochy/fletcher.git","ssh_url":"git@github.com:xhochy/fletcher.git","clone_url":"https://github.com/xhochy/fletcher.git","svn_url":"https://github.com/xhochy/fletcher","homepage":"https://fletcher.readthedocs.io/","size":118,"stargazers_count":39,"watchers_count":39,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":5,"mirror_url":null,"archived":false,"open_issues_count":12,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit","node_id":"MDc6TGljZW5zZTEz"},"forks":5,"open_issues":12,"watchers":39,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/xhochy/fletcher/pulls/55"},"html":{"href":"https://github.com/xhochy/fletcher/pull/55"},"issue":{"href":"https://api.github.com/repos/xhochy/fletcher/issues/55"},"comments":{"href":"https://api.github.com/repos/xhochy/fletcher/issues/55/comments"},"review_comments":{"href":"https://api.github.com/repos/xhochy/fletcher/pulls/55/comments"},"review_comment":{"href":"https://api.github.com/repos/xhochy/fletcher/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/xhochy/fletcher/pulls/55/commits"},"statuses":{"href":"https://api.github.com/repos/xhochy/fletcher/statuses/ee020c0968b5b96a92f4214e9a85377d4ab581d8"}},"author_association":"COLLABORATOR"}}
{"actor":{"display_login":"fjetter"}}