{"action":"created","comment":{"url":"https://api.github.com/repos/quantopian/zipline/pulls/comments/54578460","id":54578460,"diff_hunk":"@@ -0,0 +1,240 @@\n+# Copyright 2016 Quantopian, Inc.\n+#\n+# Licensed under the Apache License, Version 2.0 (the \"License\");\n+# you may not use this file except in compliance with the License.\n+# You may obtain a copy of the License at\n+#\n+#     http://www.apache.org/licenses/LICENSE-2.0\n+#\n+# Unless required by applicable law or agreed to in writing, software\n+# distributed under the License is distributed on an \"AS IS\" BASIS,\n+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+# See the License for the specific language governing permissions and\n+# limitations under the License.\n+\n+from numpy import dtype, around\n+\n+from six import iteritems\n+\n+from zipline.pipeline.data.equity_pricing import USEquityPricing\n+from zipline.lib._float64window import AdjustedArrayWindow as Float64Window\n+from zipline.lib._int64window import AdjustedArrayWindow as Int64Window\n+from zipline.lib.adjustment import Float64Multiply\n+\n+\n+class SlidingWindow(object):\n+    \"\"\"\n+    Wrapper around an AdjustedArrayWindow which supports monotonically\n+    increasing (by datetime) requests for a sized window of data.\n+\n+    Parameters\n+    ----------\n+    window : AdjustedArrayWindow\n+       Window of pricing data with prefetched values beyond the current\n+       simulation dt.\n+    cal_start : int\n+       Index in the overall calendar at which the window starts.\n+    cal_end : int\n+       Index in the overall calendar at which the window\n+    \"\"\"\n+\n+    def __init__(self, window, cal_start, cal_end):\n+        self.window = window\n+        self.cal_start = cal_start\n+        self.cal_end = cal_end\n+        self.current = around(next(window), 3)\n+        self.most_recent_ix = self.cal_start\n+\n+    def get(self, end_ix):\n+        \"\"\"\n+        Returns\n+        -------\n+        out : A np.ndarray of the equity pricing up to end_ix after adjustments\n+              and rounding have been applied.\n+        \"\"\"\n+        if end_ix < self.most_recent_ix:\n+            raise Exception('Can not access data after window has passed.')\n+\n+        anchor = end_ix - self.cal_start + 1\n+        while self.window.anchor < anchor:\n+            self.current = around(next(self.window), 3)\n+\n+        self.most_recent_ix = end_ix\n+        return self.current\n+\n+\n+class USEquityHistoryLoader(object):\n+    \"\"\"\n+    Loader for sliding history windows of adjusted US Equity Pricing data.\n+\n+    Parameters\n+    ----------\n+    daily_reader : DailyBarReader\n+        Reader for daily bars.\n+    adjustment_reader : SQLiteAdjustmentReader\n+        Reader for adjustment data.\n+    \"\"\"\n+\n+    def __init__(self, daily_reader, adjustment_reader):\n+        self._daily_reader = daily_reader\n+        self._calendar = daily_reader._calendar\n+        self._adjustments_reader = adjustment_reader\n+        self._daily_window_blocks = {}\n+\n+        self._prefetch_length = 40\n+\n+    def _get_adjustments_in_range(self, assets, days, field):\n+        \"\"\"\n+        Get the Float64Multiply objects to pass to an AdjustedArrayWindow.\n+\n+        For the use of AdjustedAraryWindow in the loader, which looks back","path":"zipline/data/us_equity_loader.py","position":90,"original_position":90,"commit_id":"e0d19d6f20a34568beb08928fa9bcb46231ff413","original_commit_id":"e0d19d6f20a34568beb08928fa9bcb46231ff413","user":{"login":"jbredeche","id":487622,"avatar_url":"https://avatars.githubusercontent.com/u/487622?v=3","gravatar_id":"","url":"https://api.github.com/users/jbredeche","html_url":"https://github.com/jbredeche","followers_url":"https://api.github.com/users/jbredeche/followers","following_url":"https://api.github.com/users/jbredeche/following{/other_user}","gists_url":"https://api.github.com/users/jbredeche/gists{/gist_id}","starred_url":"https://api.github.com/users/jbredeche/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jbredeche/subscriptions","organizations_url":"https://api.github.com/users/jbredeche/orgs","repos_url":"https://api.github.com/users/jbredeche/repos","events_url":"https://api.github.com/users/jbredeche/events{/privacy}","received_events_url":"https://api.github.com/users/jbredeche/received_events","type":"User","site_admin":false},"body":"typo - AdjustedArrayWindow","created_at":"2016-03-01T15:03:51Z","updated_at":"2016-03-01T15:03:51Z","html_url":"https://github.com/quantopian/zipline/pull/1019#discussion_r54578460","pull_request_url":"https://api.github.com/repos/quantopian/zipline/pulls/1019","_links":{"self":{"href":"https://api.github.com/repos/quantopian/zipline/pulls/comments/54578460"},"html":{"href":"https://github.com/quantopian/zipline/pull/1019#discussion_r54578460"},"pull_request":{"href":"https://api.github.com/repos/quantopian/zipline/pulls/1019"}}},"pull_request":{"url":"https://api.github.com/repos/quantopian/zipline/pulls/1019","id":61137619,"html_url":"https://github.com/quantopian/zipline/pull/1019","diff_url":"https://github.com/quantopian/zipline/pull/1019.diff","patch_url":"https://github.com/quantopian/zipline/pull/1019.patch","issue_url":"https://api.github.com/repos/quantopian/zipline/issues/1019","number":1019,"state":"open","locked":false,"title":"PERF: Prefetch daily history windows.","user":{"login":"ehebert","id":185744,"avatar_url":"https://avatars.githubusercontent.com/u/185744?v=3","gravatar_id":"","url":"https://api.github.com/users/ehebert","html_url":"https://github.com/ehebert","followers_url":"https://api.github.com/users/ehebert/followers","following_url":"https://api.github.com/users/ehebert/following{/other_user}","gists_url":"https://api.github.com/users/ehebert/gists{/gist_id}","starred_url":"https://api.github.com/users/ehebert/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ehebert/subscriptions","organizations_url":"https://api.github.com/users/ehebert/orgs","repos_url":"https://api.github.com/users/ehebert/repos","events_url":"https://api.github.com/users/ehebert/events{/privacy}","received_events_url":"https://api.github.com/users/ehebert/received_events","type":"User","site_admin":false},"body":"Use AdjustedArrayWindow to prefetch daily history windows, advancing the\r\nwindow forward on each call for a new end date of the window.\r\n\r\nLocally, on an algorithm with 6 daily windows this speeds up the daily\r\n1d history from 300ms -> 200ms per day, saving about 33%.\r\n\r\nThe main bottlenecks this addresses is:\r\n- Re-reading the data from the daily bar bcolz files every simulation\r\nday.\r\n- Remove need to do all adjustments and rounding every day.\r\n\r\nAlso, changed the `load_raw_arrays` in the Panel daily bar reader to\r\nprovide nans for missing days, to match the behavior of the bcolz\r\nloader. (Discovered when running test_examples.)\r\n\r\n# Other Notes\r\n\r\n- The repeated calculation of `size` based on the passed `dts` to history makes me think the interface is not 100% correct here. Perhaps we should be using last_dt and size as the parameter instead? Which would match the actual history calls parameters of `(assets, bar_count, '1d', fields)` more closely.\r\n\r\n- This is on the path of speeding up history calls on the lazy-mainline branch, after this the next steps are most likely:\r\n-- addressing `get_minute_window_for_equities` when called in minute mode and with `1d` \r\n-- Doing a prefetch on minute data.","created_at":"2016-02-29T20:16:43Z","updated_at":"2016-03-01T15:03:51Z","closed_at":null,"merged_at":null,"merge_commit_sha":"716f867d52a0b758388b46693a1abd49683efb7d","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/quantopian/zipline/pulls/1019/commits","review_comments_url":"https://api.github.com/repos/quantopian/zipline/pulls/1019/comments","review_comment_url":"https://api.github.com/repos/quantopian/zipline/pulls/comments{/number}","comments_url":"https://api.github.com/repos/quantopian/zipline/issues/1019/comments","statuses_url":"https://api.github.com/repos/quantopian/zipline/statuses/e0d19d6f20a34568beb08928fa9bcb46231ff413","head":{"label":"quantopian:lazy-cache-daily-window","ref":"lazy-cache-daily-window","sha":"e0d19d6f20a34568beb08928fa9bcb46231ff413","user":{"login":"quantopian","id":1393215,"avatar_url":"https://avatars.githubusercontent.com/u/1393215?v=3","gravatar_id":"","url":"https://api.github.com/users/quantopian","html_url":"https://github.com/quantopian","followers_url":"https://api.github.com/users/quantopian/followers","following_url":"https://api.github.com/users/quantopian/following{/other_user}","gists_url":"https://api.github.com/users/quantopian/gists{/gist_id}","starred_url":"https://api.github.com/users/quantopian/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/quantopian/subscriptions","organizations_url":"https://api.github.com/users/quantopian/orgs","repos_url":"https://api.github.com/users/quantopian/repos","events_url":"https://api.github.com/users/quantopian/events{/privacy}","received_events_url":"https://api.github.com/users/quantopian/received_events","type":"Organization","site_admin":false},"repo":{"id":6297520,"name":"zipline","full_name":"quantopian/zipline","owner":{"login":"quantopian","id":1393215,"avatar_url":"https://avatars.githubusercontent.com/u/1393215?v=3","gravatar_id":"","url":"https://api.github.com/users/quantopian","html_url":"https://github.com/quantopian","followers_url":"https://api.github.com/users/quantopian/followers","following_url":"https://api.github.com/users/quantopian/following{/other_user}","gists_url":"https://api.github.com/users/quantopian/gists{/gist_id}","starred_url":"https://api.github.com/users/quantopian/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/quantopian/subscriptions","organizations_url":"https://api.github.com/users/quantopian/orgs","repos_url":"https://api.github.com/users/quantopian/repos","events_url":"https://api.github.com/users/quantopian/events{/privacy}","received_events_url":"https://api.github.com/users/quantopian/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/quantopian/zipline","description":"Zipline, a Pythonic Algorithmic Trading Library","fork":false,"url":"https://api.github.com/repos/quantopian/zipline","forks_url":"https://api.github.com/repos/quantopian/zipline/forks","keys_url":"https://api.github.com/repos/quantopian/zipline/keys{/key_id}","collaborators_url":"https://api.github.com/repos/quantopian/zipline/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/quantopian/zipline/teams","hooks_url":"https://api.github.com/repos/quantopian/zipline/hooks","issue_events_url":"https://api.github.com/repos/quantopian/zipline/issues/events{/number}","events_url":"https://api.github.com/repos/quantopian/zipline/events","assignees_url":"https://api.github.com/repos/quantopian/zipline/assignees{/user}","branches_url":"https://api.github.com/repos/quantopian/zipline/branches{/branch}","tags_url":"https://api.github.com/repos/quantopian/zipline/tags","blobs_url":"https://api.github.com/repos/quantopian/zipline/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/quantopian/zipline/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/quantopian/zipline/git/refs{/sha}","trees_url":"https://api.github.com/repos/quantopian/zipline/git/trees{/sha}","statuses_url":"https://api.github.com/repos/quantopian/zipline/statuses/{sha}","languages_url":"https://api.github.com/repos/quantopian/zipline/languages","stargazers_url":"https://api.github.com/repos/quantopian/zipline/stargazers","contributors_url":"https://api.github.com/repos/quantopian/zipline/contributors","subscribers_url":"https://api.github.com/repos/quantopian/zipline/subscribers","subscription_url":"https://api.github.com/repos/quantopian/zipline/subscription","commits_url":"https://api.github.com/repos/quantopian/zipline/commits{/sha}","git_commits_url":"https://api.github.com/repos/quantopian/zipline/git/commits{/sha}","comments_url":"https://api.github.com/repos/quantopian/zipline/comments{/number}","issue_comment_url":"https://api.github.com/repos/quantopian/zipline/issues/comments{/number}","contents_url":"https://api.github.com/repos/quantopian/zipline/contents/{+path}","compare_url":"https://api.github.com/repos/quantopian/zipline/compare/{base}...{head}","merges_url":"https://api.github.com/repos/quantopian/zipline/merges","archive_url":"https://api.github.com/repos/quantopian/zipline/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/quantopian/zipline/downloads","issues_url":"https://api.github.com/repos/quantopian/zipline/issues{/number}","pulls_url":"https://api.github.com/repos/quantopian/zipline/pulls{/number}","milestones_url":"https://api.github.com/repos/quantopian/zipline/milestones{/number}","notifications_url":"https://api.github.com/repos/quantopian/zipline/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/quantopian/zipline/labels{/name}","releases_url":"https://api.github.com/repos/quantopian/zipline/releases{/id}","deployments_url":"https://api.github.com/repos/quantopian/zipline/deployments","created_at":"2012-10-19T15:50:29Z","updated_at":"2016-02-29T07:57:51Z","pushed_at":"2016-03-01T14:46:20Z","git_url":"git://github.com/quantopian/zipline.git","ssh_url":"git@github.com:quantopian/zipline.git","clone_url":"https://github.com/quantopian/zipline.git","svn_url":"https://github.com/quantopian/zipline","homepage":"","size":15620,"stargazers_count":2937,"watchers_count":2937,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":856,"mirror_url":null,"open_issues_count":160,"forks":856,"open_issues":160,"watchers":2937,"default_branch":"master"}},"base":{"label":"quantopian:lazy-mainline","ref":"lazy-mainline","sha":"b0a03cd403542ba587719e864e3fbb159d0e738d","user":{"login":"quantopian","id":1393215,"avatar_url":"https://avatars.githubusercontent.com/u/1393215?v=3","gravatar_id":"","url":"https://api.github.com/users/quantopian","html_url":"https://github.com/quantopian","followers_url":"https://api.github.com/users/quantopian/followers","following_url":"https://api.github.com/users/quantopian/following{/other_user}","gists_url":"https://api.github.com/users/quantopian/gists{/gist_id}","starred_url":"https://api.github.com/users/quantopian/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/quantopian/subscriptions","organizations_url":"https://api.github.com/users/quantopian/orgs","repos_url":"https://api.github.com/users/quantopian/repos","events_url":"https://api.github.com/users/quantopian/events{/privacy}","received_events_url":"https://api.github.com/users/quantopian/received_events","type":"Organization","site_admin":false},"repo":{"id":6297520,"name":"zipline","full_name":"quantopian/zipline","owner":{"login":"quantopian","id":1393215,"avatar_url":"https://avatars.githubusercontent.com/u/1393215?v=3","gravatar_id":"","url":"https://api.github.com/users/quantopian","html_url":"https://github.com/quantopian","followers_url":"https://api.github.com/users/quantopian/followers","following_url":"https://api.github.com/users/quantopian/following{/other_user}","gists_url":"https://api.github.com/users/quantopian/gists{/gist_id}","starred_url":"https://api.github.com/users/quantopian/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/quantopian/subscriptions","organizations_url":"https://api.github.com/users/quantopian/orgs","repos_url":"https://api.github.com/users/quantopian/repos","events_url":"https://api.github.com/users/quantopian/events{/privacy}","received_events_url":"https://api.github.com/users/quantopian/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/quantopian/zipline","description":"Zipline, a Pythonic Algorithmic Trading Library","fork":false,"url":"https://api.github.com/repos/quantopian/zipline","forks_url":"https://api.github.com/repos/quantopian/zipline/forks","keys_url":"https://api.github.com/repos/quantopian/zipline/keys{/key_id}","collaborators_url":"https://api.github.com/repos/quantopian/zipline/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/quantopian/zipline/teams","hooks_url":"https://api.github.com/repos/quantopian/zipline/hooks","issue_events_url":"https://api.github.com/repos/quantopian/zipline/issues/events{/number}","events_url":"https://api.github.com/repos/quantopian/zipline/events","assignees_url":"https://api.github.com/repos/quantopian/zipline/assignees{/user}","branches_url":"https://api.github.com/repos/quantopian/zipline/branches{/branch}","tags_url":"https://api.github.com/repos/quantopian/zipline/tags","blobs_url":"https://api.github.com/repos/quantopian/zipline/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/quantopian/zipline/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/quantopian/zipline/git/refs{/sha}","trees_url":"https://api.github.com/repos/quantopian/zipline/git/trees{/sha}","statuses_url":"https://api.github.com/repos/quantopian/zipline/statuses/{sha}","languages_url":"https://api.github.com/repos/quantopian/zipline/languages","stargazers_url":"https://api.github.com/repos/quantopian/zipline/stargazers","contributors_url":"https://api.github.com/repos/quantopian/zipline/contributors","subscribers_url":"https://api.github.com/repos/quantopian/zipline/subscribers","subscription_url":"https://api.github.com/repos/quantopian/zipline/subscription","commits_url":"https://api.github.com/repos/quantopian/zipline/commits{/sha}","git_commits_url":"https://api.github.com/repos/quantopian/zipline/git/commits{/sha}","comments_url":"https://api.github.com/repos/quantopian/zipline/comments{/number}","issue_comment_url":"https://api.github.com/repos/quantopian/zipline/issues/comments{/number}","contents_url":"https://api.github.com/repos/quantopian/zipline/contents/{+path}","compare_url":"https://api.github.com/repos/quantopian/zipline/compare/{base}...{head}","merges_url":"https://api.github.com/repos/quantopian/zipline/merges","archive_url":"https://api.github.com/repos/quantopian/zipline/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/quantopian/zipline/downloads","issues_url":"https://api.github.com/repos/quantopian/zipline/issues{/number}","pulls_url":"https://api.github.com/repos/quantopian/zipline/pulls{/number}","milestones_url":"https://api.github.com/repos/quantopian/zipline/milestones{/number}","notifications_url":"https://api.github.com/repos/quantopian/zipline/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/quantopian/zipline/labels{/name}","releases_url":"https://api.github.com/repos/quantopian/zipline/releases{/id}","deployments_url":"https://api.github.com/repos/quantopian/zipline/deployments","created_at":"2012-10-19T15:50:29Z","updated_at":"2016-02-29T07:57:51Z","pushed_at":"2016-03-01T14:46:20Z","git_url":"git://github.com/quantopian/zipline.git","ssh_url":"git@github.com:quantopian/zipline.git","clone_url":"https://github.com/quantopian/zipline.git","svn_url":"https://github.com/quantopian/zipline","homepage":"","size":15620,"stargazers_count":2937,"watchers_count":2937,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":856,"mirror_url":null,"open_issues_count":160,"forks":856,"open_issues":160,"watchers":2937,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/quantopian/zipline/pulls/1019"},"html":{"href":"https://github.com/quantopian/zipline/pull/1019"},"issue":{"href":"https://api.github.com/repos/quantopian/zipline/issues/1019"},"comments":{"href":"https://api.github.com/repos/quantopian/zipline/issues/1019/comments"},"review_comments":{"href":"https://api.github.com/repos/quantopian/zipline/pulls/1019/comments"},"review_comment":{"href":"https://api.github.com/repos/quantopian/zipline/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/quantopian/zipline/pulls/1019/commits"},"statuses":{"href":"https://api.github.com/repos/quantopian/zipline/statuses/e0d19d6f20a34568beb08928fa9bcb46231ff413"}}}}