{"action":"created","comment":{"url":"https://api.github.com/repos/quantopian/zipline/pulls/comments/50306254","id":50306254,"diff_hunk":"@@ -11,27 +11,402 @@\n # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n # See the License for the specific language governing permissions and\n # limitations under the License.\n+from textwrap import dedent\n+\n import bcolz\n+from bcolz import ctable\n+import numpy as np\n+from os.path import join\n import json\n import os\n import pandas as pd\n \n MINUTES_PER_DAY = 390\n-\n+DEFAULT_EXPECTEDLEN = 390 * 252 * 15\n+OHLC_RATIO = 1000\n METADATA_FILENAME = 'metadata.json'\n \n \n+class BcolzMinuteOverlappingData(Exception):\n+    pass\n+\n+\n+def _calc_minute_index(market_opens, minutes_per_day):\n+    minutes = np.zeros(len(market_opens) * minutes_per_day,\n+                       dtype='datetime64[ns]')\n+    deltas = np.arange(0, minutes_per_day, dtype='timedelta64[m]')\n+    for i, market_open in enumerate(market_opens):\n+        start = market_open.asm8\n+        minute_values = start + deltas\n+        start_ix = minutes_per_day * i\n+        end_ix = start_ix + minutes_per_day\n+        minutes[start_ix:end_ix] = minute_values\n+    return pd.to_datetime(minutes, utc=True, box=True)\n+\n+\n+def _sid_subdir_path(sid):\n+    \"\"\"\n+    Format subdir path to limit the number directories in any given\n+    subdirectory to 100.\n+\n+    The number in each directory is designed to support at least 100000\n+    equities.\n+\n+    Parameters:\n+    -----------\n+    sid - int","path":"zipline/data/us_equity_minutes.py","position":54,"original_position":54,"commit_id":"f6453339f22663be961d833f71d61d92aaf5920c","original_commit_id":"f6453339f22663be961d833f71d61d92aaf5920c","user":{"login":"ehebert","id":185744,"avatar_url":"https://avatars.githubusercontent.com/u/185744?v=3","gravatar_id":"","url":"https://api.github.com/users/ehebert","html_url":"https://github.com/ehebert","followers_url":"https://api.github.com/users/ehebert/followers","following_url":"https://api.github.com/users/ehebert/following{/other_user}","gists_url":"https://api.github.com/users/ehebert/gists{/gist_id}","starred_url":"https://api.github.com/users/ehebert/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ehebert/subscriptions","organizations_url":"https://api.github.com/users/ehebert/orgs","repos_url":"https://api.github.com/users/ehebert/repos","events_url":"https://api.github.com/users/ehebert/events{/privacy}","received_events_url":"https://api.github.com/users/ehebert/received_events","type":"User","site_admin":false},"body":"Will fix, I believe I used a ':' most everywhere else in this module.","created_at":"2016-01-20T19:36:26Z","updated_at":"2016-01-20T19:36:26Z","html_url":"https://github.com/quantopian/zipline/pull/960#discussion_r50306254","pull_request_url":"https://api.github.com/repos/quantopian/zipline/pulls/960","_links":{"self":{"href":"https://api.github.com/repos/quantopian/zipline/pulls/comments/50306254"},"html":{"href":"https://github.com/quantopian/zipline/pull/960#discussion_r50306254"},"pull_request":{"href":"https://api.github.com/repos/quantopian/zipline/pulls/960"}}},"pull_request":{"url":"https://api.github.com/repos/quantopian/zipline/pulls/960","id":56655206,"html_url":"https://github.com/quantopian/zipline/pull/960","diff_url":"https://github.com/quantopian/zipline/pull/960.diff","patch_url":"https://github.com/quantopian/zipline/pull/960.patch","issue_url":"https://api.github.com/repos/quantopian/zipline/issues/960","number":960,"state":"open","locked":false,"title":"ENH: Add writer for equity minute bcolz format.","user":{"login":"ehebert","id":185744,"avatar_url":"https://avatars.githubusercontent.com/u/185744?v=3","gravatar_id":"","url":"https://api.github.com/users/ehebert","html_url":"https://github.com/ehebert","followers_url":"https://api.github.com/users/ehebert/followers","following_url":"https://api.github.com/users/ehebert/following{/other_user}","gists_url":"https://api.github.com/users/ehebert/gists{/gist_id}","starred_url":"https://api.github.com/users/ehebert/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ehebert/subscriptions","organizations_url":"https://api.github.com/users/ehebert/orgs","repos_url":"https://api.github.com/users/ehebert/repos","events_url":"https://api.github.com/users/ehebert/events{/privacy}","received_events_url":"https://api.github.com/users/ehebert/received_events","type":"User","site_admin":false},"body":"Implement a writer for minute data into a format comprised of multiple\r\nctables, one for each individual equity, with a common 'index' shared by\r\nall ctables where a given a dt maps to the same array index for all\r\nequities and fields.\r\n\r\nThis format is pulled from the lazy-mainline/Q2.0 branch, with some\r\nchanges to the interface.\r\n\r\nAdd basic retrieval of values at a given dt to reader. Not yet used by\r\nZipline simulations, but added to support unit tests.\r\n\r\n# Other Notes\r\n\r\nQuestions:\r\n- Currently the main entry point into the writer is `write(sid, days, df)`, where the df is a DataFrame for just the given sid. \r\n-- Providing `days` does feel redundant, when that information could be re-constructed from the `df` and the writer's `self._trading_days`.\r\n-- `sid` could be also be redundant, if the df were formatted with a `MultiIndex` of (sid, dt), then the sids could be iterated over using `get_level_values`.\r\n\r\n@ssanderson I took in a lot of notes from our in person conversation the other day.\r\nIncluding taking in market_opens as a parameter, and then writing out the resulting index to free the writer/reader combo from requiring a `TradingEnvironment` or `tradingcalendar` module.\r\n\r\nWhere I did diverge from our discussion was:\r\n- The writer class no longer requires subclassing to implement different sources, instead the `write` method takes a DataFrame with the OHLCV data. This change resulted from using the writer with our existing and incoming internal dataloader. I found having a writer which was agnostic of its input source more flexible to work with when wiring up different sources, and I believe is a more adoptable public API.\r\n\r\nFor some background on the choice of having the Writer agnostic to the input, I looked to the following:\r\n- https://docs.python.org/2/library/csv.html\r\n- https://golang.org/pkg/bufio/#example_Writer\r\n- http://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.to_excel.html\r\n\r\n- The responsibility of the writer is just writing individual sid data, without adding the 'bring data up to date' logic that our internal loader provides. This change allowed dropping the writer into our existing internal loader. If/when we do make that logic a public API, I believe we should make a separate `Loader` (or some other name, `DataSet`/`DataSetManager`, I am not yet sure), which takes the writer as one of its components.\r\n\r\n@jbredeche this implementation differs from our implementation in lazy-mainline in the following ways.\r\n(For reference, https://github.com/quantopian/zipline/blob/lazy-mainline/zipline/data/us_equity_minutes.py)\r\n- The aforementioned change to no longer using subclasses to get the data, instead passing in the DataFrame independent of the data source.\r\n- Making the directory as part of the constructor. This change was 1) based on the above list of other Reader/Writers, 2) was more natural when using the writer with our existing internal loader 3) Allows more flexibility if we ever need to implement another writer that does not write to a directory, e.g. if a Postgres or SQLite writer is implemented then `write(sid, days, df)` would also work with a writer of that type.\r\n\r\n- Allowing multiple passes on writing data, which is a requirement of our internal writers which read the source data in subgroups of days and equities. (Instead of only supporting reading all data for an equity at once.)","created_at":"2016-01-20T19:29:43Z","updated_at":"2016-01-20T19:36:26Z","closed_at":null,"merged_at":null,"merge_commit_sha":"e31ce1fdf9a7846ea411ce60646272fba7a6c91e","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/quantopian/zipline/pulls/960/commits","review_comments_url":"https://api.github.com/repos/quantopian/zipline/pulls/960/comments","review_comment_url":"https://api.github.com/repos/quantopian/zipline/pulls/comments{/number}","comments_url":"https://api.github.com/repos/quantopian/zipline/issues/960/comments","statuses_url":"https://api.github.com/repos/quantopian/zipline/statuses/f6453339f22663be961d833f71d61d92aaf5920c","head":{"label":"quantopian:minute-writer-for-one-per-sid","ref":"minute-writer-for-one-per-sid","sha":"f6453339f22663be961d833f71d61d92aaf5920c","user":{"login":"quantopian","id":1393215,"avatar_url":"https://avatars.githubusercontent.com/u/1393215?v=3","gravatar_id":"","url":"https://api.github.com/users/quantopian","html_url":"https://github.com/quantopian","followers_url":"https://api.github.com/users/quantopian/followers","following_url":"https://api.github.com/users/quantopian/following{/other_user}","gists_url":"https://api.github.com/users/quantopian/gists{/gist_id}","starred_url":"https://api.github.com/users/quantopian/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/quantopian/subscriptions","organizations_url":"https://api.github.com/users/quantopian/orgs","repos_url":"https://api.github.com/users/quantopian/repos","events_url":"https://api.github.com/users/quantopian/events{/privacy}","received_events_url":"https://api.github.com/users/quantopian/received_events","type":"Organization","site_admin":false},"repo":{"id":6297520,"name":"zipline","full_name":"quantopian/zipline","owner":{"login":"quantopian","id":1393215,"avatar_url":"https://avatars.githubusercontent.com/u/1393215?v=3","gravatar_id":"","url":"https://api.github.com/users/quantopian","html_url":"https://github.com/quantopian","followers_url":"https://api.github.com/users/quantopian/followers","following_url":"https://api.github.com/users/quantopian/following{/other_user}","gists_url":"https://api.github.com/users/quantopian/gists{/gist_id}","starred_url":"https://api.github.com/users/quantopian/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/quantopian/subscriptions","organizations_url":"https://api.github.com/users/quantopian/orgs","repos_url":"https://api.github.com/users/quantopian/repos","events_url":"https://api.github.com/users/quantopian/events{/privacy}","received_events_url":"https://api.github.com/users/quantopian/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/quantopian/zipline","description":"Zipline, a Pythonic Algorithmic Trading Library","fork":false,"url":"https://api.github.com/repos/quantopian/zipline","forks_url":"https://api.github.com/repos/quantopian/zipline/forks","keys_url":"https://api.github.com/repos/quantopian/zipline/keys{/key_id}","collaborators_url":"https://api.github.com/repos/quantopian/zipline/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/quantopian/zipline/teams","hooks_url":"https://api.github.com/repos/quantopian/zipline/hooks","issue_events_url":"https://api.github.com/repos/quantopian/zipline/issues/events{/number}","events_url":"https://api.github.com/repos/quantopian/zipline/events","assignees_url":"https://api.github.com/repos/quantopian/zipline/assignees{/user}","branches_url":"https://api.github.com/repos/quantopian/zipline/branches{/branch}","tags_url":"https://api.github.com/repos/quantopian/zipline/tags","blobs_url":"https://api.github.com/repos/quantopian/zipline/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/quantopian/zipline/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/quantopian/zipline/git/refs{/sha}","trees_url":"https://api.github.com/repos/quantopian/zipline/git/trees{/sha}","statuses_url":"https://api.github.com/repos/quantopian/zipline/statuses/{sha}","languages_url":"https://api.github.com/repos/quantopian/zipline/languages","stargazers_url":"https://api.github.com/repos/quantopian/zipline/stargazers","contributors_url":"https://api.github.com/repos/quantopian/zipline/contributors","subscribers_url":"https://api.github.com/repos/quantopian/zipline/subscribers","subscription_url":"https://api.github.com/repos/quantopian/zipline/subscription","commits_url":"https://api.github.com/repos/quantopian/zipline/commits{/sha}","git_commits_url":"https://api.github.com/repos/quantopian/zipline/git/commits{/sha}","comments_url":"https://api.github.com/repos/quantopian/zipline/comments{/number}","issue_comment_url":"https://api.github.com/repos/quantopian/zipline/issues/comments{/number}","contents_url":"https://api.github.com/repos/quantopian/zipline/contents/{+path}","compare_url":"https://api.github.com/repos/quantopian/zipline/compare/{base}...{head}","merges_url":"https://api.github.com/repos/quantopian/zipline/merges","archive_url":"https://api.github.com/repos/quantopian/zipline/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/quantopian/zipline/downloads","issues_url":"https://api.github.com/repos/quantopian/zipline/issues{/number}","pulls_url":"https://api.github.com/repos/quantopian/zipline/pulls{/number}","milestones_url":"https://api.github.com/repos/quantopian/zipline/milestones{/number}","notifications_url":"https://api.github.com/repos/quantopian/zipline/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/quantopian/zipline/labels{/name}","releases_url":"https://api.github.com/repos/quantopian/zipline/releases{/id}","created_at":"2012-10-19T15:50:29Z","updated_at":"2016-01-20T17:03:33Z","pushed_at":"2016-01-20T19:29:44Z","git_url":"git://github.com/quantopian/zipline.git","ssh_url":"git@github.com:quantopian/zipline.git","clone_url":"https://github.com/quantopian/zipline.git","svn_url":"https://github.com/quantopian/zipline","homepage":"","size":14752,"stargazers_count":2826,"watchers_count":2826,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":818,"mirror_url":null,"open_issues_count":158,"forks":818,"open_issues":158,"watchers":2826,"default_branch":"master"}},"base":{"label":"quantopian:master","ref":"master","sha":"b16e2a7a07aa86df8264595fd3c1b32ff652c412","user":{"login":"quantopian","id":1393215,"avatar_url":"https://avatars.githubusercontent.com/u/1393215?v=3","gravatar_id":"","url":"https://api.github.com/users/quantopian","html_url":"https://github.com/quantopian","followers_url":"https://api.github.com/users/quantopian/followers","following_url":"https://api.github.com/users/quantopian/following{/other_user}","gists_url":"https://api.github.com/users/quantopian/gists{/gist_id}","starred_url":"https://api.github.com/users/quantopian/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/quantopian/subscriptions","organizations_url":"https://api.github.com/users/quantopian/orgs","repos_url":"https://api.github.com/users/quantopian/repos","events_url":"https://api.github.com/users/quantopian/events{/privacy}","received_events_url":"https://api.github.com/users/quantopian/received_events","type":"Organization","site_admin":false},"repo":{"id":6297520,"name":"zipline","full_name":"quantopian/zipline","owner":{"login":"quantopian","id":1393215,"avatar_url":"https://avatars.githubusercontent.com/u/1393215?v=3","gravatar_id":"","url":"https://api.github.com/users/quantopian","html_url":"https://github.com/quantopian","followers_url":"https://api.github.com/users/quantopian/followers","following_url":"https://api.github.com/users/quantopian/following{/other_user}","gists_url":"https://api.github.com/users/quantopian/gists{/gist_id}","starred_url":"https://api.github.com/users/quantopian/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/quantopian/subscriptions","organizations_url":"https://api.github.com/users/quantopian/orgs","repos_url":"https://api.github.com/users/quantopian/repos","events_url":"https://api.github.com/users/quantopian/events{/privacy}","received_events_url":"https://api.github.com/users/quantopian/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/quantopian/zipline","description":"Zipline, a Pythonic Algorithmic Trading Library","fork":false,"url":"https://api.github.com/repos/quantopian/zipline","forks_url":"https://api.github.com/repos/quantopian/zipline/forks","keys_url":"https://api.github.com/repos/quantopian/zipline/keys{/key_id}","collaborators_url":"https://api.github.com/repos/quantopian/zipline/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/quantopian/zipline/teams","hooks_url":"https://api.github.com/repos/quantopian/zipline/hooks","issue_events_url":"https://api.github.com/repos/quantopian/zipline/issues/events{/number}","events_url":"https://api.github.com/repos/quantopian/zipline/events","assignees_url":"https://api.github.com/repos/quantopian/zipline/assignees{/user}","branches_url":"https://api.github.com/repos/quantopian/zipline/branches{/branch}","tags_url":"https://api.github.com/repos/quantopian/zipline/tags","blobs_url":"https://api.github.com/repos/quantopian/zipline/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/quantopian/zipline/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/quantopian/zipline/git/refs{/sha}","trees_url":"https://api.github.com/repos/quantopian/zipline/git/trees{/sha}","statuses_url":"https://api.github.com/repos/quantopian/zipline/statuses/{sha}","languages_url":"https://api.github.com/repos/quantopian/zipline/languages","stargazers_url":"https://api.github.com/repos/quantopian/zipline/stargazers","contributors_url":"https://api.github.com/repos/quantopian/zipline/contributors","subscribers_url":"https://api.github.com/repos/quantopian/zipline/subscribers","subscription_url":"https://api.github.com/repos/quantopian/zipline/subscription","commits_url":"https://api.github.com/repos/quantopian/zipline/commits{/sha}","git_commits_url":"https://api.github.com/repos/quantopian/zipline/git/commits{/sha}","comments_url":"https://api.github.com/repos/quantopian/zipline/comments{/number}","issue_comment_url":"https://api.github.com/repos/quantopian/zipline/issues/comments{/number}","contents_url":"https://api.github.com/repos/quantopian/zipline/contents/{+path}","compare_url":"https://api.github.com/repos/quantopian/zipline/compare/{base}...{head}","merges_url":"https://api.github.com/repos/quantopian/zipline/merges","archive_url":"https://api.github.com/repos/quantopian/zipline/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/quantopian/zipline/downloads","issues_url":"https://api.github.com/repos/quantopian/zipline/issues{/number}","pulls_url":"https://api.github.com/repos/quantopian/zipline/pulls{/number}","milestones_url":"https://api.github.com/repos/quantopian/zipline/milestones{/number}","notifications_url":"https://api.github.com/repos/quantopian/zipline/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/quantopian/zipline/labels{/name}","releases_url":"https://api.github.com/repos/quantopian/zipline/releases{/id}","created_at":"2012-10-19T15:50:29Z","updated_at":"2016-01-20T17:03:33Z","pushed_at":"2016-01-20T19:29:44Z","git_url":"git://github.com/quantopian/zipline.git","ssh_url":"git@github.com:quantopian/zipline.git","clone_url":"https://github.com/quantopian/zipline.git","svn_url":"https://github.com/quantopian/zipline","homepage":"","size":14752,"stargazers_count":2826,"watchers_count":2826,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":818,"mirror_url":null,"open_issues_count":158,"forks":818,"open_issues":158,"watchers":2826,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/quantopian/zipline/pulls/960"},"html":{"href":"https://github.com/quantopian/zipline/pull/960"},"issue":{"href":"https://api.github.com/repos/quantopian/zipline/issues/960"},"comments":{"href":"https://api.github.com/repos/quantopian/zipline/issues/960/comments"},"review_comments":{"href":"https://api.github.com/repos/quantopian/zipline/pulls/960/comments"},"review_comment":{"href":"https://api.github.com/repos/quantopian/zipline/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/quantopian/zipline/pulls/960/commits"},"statuses":{"href":"https://api.github.com/repos/quantopian/zipline/statuses/f6453339f22663be961d833f71d61d92aaf5920c"}}}}