{"action":"created","comment":{"url":"https://api.github.com/repos/quantopian/zipline/pulls/comments/62123139","id":62123139,"diff_hunk":"@@ -291,8 +296,30 @@ def quandl_bundle(environ,\n             dividends,\n             environ.get('QUANDL_DOWNLOAD_ATTEMPTS', 5),\n         ),\n+        show_progress=show_progress,\n     )\n     adjustment_writer.write(\n         splits=pd.concat(splits, ignore_index=True),\n         dividends=pd.concat(dividends, ignore_index=True),\n     )\n+\n+\n+QUANTOPIAN_QUANDL_URL = (\n+    'https://s3.amazonaws.com/quantopian-public-zipline-data/quandl'\n+)\n+\n+\n+@bundles.register('quantopian-quandl', create_writers=False)\n+def quantopian_quandl_bundle(environ,\n+                             asset_db_writer,\n+                             minute_bar_writer,\n+                             daily_bar_writer,\n+                             adjustment_writer,\n+                             calendar,\n+                             cache,\n+                             show_progress,\n+                             output_dir):\n+    # use closing for py2 compat\n+    with closing(urlopen(QUANTOPIAN_QUANDL_URL)) as f, \\\n+            tarfile.open('r', fileobj=BytesIO(f.read())) as tar:\n+        tar.extractall(output_dir)","path":"zipline/data/bundles/quandl.py","position":67,"original_position":67,"commit_id":"fc04cd08193ed057c076e7a09d33f674f1b47545","original_commit_id":"fc04cd08193ed057c076e7a09d33f674f1b47545","user":{"login":"richafrank","id":2123660,"avatar_url":"https://avatars.githubusercontent.com/u/2123660?v=3","gravatar_id":"","url":"https://api.github.com/users/richafrank","html_url":"https://github.com/richafrank","followers_url":"https://api.github.com/users/richafrank/followers","following_url":"https://api.github.com/users/richafrank/following{/other_user}","gists_url":"https://api.github.com/users/richafrank/gists{/gist_id}","starred_url":"https://api.github.com/users/richafrank/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/richafrank/subscriptions","organizations_url":"https://api.github.com/users/richafrank/orgs","repos_url":"https://api.github.com/users/richafrank/repos","events_url":"https://api.github.com/users/richafrank/events{/privacy}","received_events_url":"https://api.github.com/users/richafrank/received_events","type":"User","site_admin":false},"body":"Since this isn't instantaneous, would it be possible to show progress?","created_at":"2016-05-04T22:06:44Z","updated_at":"2016-05-04T22:06:44Z","html_url":"https://github.com/quantopian/zipline/pull/1178#discussion_r62123139","pull_request_url":"https://api.github.com/repos/quantopian/zipline/pulls/1178","_links":{"self":{"href":"https://api.github.com/repos/quantopian/zipline/pulls/comments/62123139"},"html":{"href":"https://github.com/quantopian/zipline/pull/1178#discussion_r62123139"},"pull_request":{"href":"https://api.github.com/repos/quantopian/zipline/pulls/1178"}}},"pull_request":{"url":"https://api.github.com/repos/quantopian/zipline/pulls/1178","id":68807707,"html_url":"https://github.com/quantopian/zipline/pull/1178","diff_url":"https://github.com/quantopian/zipline/pull/1178.diff","patch_url":"https://github.com/quantopian/zipline/pull/1178.patch","issue_url":"https://api.github.com/repos/quantopian/zipline/issues/1178","number":1178,"state":"open","locked":false,"title":"ENH: Adds quantopian-quandl bundle as new default.","user":{"login":"llllllllll","id":3064397,"avatar_url":"https://avatars.githubusercontent.com/u/3064397?v=3","gravatar_id":"","url":"https://api.github.com/users/llllllllll","html_url":"https://github.com/llllllllll","followers_url":"https://api.github.com/users/llllllllll/followers","following_url":"https://api.github.com/users/llllllllll/following{/other_user}","gists_url":"https://api.github.com/users/llllllllll/gists{/gist_id}","starred_url":"https://api.github.com/users/llllllllll/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/llllllllll/subscriptions","organizations_url":"https://api.github.com/users/llllllllll/orgs","repos_url":"https://api.github.com/users/llllllllll/repos","events_url":"https://api.github.com/users/llllllllll/events{/privacy}","received_events_url":"https://api.github.com/users/llllllllll/received_events","type":"User","site_admin":false},"body":"This data bundle will use the quantopian mirror of the quandl WIKI data\r\ninstead of downloading from quandl directly. This dramatically improves\r\nthe speed because we do not pay the rate limiting for quandl and we can\r\nsend the data in the format zipline expects.\r\n\r\nNOTE: I need to rebuild the quandl set in s3. To test locally you can change the path to point to `s3://.../test/` which is a smaller sample of data which works. I will run the load overnight tonight.","created_at":"2016-05-04T03:46:19Z","updated_at":"2016-05-04T22:06:44Z","closed_at":null,"merged_at":null,"merge_commit_sha":"f1f15ec2c152bb1a6e69021ae776fb2a26488a90","assignee":{"login":"richafrank","id":2123660,"avatar_url":"https://avatars.githubusercontent.com/u/2123660?v=3","gravatar_id":"","url":"https://api.github.com/users/richafrank","html_url":"https://github.com/richafrank","followers_url":"https://api.github.com/users/richafrank/followers","following_url":"https://api.github.com/users/richafrank/following{/other_user}","gists_url":"https://api.github.com/users/richafrank/gists{/gist_id}","starred_url":"https://api.github.com/users/richafrank/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/richafrank/subscriptions","organizations_url":"https://api.github.com/users/richafrank/orgs","repos_url":"https://api.github.com/users/richafrank/repos","events_url":"https://api.github.com/users/richafrank/events{/privacy}","received_events_url":"https://api.github.com/users/richafrank/received_events","type":"User","site_admin":false},"milestone":null,"commits_url":"https://api.github.com/repos/quantopian/zipline/pulls/1178/commits","review_comments_url":"https://api.github.com/repos/quantopian/zipline/pulls/1178/comments","review_comment_url":"https://api.github.com/repos/quantopian/zipline/pulls/comments{/number}","comments_url":"https://api.github.com/repos/quantopian/zipline/issues/1178/comments","statuses_url":"https://api.github.com/repos/quantopian/zipline/statuses/fc04cd08193ed057c076e7a09d33f674f1b47545","head":{"label":"quantopian:quantopian-quandl","ref":"quantopian-quandl","sha":"fc04cd08193ed057c076e7a09d33f674f1b47545","user":{"login":"quantopian","id":1393215,"avatar_url":"https://avatars.githubusercontent.com/u/1393215?v=3","gravatar_id":"","url":"https://api.github.com/users/quantopian","html_url":"https://github.com/quantopian","followers_url":"https://api.github.com/users/quantopian/followers","following_url":"https://api.github.com/users/quantopian/following{/other_user}","gists_url":"https://api.github.com/users/quantopian/gists{/gist_id}","starred_url":"https://api.github.com/users/quantopian/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/quantopian/subscriptions","organizations_url":"https://api.github.com/users/quantopian/orgs","repos_url":"https://api.github.com/users/quantopian/repos","events_url":"https://api.github.com/users/quantopian/events{/privacy}","received_events_url":"https://api.github.com/users/quantopian/received_events","type":"Organization","site_admin":false},"repo":{"id":6297520,"name":"zipline","full_name":"quantopian/zipline","owner":{"login":"quantopian","id":1393215,"avatar_url":"https://avatars.githubusercontent.com/u/1393215?v=3","gravatar_id":"","url":"https://api.github.com/users/quantopian","html_url":"https://github.com/quantopian","followers_url":"https://api.github.com/users/quantopian/followers","following_url":"https://api.github.com/users/quantopian/following{/other_user}","gists_url":"https://api.github.com/users/quantopian/gists{/gist_id}","starred_url":"https://api.github.com/users/quantopian/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/quantopian/subscriptions","organizations_url":"https://api.github.com/users/quantopian/orgs","repos_url":"https://api.github.com/users/quantopian/repos","events_url":"https://api.github.com/users/quantopian/events{/privacy}","received_events_url":"https://api.github.com/users/quantopian/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/quantopian/zipline","description":"Zipline, a Pythonic Algorithmic Trading Library","fork":false,"url":"https://api.github.com/repos/quantopian/zipline","forks_url":"https://api.github.com/repos/quantopian/zipline/forks","keys_url":"https://api.github.com/repos/quantopian/zipline/keys{/key_id}","collaborators_url":"https://api.github.com/repos/quantopian/zipline/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/quantopian/zipline/teams","hooks_url":"https://api.github.com/repos/quantopian/zipline/hooks","issue_events_url":"https://api.github.com/repos/quantopian/zipline/issues/events{/number}","events_url":"https://api.github.com/repos/quantopian/zipline/events","assignees_url":"https://api.github.com/repos/quantopian/zipline/assignees{/user}","branches_url":"https://api.github.com/repos/quantopian/zipline/branches{/branch}","tags_url":"https://api.github.com/repos/quantopian/zipline/tags","blobs_url":"https://api.github.com/repos/quantopian/zipline/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/quantopian/zipline/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/quantopian/zipline/git/refs{/sha}","trees_url":"https://api.github.com/repos/quantopian/zipline/git/trees{/sha}","statuses_url":"https://api.github.com/repos/quantopian/zipline/statuses/{sha}","languages_url":"https://api.github.com/repos/quantopian/zipline/languages","stargazers_url":"https://api.github.com/repos/quantopian/zipline/stargazers","contributors_url":"https://api.github.com/repos/quantopian/zipline/contributors","subscribers_url":"https://api.github.com/repos/quantopian/zipline/subscribers","subscription_url":"https://api.github.com/repos/quantopian/zipline/subscription","commits_url":"https://api.github.com/repos/quantopian/zipline/commits{/sha}","git_commits_url":"https://api.github.com/repos/quantopian/zipline/git/commits{/sha}","comments_url":"https://api.github.com/repos/quantopian/zipline/comments{/number}","issue_comment_url":"https://api.github.com/repos/quantopian/zipline/issues/comments{/number}","contents_url":"https://api.github.com/repos/quantopian/zipline/contents/{+path}","compare_url":"https://api.github.com/repos/quantopian/zipline/compare/{base}...{head}","merges_url":"https://api.github.com/repos/quantopian/zipline/merges","archive_url":"https://api.github.com/repos/quantopian/zipline/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/quantopian/zipline/downloads","issues_url":"https://api.github.com/repos/quantopian/zipline/issues{/number}","pulls_url":"https://api.github.com/repos/quantopian/zipline/pulls{/number}","milestones_url":"https://api.github.com/repos/quantopian/zipline/milestones{/number}","notifications_url":"https://api.github.com/repos/quantopian/zipline/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/quantopian/zipline/labels{/name}","releases_url":"https://api.github.com/repos/quantopian/zipline/releases{/id}","deployments_url":"https://api.github.com/repos/quantopian/zipline/deployments","created_at":"2012-10-19T15:50:29Z","updated_at":"2016-05-04T20:31:23Z","pushed_at":"2016-05-04T21:49:29Z","git_url":"git://github.com/quantopian/zipline.git","ssh_url":"git@github.com:quantopian/zipline.git","clone_url":"https://github.com/quantopian/zipline.git","svn_url":"https://github.com/quantopian/zipline","homepage":"","size":37169,"stargazers_count":3162,"watchers_count":3162,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":949,"mirror_url":null,"open_issues_count":155,"forks":949,"open_issues":155,"watchers":3162,"default_branch":"master"}},"base":{"label":"quantopian:master","ref":"master","sha":"8756bf2c91c7499da9b27e57e5c8ed8050013b12","user":{"login":"quantopian","id":1393215,"avatar_url":"https://avatars.githubusercontent.com/u/1393215?v=3","gravatar_id":"","url":"https://api.github.com/users/quantopian","html_url":"https://github.com/quantopian","followers_url":"https://api.github.com/users/quantopian/followers","following_url":"https://api.github.com/users/quantopian/following{/other_user}","gists_url":"https://api.github.com/users/quantopian/gists{/gist_id}","starred_url":"https://api.github.com/users/quantopian/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/quantopian/subscriptions","organizations_url":"https://api.github.com/users/quantopian/orgs","repos_url":"https://api.github.com/users/quantopian/repos","events_url":"https://api.github.com/users/quantopian/events{/privacy}","received_events_url":"https://api.github.com/users/quantopian/received_events","type":"Organization","site_admin":false},"repo":{"id":6297520,"name":"zipline","full_name":"quantopian/zipline","owner":{"login":"quantopian","id":1393215,"avatar_url":"https://avatars.githubusercontent.com/u/1393215?v=3","gravatar_id":"","url":"https://api.github.com/users/quantopian","html_url":"https://github.com/quantopian","followers_url":"https://api.github.com/users/quantopian/followers","following_url":"https://api.github.com/users/quantopian/following{/other_user}","gists_url":"https://api.github.com/users/quantopian/gists{/gist_id}","starred_url":"https://api.github.com/users/quantopian/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/quantopian/subscriptions","organizations_url":"https://api.github.com/users/quantopian/orgs","repos_url":"https://api.github.com/users/quantopian/repos","events_url":"https://api.github.com/users/quantopian/events{/privacy}","received_events_url":"https://api.github.com/users/quantopian/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/quantopian/zipline","description":"Zipline, a Pythonic Algorithmic Trading Library","fork":false,"url":"https://api.github.com/repos/quantopian/zipline","forks_url":"https://api.github.com/repos/quantopian/zipline/forks","keys_url":"https://api.github.com/repos/quantopian/zipline/keys{/key_id}","collaborators_url":"https://api.github.com/repos/quantopian/zipline/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/quantopian/zipline/teams","hooks_url":"https://api.github.com/repos/quantopian/zipline/hooks","issue_events_url":"https://api.github.com/repos/quantopian/zipline/issues/events{/number}","events_url":"https://api.github.com/repos/quantopian/zipline/events","assignees_url":"https://api.github.com/repos/quantopian/zipline/assignees{/user}","branches_url":"https://api.github.com/repos/quantopian/zipline/branches{/branch}","tags_url":"https://api.github.com/repos/quantopian/zipline/tags","blobs_url":"https://api.github.com/repos/quantopian/zipline/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/quantopian/zipline/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/quantopian/zipline/git/refs{/sha}","trees_url":"https://api.github.com/repos/quantopian/zipline/git/trees{/sha}","statuses_url":"https://api.github.com/repos/quantopian/zipline/statuses/{sha}","languages_url":"https://api.github.com/repos/quantopian/zipline/languages","stargazers_url":"https://api.github.com/repos/quantopian/zipline/stargazers","contributors_url":"https://api.github.com/repos/quantopian/zipline/contributors","subscribers_url":"https://api.github.com/repos/quantopian/zipline/subscribers","subscription_url":"https://api.github.com/repos/quantopian/zipline/subscription","commits_url":"https://api.github.com/repos/quantopian/zipline/commits{/sha}","git_commits_url":"https://api.github.com/repos/quantopian/zipline/git/commits{/sha}","comments_url":"https://api.github.com/repos/quantopian/zipline/comments{/number}","issue_comment_url":"https://api.github.com/repos/quantopian/zipline/issues/comments{/number}","contents_url":"https://api.github.com/repos/quantopian/zipline/contents/{+path}","compare_url":"https://api.github.com/repos/quantopian/zipline/compare/{base}...{head}","merges_url":"https://api.github.com/repos/quantopian/zipline/merges","archive_url":"https://api.github.com/repos/quantopian/zipline/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/quantopian/zipline/downloads","issues_url":"https://api.github.com/repos/quantopian/zipline/issues{/number}","pulls_url":"https://api.github.com/repos/quantopian/zipline/pulls{/number}","milestones_url":"https://api.github.com/repos/quantopian/zipline/milestones{/number}","notifications_url":"https://api.github.com/repos/quantopian/zipline/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/quantopian/zipline/labels{/name}","releases_url":"https://api.github.com/repos/quantopian/zipline/releases{/id}","deployments_url":"https://api.github.com/repos/quantopian/zipline/deployments","created_at":"2012-10-19T15:50:29Z","updated_at":"2016-05-04T20:31:23Z","pushed_at":"2016-05-04T21:49:29Z","git_url":"git://github.com/quantopian/zipline.git","ssh_url":"git@github.com:quantopian/zipline.git","clone_url":"https://github.com/quantopian/zipline.git","svn_url":"https://github.com/quantopian/zipline","homepage":"","size":37169,"stargazers_count":3162,"watchers_count":3162,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":949,"mirror_url":null,"open_issues_count":155,"forks":949,"open_issues":155,"watchers":3162,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/quantopian/zipline/pulls/1178"},"html":{"href":"https://github.com/quantopian/zipline/pull/1178"},"issue":{"href":"https://api.github.com/repos/quantopian/zipline/issues/1178"},"comments":{"href":"https://api.github.com/repos/quantopian/zipline/issues/1178/comments"},"review_comments":{"href":"https://api.github.com/repos/quantopian/zipline/pulls/1178/comments"},"review_comment":{"href":"https://api.github.com/repos/quantopian/zipline/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/quantopian/zipline/pulls/1178/commits"},"statuses":{"href":"https://api.github.com/repos/quantopian/zipline/statuses/fc04cd08193ed057c076e7a09d33f674f1b47545"}}}}