{"action":"created","comment":{"url":"https://api.github.com/repos/codalab/codalab-cli/pulls/comments/56014969","id":56014969,"diff_hunk":"@@ -687,6 +693,64 @@ def remove_data_hash_references(self, uuids):\n         with self.engine.begin() as connection:\n             connection.execute(cl_bundle.update().where(cl_bundle.c.uuid.in_(uuids)).values({'data_hash': None}))\n \n+    def update_bundle_contents_index(self, uuid, index):\n+        rows = []\n+        def recurse(entry, parent_path):\n+            entry['bundle_uuid'] = uuid\n+            entry['path'] = os.path.join(parent_path, entry['name'])\n+            del entry['name']\n+            if 'contents' in entry:\n+                for child_entry in entry['contents']:\n+                    recurse(child_entry, entry['path'])\n+                del entry['contents']\n+            if 'link' not in entry:\n+                entry['link'] = None\n+            rows.append(entry)\n+        index_copy = copy.deepcopy(index)\n+        if index_copy['name'] != uuid:\n+            raise UsageError(\n+                'Valid indices have the directory with the name as the UUID '\n+                'being the top-level directory.')\n+        # Clear the top-level name to save space in the database.\n+        index_copy['name'] = ''\n+        recurse(index_copy, '/')\n+        with self.engine.begin() as connection:\n+            connection.execute(cl_bundle_contents_index.delete().where(","path":"codalab/model/bundle_model.py","position":55,"original_position":55,"commit_id":"fbf3e42399c16b89468d2e94618dd7684f23cea7","original_commit_id":"fbf3e42399c16b89468d2e94618dd7684f23cea7","user":{"login":"percyliang","id":435384,"avatar_url":"https://avatars.githubusercontent.com/u/435384?v=3","gravatar_id":"","url":"https://api.github.com/users/percyliang","html_url":"https://github.com/percyliang","followers_url":"https://api.github.com/users/percyliang/followers","following_url":"https://api.github.com/users/percyliang/following{/other_user}","gists_url":"https://api.github.com/users/percyliang/gists{/gist_id}","starred_url":"https://api.github.com/users/percyliang/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/percyliang/subscriptions","organizations_url":"https://api.github.com/users/percyliang/orgs","repos_url":"https://api.github.com/users/percyliang/repos","events_url":"https://api.github.com/users/percyliang/events{/privacy}","received_events_url":"https://api.github.com/users/percyliang/received_events","type":"User","site_admin":false},"body":"I'm a bit worried about bundles that have lots and lots of files...this might get pretty slow...\r\nWe don't have a cap on the number of files (maybe we should at least for intermediate updates).  In the past, a user has created 100K files.","created_at":"2016-03-14T15:07:02Z","updated_at":"2016-03-14T15:07:02Z","html_url":"https://github.com/codalab/codalab-cli/pull/393#discussion_r56014969","pull_request_url":"https://api.github.com/repos/codalab/codalab-cli/pulls/393","_links":{"self":{"href":"https://api.github.com/repos/codalab/codalab-cli/pulls/comments/56014969"},"html":{"href":"https://github.com/codalab/codalab-cli/pull/393#discussion_r56014969"},"pull_request":{"href":"https://api.github.com/repos/codalab/codalab-cli/pulls/393"}}},"pull_request":{"url":"https://api.github.com/repos/codalab/codalab-cli/pulls/393","id":62688653,"html_url":"https://github.com/codalab/codalab-cli/pull/393","diff_url":"https://github.com/codalab/codalab-cli/pull/393.diff","patch_url":"https://github.com/codalab/codalab-cli/pull/393.patch","issue_url":"https://api.github.com/repos/codalab/codalab-cli/issues/393","number":393,"state":"open","locked":false,"title":"Bundle contents index.","user":{"login":"klopyrev","id":5249090,"avatar_url":"https://avatars.githubusercontent.com/u/5249090?v=3","gravatar_id":"","url":"https://api.github.com/users/klopyrev","html_url":"https://github.com/klopyrev","followers_url":"https://api.github.com/users/klopyrev/followers","following_url":"https://api.github.com/users/klopyrev/following{/other_user}","gists_url":"https://api.github.com/users/klopyrev/gists{/gist_id}","starred_url":"https://api.github.com/users/klopyrev/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/klopyrev/subscriptions","organizations_url":"https://api.github.com/users/klopyrev/orgs","repos_url":"https://api.github.com/users/klopyrev/repos","events_url":"https://api.github.com/users/klopyrev/events{/privacy}","received_events_url":"https://api.github.com/users/klopyrev/received_events","type":"User","site_admin":false},"body":"Add an index of the bundle contents. Once the worker system is launched, this index will be used, instead of calling path_util.get_info. The worker updates this index pretty often, so it should be fresh.\r\n\r\nI'm also moving the get_info method to worker.file_util since it'll be used by the worker.\r\n\r\nAdd a script to index the current contents of the bundle store. This will be used to launch the new worker system.\r\n\r\n@percyliang \r\n\r\nNote: since this change depends on the download_manager change, I've done a trick where I made the pull requests again the download_manager branch in codalab/codalab-cli that is identical to the one in klopyrev/codalab-cli. Once the download_manager change is submitted, I'll fix the pull request to be for the codalab/codalab-cli develop branch.","created_at":"2016-03-14T00:09:10Z","updated_at":"2016-03-14T15:07:02Z","closed_at":null,"merged_at":null,"merge_commit_sha":"436176f00df29e4cd0187deacc8f54f868a8ec0d","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/codalab/codalab-cli/pulls/393/commits","review_comments_url":"https://api.github.com/repos/codalab/codalab-cli/pulls/393/comments","review_comment_url":"https://api.github.com/repos/codalab/codalab-cli/pulls/comments{/number}","comments_url":"https://api.github.com/repos/codalab/codalab-cli/issues/393/comments","statuses_url":"https://api.github.com/repos/codalab/codalab-cli/statuses/fbf3e42399c16b89468d2e94618dd7684f23cea7","head":{"label":"klopyrev:contents_index","ref":"contents_index","sha":"fbf3e42399c16b89468d2e94618dd7684f23cea7","user":{"login":"klopyrev","id":5249090,"avatar_url":"https://avatars.githubusercontent.com/u/5249090?v=3","gravatar_id":"","url":"https://api.github.com/users/klopyrev","html_url":"https://github.com/klopyrev","followers_url":"https://api.github.com/users/klopyrev/followers","following_url":"https://api.github.com/users/klopyrev/following{/other_user}","gists_url":"https://api.github.com/users/klopyrev/gists{/gist_id}","starred_url":"https://api.github.com/users/klopyrev/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/klopyrev/subscriptions","organizations_url":"https://api.github.com/users/klopyrev/orgs","repos_url":"https://api.github.com/users/klopyrev/repos","events_url":"https://api.github.com/users/klopyrev/events{/privacy}","received_events_url":"https://api.github.com/users/klopyrev/received_events","type":"User","site_admin":false},"repo":{"id":49246702,"name":"codalab-cli","full_name":"klopyrev/codalab-cli","owner":{"login":"klopyrev","id":5249090,"avatar_url":"https://avatars.githubusercontent.com/u/5249090?v=3","gravatar_id":"","url":"https://api.github.com/users/klopyrev","html_url":"https://github.com/klopyrev","followers_url":"https://api.github.com/users/klopyrev/followers","following_url":"https://api.github.com/users/klopyrev/following{/other_user}","gists_url":"https://api.github.com/users/klopyrev/gists{/gist_id}","starred_url":"https://api.github.com/users/klopyrev/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/klopyrev/subscriptions","organizations_url":"https://api.github.com/users/klopyrev/orgs","repos_url":"https://api.github.com/users/klopyrev/repos","events_url":"https://api.github.com/users/klopyrev/events{/privacy}","received_events_url":"https://api.github.com/users/klopyrev/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/klopyrev/codalab-cli","description":"Bundle service and command-line interface for supporting worksheets in CodaLab.","fork":true,"url":"https://api.github.com/repos/klopyrev/codalab-cli","forks_url":"https://api.github.com/repos/klopyrev/codalab-cli/forks","keys_url":"https://api.github.com/repos/klopyrev/codalab-cli/keys{/key_id}","collaborators_url":"https://api.github.com/repos/klopyrev/codalab-cli/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/klopyrev/codalab-cli/teams","hooks_url":"https://api.github.com/repos/klopyrev/codalab-cli/hooks","issue_events_url":"https://api.github.com/repos/klopyrev/codalab-cli/issues/events{/number}","events_url":"https://api.github.com/repos/klopyrev/codalab-cli/events","assignees_url":"https://api.github.com/repos/klopyrev/codalab-cli/assignees{/user}","branches_url":"https://api.github.com/repos/klopyrev/codalab-cli/branches{/branch}","tags_url":"https://api.github.com/repos/klopyrev/codalab-cli/tags","blobs_url":"https://api.github.com/repos/klopyrev/codalab-cli/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/klopyrev/codalab-cli/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/klopyrev/codalab-cli/git/refs{/sha}","trees_url":"https://api.github.com/repos/klopyrev/codalab-cli/git/trees{/sha}","statuses_url":"https://api.github.com/repos/klopyrev/codalab-cli/statuses/{sha}","languages_url":"https://api.github.com/repos/klopyrev/codalab-cli/languages","stargazers_url":"https://api.github.com/repos/klopyrev/codalab-cli/stargazers","contributors_url":"https://api.github.com/repos/klopyrev/codalab-cli/contributors","subscribers_url":"https://api.github.com/repos/klopyrev/codalab-cli/subscribers","subscription_url":"https://api.github.com/repos/klopyrev/codalab-cli/subscription","commits_url":"https://api.github.com/repos/klopyrev/codalab-cli/commits{/sha}","git_commits_url":"https://api.github.com/repos/klopyrev/codalab-cli/git/commits{/sha}","comments_url":"https://api.github.com/repos/klopyrev/codalab-cli/comments{/number}","issue_comment_url":"https://api.github.com/repos/klopyrev/codalab-cli/issues/comments{/number}","contents_url":"https://api.github.com/repos/klopyrev/codalab-cli/contents/{+path}","compare_url":"https://api.github.com/repos/klopyrev/codalab-cli/compare/{base}...{head}","merges_url":"https://api.github.com/repos/klopyrev/codalab-cli/merges","archive_url":"https://api.github.com/repos/klopyrev/codalab-cli/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/klopyrev/codalab-cli/downloads","issues_url":"https://api.github.com/repos/klopyrev/codalab-cli/issues{/number}","pulls_url":"https://api.github.com/repos/klopyrev/codalab-cli/pulls{/number}","milestones_url":"https://api.github.com/repos/klopyrev/codalab-cli/milestones{/number}","notifications_url":"https://api.github.com/repos/klopyrev/codalab-cli/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/klopyrev/codalab-cli/labels{/name}","releases_url":"https://api.github.com/repos/klopyrev/codalab-cli/releases{/id}","deployments_url":"https://api.github.com/repos/klopyrev/codalab-cli/deployments","created_at":"2016-01-08T03:23:44Z","updated_at":"2016-02-11T03:55:46Z","pushed_at":"2016-03-14T03:47:58Z","git_url":"git://github.com/klopyrev/codalab-cli.git","ssh_url":"git@github.com:klopyrev/codalab-cli.git","clone_url":"https://github.com/klopyrev/codalab-cli.git","svn_url":"https://github.com/klopyrev/codalab-cli","homepage":null,"size":2449,"stargazers_count":1,"watchers_count":1,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":1,"default_branch":"master"}},"base":{"label":"codalab:download_manager","ref":"download_manager","sha":"91ce2ccc03129501f8ea759a16dbdadafb205393","user":{"login":"codalab","id":4633552,"avatar_url":"https://avatars.githubusercontent.com/u/4633552?v=3","gravatar_id":"","url":"https://api.github.com/users/codalab","html_url":"https://github.com/codalab","followers_url":"https://api.github.com/users/codalab/followers","following_url":"https://api.github.com/users/codalab/following{/other_user}","gists_url":"https://api.github.com/users/codalab/gists{/gist_id}","starred_url":"https://api.github.com/users/codalab/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/codalab/subscriptions","organizations_url":"https://api.github.com/users/codalab/orgs","repos_url":"https://api.github.com/users/codalab/repos","events_url":"https://api.github.com/users/codalab/events{/privacy}","received_events_url":"https://api.github.com/users/codalab/received_events","type":"User","site_admin":false},"repo":{"id":27352490,"name":"codalab-cli","full_name":"codalab/codalab-cli","owner":{"login":"codalab","id":4633552,"avatar_url":"https://avatars.githubusercontent.com/u/4633552?v=3","gravatar_id":"","url":"https://api.github.com/users/codalab","html_url":"https://github.com/codalab","followers_url":"https://api.github.com/users/codalab/followers","following_url":"https://api.github.com/users/codalab/following{/other_user}","gists_url":"https://api.github.com/users/codalab/gists{/gist_id}","starred_url":"https://api.github.com/users/codalab/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/codalab/subscriptions","organizations_url":"https://api.github.com/users/codalab/orgs","repos_url":"https://api.github.com/users/codalab/repos","events_url":"https://api.github.com/users/codalab/events{/privacy}","received_events_url":"https://api.github.com/users/codalab/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/codalab/codalab-cli","description":"Bundle service and command-line interface for supporting worksheets in CodaLab.","fork":false,"url":"https://api.github.com/repos/codalab/codalab-cli","forks_url":"https://api.github.com/repos/codalab/codalab-cli/forks","keys_url":"https://api.github.com/repos/codalab/codalab-cli/keys{/key_id}","collaborators_url":"https://api.github.com/repos/codalab/codalab-cli/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/codalab/codalab-cli/teams","hooks_url":"https://api.github.com/repos/codalab/codalab-cli/hooks","issue_events_url":"https://api.github.com/repos/codalab/codalab-cli/issues/events{/number}","events_url":"https://api.github.com/repos/codalab/codalab-cli/events","assignees_url":"https://api.github.com/repos/codalab/codalab-cli/assignees{/user}","branches_url":"https://api.github.com/repos/codalab/codalab-cli/branches{/branch}","tags_url":"https://api.github.com/repos/codalab/codalab-cli/tags","blobs_url":"https://api.github.com/repos/codalab/codalab-cli/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/codalab/codalab-cli/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/codalab/codalab-cli/git/refs{/sha}","trees_url":"https://api.github.com/repos/codalab/codalab-cli/git/trees{/sha}","statuses_url":"https://api.github.com/repos/codalab/codalab-cli/statuses/{sha}","languages_url":"https://api.github.com/repos/codalab/codalab-cli/languages","stargazers_url":"https://api.github.com/repos/codalab/codalab-cli/stargazers","contributors_url":"https://api.github.com/repos/codalab/codalab-cli/contributors","subscribers_url":"https://api.github.com/repos/codalab/codalab-cli/subscribers","subscription_url":"https://api.github.com/repos/codalab/codalab-cli/subscription","commits_url":"https://api.github.com/repos/codalab/codalab-cli/commits{/sha}","git_commits_url":"https://api.github.com/repos/codalab/codalab-cli/git/commits{/sha}","comments_url":"https://api.github.com/repos/codalab/codalab-cli/comments{/number}","issue_comment_url":"https://api.github.com/repos/codalab/codalab-cli/issues/comments{/number}","contents_url":"https://api.github.com/repos/codalab/codalab-cli/contents/{+path}","compare_url":"https://api.github.com/repos/codalab/codalab-cli/compare/{base}...{head}","merges_url":"https://api.github.com/repos/codalab/codalab-cli/merges","archive_url":"https://api.github.com/repos/codalab/codalab-cli/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/codalab/codalab-cli/downloads","issues_url":"https://api.github.com/repos/codalab/codalab-cli/issues{/number}","pulls_url":"https://api.github.com/repos/codalab/codalab-cli/pulls{/number}","milestones_url":"https://api.github.com/repos/codalab/codalab-cli/milestones{/number}","notifications_url":"https://api.github.com/repos/codalab/codalab-cli/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/codalab/codalab-cli/labels{/name}","releases_url":"https://api.github.com/repos/codalab/codalab-cli/releases{/id}","deployments_url":"https://api.github.com/repos/codalab/codalab-cli/deployments","created_at":"2014-11-30T22:33:18Z","updated_at":"2016-01-19T21:16:59Z","pushed_at":"2016-03-14T03:52:58Z","git_url":"git://github.com/codalab/codalab-cli.git","ssh_url":"git@github.com:codalab/codalab-cli.git","clone_url":"https://github.com/codalab/codalab-cli.git","svn_url":"https://github.com/codalab/codalab-cli","homepage":null,"size":2453,"stargazers_count":10,"watchers_count":10,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":11,"mirror_url":null,"open_issues_count":58,"forks":11,"open_issues":58,"watchers":10,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/codalab/codalab-cli/pulls/393"},"html":{"href":"https://github.com/codalab/codalab-cli/pull/393"},"issue":{"href":"https://api.github.com/repos/codalab/codalab-cli/issues/393"},"comments":{"href":"https://api.github.com/repos/codalab/codalab-cli/issues/393/comments"},"review_comments":{"href":"https://api.github.com/repos/codalab/codalab-cli/pulls/393/comments"},"review_comment":{"href":"https://api.github.com/repos/codalab/codalab-cli/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/codalab/codalab-cli/pulls/393/commits"},"statuses":{"href":"https://api.github.com/repos/codalab/codalab-cli/statuses/fbf3e42399c16b89468d2e94618dd7684f23cea7"}}}}