{"action":"created","comment":{"url":"https://api.github.com/repos/borgbackup/borg/pulls/comments/79291478","id":79291478,"diff_hunk":"@@ -724,9 +724,51 @@ def __contains__(self, id):\n         return id in self.index\n \n     def list(self, limit=None, marker=None):\n+        \"\"\"\n+        list <limit> IDs starting from after id <marker> - in on-disk order, so that a client\n+        fetching data in this order does linear reads and reuses stuff from disk cache.\n+        \"\"\"\n+        if limit is not None and limit < 1:\n+            raise ValueError('please use limit > 0 or limit = None')\n         if not self.index:\n-            self.index = self.open_index(self.get_transaction_id())\n-        return [id_ for id_, _ in islice(self.index.iteritems(marker=marker), limit)]\n+            transaction_id = self.get_transaction_id()\n+            self.index = self.open_index(transaction_id)\n+        # create a second index object - use it to discover object IDs we can not discover on-disk\n+        # (either because they are not there, just in the index, or because they have IntegrityErrors).\n+        if marker is None:\n+            transaction_id = self.get_transaction_id()\n+            self.remaining = self.open_index(transaction_id)\n+        if marker is None:","path":"src/borg/repository.py","position":29,"original_position":29,"commit_id":"6f2d28b3e8efa6221927be56d8568c2ea65ef1ac","original_commit_id":"6f2d28b3e8efa6221927be56d8568c2ea65ef1ac","user":{"login":"ThomasWaldmann","id":356103,"avatar_url":"https://avatars.githubusercontent.com/u/356103?v=3","gravatar_id":"","url":"https://api.github.com/users/ThomasWaldmann","html_url":"https://github.com/ThomasWaldmann","followers_url":"https://api.github.com/users/ThomasWaldmann/followers","following_url":"https://api.github.com/users/ThomasWaldmann/following{/other_user}","gists_url":"https://api.github.com/users/ThomasWaldmann/gists{/gist_id}","starred_url":"https://api.github.com/users/ThomasWaldmann/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ThomasWaldmann/subscriptions","organizations_url":"https://api.github.com/users/ThomasWaldmann/orgs","repos_url":"https://api.github.com/users/ThomasWaldmann/repos","events_url":"https://api.github.com/users/ThomasWaldmann/events{/privacy}","received_events_url":"https://api.github.com/users/ThomasWaldmann/received_events","type":"User","site_admin":false},"body":"i noticed that also but kept it because the blocks have different concerns.","created_at":"2016-09-17T20:33:11Z","updated_at":"2016-09-17T20:33:11Z","html_url":"https://github.com/borgbackup/borg/pull/1607#discussion_r79291478","pull_request_url":"https://api.github.com/repos/borgbackup/borg/pulls/1607","_links":{"self":{"href":"https://api.github.com/repos/borgbackup/borg/pulls/comments/79291478"},"html":{"href":"https://github.com/borgbackup/borg/pull/1607#discussion_r79291478"},"pull_request":{"href":"https://api.github.com/repos/borgbackup/borg/pulls/1607"}}},"pull_request":{"url":"https://api.github.com/repos/borgbackup/borg/pulls/1607","id":85504750,"html_url":"https://github.com/borgbackup/borg/pull/1607","diff_url":"https://github.com/borgbackup/borg/pull/1607.diff","patch_url":"https://github.com/borgbackup/borg/pull/1607.patch","issue_url":"https://api.github.com/repos/borgbackup/borg/issues/1607","number":1607,"state":"open","locked":false,"title":"borg check --verify-data tuning","user":{"login":"ThomasWaldmann","id":356103,"avatar_url":"https://avatars.githubusercontent.com/u/356103?v=3","gravatar_id":"","url":"https://api.github.com/users/ThomasWaldmann","html_url":"https://github.com/ThomasWaldmann","followers_url":"https://api.github.com/users/ThomasWaldmann/followers","following_url":"https://api.github.com/users/ThomasWaldmann/following{/other_user}","gists_url":"https://api.github.com/users/ThomasWaldmann/gists{/gist_id}","starred_url":"https://api.github.com/users/ThomasWaldmann/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ThomasWaldmann/subscriptions","organizations_url":"https://api.github.com/users/ThomasWaldmann/orgs","repos_url":"https://api.github.com/users/ThomasWaldmann/repos","events_url":"https://api.github.com/users/ThomasWaldmann/events{/privacy}","received_events_url":"https://api.github.com/users/ThomasWaldmann/received_events","type":"User","site_admin":false},"body":"please review esp. the on-disk order commit - it's way more code as we had before and has to deal with some difficulties.","created_at":"2016-09-16T00:50:27Z","updated_at":"2016-09-17T20:33:11Z","closed_at":null,"merged_at":null,"merge_commit_sha":"1ef0a720c066db43cbcac6d6e2f7110f0f7eaae8","assignee":null,"assignees":[],"milestone":{"url":"https://api.github.com/repos/borgbackup/borg/milestones/31","html_url":"https://github.com/borgbackup/borg/milestones/1.1.0b2","labels_url":"https://api.github.com/repos/borgbackup/borg/milestones/31/labels","id":1967454,"number":31,"title":"1.1.0b2","description":"","creator":{"login":"ThomasWaldmann","id":356103,"avatar_url":"https://avatars.githubusercontent.com/u/356103?v=3","gravatar_id":"","url":"https://api.github.com/users/ThomasWaldmann","html_url":"https://github.com/ThomasWaldmann","followers_url":"https://api.github.com/users/ThomasWaldmann/followers","following_url":"https://api.github.com/users/ThomasWaldmann/following{/other_user}","gists_url":"https://api.github.com/users/ThomasWaldmann/gists{/gist_id}","starred_url":"https://api.github.com/users/ThomasWaldmann/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ThomasWaldmann/subscriptions","organizations_url":"https://api.github.com/users/ThomasWaldmann/orgs","repos_url":"https://api.github.com/users/ThomasWaldmann/repos","events_url":"https://api.github.com/users/ThomasWaldmann/events{/privacy}","received_events_url":"https://api.github.com/users/ThomasWaldmann/received_events","type":"User","site_admin":false},"open_issues":6,"closed_issues":5,"state":"open","created_at":"2016-08-27T20:49:21Z","updated_at":"2016-09-16T17:52:25Z","due_on":null,"closed_at":null},"commits_url":"https://api.github.com/repos/borgbackup/borg/pulls/1607/commits","review_comments_url":"https://api.github.com/repos/borgbackup/borg/pulls/1607/comments","review_comment_url":"https://api.github.com/repos/borgbackup/borg/pulls/comments{/number}","comments_url":"https://api.github.com/repos/borgbackup/borg/issues/1607/comments","statuses_url":"https://api.github.com/repos/borgbackup/borg/statuses/6f2d28b3e8efa6221927be56d8568c2ea65ef1ac","head":{"label":"ThomasWaldmann:tune-check-verify-data","ref":"tune-check-verify-data","sha":"6f2d28b3e8efa6221927be56d8568c2ea65ef1ac","user":{"login":"ThomasWaldmann","id":356103,"avatar_url":"https://avatars.githubusercontent.com/u/356103?v=3","gravatar_id":"","url":"https://api.github.com/users/ThomasWaldmann","html_url":"https://github.com/ThomasWaldmann","followers_url":"https://api.github.com/users/ThomasWaldmann/followers","following_url":"https://api.github.com/users/ThomasWaldmann/following{/other_user}","gists_url":"https://api.github.com/users/ThomasWaldmann/gists{/gist_id}","starred_url":"https://api.github.com/users/ThomasWaldmann/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ThomasWaldmann/subscriptions","organizations_url":"https://api.github.com/users/ThomasWaldmann/orgs","repos_url":"https://api.github.com/users/ThomasWaldmann/repos","events_url":"https://api.github.com/users/ThomasWaldmann/events{/privacy}","received_events_url":"https://api.github.com/users/ThomasWaldmann/received_events","type":"User","site_admin":false},"repo":{"id":35605377,"name":"borg","full_name":"ThomasWaldmann/borg","owner":{"login":"ThomasWaldmann","id":356103,"avatar_url":"https://avatars.githubusercontent.com/u/356103?v=3","gravatar_id":"","url":"https://api.github.com/users/ThomasWaldmann","html_url":"https://github.com/ThomasWaldmann","followers_url":"https://api.github.com/users/ThomasWaldmann/followers","following_url":"https://api.github.com/users/ThomasWaldmann/following{/other_user}","gists_url":"https://api.github.com/users/ThomasWaldmann/gists{/gist_id}","starred_url":"https://api.github.com/users/ThomasWaldmann/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ThomasWaldmann/subscriptions","organizations_url":"https://api.github.com/users/ThomasWaldmann/orgs","repos_url":"https://api.github.com/users/ThomasWaldmann/repos","events_url":"https://api.github.com/users/ThomasWaldmann/events{/privacy}","received_events_url":"https://api.github.com/users/ThomasWaldmann/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/ThomasWaldmann/borg","description":"Backup Software","fork":true,"url":"https://api.github.com/repos/ThomasWaldmann/borg","forks_url":"https://api.github.com/repos/ThomasWaldmann/borg/forks","keys_url":"https://api.github.com/repos/ThomasWaldmann/borg/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ThomasWaldmann/borg/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ThomasWaldmann/borg/teams","hooks_url":"https://api.github.com/repos/ThomasWaldmann/borg/hooks","issue_events_url":"https://api.github.com/repos/ThomasWaldmann/borg/issues/events{/number}","events_url":"https://api.github.com/repos/ThomasWaldmann/borg/events","assignees_url":"https://api.github.com/repos/ThomasWaldmann/borg/assignees{/user}","branches_url":"https://api.github.com/repos/ThomasWaldmann/borg/branches{/branch}","tags_url":"https://api.github.com/repos/ThomasWaldmann/borg/tags","blobs_url":"https://api.github.com/repos/ThomasWaldmann/borg/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ThomasWaldmann/borg/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ThomasWaldmann/borg/git/refs{/sha}","trees_url":"https://api.github.com/repos/ThomasWaldmann/borg/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ThomasWaldmann/borg/statuses/{sha}","languages_url":"https://api.github.com/repos/ThomasWaldmann/borg/languages","stargazers_url":"https://api.github.com/repos/ThomasWaldmann/borg/stargazers","contributors_url":"https://api.github.com/repos/ThomasWaldmann/borg/contributors","subscribers_url":"https://api.github.com/repos/ThomasWaldmann/borg/subscribers","subscription_url":"https://api.github.com/repos/ThomasWaldmann/borg/subscription","commits_url":"https://api.github.com/repos/ThomasWaldmann/borg/commits{/sha}","git_commits_url":"https://api.github.com/repos/ThomasWaldmann/borg/git/commits{/sha}","comments_url":"https://api.github.com/repos/ThomasWaldmann/borg/comments{/number}","issue_comment_url":"https://api.github.com/repos/ThomasWaldmann/borg/issues/comments{/number}","contents_url":"https://api.github.com/repos/ThomasWaldmann/borg/contents/{+path}","compare_url":"https://api.github.com/repos/ThomasWaldmann/borg/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ThomasWaldmann/borg/merges","archive_url":"https://api.github.com/repos/ThomasWaldmann/borg/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ThomasWaldmann/borg/downloads","issues_url":"https://api.github.com/repos/ThomasWaldmann/borg/issues{/number}","pulls_url":"https://api.github.com/repos/ThomasWaldmann/borg/pulls{/number}","milestones_url":"https://api.github.com/repos/ThomasWaldmann/borg/milestones{/number}","notifications_url":"https://api.github.com/repos/ThomasWaldmann/borg/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ThomasWaldmann/borg/labels{/name}","releases_url":"https://api.github.com/repos/ThomasWaldmann/borg/releases{/id}","deployments_url":"https://api.github.com/repos/ThomasWaldmann/borg/deployments","created_at":"2015-05-14T10:27:42Z","updated_at":"2016-01-11T22:17:17Z","pushed_at":"2016-09-17T19:37:31Z","git_url":"git://github.com/ThomasWaldmann/borg.git","ssh_url":"git@github.com:ThomasWaldmann/borg.git","clone_url":"https://github.com/ThomasWaldmann/borg.git","svn_url":"https://github.com/ThomasWaldmann/borg","homepage":null,"size":5477,"stargazers_count":2,"watchers_count":2,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":0,"forks":1,"open_issues":0,"watchers":2,"default_branch":"master"}},"base":{"label":"borgbackup:master","ref":"master","sha":"765b8f38d68aa2b623fe1fac0030c0c478f4a863","user":{"login":"borgbackup","id":12418060,"avatar_url":"https://avatars.githubusercontent.com/u/12418060?v=3","gravatar_id":"","url":"https://api.github.com/users/borgbackup","html_url":"https://github.com/borgbackup","followers_url":"https://api.github.com/users/borgbackup/followers","following_url":"https://api.github.com/users/borgbackup/following{/other_user}","gists_url":"https://api.github.com/users/borgbackup/gists{/gist_id}","starred_url":"https://api.github.com/users/borgbackup/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/borgbackup/subscriptions","organizations_url":"https://api.github.com/users/borgbackup/orgs","repos_url":"https://api.github.com/users/borgbackup/repos","events_url":"https://api.github.com/users/borgbackup/events{/privacy}","received_events_url":"https://api.github.com/users/borgbackup/received_events","type":"Organization","site_admin":false},"repo":{"id":35517126,"name":"borg","full_name":"borgbackup/borg","owner":{"login":"borgbackup","id":12418060,"avatar_url":"https://avatars.githubusercontent.com/u/12418060?v=3","gravatar_id":"","url":"https://api.github.com/users/borgbackup","html_url":"https://github.com/borgbackup","followers_url":"https://api.github.com/users/borgbackup/followers","following_url":"https://api.github.com/users/borgbackup/following{/other_user}","gists_url":"https://api.github.com/users/borgbackup/gists{/gist_id}","starred_url":"https://api.github.com/users/borgbackup/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/borgbackup/subscriptions","organizations_url":"https://api.github.com/users/borgbackup/orgs","repos_url":"https://api.github.com/users/borgbackup/repos","events_url":"https://api.github.com/users/borgbackup/events{/privacy}","received_events_url":"https://api.github.com/users/borgbackup/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/borgbackup/borg","description":"Deduplicating backup program with compression and authenticated encryption.","fork":false,"url":"https://api.github.com/repos/borgbackup/borg","forks_url":"https://api.github.com/repos/borgbackup/borg/forks","keys_url":"https://api.github.com/repos/borgbackup/borg/keys{/key_id}","collaborators_url":"https://api.github.com/repos/borgbackup/borg/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/borgbackup/borg/teams","hooks_url":"https://api.github.com/repos/borgbackup/borg/hooks","issue_events_url":"https://api.github.com/repos/borgbackup/borg/issues/events{/number}","events_url":"https://api.github.com/repos/borgbackup/borg/events","assignees_url":"https://api.github.com/repos/borgbackup/borg/assignees{/user}","branches_url":"https://api.github.com/repos/borgbackup/borg/branches{/branch}","tags_url":"https://api.github.com/repos/borgbackup/borg/tags","blobs_url":"https://api.github.com/repos/borgbackup/borg/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/borgbackup/borg/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/borgbackup/borg/git/refs{/sha}","trees_url":"https://api.github.com/repos/borgbackup/borg/git/trees{/sha}","statuses_url":"https://api.github.com/repos/borgbackup/borg/statuses/{sha}","languages_url":"https://api.github.com/repos/borgbackup/borg/languages","stargazers_url":"https://api.github.com/repos/borgbackup/borg/stargazers","contributors_url":"https://api.github.com/repos/borgbackup/borg/contributors","subscribers_url":"https://api.github.com/repos/borgbackup/borg/subscribers","subscription_url":"https://api.github.com/repos/borgbackup/borg/subscription","commits_url":"https://api.github.com/repos/borgbackup/borg/commits{/sha}","git_commits_url":"https://api.github.com/repos/borgbackup/borg/git/commits{/sha}","comments_url":"https://api.github.com/repos/borgbackup/borg/comments{/number}","issue_comment_url":"https://api.github.com/repos/borgbackup/borg/issues/comments{/number}","contents_url":"https://api.github.com/repos/borgbackup/borg/contents/{+path}","compare_url":"https://api.github.com/repos/borgbackup/borg/compare/{base}...{head}","merges_url":"https://api.github.com/repos/borgbackup/borg/merges","archive_url":"https://api.github.com/repos/borgbackup/borg/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/borgbackup/borg/downloads","issues_url":"https://api.github.com/repos/borgbackup/borg/issues{/number}","pulls_url":"https://api.github.com/repos/borgbackup/borg/pulls{/number}","milestones_url":"https://api.github.com/repos/borgbackup/borg/milestones{/number}","notifications_url":"https://api.github.com/repos/borgbackup/borg/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/borgbackup/borg/labels{/name}","releases_url":"https://api.github.com/repos/borgbackup/borg/releases{/id}","deployments_url":"https://api.github.com/repos/borgbackup/borg/deployments","created_at":"2015-05-12T23:10:47Z","updated_at":"2016-09-17T15:30:04Z","pushed_at":"2016-09-17T19:37:31Z","git_url":"git://github.com/borgbackup/borg.git","ssh_url":"git@github.com:borgbackup/borg.git","clone_url":"https://github.com/borgbackup/borg.git","svn_url":"https://github.com/borgbackup/borg","homepage":"https://borgbackup.readthedocs.io/","size":6578,"stargazers_count":1496,"watchers_count":1496,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":140,"mirror_url":null,"open_issues_count":216,"forks":140,"open_issues":216,"watchers":1496,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/borgbackup/borg/pulls/1607"},"html":{"href":"https://github.com/borgbackup/borg/pull/1607"},"issue":{"href":"https://api.github.com/repos/borgbackup/borg/issues/1607"},"comments":{"href":"https://api.github.com/repos/borgbackup/borg/issues/1607/comments"},"review_comments":{"href":"https://api.github.com/repos/borgbackup/borg/pulls/1607/comments"},"review_comment":{"href":"https://api.github.com/repos/borgbackup/borg/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/borgbackup/borg/pulls/1607/commits"},"statuses":{"href":"https://api.github.com/repos/borgbackup/borg/statuses/6f2d28b3e8efa6221927be56d8568c2ea65ef1ac"}}}}
{"actor":{"display_login":"ThomasWaldmann"}}