{"action":"created","comment":{"url":"https://api.github.com/repos/apache/incubator-pulsar/pulls/comments/187867412","pull_request_review_id":119730806,"id":187867412,"diff_hunk":"@@ -60,28 +80,119 @@ public static S3ManagedLedgerOffloader create(ServiceConfiguration conf,\n         } else {\n             builder.setRegion(region);\n         }\n-        return new S3ManagedLedgerOffloader(builder.build(), bucket, scheduler);\n+        return new S3ManagedLedgerOffloader(builder.build(), bucket, scheduler, maxBlockSize);\n     }\n \n-    S3ManagedLedgerOffloader(AmazonS3 s3client, String bucket, ScheduledExecutorService scheduler) {\n+    S3ManagedLedgerOffloader(AmazonS3 s3client, String bucket, ScheduledExecutorService scheduler, int maxBlockSize) {\n         this.s3client = s3client;\n         this.bucket = bucket;\n         this.scheduler = scheduler;\n+        this.maxBlockSize = maxBlockSize;\n     }\n \n+    static String dataBlockOffloadKey(ReadHandle readHandle, UUID uuid) {\n+        return String.format(\"ledger-%d-%s\", readHandle.getId(), uuid.toString());\n+    }\n+\n+    static String indexBlockOffloadKey(ReadHandle readHandle, UUID uuid) {\n+        return String.format(\"ledger-%d-%s-index\", readHandle.getId(), uuid.toString());\n+    }\n+\n+    // upload DataBlock to s3 using MultiPartUpload, and indexBlock in a new Block,\n     @Override\n-    public CompletableFuture<Void> offload(ReadHandle ledger,\n-                                           UUID uid,\n+    public CompletableFuture<Void> offload(ReadHandle readHandle,\n+                                           UUID uuid,\n                                            Map<String, String> extraMetadata) {\n         CompletableFuture<Void> promise = new CompletableFuture<>();\n         scheduler.submit(() -> {\n-                try {\n-                    s3client.putObject(bucket, uid.toString(), uid.toString());\n-                    promise.complete(null);\n-                } catch (Throwable t) {\n-                    promise.completeExceptionally(t);\n+            OffloadIndexBlockBuilder indexBuilder = OffloadIndexBlockBuilder.create()\n+                .withMetadata(readHandle.getLedgerMetadata());\n+            String dataBlockKey = dataBlockOffloadKey(readHandle, uuid);\n+            String indexBlockKey = indexBlockOffloadKey(readHandle, uuid);\n+            InitiateMultipartUploadRequest dataBlockReq = new InitiateMultipartUploadRequest(bucket, dataBlockKey);\n+            InitiateMultipartUploadResult dataBlockRes = null;\n+\n+            // init multi part upload for data block.\n+            try {\n+                dataBlockRes = s3client.initiateMultipartUpload(dataBlockReq);\n+            } catch (Throwable t) {\n+                if (dataBlockRes != null) {","path":"pulsar-broker/src/main/java/org/apache/pulsar/broker/s3offload/S3ManagedLedgerOffloader.java","position":null,"original_position":107,"commit_id":"ede921fbb7fc22eda9de0edb88299f2de3cf82e9","original_commit_id":"df26b3c05bec1c5b53c717854908680163e32525","user":{"login":"zhaijack","id":26834121,"avatar_url":"https://avatars0.githubusercontent.com/u/26834121?v=4","gravatar_id":"","url":"https://api.github.com/users/zhaijack","html_url":"https://github.com/zhaijack","followers_url":"https://api.github.com/users/zhaijack/followers","following_url":"https://api.github.com/users/zhaijack/following{/other_user}","gists_url":"https://api.github.com/users/zhaijack/gists{/gist_id}","starred_url":"https://api.github.com/users/zhaijack/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/zhaijack/subscriptions","organizations_url":"https://api.github.com/users/zhaijack/orgs","repos_url":"https://api.github.com/users/zhaijack/repos","events_url":"https://api.github.com/users/zhaijack/events{/privacy}","received_events_url":"https://api.github.com/users/zhaijack/received_events","type":"User","site_admin":false},"body":"Thanks, this is to avoid findify.s3mock error, where delete or abort an not exist target will cause error. will change it.","created_at":"2018-05-14T08:14:00Z","updated_at":"2018-05-14T08:14:00Z","html_url":"https://github.com/apache/incubator-pulsar/pull/1746#discussion_r187867412","pull_request_url":"https://api.github.com/repos/apache/incubator-pulsar/pulls/1746","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/apache/incubator-pulsar/pulls/comments/187867412"},"html":{"href":"https://github.com/apache/incubator-pulsar/pull/1746#discussion_r187867412"},"pull_request":{"href":"https://api.github.com/repos/apache/incubator-pulsar/pulls/1746"}},"in_reply_to_id":187851866},"pull_request":{"url":"https://api.github.com/repos/apache/incubator-pulsar/pulls/1746","id":186576064,"html_url":"https://github.com/apache/incubator-pulsar/pull/1746","diff_url":"https://github.com/apache/incubator-pulsar/pull/1746.diff","patch_url":"https://github.com/apache/incubator-pulsar/pull/1746.patch","issue_url":"https://api.github.com/repos/apache/incubator-pulsar/issues/1746","number":1746,"state":"open","locked":false,"title":"PIP-17: impl offload() for S3ManagedLedgerOffloader","user":{"login":"zhaijack","id":26834121,"avatar_url":"https://avatars0.githubusercontent.com/u/26834121?v=4","gravatar_id":"","url":"https://api.github.com/users/zhaijack","html_url":"https://github.com/zhaijack","followers_url":"https://api.github.com/users/zhaijack/followers","following_url":"https://api.github.com/users/zhaijack/following{/other_user}","gists_url":"https://api.github.com/users/zhaijack/gists{/gist_id}","starred_url":"https://api.github.com/users/zhaijack/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/zhaijack/subscriptions","organizations_url":"https://api.github.com/users/zhaijack/orgs","repos_url":"https://api.github.com/users/zhaijack/repos","events_url":"https://api.github.com/users/zhaijack/events{/privacy}","received_events_url":"https://api.github.com/users/zhaijack/received_events","type":"User","site_admin":false},"body":"Provide a implementation for offload in S3ManagedLedgerOffloader.\r\n\r\nMaster issue #1511.","created_at":"2018-05-08T09:28:50Z","updated_at":"2018-05-14T08:14:00Z","closed_at":null,"merged_at":null,"merge_commit_sha":"8f5ca0a6ad7c086225da97e1a3cc700fb9c50b33","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/apache/incubator-pulsar/pulls/1746/commits","review_comments_url":"https://api.github.com/repos/apache/incubator-pulsar/pulls/1746/comments","review_comment_url":"https://api.github.com/repos/apache/incubator-pulsar/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/incubator-pulsar/issues/1746/comments","statuses_url":"https://api.github.com/repos/apache/incubator-pulsar/statuses/ede921fbb7fc22eda9de0edb88299f2de3cf82e9","head":{"label":"zhaijack:s3-write","ref":"s3-write","sha":"ede921fbb7fc22eda9de0edb88299f2de3cf82e9","user":{"login":"zhaijack","id":26834121,"avatar_url":"https://avatars0.githubusercontent.com/u/26834121?v=4","gravatar_id":"","url":"https://api.github.com/users/zhaijack","html_url":"https://github.com/zhaijack","followers_url":"https://api.github.com/users/zhaijack/followers","following_url":"https://api.github.com/users/zhaijack/following{/other_user}","gists_url":"https://api.github.com/users/zhaijack/gists{/gist_id}","starred_url":"https://api.github.com/users/zhaijack/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/zhaijack/subscriptions","organizations_url":"https://api.github.com/users/zhaijack/orgs","repos_url":"https://api.github.com/users/zhaijack/repos","events_url":"https://api.github.com/users/zhaijack/events{/privacy}","received_events_url":"https://api.github.com/users/zhaijack/received_events","type":"User","site_admin":false},"repo":{"id":89754280,"name":"pulsar","full_name":"zhaijack/pulsar","owner":{"login":"zhaijack","id":26834121,"avatar_url":"https://avatars0.githubusercontent.com/u/26834121?v=4","gravatar_id":"","url":"https://api.github.com/users/zhaijack","html_url":"https://github.com/zhaijack","followers_url":"https://api.github.com/users/zhaijack/followers","following_url":"https://api.github.com/users/zhaijack/following{/other_user}","gists_url":"https://api.github.com/users/zhaijack/gists{/gist_id}","starred_url":"https://api.github.com/users/zhaijack/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/zhaijack/subscriptions","organizations_url":"https://api.github.com/users/zhaijack/orgs","repos_url":"https://api.github.com/users/zhaijack/repos","events_url":"https://api.github.com/users/zhaijack/events{/privacy}","received_events_url":"https://api.github.com/users/zhaijack/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/zhaijack/pulsar","description":"Pulsar - distributed pub-sub messaging system","fork":true,"url":"https://api.github.com/repos/zhaijack/pulsar","forks_url":"https://api.github.com/repos/zhaijack/pulsar/forks","keys_url":"https://api.github.com/repos/zhaijack/pulsar/keys{/key_id}","collaborators_url":"https://api.github.com/repos/zhaijack/pulsar/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/zhaijack/pulsar/teams","hooks_url":"https://api.github.com/repos/zhaijack/pulsar/hooks","issue_events_url":"https://api.github.com/repos/zhaijack/pulsar/issues/events{/number}","events_url":"https://api.github.com/repos/zhaijack/pulsar/events","assignees_url":"https://api.github.com/repos/zhaijack/pulsar/assignees{/user}","branches_url":"https://api.github.com/repos/zhaijack/pulsar/branches{/branch}","tags_url":"https://api.github.com/repos/zhaijack/pulsar/tags","blobs_url":"https://api.github.com/repos/zhaijack/pulsar/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/zhaijack/pulsar/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/zhaijack/pulsar/git/refs{/sha}","trees_url":"https://api.github.com/repos/zhaijack/pulsar/git/trees{/sha}","statuses_url":"https://api.github.com/repos/zhaijack/pulsar/statuses/{sha}","languages_url":"https://api.github.com/repos/zhaijack/pulsar/languages","stargazers_url":"https://api.github.com/repos/zhaijack/pulsar/stargazers","contributors_url":"https://api.github.com/repos/zhaijack/pulsar/contributors","subscribers_url":"https://api.github.com/repos/zhaijack/pulsar/subscribers","subscription_url":"https://api.github.com/repos/zhaijack/pulsar/subscription","commits_url":"https://api.github.com/repos/zhaijack/pulsar/commits{/sha}","git_commits_url":"https://api.github.com/repos/zhaijack/pulsar/git/commits{/sha}","comments_url":"https://api.github.com/repos/zhaijack/pulsar/comments{/number}","issue_comment_url":"https://api.github.com/repos/zhaijack/pulsar/issues/comments{/number}","contents_url":"https://api.github.com/repos/zhaijack/pulsar/contents/{+path}","compare_url":"https://api.github.com/repos/zhaijack/pulsar/compare/{base}...{head}","merges_url":"https://api.github.com/repos/zhaijack/pulsar/merges","archive_url":"https://api.github.com/repos/zhaijack/pulsar/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/zhaijack/pulsar/downloads","issues_url":"https://api.github.com/repos/zhaijack/pulsar/issues{/number}","pulls_url":"https://api.github.com/repos/zhaijack/pulsar/pulls{/number}","milestones_url":"https://api.github.com/repos/zhaijack/pulsar/milestones{/number}","notifications_url":"https://api.github.com/repos/zhaijack/pulsar/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/zhaijack/pulsar/labels{/name}","releases_url":"https://api.github.com/repos/zhaijack/pulsar/releases{/id}","deployments_url":"https://api.github.com/repos/zhaijack/pulsar/deployments","created_at":"2017-04-29T00:27:10Z","updated_at":"2018-05-10T07:18:04Z","pushed_at":"2018-05-14T08:12:13Z","git_url":"git://github.com/zhaijack/pulsar.git","ssh_url":"git@github.com:zhaijack/pulsar.git","clone_url":"https://github.com/zhaijack/pulsar.git","svn_url":"https://github.com/zhaijack/pulsar","homepage":"","size":22849,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0"},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"cd973361a753eb32d7cd62a4dc52dc4d1e65474d","user":{"login":"apache","id":47359,"avatar_url":"https://avatars0.githubusercontent.com/u/47359?v=4","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":62117812,"name":"incubator-pulsar","full_name":"apache/incubator-pulsar","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars0.githubusercontent.com/u/47359?v=4","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/incubator-pulsar","description":"Pulsar - distributed pub-sub messaging system","fork":false,"url":"https://api.github.com/repos/apache/incubator-pulsar","forks_url":"https://api.github.com/repos/apache/incubator-pulsar/forks","keys_url":"https://api.github.com/repos/apache/incubator-pulsar/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/incubator-pulsar/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/incubator-pulsar/teams","hooks_url":"https://api.github.com/repos/apache/incubator-pulsar/hooks","issue_events_url":"https://api.github.com/repos/apache/incubator-pulsar/issues/events{/number}","events_url":"https://api.github.com/repos/apache/incubator-pulsar/events","assignees_url":"https://api.github.com/repos/apache/incubator-pulsar/assignees{/user}","branches_url":"https://api.github.com/repos/apache/incubator-pulsar/branches{/branch}","tags_url":"https://api.github.com/repos/apache/incubator-pulsar/tags","blobs_url":"https://api.github.com/repos/apache/incubator-pulsar/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/incubator-pulsar/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/incubator-pulsar/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/incubator-pulsar/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/incubator-pulsar/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/incubator-pulsar/languages","stargazers_url":"https://api.github.com/repos/apache/incubator-pulsar/stargazers","contributors_url":"https://api.github.com/repos/apache/incubator-pulsar/contributors","subscribers_url":"https://api.github.com/repos/apache/incubator-pulsar/subscribers","subscription_url":"https://api.github.com/repos/apache/incubator-pulsar/subscription","commits_url":"https://api.github.com/repos/apache/incubator-pulsar/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/incubator-pulsar/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/incubator-pulsar/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/incubator-pulsar/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/incubator-pulsar/contents/{+path}","compare_url":"https://api.github.com/repos/apache/incubator-pulsar/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/incubator-pulsar/merges","archive_url":"https://api.github.com/repos/apache/incubator-pulsar/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/incubator-pulsar/downloads","issues_url":"https://api.github.com/repos/apache/incubator-pulsar/issues{/number}","pulls_url":"https://api.github.com/repos/apache/incubator-pulsar/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/incubator-pulsar/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/incubator-pulsar/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/incubator-pulsar/labels{/name}","releases_url":"https://api.github.com/repos/apache/incubator-pulsar/releases{/id}","deployments_url":"https://api.github.com/repos/apache/incubator-pulsar/deployments","created_at":"2016-06-28T07:00:03Z","updated_at":"2018-05-14T06:02:35Z","pushed_at":"2018-05-14T08:12:15Z","git_url":"git://github.com/apache/incubator-pulsar.git","ssh_url":"git@github.com:apache/incubator-pulsar.git","clone_url":"https://github.com/apache/incubator-pulsar.git","svn_url":"https://github.com/apache/incubator-pulsar","homepage":"https://pulsar.incubator.apache.org","size":29826,"stargazers_count":1689,"watchers_count":1689,"language":"Java","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":357,"mirror_url":null,"archived":false,"open_issues_count":159,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0"},"forks":357,"open_issues":159,"watchers":1689,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/incubator-pulsar/pulls/1746"},"html":{"href":"https://github.com/apache/incubator-pulsar/pull/1746"},"issue":{"href":"https://api.github.com/repos/apache/incubator-pulsar/issues/1746"},"comments":{"href":"https://api.github.com/repos/apache/incubator-pulsar/issues/1746/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/incubator-pulsar/pulls/1746/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/incubator-pulsar/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/incubator-pulsar/pulls/1746/commits"},"statuses":{"href":"https://api.github.com/repos/apache/incubator-pulsar/statuses/ede921fbb7fc22eda9de0edb88299f2de3cf82e9"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"zhaijack"}}