{"action":"created","comment":{"url":"https://api.github.com/repos/apache/apex-malhar/pulls/comments/88856749","pull_request_review_id":9415416,"id":88856749,"diff_hunk":"@@ -0,0 +1,302 @@\n+/**\n+ * Licensed to the Apache Software Foundation (ASF) under one\n+ * or more contributor license agreements.  See the NOTICE file\n+ * distributed with this work for additional information\n+ * regarding copyright ownership.  The ASF licenses this file\n+ * to you under the Apache License, Version 2.0 (the\n+ * \"License\"); you may not use this file except in compliance\n+ * with the License.  You may obtain a copy of the License at\n+ *\n+ *   http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing,\n+ * software distributed under the License is distributed on an\n+ * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n+ * KIND, either express or implied.  See the License for the\n+ * specific language governing permissions and limitations\n+ * under the License.\n+ */\n+package org.apache.apex.malhar.lib.fs.s3output;\n+\n+import java.io.IOException;\n+import java.util.ArrayList;\n+import java.util.HashMap;\n+import java.util.List;\n+import java.util.Map;\n+\n+import javax.validation.constraints.NotNull;\n+\n+import org.slf4j.Logger;\n+import org.slf4j.LoggerFactory;\n+\n+import org.apache.apex.malhar.lib.wal.FSWindowDataManager;\n+import org.apache.apex.malhar.lib.wal.WindowDataManager;\n+import org.apache.hadoop.classification.InterfaceStability;\n+\n+import com.amazonaws.auth.BasicAWSCredentials;\n+import com.amazonaws.services.s3.AmazonS3;\n+import com.amazonaws.services.s3.AmazonS3Client;\n+import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;\n+import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;\n+import com.amazonaws.services.s3.model.PartETag;\n+import com.esotericsoftware.kryo.serializers.FieldSerializer;\n+import com.esotericsoftware.kryo.serializers.JavaSerializer;\n+import com.google.common.base.Preconditions;\n+\n+import com.datatorrent.api.Context;\n+import com.datatorrent.api.DefaultInputPort;\n+import com.datatorrent.api.Operator;\n+\n+/**\n+ * This operator can be used to merge the S3 blocks into a file. This operator will request for\n+ * S3 CompleteMultipartUploadRequest once all the blocks are uploaded using multi-part feature.\n+ */\n+\n+@InterfaceStability.Evolving\n+public class S3FileMerger implements Operator, Operator.CheckpointNotificationListener\n+{\n+  private static final Logger LOG = LoggerFactory.getLogger(S3FileMerger.class);\n+  @NotNull\n+  private String bucketName;\n+  @NotNull\n+  private String accessKey;\n+  @NotNull\n+  private String secretAccessKey;\n+  private String endPoint;\n+  protected List<String> uploadedFiles = new ArrayList<>();\n+  private WindowDataManager windowDataManager = new FSWindowDataManager();\n+  @FieldSerializer.Bind(JavaSerializer.class)\n+  private Map<String, List<PartETag>> uploadParts = new HashMap<>();\n+  private Map<String, S3InitiateFileUpload.UploadFileMetadata> fileMetadatas = new HashMap<>();\n+  protected transient long currentWindowId;\n+  protected transient AmazonS3 s3Client;\n+\n+  /**\n+   * Input port to receive UploadBlockMetadata\n+   */\n+  public final transient DefaultInputPort<S3BlockUpload.UploadBlockMetadata> uploadMetadataInput = new DefaultInputPort<S3BlockUpload.UploadBlockMetadata>()\n+  {\n+    @Override\n+    public void process(S3BlockUpload.UploadBlockMetadata tuple)\n+    {\n+      processUploadBlock(tuple);\n+    }\n+  };\n+\n+  /**\n+   * Process to merge the uploaded block into a file.\n+   * @param tuple uploaded block meta data\n+   */\n+  protected void processUploadBlock(S3BlockUpload.UploadBlockMetadata tuple)\n+  {\n+    List<PartETag> listOfUploads = uploadParts.get(tuple.getKeyName());\n+    if (listOfUploads == null) {\n+      listOfUploads = new ArrayList<>();\n+      uploadParts.put(tuple.getKeyName(), listOfUploads);\n+    }\n+    listOfUploads.add(tuple.getPartETag());\n+    if (fileMetadatas.get(tuple.getKeyName()) != null) {\n+      emitFileMerge(tuple.getKeyName());\n+    }\n+  }\n+\n+  /**\n+   * Input port to receive UploadFileMetadata\n+   */\n+  public final transient DefaultInputPort<S3InitiateFileUpload.UploadFileMetadata> filesMetadataInput = new DefaultInputPort<S3InitiateFileUpload.UploadFileMetadata>()\n+  {\n+    @Override\n+    public void process(S3InitiateFileUpload.UploadFileMetadata tuple)\n+    {\n+      processFileMetadata(tuple);\n+    }\n+  };\n+\n+  /**\n+   * Process to merge the uploaded blocks for the given file metadata.\n+   * @param tuple file metadata\n+   */\n+  protected void processFileMetadata(S3InitiateFileUpload.UploadFileMetadata tuple)\n+  {\n+    String keyName = tuple.getKeyName();\n+    fileMetadatas.put(keyName, tuple);\n+    if (uploadParts.get(keyName) != null) {\n+      emitFileMerge(keyName);\n+    }\n+  }\n+\n+  @Override\n+  public void setup(Context.OperatorContext context)\n+  {\n+    windowDataManager.setup(context);\n+    s3Client = createClient();\n+  }\n+\n+  /**\n+   * Create AmazonS3 client using AWS credentials\n+   * @return AmazonS3\n+   */\n+  protected AmazonS3 createClient()\n+  {\n+    AmazonS3 client = new AmazonS3Client(new BasicAWSCredentials(accessKey, secretAccessKey));\n+    if (endPoint != null) {\n+      client.setEndpoint(endPoint);\n+    }\n+    return client;\n+  }\n+\n+\n+  @Override\n+  public void beginWindow(long windowId)\n+  {\n+    currentWindowId = windowId;\n+  }\n+\n+  @Override\n+  public void endWindow()\n+  {\n+    if (currentWindowId > windowDataManager.getLargestCompletedWindow()) {\n+      try {\n+        windowDataManager.save(\"UploadedFiles\", currentWindowId);\n+      } catch (IOException e) {\n+        throw new RuntimeException(\"Unable to save recovery\", e);\n+      }\n+    }\n+  }\n+\n+  @Override\n+  public void teardown()\n+  {\n+    windowDataManager.teardown();\n+  }\n+\n+  /**\n+   * Send the CompleteMultipartUploadRequest to S3 if all the blocks of a file are uploaded into S3.\n+   * @param keyName file to upload into S3\n+   */\n+  private void emitFileMerge(String keyName)","path":"library/src/main/java/org/apache/apex/malhar/lib/fs/s3output/S3FileMerger.java","position":177,"original_position":177,"commit_id":"24fb5638ecb6f0e45edb5d5f640b220ad9372fcc","original_commit_id":"24fb5638ecb6f0e45edb5d5f640b220ad9372fcc","user":{"login":"chaithu14","id":8510691,"avatar_url":"https://avatars.githubusercontent.com/u/8510691?v=3","gravatar_id":"","url":"https://api.github.com/users/chaithu14","html_url":"https://github.com/chaithu14","followers_url":"https://api.github.com/users/chaithu14/followers","following_url":"https://api.github.com/users/chaithu14/following{/other_user}","gists_url":"https://api.github.com/users/chaithu14/gists{/gist_id}","starred_url":"https://api.github.com/users/chaithu14/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/chaithu14/subscriptions","organizations_url":"https://api.github.com/users/chaithu14/orgs","repos_url":"https://api.github.com/users/chaithu14/repos","events_url":"https://api.github.com/users/chaithu14/events{/privacy}","received_events_url":"https://api.github.com/users/chaithu14/received_events","type":"User","site_admin":false},"body":"Done.","created_at":"2016-11-21T09:36:47Z","updated_at":"2016-11-21T09:36:47Z","html_url":"https://github.com/apache/apex-malhar/pull/483#discussion_r88856749","pull_request_url":"https://api.github.com/repos/apache/apex-malhar/pulls/483","_links":{"self":{"href":"https://api.github.com/repos/apache/apex-malhar/pulls/comments/88856749"},"html":{"href":"https://github.com/apache/apex-malhar/pull/483#discussion_r88856749"},"pull_request":{"href":"https://api.github.com/repos/apache/apex-malhar/pulls/483"}}},"pull_request":{"url":"https://api.github.com/repos/apache/apex-malhar/pulls/483","id":92333996,"html_url":"https://github.com/apache/apex-malhar/pull/483","diff_url":"https://github.com/apache/apex-malhar/pull/483.diff","patch_url":"https://github.com/apache/apex-malhar/pull/483.patch","issue_url":"https://api.github.com/repos/apache/apex-malhar/issues/483","number":483,"state":"open","locked":false,"title":"APEXMALHAR-2022 Developed S3 Output Module","user":{"login":"chaithu14","id":8510691,"avatar_url":"https://avatars.githubusercontent.com/u/8510691?v=3","gravatar_id":"","url":"https://api.github.com/users/chaithu14","html_url":"https://github.com/chaithu14","followers_url":"https://api.github.com/users/chaithu14/followers","following_url":"https://api.github.com/users/chaithu14/following{/other_user}","gists_url":"https://api.github.com/users/chaithu14/gists{/gist_id}","starred_url":"https://api.github.com/users/chaithu14/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/chaithu14/subscriptions","organizations_url":"https://api.github.com/users/chaithu14/orgs","repos_url":"https://api.github.com/users/chaithu14/repos","events_url":"https://api.github.com/users/chaithu14/events{/privacy}","received_events_url":"https://api.github.com/users/chaithu14/received_events","type":"User","site_admin":false},"body":"","created_at":"2016-11-04T12:49:25Z","updated_at":"2016-11-21T09:36:47Z","closed_at":null,"merged_at":null,"merge_commit_sha":"28d4b841e40d246113bb12c00056e7870cc176cf","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/apache/apex-malhar/pulls/483/commits","review_comments_url":"https://api.github.com/repos/apache/apex-malhar/pulls/483/comments","review_comment_url":"https://api.github.com/repos/apache/apex-malhar/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/apex-malhar/issues/483/comments","statuses_url":"https://api.github.com/repos/apache/apex-malhar/statuses/24fb5638ecb6f0e45edb5d5f640b220ad9372fcc","head":{"label":"chaithu14:APEXMALHAR-2022-S3Output-multiPart","ref":"APEXMALHAR-2022-S3Output-multiPart","sha":"24fb5638ecb6f0e45edb5d5f640b220ad9372fcc","user":{"login":"chaithu14","id":8510691,"avatar_url":"https://avatars.githubusercontent.com/u/8510691?v=3","gravatar_id":"","url":"https://api.github.com/users/chaithu14","html_url":"https://github.com/chaithu14","followers_url":"https://api.github.com/users/chaithu14/followers","following_url":"https://api.github.com/users/chaithu14/following{/other_user}","gists_url":"https://api.github.com/users/chaithu14/gists{/gist_id}","starred_url":"https://api.github.com/users/chaithu14/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/chaithu14/subscriptions","organizations_url":"https://api.github.com/users/chaithu14/orgs","repos_url":"https://api.github.com/users/chaithu14/repos","events_url":"https://api.github.com/users/chaithu14/events{/privacy}","received_events_url":"https://api.github.com/users/chaithu14/received_events","type":"User","site_admin":false},"repo":{"id":43864822,"name":"incubator-apex-malhar","full_name":"chaithu14/incubator-apex-malhar","owner":{"login":"chaithu14","id":8510691,"avatar_url":"https://avatars.githubusercontent.com/u/8510691?v=3","gravatar_id":"","url":"https://api.github.com/users/chaithu14","html_url":"https://github.com/chaithu14","followers_url":"https://api.github.com/users/chaithu14/followers","following_url":"https://api.github.com/users/chaithu14/following{/other_user}","gists_url":"https://api.github.com/users/chaithu14/gists{/gist_id}","starred_url":"https://api.github.com/users/chaithu14/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/chaithu14/subscriptions","organizations_url":"https://api.github.com/users/chaithu14/orgs","repos_url":"https://api.github.com/users/chaithu14/repos","events_url":"https://api.github.com/users/chaithu14/events{/privacy}","received_events_url":"https://api.github.com/users/chaithu14/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/chaithu14/incubator-apex-malhar","description":"Mirror of Apache Apex malhar (Incubating)","fork":true,"url":"https://api.github.com/repos/chaithu14/incubator-apex-malhar","forks_url":"https://api.github.com/repos/chaithu14/incubator-apex-malhar/forks","keys_url":"https://api.github.com/repos/chaithu14/incubator-apex-malhar/keys{/key_id}","collaborators_url":"https://api.github.com/repos/chaithu14/incubator-apex-malhar/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/chaithu14/incubator-apex-malhar/teams","hooks_url":"https://api.github.com/repos/chaithu14/incubator-apex-malhar/hooks","issue_events_url":"https://api.github.com/repos/chaithu14/incubator-apex-malhar/issues/events{/number}","events_url":"https://api.github.com/repos/chaithu14/incubator-apex-malhar/events","assignees_url":"https://api.github.com/repos/chaithu14/incubator-apex-malhar/assignees{/user}","branches_url":"https://api.github.com/repos/chaithu14/incubator-apex-malhar/branches{/branch}","tags_url":"https://api.github.com/repos/chaithu14/incubator-apex-malhar/tags","blobs_url":"https://api.github.com/repos/chaithu14/incubator-apex-malhar/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/chaithu14/incubator-apex-malhar/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/chaithu14/incubator-apex-malhar/git/refs{/sha}","trees_url":"https://api.github.com/repos/chaithu14/incubator-apex-malhar/git/trees{/sha}","statuses_url":"https://api.github.com/repos/chaithu14/incubator-apex-malhar/statuses/{sha}","languages_url":"https://api.github.com/repos/chaithu14/incubator-apex-malhar/languages","stargazers_url":"https://api.github.com/repos/chaithu14/incubator-apex-malhar/stargazers","contributors_url":"https://api.github.com/repos/chaithu14/incubator-apex-malhar/contributors","subscribers_url":"https://api.github.com/repos/chaithu14/incubator-apex-malhar/subscribers","subscription_url":"https://api.github.com/repos/chaithu14/incubator-apex-malhar/subscription","commits_url":"https://api.github.com/repos/chaithu14/incubator-apex-malhar/commits{/sha}","git_commits_url":"https://api.github.com/repos/chaithu14/incubator-apex-malhar/git/commits{/sha}","comments_url":"https://api.github.com/repos/chaithu14/incubator-apex-malhar/comments{/number}","issue_comment_url":"https://api.github.com/repos/chaithu14/incubator-apex-malhar/issues/comments{/number}","contents_url":"https://api.github.com/repos/chaithu14/incubator-apex-malhar/contents/{+path}","compare_url":"https://api.github.com/repos/chaithu14/incubator-apex-malhar/compare/{base}...{head}","merges_url":"https://api.github.com/repos/chaithu14/incubator-apex-malhar/merges","archive_url":"https://api.github.com/repos/chaithu14/incubator-apex-malhar/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/chaithu14/incubator-apex-malhar/downloads","issues_url":"https://api.github.com/repos/chaithu14/incubator-apex-malhar/issues{/number}","pulls_url":"https://api.github.com/repos/chaithu14/incubator-apex-malhar/pulls{/number}","milestones_url":"https://api.github.com/repos/chaithu14/incubator-apex-malhar/milestones{/number}","notifications_url":"https://api.github.com/repos/chaithu14/incubator-apex-malhar/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/chaithu14/incubator-apex-malhar/labels{/name}","releases_url":"https://api.github.com/repos/chaithu14/incubator-apex-malhar/releases{/id}","deployments_url":"https://api.github.com/repos/chaithu14/incubator-apex-malhar/deployments","created_at":"2015-10-08T05:25:25Z","updated_at":"2015-10-08T05:25:36Z","pushed_at":"2016-11-18T11:44:15Z","git_url":"git://github.com/chaithu14/incubator-apex-malhar.git","ssh_url":"git@github.com:chaithu14/incubator-apex-malhar.git","clone_url":"https://github.com/chaithu14/incubator-apex-malhar.git","svn_url":"https://github.com/chaithu14/incubator-apex-malhar","homepage":null,"size":88128,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"b43818bce0da0ee94458b983061c726724c3bf14","user":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":41348334,"name":"apex-malhar","full_name":"apache/apex-malhar","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/apex-malhar","description":"Mirror of Apache Apex malhar","fork":false,"url":"https://api.github.com/repos/apache/apex-malhar","forks_url":"https://api.github.com/repos/apache/apex-malhar/forks","keys_url":"https://api.github.com/repos/apache/apex-malhar/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/apex-malhar/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/apex-malhar/teams","hooks_url":"https://api.github.com/repos/apache/apex-malhar/hooks","issue_events_url":"https://api.github.com/repos/apache/apex-malhar/issues/events{/number}","events_url":"https://api.github.com/repos/apache/apex-malhar/events","assignees_url":"https://api.github.com/repos/apache/apex-malhar/assignees{/user}","branches_url":"https://api.github.com/repos/apache/apex-malhar/branches{/branch}","tags_url":"https://api.github.com/repos/apache/apex-malhar/tags","blobs_url":"https://api.github.com/repos/apache/apex-malhar/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/apex-malhar/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/apex-malhar/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/apex-malhar/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/apex-malhar/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/apex-malhar/languages","stargazers_url":"https://api.github.com/repos/apache/apex-malhar/stargazers","contributors_url":"https://api.github.com/repos/apache/apex-malhar/contributors","subscribers_url":"https://api.github.com/repos/apache/apex-malhar/subscribers","subscription_url":"https://api.github.com/repos/apache/apex-malhar/subscription","commits_url":"https://api.github.com/repos/apache/apex-malhar/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/apex-malhar/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/apex-malhar/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/apex-malhar/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/apex-malhar/contents/{+path}","compare_url":"https://api.github.com/repos/apache/apex-malhar/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/apex-malhar/merges","archive_url":"https://api.github.com/repos/apache/apex-malhar/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/apex-malhar/downloads","issues_url":"https://api.github.com/repos/apache/apex-malhar/issues{/number}","pulls_url":"https://api.github.com/repos/apache/apex-malhar/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/apex-malhar/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/apex-malhar/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/apex-malhar/labels{/name}","releases_url":"https://api.github.com/repos/apache/apex-malhar/releases{/id}","deployments_url":"https://api.github.com/repos/apache/apex-malhar/deployments","created_at":"2015-08-25T07:00:05Z","updated_at":"2016-11-01T10:48:51Z","pushed_at":"2016-11-21T08:50:37Z","git_url":"git://github.com/apache/apex-malhar.git","ssh_url":"git@github.com:apache/apex-malhar.git","clone_url":"https://github.com/apache/apex-malhar.git","svn_url":"https://github.com/apache/apex-malhar","homepage":"","size":99545,"stargazers_count":66,"watchers_count":66,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":true,"forks_count":110,"mirror_url":"git://git.apache.org/incubator-apex-malhar.git","open_issues_count":32,"forks":110,"open_issues":32,"watchers":66,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/apex-malhar/pulls/483"},"html":{"href":"https://github.com/apache/apex-malhar/pull/483"},"issue":{"href":"https://api.github.com/repos/apache/apex-malhar/issues/483"},"comments":{"href":"https://api.github.com/repos/apache/apex-malhar/issues/483/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/apex-malhar/pulls/483/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/apex-malhar/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/apex-malhar/pulls/483/commits"},"statuses":{"href":"https://api.github.com/repos/apache/apex-malhar/statuses/24fb5638ecb6f0e45edb5d5f640b220ad9372fcc"}}}}
{"actor":{"display_login":"chaithu14"}}