{"action":"created","comment":{"url":"https://api.github.com/repos/metamx/java-util/pulls/comments/81612646","pull_request_review_id":2580009,"id":81612646,"diff_hunk":"@@ -195,13 +206,95 @@ public void close() throws IOException\n               String.format(\"Expected [%,d] bytes, only saw [%,d], potential corruption?\", size, bytesWritten)\n           );\n         }\n+        mergeWithSmoosher();\n+      }\n+    };\n+  }\n+\n+  private void mergeWithSmoosher() throws IOException\n+  {\n+    //get processed elements from the stack and write.\n+    List<File> fileToProcess = new ArrayList<>(completedFiles);\n+    completedFiles = Lists.newArrayList();\n+    for(File file: fileToProcess)\n+    {\n+      add(file);\n+      file.delete();\n+    }\n+  }\n+\n+  private SmooshedWriter delegateSmooshedWriter(final String name,final long size) throws IOException\n+  {\t\n+    final File tmpFile = new File(baseDir, name);\n+    filesInProcess.add(tmpFile);\n+    return new SmooshedWriter () {\n+      private int currOffset = 0;\n+      private boolean open = true;\n+      private final FileOutputStream out = new FileOutputStream(tmpFile);\n+\n+      @Override\n+      public void close() throws IOException\n+      {\n+        open = false;\n+        out.close();\n+        completedFiles.add(tmpFile);\n+        filesInProcess.remove(tmpFile);\n+\n+        if(!writerCurrentlyInUse ) {\n+          mergeWithSmoosher();\n+        }\n+      }\n+      public int bytesLeft()\n+      {\n+        return (int) (size - currOffset);\n+      }\n+\n+      @Override\n+      public int write(ByteBuffer buffer) throws IOException\n+      {\n+        WritableByteChannel channel = Channels.newChannel(out);\n+        int bytesWritten = channel.write(buffer);\n+        addToOffset(bytesWritten);\n+        return bytesWritten;\n+      }\n+\n+      @Override\n+      public int write(InputStream in) throws IOException\n+      {\n+        int bytesWritten = Ints.checkedCast(ByteStreams.copy(in, out));\n+        addToOffset(bytesWritten);\n+        return bytesWritten;\n+      }\n+\n+      private void addToOffset(int numBytesWritten)\n+      {\n+        if (numBytesWritten > bytesLeft()) {\n+          throw new ISE(\"Wrote more bytes[%,d] than available[%,d]. Don't do that.\", numBytesWritten, bytesLeft());\n+        }\n+        currOffset += numBytesWritten;\n+      }\n+\n+      @Override\n+      public boolean isOpen() {\n+        return open;\n       }\n     };\n+\n   }\n \n   @Override\n   public void close() throws IOException\n   {\n+    //book keeping checks on created file.\n+    if(!completedFiles.isEmpty())","path":"src/main/java/com/metamx/common/io/smoosh/FileSmoosher.java","position":null,"original_position":135,"commit_id":"b060cd6917cbd28ac7516ab0150cb4e06d025e7c","original_commit_id":"034a3746f2e34aab634ba765ca938f1d3c93fed1","user":{"login":"akashdw","id":5886276,"avatar_url":"https://avatars.githubusercontent.com/u/5886276?v=3","gravatar_id":"","url":"https://api.github.com/users/akashdw","html_url":"https://github.com/akashdw","followers_url":"https://api.github.com/users/akashdw/followers","following_url":"https://api.github.com/users/akashdw/following{/other_user}","gists_url":"https://api.github.com/users/akashdw/gists{/gist_id}","starred_url":"https://api.github.com/users/akashdw/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/akashdw/subscriptions","organizations_url":"https://api.github.com/users/akashdw/orgs","repos_url":"https://api.github.com/users/akashdw/repos","events_url":"https://api.github.com/users/akashdw/events{/privacy}","received_events_url":"https://api.github.com/users/akashdw/received_events","type":"User","site_admin":false},"body":"fixed.","created_at":"2016-10-03T18:39:04Z","updated_at":"2016-10-03T18:39:04Z","html_url":"https://github.com/metamx/java-util/pull/55#discussion_r81612646","pull_request_url":"https://api.github.com/repos/metamx/java-util/pulls/55","_links":{"self":{"href":"https://api.github.com/repos/metamx/java-util/pulls/comments/81612646"},"html":{"href":"https://github.com/metamx/java-util/pull/55#discussion_r81612646"},"pull_request":{"href":"https://api.github.com/repos/metamx/java-util/pulls/55"}}},"pull_request":{"url":"https://api.github.com/repos/metamx/java-util/pulls/55","id":86800512,"html_url":"https://github.com/metamx/java-util/pull/55","diff_url":"https://github.com/metamx/java-util/pull/55.diff","patch_url":"https://github.com/metamx/java-util/pull/55.patch","issue_url":"https://api.github.com/repos/metamx/java-util/issues/55","number":55,"state":"open","locked":false,"title":"File smoosher changes","user":{"login":"akashdw","id":5886276,"avatar_url":"https://avatars.githubusercontent.com/u/5886276?v=3","gravatar_id":"","url":"https://api.github.com/users/akashdw","html_url":"https://github.com/akashdw","followers_url":"https://api.github.com/users/akashdw/followers","following_url":"https://api.github.com/users/akashdw/following{/other_user}","gists_url":"https://api.github.com/users/akashdw/gists{/gist_id}","starred_url":"https://api.github.com/users/akashdw/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/akashdw/subscriptions","organizations_url":"https://api.github.com/users/akashdw/orgs","repos_url":"https://api.github.com/users/akashdw/repos","events_url":"https://api.github.com/users/akashdw/events{/privacy}","received_events_url":"https://api.github.com/users/akashdw/received_events","type":"User","site_admin":false},"body":"This pr is to support `FileSmoosher` changes proposed in https://github.com/druid-io/druid/issues/3513 .\r\n\r\n`FileSmoosher` detect when it already has an OutputStream open and redirect newly opened OutputStreams to new files on the file system. When any of the open OutputStream objects are closed, they will also check to see if any of the other files have been closed in the meantime. If there are OutputStreams that have been closed, they will be copied on to the main smoosh file and the extra underlying file will be cleaned up. This has the downside of introducing an extra copy to these files, but it given that the strategy will only be used when absolutely necessary, this shouldn’t result in any noticeable performance degradation during indexing.","created_at":"2016-09-26T19:08:18Z","updated_at":"2016-10-03T18:39:04Z","closed_at":null,"merged_at":null,"merge_commit_sha":"2604b3ba4aa2de83d03cd719e97b6f12419d6a22","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/metamx/java-util/pulls/55/commits","review_comments_url":"https://api.github.com/repos/metamx/java-util/pulls/55/comments","review_comment_url":"https://api.github.com/repos/metamx/java-util/pulls/comments{/number}","comments_url":"https://api.github.com/repos/metamx/java-util/issues/55/comments","statuses_url":"https://api.github.com/repos/metamx/java-util/statuses/b060cd6917cbd28ac7516ab0150cb4e06d025e7c","head":{"label":"akashdw:file_smoosher_changes","ref":"file_smoosher_changes","sha":"b060cd6917cbd28ac7516ab0150cb4e06d025e7c","user":{"login":"akashdw","id":5886276,"avatar_url":"https://avatars.githubusercontent.com/u/5886276?v=3","gravatar_id":"","url":"https://api.github.com/users/akashdw","html_url":"https://github.com/akashdw","followers_url":"https://api.github.com/users/akashdw/followers","following_url":"https://api.github.com/users/akashdw/following{/other_user}","gists_url":"https://api.github.com/users/akashdw/gists{/gist_id}","starred_url":"https://api.github.com/users/akashdw/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/akashdw/subscriptions","organizations_url":"https://api.github.com/users/akashdw/orgs","repos_url":"https://api.github.com/users/akashdw/repos","events_url":"https://api.github.com/users/akashdw/events{/privacy}","received_events_url":"https://api.github.com/users/akashdw/received_events","type":"User","site_admin":false},"repo":{"id":65037987,"name":"java-util","full_name":"akashdw/java-util","owner":{"login":"akashdw","id":5886276,"avatar_url":"https://avatars.githubusercontent.com/u/5886276?v=3","gravatar_id":"","url":"https://api.github.com/users/akashdw","html_url":"https://github.com/akashdw","followers_url":"https://api.github.com/users/akashdw/followers","following_url":"https://api.github.com/users/akashdw/following{/other_user}","gists_url":"https://api.github.com/users/akashdw/gists{/gist_id}","starred_url":"https://api.github.com/users/akashdw/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/akashdw/subscriptions","organizations_url":"https://api.github.com/users/akashdw/orgs","repos_url":"https://api.github.com/users/akashdw/repos","events_url":"https://api.github.com/users/akashdw/events{/privacy}","received_events_url":"https://api.github.com/users/akashdw/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/akashdw/java-util","description":"Utility code for java and jvm-based languages","fork":true,"url":"https://api.github.com/repos/akashdw/java-util","forks_url":"https://api.github.com/repos/akashdw/java-util/forks","keys_url":"https://api.github.com/repos/akashdw/java-util/keys{/key_id}","collaborators_url":"https://api.github.com/repos/akashdw/java-util/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/akashdw/java-util/teams","hooks_url":"https://api.github.com/repos/akashdw/java-util/hooks","issue_events_url":"https://api.github.com/repos/akashdw/java-util/issues/events{/number}","events_url":"https://api.github.com/repos/akashdw/java-util/events","assignees_url":"https://api.github.com/repos/akashdw/java-util/assignees{/user}","branches_url":"https://api.github.com/repos/akashdw/java-util/branches{/branch}","tags_url":"https://api.github.com/repos/akashdw/java-util/tags","blobs_url":"https://api.github.com/repos/akashdw/java-util/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/akashdw/java-util/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/akashdw/java-util/git/refs{/sha}","trees_url":"https://api.github.com/repos/akashdw/java-util/git/trees{/sha}","statuses_url":"https://api.github.com/repos/akashdw/java-util/statuses/{sha}","languages_url":"https://api.github.com/repos/akashdw/java-util/languages","stargazers_url":"https://api.github.com/repos/akashdw/java-util/stargazers","contributors_url":"https://api.github.com/repos/akashdw/java-util/contributors","subscribers_url":"https://api.github.com/repos/akashdw/java-util/subscribers","subscription_url":"https://api.github.com/repos/akashdw/java-util/subscription","commits_url":"https://api.github.com/repos/akashdw/java-util/commits{/sha}","git_commits_url":"https://api.github.com/repos/akashdw/java-util/git/commits{/sha}","comments_url":"https://api.github.com/repos/akashdw/java-util/comments{/number}","issue_comment_url":"https://api.github.com/repos/akashdw/java-util/issues/comments{/number}","contents_url":"https://api.github.com/repos/akashdw/java-util/contents/{+path}","compare_url":"https://api.github.com/repos/akashdw/java-util/compare/{base}...{head}","merges_url":"https://api.github.com/repos/akashdw/java-util/merges","archive_url":"https://api.github.com/repos/akashdw/java-util/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/akashdw/java-util/downloads","issues_url":"https://api.github.com/repos/akashdw/java-util/issues{/number}","pulls_url":"https://api.github.com/repos/akashdw/java-util/pulls{/number}","milestones_url":"https://api.github.com/repos/akashdw/java-util/milestones{/number}","notifications_url":"https://api.github.com/repos/akashdw/java-util/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/akashdw/java-util/labels{/name}","releases_url":"https://api.github.com/repos/akashdw/java-util/releases{/id}","deployments_url":"https://api.github.com/repos/akashdw/java-util/deployments","created_at":"2016-08-05T17:48:11Z","updated_at":"2016-08-05T17:48:13Z","pushed_at":"2016-10-03T18:36:26Z","git_url":"git://github.com/akashdw/java-util.git","ssh_url":"git@github.com:akashdw/java-util.git","clone_url":"https://github.com/akashdw/java-util.git","svn_url":"https://github.com/akashdw/java-util","homepage":"","size":647,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"metamx:master","ref":"master","sha":"5d81786f63742187215dcb9aeb19b6ba1b3b18a2","user":{"login":"metamx","id":299329,"avatar_url":"https://avatars.githubusercontent.com/u/299329?v=3","gravatar_id":"","url":"https://api.github.com/users/metamx","html_url":"https://github.com/metamx","followers_url":"https://api.github.com/users/metamx/followers","following_url":"https://api.github.com/users/metamx/following{/other_user}","gists_url":"https://api.github.com/users/metamx/gists{/gist_id}","starred_url":"https://api.github.com/users/metamx/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/metamx/subscriptions","organizations_url":"https://api.github.com/users/metamx/orgs","repos_url":"https://api.github.com/users/metamx/repos","events_url":"https://api.github.com/users/metamx/events{/privacy}","received_events_url":"https://api.github.com/users/metamx/received_events","type":"Organization","site_admin":false},"repo":{"id":2622329,"name":"java-util","full_name":"metamx/java-util","owner":{"login":"metamx","id":299329,"avatar_url":"https://avatars.githubusercontent.com/u/299329?v=3","gravatar_id":"","url":"https://api.github.com/users/metamx","html_url":"https://github.com/metamx","followers_url":"https://api.github.com/users/metamx/followers","following_url":"https://api.github.com/users/metamx/following{/other_user}","gists_url":"https://api.github.com/users/metamx/gists{/gist_id}","starred_url":"https://api.github.com/users/metamx/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/metamx/subscriptions","organizations_url":"https://api.github.com/users/metamx/orgs","repos_url":"https://api.github.com/users/metamx/repos","events_url":"https://api.github.com/users/metamx/events{/privacy}","received_events_url":"https://api.github.com/users/metamx/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/metamx/java-util","description":"Utility code for java and jvm-based languages","fork":false,"url":"https://api.github.com/repos/metamx/java-util","forks_url":"https://api.github.com/repos/metamx/java-util/forks","keys_url":"https://api.github.com/repos/metamx/java-util/keys{/key_id}","collaborators_url":"https://api.github.com/repos/metamx/java-util/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/metamx/java-util/teams","hooks_url":"https://api.github.com/repos/metamx/java-util/hooks","issue_events_url":"https://api.github.com/repos/metamx/java-util/issues/events{/number}","events_url":"https://api.github.com/repos/metamx/java-util/events","assignees_url":"https://api.github.com/repos/metamx/java-util/assignees{/user}","branches_url":"https://api.github.com/repos/metamx/java-util/branches{/branch}","tags_url":"https://api.github.com/repos/metamx/java-util/tags","blobs_url":"https://api.github.com/repos/metamx/java-util/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/metamx/java-util/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/metamx/java-util/git/refs{/sha}","trees_url":"https://api.github.com/repos/metamx/java-util/git/trees{/sha}","statuses_url":"https://api.github.com/repos/metamx/java-util/statuses/{sha}","languages_url":"https://api.github.com/repos/metamx/java-util/languages","stargazers_url":"https://api.github.com/repos/metamx/java-util/stargazers","contributors_url":"https://api.github.com/repos/metamx/java-util/contributors","subscribers_url":"https://api.github.com/repos/metamx/java-util/subscribers","subscription_url":"https://api.github.com/repos/metamx/java-util/subscription","commits_url":"https://api.github.com/repos/metamx/java-util/commits{/sha}","git_commits_url":"https://api.github.com/repos/metamx/java-util/git/commits{/sha}","comments_url":"https://api.github.com/repos/metamx/java-util/comments{/number}","issue_comment_url":"https://api.github.com/repos/metamx/java-util/issues/comments{/number}","contents_url":"https://api.github.com/repos/metamx/java-util/contents/{+path}","compare_url":"https://api.github.com/repos/metamx/java-util/compare/{base}...{head}","merges_url":"https://api.github.com/repos/metamx/java-util/merges","archive_url":"https://api.github.com/repos/metamx/java-util/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/metamx/java-util/downloads","issues_url":"https://api.github.com/repos/metamx/java-util/issues{/number}","pulls_url":"https://api.github.com/repos/metamx/java-util/pulls{/number}","milestones_url":"https://api.github.com/repos/metamx/java-util/milestones{/number}","notifications_url":"https://api.github.com/repos/metamx/java-util/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/metamx/java-util/labels{/name}","releases_url":"https://api.github.com/repos/metamx/java-util/releases{/id}","deployments_url":"https://api.github.com/repos/metamx/java-util/deployments","created_at":"2011-10-21T18:41:10Z","updated_at":"2016-09-07T10:01:42Z","pushed_at":"2016-10-03T18:36:26Z","git_url":"git://github.com/metamx/java-util.git","ssh_url":"git@github.com:metamx/java-util.git","clone_url":"https://github.com/metamx/java-util.git","svn_url":"https://github.com/metamx/java-util","homepage":"","size":929,"stargazers_count":30,"watchers_count":30,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":36,"mirror_url":null,"open_issues_count":2,"forks":36,"open_issues":2,"watchers":30,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/metamx/java-util/pulls/55"},"html":{"href":"https://github.com/metamx/java-util/pull/55"},"issue":{"href":"https://api.github.com/repos/metamx/java-util/issues/55"},"comments":{"href":"https://api.github.com/repos/metamx/java-util/issues/55/comments"},"review_comments":{"href":"https://api.github.com/repos/metamx/java-util/pulls/55/comments"},"review_comment":{"href":"https://api.github.com/repos/metamx/java-util/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/metamx/java-util/pulls/55/commits"},"statuses":{"href":"https://api.github.com/repos/metamx/java-util/statuses/b060cd6917cbd28ac7516ab0150cb4e06d025e7c"}}}}
{"actor":{"display_login":"akashdw"}}