{"action":"created","comment":{"url":"https://api.github.com/repos/apache/cassandra/pulls/comments/205297956","pull_request_review_id":140541696,"id":205297956,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDIwNTI5Nzk1Ng==","diff_hunk":"@@ -0,0 +1,109 @@\n+/*\n+ * Licensed to the Apache Software Foundation (ASF) under one\n+ * or more contributor license agreements.  See the NOTICE file\n+ * distributed with this work for additional information\n+ * regarding copyright ownership.  The ASF licenses this file\n+ * to you under the Apache License, Version 2.0 (the\n+ * \"License\"); you may not use this file except in compliance\n+ * with the License.  You may obtain a copy of the License at\n+ *\n+ *     http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+\n+package org.apache.cassandra.db.streaming;\n+\n+import java.io.IOException;\n+import java.io.RandomAccessFile;\n+import java.nio.channels.FileChannel;\n+import java.util.List;\n+\n+import org.slf4j.Logger;\n+import org.slf4j.LoggerFactory;\n+\n+import org.apache.cassandra.io.sstable.Component;\n+import org.apache.cassandra.io.sstable.format.SSTableReader;\n+import org.apache.cassandra.io.util.DataOutputStreamPlus;\n+import org.apache.cassandra.net.async.ByteBufDataOutputStreamPlus;\n+import org.apache.cassandra.streaming.ProgressInfo;\n+import org.apache.cassandra.streaming.StreamManager;\n+import org.apache.cassandra.streaming.StreamSession;\n+import org.apache.cassandra.utils.FBUtilities;\n+\n+import static org.apache.cassandra.streaming.StreamManager.StreamRateLimiter;\n+\n+/**\n+ * CassandraBlockStreamWriter streams the entire SSTable to given channel.\n+ */\n+public class CassandraBlockStreamWriter implements IStreamWriter\n+{\n+    private static final Logger logger = LoggerFactory.getLogger(CassandraBlockStreamWriter.class);\n+\n+    private final SSTableReader sstable;\n+    private final ComponentManifest manifest;\n+    private final StreamSession session;\n+    private final StreamRateLimiter limiter;\n+\n+    public CassandraBlockStreamWriter(SSTableReader sstable, StreamSession session, ComponentManifest manifest)\n+    {\n+        this.session = session;\n+        this.sstable = sstable;\n+        this.manifest = manifest;\n+        this.limiter =  StreamManager.getRateLimiter(session.peer);\n+    }\n+\n+    /**\n+     * Stream the entire file to given channel.\n+     * <p>\n+     *\n+     * @param output where this writes data to\n+     * @throws IOException on any I/O error\n+     */\n+    @Override\n+    public void write(DataOutputStreamPlus output) throws IOException\n+    {\n+        long totalSize = manifest.getTotalSize();\n+        logger.debug(\"[Stream #{}] Start streaming sstable {} to {}, repairedAt = {}, totalSize = {}\", session.planId(),\n+                     sstable.getFilename(), session.peer, sstable.getSSTableMetadata().repairedAt, totalSize);\n+\n+        long progress = 0L;\n+        ByteBufDataOutputStreamPlus byteBufDataOutputStreamPlus = (ByteBufDataOutputStreamPlus) output;\n+\n+        for (Component component : manifest.getComponents())\n+        {\n+            @SuppressWarnings(\"resource\") // this is closed after the file is transferred by ByteBufDataOutputStreamPlus\n+            FileChannel in = new RandomAccessFile(sstable.descriptor.filenameFor(component), \"r\").getChannel();\n+\n+            // Total Length to transmit for this file\n+            long length = in.size();\n+\n+            // tracks write progress\n+            long bytesRead = 0;\n+            logger.debug(\"[Stream #{}] Block streaming {}.{} gen {} component {} size {}\", session.planId(),\n+                        sstable.getKeyspaceName(), sstable.getColumnFamilyName(), sstable.descriptor.generation, component, length);\n+\n+            bytesRead += byteBufDataOutputStreamPlus.writeToChannel(in, limiter);\n+            progress += bytesRead;\n+\n+            session.progress(sstable.descriptor.filenameFor(component), ProgressInfo.Direction.OUT, bytesRead,\n+                             length);\n+\n+            logger.debug(\"[Stream #{}] Finished block streaming {}.{} gen {} component {} to {}, xfered = {}, length = {}, totalSize = {}\",\n+                         session.planId(), sstable.getKeyspaceName(), sstable.getColumnFamilyName(),\n+                         sstable.descriptor.generation, component, session.peer, FBUtilities.prettyPrintMemory(bytesRead),\n+                         FBUtilities.prettyPrintMemory(length), FBUtilities.prettyPrintMemory(totalSize));\n+\n+            byteBufDataOutputStreamPlus.flush();","path":"src/java/org/apache/cassandra/db/streaming/CassandraBlockStreamWriter.java","position":101,"original_position":101,"commit_id":"efa97da9358130c9cdfd6e77f0873f723518cbe3","original_commit_id":"efa97da9358130c9cdfd6e77f0873f723518cbe3","user":{"login":"dineshjoshi","id":789885,"node_id":"MDQ6VXNlcjc4OTg4NQ==","avatar_url":"https://avatars3.githubusercontent.com/u/789885?v=4","gravatar_id":"","url":"https://api.github.com/users/dineshjoshi","html_url":"https://github.com/dineshjoshi","followers_url":"https://api.github.com/users/dineshjoshi/followers","following_url":"https://api.github.com/users/dineshjoshi/following{/other_user}","gists_url":"https://api.github.com/users/dineshjoshi/gists{/gist_id}","starred_url":"https://api.github.com/users/dineshjoshi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dineshjoshi/subscriptions","organizations_url":"https://api.github.com/users/dineshjoshi/orgs","repos_url":"https://api.github.com/users/dineshjoshi/repos","events_url":"https://api.github.com/users/dineshjoshi/events{/privacy}","received_events_url":"https://api.github.com/users/dineshjoshi/received_events","type":"User","site_admin":false},"body":"I just have it as a good measure to ensure that when `write()` returns, the stream is flushed. I have moved it outside the loop.","created_at":"2018-07-26T00:07:38Z","updated_at":"2018-07-26T00:07:39Z","html_url":"https://github.com/apache/cassandra/pull/239#discussion_r205297956","pull_request_url":"https://api.github.com/repos/apache/cassandra/pulls/239","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/apache/cassandra/pulls/comments/205297956"},"html":{"href":"https://github.com/apache/cassandra/pull/239#discussion_r205297956"},"pull_request":{"href":"https://api.github.com/repos/apache/cassandra/pulls/239"}},"in_reply_to_id":205174453},"pull_request":{"url":"https://api.github.com/repos/apache/cassandra/pulls/239","id":199057823,"node_id":"MDExOlB1bGxSZXF1ZXN0MTk5MDU3ODIz","html_url":"https://github.com/apache/cassandra/pull/239","diff_url":"https://github.com/apache/cassandra/pull/239.diff","patch_url":"https://github.com/apache/cassandra/pull/239.patch","issue_url":"https://api.github.com/repos/apache/cassandra/issues/239","number":239,"state":"open","locked":false,"title":"[CASSANDRA-14556] Optimize Streaming","user":{"login":"dineshjoshi","id":789885,"node_id":"MDQ6VXNlcjc4OTg4NQ==","avatar_url":"https://avatars3.githubusercontent.com/u/789885?v=4","gravatar_id":"","url":"https://api.github.com/users/dineshjoshi","html_url":"https://github.com/dineshjoshi","followers_url":"https://api.github.com/users/dineshjoshi/followers","following_url":"https://api.github.com/users/dineshjoshi/following{/other_user}","gists_url":"https://api.github.com/users/dineshjoshi/gists{/gist_id}","starred_url":"https://api.github.com/users/dineshjoshi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dineshjoshi/subscriptions","organizations_url":"https://api.github.com/users/dineshjoshi/orgs","repos_url":"https://api.github.com/users/dineshjoshi/repos","events_url":"https://api.github.com/users/dineshjoshi/events{/privacy}","received_events_url":"https://api.github.com/users/dineshjoshi/received_events","type":"User","site_admin":false},"body":"DTests here - https://github.com/apache/cassandra-dtest/compare/master...dineshjoshi:faster-streaming-rev2","created_at":"2018-07-03T19:08:22Z","updated_at":"2018-07-26T00:07:39Z","closed_at":null,"merged_at":null,"merge_commit_sha":"954993d689ce9118f1d67a575e5c889f96e33dc3","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/apache/cassandra/pulls/239/commits","review_comments_url":"https://api.github.com/repos/apache/cassandra/pulls/239/comments","review_comment_url":"https://api.github.com/repos/apache/cassandra/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/cassandra/issues/239/comments","statuses_url":"https://api.github.com/repos/apache/cassandra/statuses/efa97da9358130c9cdfd6e77f0873f723518cbe3","head":{"label":"dineshjoshi:14556-trunk-optimize-streaming","ref":"14556-trunk-optimize-streaming","sha":"efa97da9358130c9cdfd6e77f0873f723518cbe3","user":{"login":"dineshjoshi","id":789885,"node_id":"MDQ6VXNlcjc4OTg4NQ==","avatar_url":"https://avatars3.githubusercontent.com/u/789885?v=4","gravatar_id":"","url":"https://api.github.com/users/dineshjoshi","html_url":"https://github.com/dineshjoshi","followers_url":"https://api.github.com/users/dineshjoshi/followers","following_url":"https://api.github.com/users/dineshjoshi/following{/other_user}","gists_url":"https://api.github.com/users/dineshjoshi/gists{/gist_id}","starred_url":"https://api.github.com/users/dineshjoshi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dineshjoshi/subscriptions","organizations_url":"https://api.github.com/users/dineshjoshi/orgs","repos_url":"https://api.github.com/users/dineshjoshi/repos","events_url":"https://api.github.com/users/dineshjoshi/events{/privacy}","received_events_url":"https://api.github.com/users/dineshjoshi/received_events","type":"User","site_admin":false},"repo":{"id":119905967,"node_id":"MDEwOlJlcG9zaXRvcnkxMTk5MDU5Njc=","name":"cassandra","full_name":"dineshjoshi/cassandra","owner":{"login":"dineshjoshi","id":789885,"node_id":"MDQ6VXNlcjc4OTg4NQ==","avatar_url":"https://avatars3.githubusercontent.com/u/789885?v=4","gravatar_id":"","url":"https://api.github.com/users/dineshjoshi","html_url":"https://github.com/dineshjoshi","followers_url":"https://api.github.com/users/dineshjoshi/followers","following_url":"https://api.github.com/users/dineshjoshi/following{/other_user}","gists_url":"https://api.github.com/users/dineshjoshi/gists{/gist_id}","starred_url":"https://api.github.com/users/dineshjoshi/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dineshjoshi/subscriptions","organizations_url":"https://api.github.com/users/dineshjoshi/orgs","repos_url":"https://api.github.com/users/dineshjoshi/repos","events_url":"https://api.github.com/users/dineshjoshi/events{/privacy}","received_events_url":"https://api.github.com/users/dineshjoshi/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/dineshjoshi/cassandra","description":"Mirror of Apache Cassandra","fork":true,"url":"https://api.github.com/repos/dineshjoshi/cassandra","forks_url":"https://api.github.com/repos/dineshjoshi/cassandra/forks","keys_url":"https://api.github.com/repos/dineshjoshi/cassandra/keys{/key_id}","collaborators_url":"https://api.github.com/repos/dineshjoshi/cassandra/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/dineshjoshi/cassandra/teams","hooks_url":"https://api.github.com/repos/dineshjoshi/cassandra/hooks","issue_events_url":"https://api.github.com/repos/dineshjoshi/cassandra/issues/events{/number}","events_url":"https://api.github.com/repos/dineshjoshi/cassandra/events","assignees_url":"https://api.github.com/repos/dineshjoshi/cassandra/assignees{/user}","branches_url":"https://api.github.com/repos/dineshjoshi/cassandra/branches{/branch}","tags_url":"https://api.github.com/repos/dineshjoshi/cassandra/tags","blobs_url":"https://api.github.com/repos/dineshjoshi/cassandra/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/dineshjoshi/cassandra/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/dineshjoshi/cassandra/git/refs{/sha}","trees_url":"https://api.github.com/repos/dineshjoshi/cassandra/git/trees{/sha}","statuses_url":"https://api.github.com/repos/dineshjoshi/cassandra/statuses/{sha}","languages_url":"https://api.github.com/repos/dineshjoshi/cassandra/languages","stargazers_url":"https://api.github.com/repos/dineshjoshi/cassandra/stargazers","contributors_url":"https://api.github.com/repos/dineshjoshi/cassandra/contributors","subscribers_url":"https://api.github.com/repos/dineshjoshi/cassandra/subscribers","subscription_url":"https://api.github.com/repos/dineshjoshi/cassandra/subscription","commits_url":"https://api.github.com/repos/dineshjoshi/cassandra/commits{/sha}","git_commits_url":"https://api.github.com/repos/dineshjoshi/cassandra/git/commits{/sha}","comments_url":"https://api.github.com/repos/dineshjoshi/cassandra/comments{/number}","issue_comment_url":"https://api.github.com/repos/dineshjoshi/cassandra/issues/comments{/number}","contents_url":"https://api.github.com/repos/dineshjoshi/cassandra/contents/{+path}","compare_url":"https://api.github.com/repos/dineshjoshi/cassandra/compare/{base}...{head}","merges_url":"https://api.github.com/repos/dineshjoshi/cassandra/merges","archive_url":"https://api.github.com/repos/dineshjoshi/cassandra/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/dineshjoshi/cassandra/downloads","issues_url":"https://api.github.com/repos/dineshjoshi/cassandra/issues{/number}","pulls_url":"https://api.github.com/repos/dineshjoshi/cassandra/pulls{/number}","milestones_url":"https://api.github.com/repos/dineshjoshi/cassandra/milestones{/number}","notifications_url":"https://api.github.com/repos/dineshjoshi/cassandra/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/dineshjoshi/cassandra/labels{/name}","releases_url":"https://api.github.com/repos/dineshjoshi/cassandra/releases{/id}","deployments_url":"https://api.github.com/repos/dineshjoshi/cassandra/deployments","created_at":"2018-02-01T23:24:31Z","updated_at":"2018-07-16T18:31:01Z","pushed_at":"2018-07-25T06:44:47Z","git_url":"git://github.com/dineshjoshi/cassandra.git","ssh_url":"git@github.com:dineshjoshi/cassandra.git","clone_url":"https://github.com/dineshjoshi/cassandra.git","svn_url":"https://github.com/dineshjoshi/cassandra","homepage":"http://cassandra.apache.org","size":268173,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":0,"open_issues":0,"watchers":0,"default_branch":"trunk"}},"base":{"label":"apache:trunk","ref":"trunk","sha":"176d4bac22c356c80e275dcb4040bc5cbd0da1c2","user":{"login":"apache","id":47359,"node_id":"MDEyOk9yZ2FuaXphdGlvbjQ3MzU5","avatar_url":"https://avatars0.githubusercontent.com/u/47359?v=4","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":206424,"node_id":"MDEwOlJlcG9zaXRvcnkyMDY0MjQ=","name":"cassandra","full_name":"apache/cassandra","owner":{"login":"apache","id":47359,"node_id":"MDEyOk9yZ2FuaXphdGlvbjQ3MzU5","avatar_url":"https://avatars0.githubusercontent.com/u/47359?v=4","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/cassandra","description":"Mirror of Apache Cassandra","fork":false,"url":"https://api.github.com/repos/apache/cassandra","forks_url":"https://api.github.com/repos/apache/cassandra/forks","keys_url":"https://api.github.com/repos/apache/cassandra/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/cassandra/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/cassandra/teams","hooks_url":"https://api.github.com/repos/apache/cassandra/hooks","issue_events_url":"https://api.github.com/repos/apache/cassandra/issues/events{/number}","events_url":"https://api.github.com/repos/apache/cassandra/events","assignees_url":"https://api.github.com/repos/apache/cassandra/assignees{/user}","branches_url":"https://api.github.com/repos/apache/cassandra/branches{/branch}","tags_url":"https://api.github.com/repos/apache/cassandra/tags","blobs_url":"https://api.github.com/repos/apache/cassandra/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/cassandra/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/cassandra/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/cassandra/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/cassandra/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/cassandra/languages","stargazers_url":"https://api.github.com/repos/apache/cassandra/stargazers","contributors_url":"https://api.github.com/repos/apache/cassandra/contributors","subscribers_url":"https://api.github.com/repos/apache/cassandra/subscribers","subscription_url":"https://api.github.com/repos/apache/cassandra/subscription","commits_url":"https://api.github.com/repos/apache/cassandra/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/cassandra/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/cassandra/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/cassandra/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/cassandra/contents/{+path}","compare_url":"https://api.github.com/repos/apache/cassandra/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/cassandra/merges","archive_url":"https://api.github.com/repos/apache/cassandra/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/cassandra/downloads","issues_url":"https://api.github.com/repos/apache/cassandra/issues{/number}","pulls_url":"https://api.github.com/repos/apache/cassandra/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/cassandra/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/cassandra/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/cassandra/labels{/name}","releases_url":"https://api.github.com/repos/apache/cassandra/releases{/id}","deployments_url":"https://api.github.com/repos/apache/cassandra/deployments","created_at":"2009-05-21T02:10:09Z","updated_at":"2018-07-25T12:44:18Z","pushed_at":"2018-07-25T06:44:49Z","git_url":"git://github.com/apache/cassandra.git","ssh_url":"git@github.com:apache/cassandra.git","clone_url":"https://github.com/apache/cassandra.git","svn_url":"https://github.com/apache/cassandra","homepage":"http://cassandra.apache.org","size":268186,"stargazers_count":4537,"watchers_count":4537,"language":"Java","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":2076,"mirror_url":null,"archived":false,"open_issues_count":84,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":2076,"open_issues":84,"watchers":4537,"default_branch":"trunk"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/cassandra/pulls/239"},"html":{"href":"https://github.com/apache/cassandra/pull/239"},"issue":{"href":"https://api.github.com/repos/apache/cassandra/issues/239"},"comments":{"href":"https://api.github.com/repos/apache/cassandra/issues/239/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/cassandra/pulls/239/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/cassandra/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/cassandra/pulls/239/commits"},"statuses":{"href":"https://api.github.com/repos/apache/cassandra/statuses/efa97da9358130c9cdfd6e77f0873f723518cbe3"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"dineshjoshi"}}