{"action":"created","comment":{"url":"https://api.github.com/repos/uber/hudi/pulls/comments/173955071","pull_request_review_id":103229373,"id":173955071,"diff_hunk":"@@ -0,0 +1,224 @@\n+/*\n+ *  Copyright (c) 2018 Uber Technologies, Inc. (hoodie-dev-group@uber.com)\n+ *\n+ *  Licensed under the Apache License, Version 2.0 (the \"License\");\n+ *  you may not use this file except in compliance with the License.\n+ *  You may obtain a copy of the License at\n+ *\n+ *           http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ *  Unless required by applicable law or agreed to in writing, software\n+ *  distributed under the License is distributed on an \"AS IS\" BASIS,\n+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ *  See the License for the specific language governing permissions and\n+ *  limitations under the License.\n+ */\n+\n+package com.uber.hoodie.func;\n+\n+import com.google.common.base.Optional;\n+import com.google.common.base.Preconditions;\n+import com.uber.hoodie.common.model.HoodieRecord;\n+import com.uber.hoodie.common.model.HoodieRecordPayload;\n+import com.uber.hoodie.config.HoodieWriteConfig;\n+import com.uber.hoodie.exception.HoodieException;\n+import com.uber.hoodie.io.HoodieIOHandle;\n+import org.apache.avro.Schema;\n+import org.apache.avro.generic.IndexedRecord;\n+import org.apache.log4j.LogManager;\n+import org.apache.log4j.Logger;\n+import org.apache.spark.util.SizeEstimator;\n+\n+import java.io.IOException;\n+import java.util.Iterator;\n+import java.util.Map;\n+import java.util.concurrent.LinkedBlockingQueue;\n+import java.util.concurrent.Semaphore;\n+import java.util.concurrent.TimeUnit;\n+import java.util.concurrent.atomic.AtomicBoolean;\n+import java.util.concurrent.atomic.AtomicLong;\n+import java.util.concurrent.atomic.AtomicReference;\n+\n+/**\n+ * Used for buffering input records. Buffer limit is controlled by {@link #bufferMemoryLimit}. It internally samples\n+ * every {@link #RECORD_SAMPLING_RATE}th record and adjusts number of records in buffer accordingly. This is done to\n+ * ensure that we don't OOM.\n+ */\n+public class BufferedIterator<K extends HoodieRecordPayload, T extends HoodieRecord<K>> implements Iterator<T> {\n+\n+  private static Logger logger = LogManager.getLogger(BufferedIterator.class);\n+  private static final int RECORD_SAMPLING_RATE = 64;","path":"hoodie-client/src/main/java/com/uber/hoodie/func/BufferedIterator.java","position":50,"original_position":50,"commit_id":"c7b2d786552aeca9649288900deff37b6791aaf4","original_commit_id":"c7b2d786552aeca9649288900deff37b6791aaf4","user":{"login":"ovj","id":899843,"avatar_url":"https://avatars2.githubusercontent.com/u/899843?v=4","gravatar_id":"","url":"https://api.github.com/users/ovj","html_url":"https://github.com/ovj","followers_url":"https://api.github.com/users/ovj/followers","following_url":"https://api.github.com/users/ovj/following{/other_user}","gists_url":"https://api.github.com/users/ovj/gists{/gist_id}","starred_url":"https://api.github.com/users/ovj/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ovj/subscriptions","organizations_url":"https://api.github.com/users/ovj/orgs","repos_url":"https://api.github.com/users/ovj/repos","events_url":"https://api.github.com/users/ovj/events{/privacy}","received_events_url":"https://api.github.com/users/ovj/received_events","type":"User","site_admin":false},"body":"let me add comments.","created_at":"2018-03-12T21:37:42Z","updated_at":"2018-03-12T21:37:43Z","html_url":"https://github.com/uber/hudi/pull/294#discussion_r173955071","pull_request_url":"https://api.github.com/repos/uber/hudi/pulls/294","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/uber/hudi/pulls/comments/173955071"},"html":{"href":"https://github.com/uber/hudi/pull/294#discussion_r173955071"},"pull_request":{"href":"https://api.github.com/repos/uber/hudi/pulls/294"}},"in_reply_to_id":161150987},"pull_request":{"url":"https://api.github.com/repos/uber/hudi/pulls/294","id":161010157,"html_url":"https://github.com/uber/hudi/pull/294","diff_url":"https://github.com/uber/hudi/pull/294.diff","patch_url":"https://github.com/uber/hudi/pull/294.patch","issue_url":"https://api.github.com/repos/uber/hudi/issues/294","number":294,"state":"open","locked":false,"title":"Parallelizing parquet write and spark's external read operation.","user":{"login":"ovj","id":899843,"avatar_url":"https://avatars2.githubusercontent.com/u/899843?v=4","gravatar_id":"","url":"https://api.github.com/users/ovj","html_url":"https://github.com/ovj","followers_url":"https://api.github.com/users/ovj/followers","following_url":"https://api.github.com/users/ovj/following{/other_user}","gists_url":"https://api.github.com/users/ovj/gists{/gist_id}","starred_url":"https://api.github.com/users/ovj/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ovj/subscriptions","organizations_url":"https://api.github.com/users/ovj/orgs","repos_url":"https://api.github.com/users/ovj/repos","events_url":"https://api.github.com/users/ovj/events{/privacy}","received_events_url":"https://api.github.com/users/ovj/received_events","type":"User","site_admin":false},"body":"For large data inserts we have noticed that time spent in reading records from spark's external sorter is almost comparable to time spent in writing records to parquet. We want to reduce the overall write time by parallelizing read and write operations.\r\n\r\nAs a part of the PR we are trying to parallelize below operations\r\n- reading records from spark's external record reader + pre computing insert value (this saves writer thread's time) (depends a lot on the complexity of the schema).\r\n- writing records to parquet file.\r\n\r\nWith these changes we are able to reduce our final parquet write stage runtime from ~1.1h to ~19min.\r\n\r\n@vinothchandar , @n3nash, @jianxu  please take a look at it.\r\n\r\nFYI @esmioley ","created_at":"2018-01-03T22:01:30Z","updated_at":"2018-03-12T21:37:43Z","closed_at":null,"merged_at":null,"merge_commit_sha":"429b5d595a27de64fb0916da4ef36413986c281a","assignee":{"login":"ovj","id":899843,"avatar_url":"https://avatars2.githubusercontent.com/u/899843?v=4","gravatar_id":"","url":"https://api.github.com/users/ovj","html_url":"https://github.com/ovj","followers_url":"https://api.github.com/users/ovj/followers","following_url":"https://api.github.com/users/ovj/following{/other_user}","gists_url":"https://api.github.com/users/ovj/gists{/gist_id}","starred_url":"https://api.github.com/users/ovj/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ovj/subscriptions","organizations_url":"https://api.github.com/users/ovj/orgs","repos_url":"https://api.github.com/users/ovj/repos","events_url":"https://api.github.com/users/ovj/events{/privacy}","received_events_url":"https://api.github.com/users/ovj/received_events","type":"User","site_admin":false},"assignees":[{"login":"ovj","id":899843,"avatar_url":"https://avatars2.githubusercontent.com/u/899843?v=4","gravatar_id":"","url":"https://api.github.com/users/ovj","html_url":"https://github.com/ovj","followers_url":"https://api.github.com/users/ovj/followers","following_url":"https://api.github.com/users/ovj/following{/other_user}","gists_url":"https://api.github.com/users/ovj/gists{/gist_id}","starred_url":"https://api.github.com/users/ovj/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ovj/subscriptions","organizations_url":"https://api.github.com/users/ovj/orgs","repos_url":"https://api.github.com/users/ovj/repos","events_url":"https://api.github.com/users/ovj/events{/privacy}","received_events_url":"https://api.github.com/users/ovj/received_events","type":"User","site_admin":false}],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/uber/hudi/pulls/294/commits","review_comments_url":"https://api.github.com/repos/uber/hudi/pulls/294/comments","review_comment_url":"https://api.github.com/repos/uber/hudi/pulls/comments{/number}","comments_url":"https://api.github.com/repos/uber/hudi/issues/294/comments","statuses_url":"https://api.github.com/repos/uber/hudi/statuses/c7b2d786552aeca9649288900deff37b6791aaf4","head":{"label":"ovj:parallel_writer_thread","ref":"parallel_writer_thread","sha":"c7b2d786552aeca9649288900deff37b6791aaf4","user":{"login":"ovj","id":899843,"avatar_url":"https://avatars2.githubusercontent.com/u/899843?v=4","gravatar_id":"","url":"https://api.github.com/users/ovj","html_url":"https://github.com/ovj","followers_url":"https://api.github.com/users/ovj/followers","following_url":"https://api.github.com/users/ovj/following{/other_user}","gists_url":"https://api.github.com/users/ovj/gists{/gist_id}","starred_url":"https://api.github.com/users/ovj/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ovj/subscriptions","organizations_url":"https://api.github.com/users/ovj/orgs","repos_url":"https://api.github.com/users/ovj/repos","events_url":"https://api.github.com/users/ovj/events{/privacy}","received_events_url":"https://api.github.com/users/ovj/received_events","type":"User","site_admin":false},"repo":{"id":84595017,"name":"hoodie","full_name":"ovj/hoodie","owner":{"login":"ovj","id":899843,"avatar_url":"https://avatars2.githubusercontent.com/u/899843?v=4","gravatar_id":"","url":"https://api.github.com/users/ovj","html_url":"https://github.com/ovj","followers_url":"https://api.github.com/users/ovj/followers","following_url":"https://api.github.com/users/ovj/following{/other_user}","gists_url":"https://api.github.com/users/ovj/gists{/gist_id}","starred_url":"https://api.github.com/users/ovj/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ovj/subscriptions","organizations_url":"https://api.github.com/users/ovj/orgs","repos_url":"https://api.github.com/users/ovj/repos","events_url":"https://api.github.com/users/ovj/events{/privacy}","received_events_url":"https://api.github.com/users/ovj/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/ovj/hoodie","description":"Spark Library for Hadoop Upserts And Incrementals","fork":true,"url":"https://api.github.com/repos/ovj/hoodie","forks_url":"https://api.github.com/repos/ovj/hoodie/forks","keys_url":"https://api.github.com/repos/ovj/hoodie/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ovj/hoodie/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ovj/hoodie/teams","hooks_url":"https://api.github.com/repos/ovj/hoodie/hooks","issue_events_url":"https://api.github.com/repos/ovj/hoodie/issues/events{/number}","events_url":"https://api.github.com/repos/ovj/hoodie/events","assignees_url":"https://api.github.com/repos/ovj/hoodie/assignees{/user}","branches_url":"https://api.github.com/repos/ovj/hoodie/branches{/branch}","tags_url":"https://api.github.com/repos/ovj/hoodie/tags","blobs_url":"https://api.github.com/repos/ovj/hoodie/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ovj/hoodie/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ovj/hoodie/git/refs{/sha}","trees_url":"https://api.github.com/repos/ovj/hoodie/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ovj/hoodie/statuses/{sha}","languages_url":"https://api.github.com/repos/ovj/hoodie/languages","stargazers_url":"https://api.github.com/repos/ovj/hoodie/stargazers","contributors_url":"https://api.github.com/repos/ovj/hoodie/contributors","subscribers_url":"https://api.github.com/repos/ovj/hoodie/subscribers","subscription_url":"https://api.github.com/repos/ovj/hoodie/subscription","commits_url":"https://api.github.com/repos/ovj/hoodie/commits{/sha}","git_commits_url":"https://api.github.com/repos/ovj/hoodie/git/commits{/sha}","comments_url":"https://api.github.com/repos/ovj/hoodie/comments{/number}","issue_comment_url":"https://api.github.com/repos/ovj/hoodie/issues/comments{/number}","contents_url":"https://api.github.com/repos/ovj/hoodie/contents/{+path}","compare_url":"https://api.github.com/repos/ovj/hoodie/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ovj/hoodie/merges","archive_url":"https://api.github.com/repos/ovj/hoodie/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ovj/hoodie/downloads","issues_url":"https://api.github.com/repos/ovj/hoodie/issues{/number}","pulls_url":"https://api.github.com/repos/ovj/hoodie/pulls{/number}","milestones_url":"https://api.github.com/repos/ovj/hoodie/milestones{/number}","notifications_url":"https://api.github.com/repos/ovj/hoodie/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ovj/hoodie/labels{/name}","releases_url":"https://api.github.com/repos/ovj/hoodie/releases{/id}","deployments_url":"https://api.github.com/repos/ovj/hoodie/deployments","created_at":"2017-03-10T19:44:28Z","updated_at":"2017-03-10T19:44:30Z","pushed_at":"2018-03-10T10:46:51Z","git_url":"git://github.com/ovj/hoodie.git","ssh_url":"git@github.com:ovj/hoodie.git","clone_url":"https://github.com/ovj/hoodie.git","svn_url":"https://github.com/ovj/hoodie","homepage":"https://uber.github.io/hoodie","size":4866,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":null,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"uber:master","ref":"master","sha":"a1c0d0dbad5f9bc3e020a83953668d00bfa574ef","user":{"login":"uber","id":538264,"avatar_url":"https://avatars3.githubusercontent.com/u/538264?v=4","gravatar_id":"","url":"https://api.github.com/users/uber","html_url":"https://github.com/uber","followers_url":"https://api.github.com/users/uber/followers","following_url":"https://api.github.com/users/uber/following{/other_user}","gists_url":"https://api.github.com/users/uber/gists{/gist_id}","starred_url":"https://api.github.com/users/uber/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/uber/subscriptions","organizations_url":"https://api.github.com/users/uber/orgs","repos_url":"https://api.github.com/users/uber/repos","events_url":"https://api.github.com/users/uber/events{/privacy}","received_events_url":"https://api.github.com/users/uber/received_events","type":"Organization","site_admin":false},"repo":{"id":76474200,"name":"hudi","full_name":"uber/hudi","owner":{"login":"uber","id":538264,"avatar_url":"https://avatars3.githubusercontent.com/u/538264?v=4","gravatar_id":"","url":"https://api.github.com/users/uber","html_url":"https://github.com/uber","followers_url":"https://api.github.com/users/uber/followers","following_url":"https://api.github.com/users/uber/following{/other_user}","gists_url":"https://api.github.com/users/uber/gists{/gist_id}","starred_url":"https://api.github.com/users/uber/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/uber/subscriptions","organizations_url":"https://api.github.com/users/uber/orgs","repos_url":"https://api.github.com/users/uber/repos","events_url":"https://api.github.com/users/uber/events{/privacy}","received_events_url":"https://api.github.com/users/uber/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/uber/hudi","description":"Spark Library for Hadoop Upserts And Incrementals","fork":false,"url":"https://api.github.com/repos/uber/hudi","forks_url":"https://api.github.com/repos/uber/hudi/forks","keys_url":"https://api.github.com/repos/uber/hudi/keys{/key_id}","collaborators_url":"https://api.github.com/repos/uber/hudi/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/uber/hudi/teams","hooks_url":"https://api.github.com/repos/uber/hudi/hooks","issue_events_url":"https://api.github.com/repos/uber/hudi/issues/events{/number}","events_url":"https://api.github.com/repos/uber/hudi/events","assignees_url":"https://api.github.com/repos/uber/hudi/assignees{/user}","branches_url":"https://api.github.com/repos/uber/hudi/branches{/branch}","tags_url":"https://api.github.com/repos/uber/hudi/tags","blobs_url":"https://api.github.com/repos/uber/hudi/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/uber/hudi/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/uber/hudi/git/refs{/sha}","trees_url":"https://api.github.com/repos/uber/hudi/git/trees{/sha}","statuses_url":"https://api.github.com/repos/uber/hudi/statuses/{sha}","languages_url":"https://api.github.com/repos/uber/hudi/languages","stargazers_url":"https://api.github.com/repos/uber/hudi/stargazers","contributors_url":"https://api.github.com/repos/uber/hudi/contributors","subscribers_url":"https://api.github.com/repos/uber/hudi/subscribers","subscription_url":"https://api.github.com/repos/uber/hudi/subscription","commits_url":"https://api.github.com/repos/uber/hudi/commits{/sha}","git_commits_url":"https://api.github.com/repos/uber/hudi/git/commits{/sha}","comments_url":"https://api.github.com/repos/uber/hudi/comments{/number}","issue_comment_url":"https://api.github.com/repos/uber/hudi/issues/comments{/number}","contents_url":"https://api.github.com/repos/uber/hudi/contents/{+path}","compare_url":"https://api.github.com/repos/uber/hudi/compare/{base}...{head}","merges_url":"https://api.github.com/repos/uber/hudi/merges","archive_url":"https://api.github.com/repos/uber/hudi/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/uber/hudi/downloads","issues_url":"https://api.github.com/repos/uber/hudi/issues{/number}","pulls_url":"https://api.github.com/repos/uber/hudi/pulls{/number}","milestones_url":"https://api.github.com/repos/uber/hudi/milestones{/number}","notifications_url":"https://api.github.com/repos/uber/hudi/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/uber/hudi/labels{/name}","releases_url":"https://api.github.com/repos/uber/hudi/releases{/id}","deployments_url":"https://api.github.com/repos/uber/hudi/deployments","created_at":"2016-12-14T15:53:41Z","updated_at":"2018-03-12T19:37:05Z","pushed_at":"2018-03-12T18:47:56Z","git_url":"git://github.com/uber/hudi.git","ssh_url":"git@github.com:uber/hudi.git","clone_url":"https://github.com/uber/hudi.git","svn_url":"https://github.com/uber/hudi","homepage":"https://uber.github.io/hudi","size":4901,"stargazers_count":136,"watchers_count":136,"language":"Java","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":43,"mirror_url":null,"archived":false,"open_issues_count":108,"license":{"key":"other","name":"Other","spdx_id":null,"url":null},"forks":43,"open_issues":108,"watchers":136,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/uber/hudi/pulls/294"},"html":{"href":"https://github.com/uber/hudi/pull/294"},"issue":{"href":"https://api.github.com/repos/uber/hudi/issues/294"},"comments":{"href":"https://api.github.com/repos/uber/hudi/issues/294/comments"},"review_comments":{"href":"https://api.github.com/repos/uber/hudi/pulls/294/comments"},"review_comment":{"href":"https://api.github.com/repos/uber/hudi/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/uber/hudi/pulls/294/commits"},"statuses":{"href":"https://api.github.com/repos/uber/hudi/statuses/c7b2d786552aeca9649288900deff37b6791aaf4"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"ovj"}}