{"action":"created","comment":{"url":"https://api.github.com/repos/apache/hive/pulls/comments/189424949","pull_request_review_id":121609086,"id":189424949,"diff_hunk":"@@ -18,21 +18,129 @@\n  */\n package org.apache.hadoop.hive.ql.parse.repl.dump.events;\n \n+import com.google.common.collect.Lists;\n+import org.apache.hadoop.fs.Path;\n+import org.apache.hadoop.hive.conf.HiveConf;\n+import org.apache.hadoop.hive.metastore.HiveMetaStore;\n import org.apache.hadoop.hive.metastore.api.NotificationEvent;\n+import org.apache.hadoop.hive.metastore.api.WriteEventInfo;\n+import org.apache.hadoop.hive.metastore.messaging.CommitTxnMessage;\n+import org.apache.hadoop.hive.ql.metadata.HiveUtils;\n+import org.apache.hadoop.hive.ql.metadata.Partition;\n+import org.apache.hadoop.hive.ql.parse.EximUtil;\n+import org.apache.hadoop.hive.ql.parse.SemanticException;\n import org.apache.hadoop.hive.ql.parse.repl.DumpType;\n import org.apache.hadoop.hive.ql.parse.repl.load.DumpMetaData;\n+import org.apache.hadoop.fs.FileSystem;\n+import java.io.BufferedWriter;\n+import java.io.IOException;\n+import java.io.OutputStreamWriter;\n+import java.util.ArrayList;\n+import java.util.List;\n \n class CommitTxnHandler extends AbstractEventHandler {\n \n   CommitTxnHandler(NotificationEvent event) {\n     super(event);\n   }\n \n+  private BufferedWriter writer(Context withinContext, Path dataPath) throws IOException {\n+    Path filesPath = new Path(dataPath, EximUtil.FILES_NAME);\n+    FileSystem fs = dataPath.getFileSystem(withinContext.hiveConf);\n+    return new BufferedWriter(new OutputStreamWriter(fs.create(filesPath)));\n+  }\n+\n+  private void writeDumpFiles(Context withinContext, Iterable<String> files, Path dataPath) throws IOException {\n+    // encoded filename/checksum of files, write into _files\n+    try (BufferedWriter fileListWriter = writer(withinContext, dataPath)) {\n+      for (String file : files) {\n+        fileListWriter.write(file + \"\\n\");\n+      }\n+    }\n+  }\n+\n+  private void createDumpFile(Context withinContext, org.apache.hadoop.hive.ql.metadata.Table qlMdTable,\n+                  List<Partition> qlPtns, List<List<String>> fileListArray) throws IOException, SemanticException {\n+    if (fileListArray == null && fileListArray.isEmpty()) {\n+      return;\n+    }\n+\n+    Path metaDataPath = new Path(withinContext.eventRoot, EximUtil.METADATA_NAME);\n+    withinContext.replicationSpec.setIsReplace(true);\n+    EximUtil.createExportDump(metaDataPath.getFileSystem(withinContext.hiveConf), metaDataPath,\n+            qlMdTable, qlPtns,\n+            withinContext.replicationSpec,\n+            withinContext.hiveConf);\n+\n+    if ((null == qlPtns) || qlPtns.isEmpty()) {\n+      Path dataPath = new Path(withinContext.eventRoot, EximUtil.DATA_PATH_NAME);\n+      writeDumpFiles(withinContext, fileListArray.get(0), dataPath);\n+    } else {\n+      for (int idx = 0; idx < qlPtns.size(); idx++) {\n+        Path dataPath = new Path(withinContext.eventRoot, qlPtns.get(idx).getName());\n+        writeDumpFiles(withinContext, fileListArray.get(idx), dataPath);\n+      }\n+    }\n+  }\n+\n   @Override\n   public void handle(Context withinContext) throws Exception {\n     LOG.info(\"Processing#{} COMMIT_TXN message : {}\", fromEventId(), event.getMessage());\n+    String payload = event.getMessage();\n+\n+    if (!withinContext.hiveConf.getBoolVar(HiveConf.ConfVars.REPL_DUMP_METADATA_ONLY)) {\n+      CommitTxnMessage commitTxnMessage = deserializer.getCommitTxnMessage(event.getMessage());\n+\n+      List<WriteEventInfo> writeEventInfoList = HiveMetaStore.HMSHandler.getMSForConf(withinContext.hiveConf).\n+              getAllWriteEventInfo(commitTxnMessage.getTxnId(), withinContext.dbName, withinContext.tableName);\n+      int numEntry = (writeEventInfoList != null ? writeEventInfoList.size() : 0);\n+      if (numEntry != 0) {\n+        commitTxnMessage.addWriteEventInfo(writeEventInfoList);\n+        payload = commitTxnMessage.toString();\n+        LOG.debug(\"payload for commit txn event : \" + payload);\n+      }\n+\n+      Context  context = null;\n+      org.apache.hadoop.hive.ql.metadata.Table qlMdTablePrev = null;\n+      org.apache.hadoop.hive.ql.metadata.Table qlMdTable = null;\n+      List<Partition> qlPtns = new ArrayList<>();\n+      List<List<String>> filesTobeAdded = new ArrayList<>();\n+\n+      for (int idx = 0; idx < numEntry; idx++) {\n+        qlMdTable = new org.apache.hadoop.hive.ql.metadata.Table(commitTxnMessage.getTableObj(idx));\n+        Path newPath = new Path(withinContext.eventRoot,\n+                HiveUtils.getDumpPath(qlMdTable.getDbName(), qlMdTable.getTableName()));\n+        context = new Context(newPath, withinContext.cmRoot,\n+                withinContext.db, withinContext.hiveConf, withinContext.replicationSpec,\n+                withinContext.dbName, withinContext.tableName);\n+        if (qlMdTablePrev == null) {\n+          qlMdTablePrev = qlMdTable;\n+        }\n+\n+        // one dump directory per table\n+        if (!qlMdTablePrev.getTableName().equals(qlMdTable.getTableName())) {\n+          createDumpFile(context, qlMdTablePrev, qlPtns, filesTobeAdded);\n+          qlPtns = new ArrayList<>();\n+          filesTobeAdded = new ArrayList<>();\n+          qlMdTablePrev = qlMdTable;","path":"ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/events/CommitTxnHandler.java","position":108,"original_position":108,"commit_id":"eb260e90835035ead56141a8415fb8db8ea87160","original_commit_id":"eb260e90835035ead56141a8415fb8db8ea87160","user":{"login":"sankarh","id":25001186,"avatar_url":"https://avatars2.githubusercontent.com/u/25001186?v=4","gravatar_id":"","url":"https://api.github.com/users/sankarh","html_url":"https://github.com/sankarh","followers_url":"https://api.github.com/users/sankarh/followers","following_url":"https://api.github.com/users/sankarh/following{/other_user}","gists_url":"https://api.github.com/users/sankarh/gists{/gist_id}","starred_url":"https://api.github.com/users/sankarh/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sankarh/subscriptions","organizations_url":"https://api.github.com/users/sankarh/orgs","repos_url":"https://api.github.com/users/sankarh/repos","events_url":"https://api.github.com/users/sankarh/events{/privacy}","received_events_url":"https://api.github.com/users/sankarh/received_events","type":"User","site_admin":false},"body":"Ok","created_at":"2018-05-19T04:45:08Z","updated_at":"2018-05-19T04:45:08Z","html_url":"https://github.com/apache/hive/pull/335#discussion_r189424949","pull_request_url":"https://api.github.com/repos/apache/hive/pulls/335","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/apache/hive/pulls/comments/189424949"},"html":{"href":"https://github.com/apache/hive/pull/335#discussion_r189424949"},"pull_request":{"href":"https://api.github.com/repos/apache/hive/pulls/335"}},"in_reply_to_id":187605064},"pull_request":{"url":"https://api.github.com/repos/apache/hive/pulls/335","id":183284048,"html_url":"https://github.com/apache/hive/pull/335","diff_url":"https://github.com/apache/hive/pull/335.diff","patch_url":"https://github.com/apache/hive/pull/335.patch","issue_url":"https://api.github.com/repos/apache/hive/issues/335","number":335,"state":"open","locked":false,"title":"HIVE-19267 : Create/Replicate ACID Write event","user":{"login":"maheshk114","id":19660171,"avatar_url":"https://avatars3.githubusercontent.com/u/19660171?v=4","gravatar_id":"","url":"https://api.github.com/users/maheshk114","html_url":"https://github.com/maheshk114","followers_url":"https://api.github.com/users/maheshk114/followers","following_url":"https://api.github.com/users/maheshk114/following{/other_user}","gists_url":"https://api.github.com/users/maheshk114/gists{/gist_id}","starred_url":"https://api.github.com/users/maheshk114/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/maheshk114/subscriptions","organizations_url":"https://api.github.com/users/maheshk114/orgs","repos_url":"https://api.github.com/users/maheshk114/repos","events_url":"https://api.github.com/users/maheshk114/events{/privacy}","received_events_url":"https://api.github.com/users/maheshk114/received_events","type":"User","site_admin":false},"body":"Replicate ACID write Events\r\nCreate new EVENT_WRITE event with related message format to log the write operations with in a txn along with data associated.\r\nLog this event when perform any writes (insert into, insert overwrite, load table, delete, update, merge, truncate) on table/partition.\r\nIf a single MERGE/UPDATE/INSERT/DELETE statement operates on multiple partitions, then need to log one event per partition.\r\nDbNotificationListener should log this type of event to special metastore table named \"MTxnWriteNotificationLog\".\r\nThis table should maintain a map of txn ID against list of tables/partitions written by given txn.\r\nThe entry for a given txn should be removed by the cleaner thread that removes the expired events from EventNotificationTable.\r\nReplicate Commit Txn operation (with writes)\r\nAdd new EVENT_COMMIT_TXN to log the metadata/data of all tables/partitions modified within the txn.\r\n\r\nSource warehouse:\r\n\r\nThis event should read the EVENT_WRITEs from \"MTxnWriteNotificationLog\" metastore table to consolidate the list of tables/partitions modified within this txn scope.\r\nBased on the list of tables/partitions modified and table Write ID, need to compute the list of delta files added by this txn.\r\nRepl dump should read this message and dump the metadata and delta files list.\r\nTarget warehouse:\r\n\r\nEnsure snapshot isolation at target for on-going read txns which shouldn't view the data replicated from committed txn. (Ensured with open and allocate write ID events).","created_at":"2018-04-22T17:23:19Z","updated_at":"2018-05-19T04:45:08Z","closed_at":null,"merged_at":null,"merge_commit_sha":"6ed989e9f36332c17993eebc5583d376808539cf","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/apache/hive/pulls/335/commits","review_comments_url":"https://api.github.com/repos/apache/hive/pulls/335/comments","review_comment_url":"https://api.github.com/repos/apache/hive/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/hive/issues/335/comments","statuses_url":"https://api.github.com/repos/apache/hive/statuses/eb260e90835035ead56141a8415fb8db8ea87160","head":{"label":"maheshk114:BUG-92690","ref":"BUG-92690","sha":"eb260e90835035ead56141a8415fb8db8ea87160","user":{"login":"maheshk114","id":19660171,"avatar_url":"https://avatars3.githubusercontent.com/u/19660171?v=4","gravatar_id":"","url":"https://api.github.com/users/maheshk114","html_url":"https://github.com/maheshk114","followers_url":"https://api.github.com/users/maheshk114/followers","following_url":"https://api.github.com/users/maheshk114/following{/other_user}","gists_url":"https://api.github.com/users/maheshk114/gists{/gist_id}","starred_url":"https://api.github.com/users/maheshk114/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/maheshk114/subscriptions","organizations_url":"https://api.github.com/users/maheshk114/orgs","repos_url":"https://api.github.com/users/maheshk114/repos","events_url":"https://api.github.com/users/maheshk114/events{/privacy}","received_events_url":"https://api.github.com/users/maheshk114/received_events","type":"User","site_admin":false},"repo":{"id":118883759,"name":"hive","full_name":"maheshk114/hive","owner":{"login":"maheshk114","id":19660171,"avatar_url":"https://avatars3.githubusercontent.com/u/19660171?v=4","gravatar_id":"","url":"https://api.github.com/users/maheshk114","html_url":"https://github.com/maheshk114","followers_url":"https://api.github.com/users/maheshk114/followers","following_url":"https://api.github.com/users/maheshk114/following{/other_user}","gists_url":"https://api.github.com/users/maheshk114/gists{/gist_id}","starred_url":"https://api.github.com/users/maheshk114/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/maheshk114/subscriptions","organizations_url":"https://api.github.com/users/maheshk114/orgs","repos_url":"https://api.github.com/users/maheshk114/repos","events_url":"https://api.github.com/users/maheshk114/events{/privacy}","received_events_url":"https://api.github.com/users/maheshk114/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/maheshk114/hive","description":"Mirror of Apache Hive","fork":true,"url":"https://api.github.com/repos/maheshk114/hive","forks_url":"https://api.github.com/repos/maheshk114/hive/forks","keys_url":"https://api.github.com/repos/maheshk114/hive/keys{/key_id}","collaborators_url":"https://api.github.com/repos/maheshk114/hive/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/maheshk114/hive/teams","hooks_url":"https://api.github.com/repos/maheshk114/hive/hooks","issue_events_url":"https://api.github.com/repos/maheshk114/hive/issues/events{/number}","events_url":"https://api.github.com/repos/maheshk114/hive/events","assignees_url":"https://api.github.com/repos/maheshk114/hive/assignees{/user}","branches_url":"https://api.github.com/repos/maheshk114/hive/branches{/branch}","tags_url":"https://api.github.com/repos/maheshk114/hive/tags","blobs_url":"https://api.github.com/repos/maheshk114/hive/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/maheshk114/hive/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/maheshk114/hive/git/refs{/sha}","trees_url":"https://api.github.com/repos/maheshk114/hive/git/trees{/sha}","statuses_url":"https://api.github.com/repos/maheshk114/hive/statuses/{sha}","languages_url":"https://api.github.com/repos/maheshk114/hive/languages","stargazers_url":"https://api.github.com/repos/maheshk114/hive/stargazers","contributors_url":"https://api.github.com/repos/maheshk114/hive/contributors","subscribers_url":"https://api.github.com/repos/maheshk114/hive/subscribers","subscription_url":"https://api.github.com/repos/maheshk114/hive/subscription","commits_url":"https://api.github.com/repos/maheshk114/hive/commits{/sha}","git_commits_url":"https://api.github.com/repos/maheshk114/hive/git/commits{/sha}","comments_url":"https://api.github.com/repos/maheshk114/hive/comments{/number}","issue_comment_url":"https://api.github.com/repos/maheshk114/hive/issues/comments{/number}","contents_url":"https://api.github.com/repos/maheshk114/hive/contents/{+path}","compare_url":"https://api.github.com/repos/maheshk114/hive/compare/{base}...{head}","merges_url":"https://api.github.com/repos/maheshk114/hive/merges","archive_url":"https://api.github.com/repos/maheshk114/hive/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/maheshk114/hive/downloads","issues_url":"https://api.github.com/repos/maheshk114/hive/issues{/number}","pulls_url":"https://api.github.com/repos/maheshk114/hive/pulls{/number}","milestones_url":"https://api.github.com/repos/maheshk114/hive/milestones{/number}","notifications_url":"https://api.github.com/repos/maheshk114/hive/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/maheshk114/hive/labels{/name}","releases_url":"https://api.github.com/repos/maheshk114/hive/releases{/id}","deployments_url":"https://api.github.com/repos/maheshk114/hive/deployments","created_at":"2018-01-25T08:17:10Z","updated_at":"2018-01-25T08:17:57Z","pushed_at":"2018-05-16T04:43:52Z","git_url":"git://github.com/maheshk114/hive.git","ssh_url":"git@github.com:maheshk114/hive.git","clone_url":"https://github.com/maheshk114/hive.git","svn_url":"https://github.com/maheshk114/hive","homepage":null,"size":395157,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0"},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"1cd5274c58eb53cae8f38d172bbbc1517f199251","user":{"login":"apache","id":47359,"avatar_url":"https://avatars0.githubusercontent.com/u/47359?v=4","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":206444,"name":"hive","full_name":"apache/hive","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars0.githubusercontent.com/u/47359?v=4","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/hive","description":"Mirror of Apache Hive","fork":false,"url":"https://api.github.com/repos/apache/hive","forks_url":"https://api.github.com/repos/apache/hive/forks","keys_url":"https://api.github.com/repos/apache/hive/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/hive/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/hive/teams","hooks_url":"https://api.github.com/repos/apache/hive/hooks","issue_events_url":"https://api.github.com/repos/apache/hive/issues/events{/number}","events_url":"https://api.github.com/repos/apache/hive/events","assignees_url":"https://api.github.com/repos/apache/hive/assignees{/user}","branches_url":"https://api.github.com/repos/apache/hive/branches{/branch}","tags_url":"https://api.github.com/repos/apache/hive/tags","blobs_url":"https://api.github.com/repos/apache/hive/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/hive/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/hive/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/hive/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/hive/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/hive/languages","stargazers_url":"https://api.github.com/repos/apache/hive/stargazers","contributors_url":"https://api.github.com/repos/apache/hive/contributors","subscribers_url":"https://api.github.com/repos/apache/hive/subscribers","subscription_url":"https://api.github.com/repos/apache/hive/subscription","commits_url":"https://api.github.com/repos/apache/hive/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/hive/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/hive/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/hive/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/hive/contents/{+path}","compare_url":"https://api.github.com/repos/apache/hive/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/hive/merges","archive_url":"https://api.github.com/repos/apache/hive/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/hive/downloads","issues_url":"https://api.github.com/repos/apache/hive/issues{/number}","pulls_url":"https://api.github.com/repos/apache/hive/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/hive/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/hive/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/hive/labels{/name}","releases_url":"https://api.github.com/repos/apache/hive/releases{/id}","deployments_url":"https://api.github.com/repos/apache/hive/deployments","created_at":"2009-05-21T02:31:01Z","updated_at":"2018-05-19T03:44:19Z","pushed_at":"2018-05-19T03:43:58Z","git_url":"git://github.com/apache/hive.git","ssh_url":"git@github.com:apache/hive.git","clone_url":"https://github.com/apache/hive.git","svn_url":"https://github.com/apache/hive","homepage":null,"size":394356,"stargazers_count":1873,"watchers_count":1873,"language":"Java","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":1960,"mirror_url":null,"archived":false,"open_issues_count":141,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0"},"forks":1960,"open_issues":141,"watchers":1873,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/hive/pulls/335"},"html":{"href":"https://github.com/apache/hive/pull/335"},"issue":{"href":"https://api.github.com/repos/apache/hive/issues/335"},"comments":{"href":"https://api.github.com/repos/apache/hive/issues/335/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/hive/pulls/335/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/hive/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/hive/pulls/335/commits"},"statuses":{"href":"https://api.github.com/repos/apache/hive/statuses/eb260e90835035ead56141a8415fb8db8ea87160"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"sankarh"}}