{"action":"created","comment":{"url":"https://api.github.com/repos/linkedin/gobblin/pulls/comments/57806183","id":57806183,"diff_hunk":"@@ -0,0 +1,84 @@\n+/*\n+ * Copyright (C) 2014-2016 LinkedIn Corp. All rights reserved.\n+ *\n+ * Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use\n+ * this file except in compliance with the License. You may obtain a copy of the\n+ * License at  http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software distributed\n+ * under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR\n+ * CONDITIONS OF ANY KIND, either express or implied.\n+ */\n+package gobblin.audit.values.auditor;\n+\n+import java.io.Closeable;\n+import java.io.IOException;\n+\n+import lombok.AllArgsConstructor;\n+import lombok.Getter;\n+\n+import org.apache.avro.generic.GenericRecord;\n+\n+import com.typesafe.config.Config;\n+\n+import gobblin.audit.values.policy.column.ColumnProjectionPolicy;\n+import gobblin.audit.values.policy.column.DefaultColumnProjectionPolicyFactory;\n+import gobblin.audit.values.policy.row.DefaultRowSelectionPolicyFactory;\n+import gobblin.audit.values.policy.row.RowSelectionPolicy;\n+import gobblin.audit.values.sink.AuditSink;\n+import gobblin.audit.values.sink.DefaultAuditSinkFactory;\n+\n+\n+/**\n+ * The class that implements value based auditing. The class captures the values of certain\n+ * columns from the rows in the dataset using the {@link ColumnProjectionPolicy}.\n+ * This is done for every row or for a sample of the rows as defined by the {@link RowSelectionPolicy}.\n+ * The selected rows are then written to the {@link AuditSink}\n+ *\n+ * {@link ValueAuditGenerator#audit(GenericRecord)} is the method that audits an inputRecord.\n+ */\n+@AllArgsConstructor\n+@Getter\n+public class ValueAuditGenerator implements Closeable {\n+\n+  private final ColumnProjectionPolicy columnProjectionPolicy;\n+  private final RowSelectionPolicy rowSelectionPolicy;\n+  private final AuditSink auditSink;\n+\n+  /**\n+   * Factory method to create a new {@link ValueAuditGenerator}\n+   * @param config job configs\n+   * @param runtimeAuditMetadata is used to pass the table specific runtime information like tablename, databaseName, snapshotName etc.\n+   *      See {@link ValueAuditRuntimeMetadata}\n+   * @return a new {@link ValueAuditGenerator}\n+   */\n+  public static ValueAuditGenerator create(Config config, ValueAuditRuntimeMetadata runtimeAuditMetadata) {\n+    ColumnProjectionPolicy columnProjectionPolicy = DefaultColumnProjectionPolicyFactory.getInstance().create(\n+        config.getConfig(DefaultColumnProjectionPolicyFactory.CONFIG_SCOPE_PREFIX),runtimeAuditMetadata.getTableMetadata());","path":"gobblin-audit/src/main/java/gobblin/audit/values/auditor/ValueAuditGenerator.java","position":57,"original_position":57,"commit_id":"4de6982a26277c7db5f10d87c646bc4a590871c6","original_commit_id":"4de6982a26277c7db5f10d87c646bc4a590871c6","user":{"login":"chavdar","id":928795,"avatar_url":"https://avatars.githubusercontent.com/u/928795?v=3","gravatar_id":"","url":"https://api.github.com/users/chavdar","html_url":"https://github.com/chavdar","followers_url":"https://api.github.com/users/chavdar/followers","following_url":"https://api.github.com/users/chavdar/following{/other_user}","gists_url":"https://api.github.com/users/chavdar/gists{/gist_id}","starred_url":"https://api.github.com/users/chavdar/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/chavdar/subscriptions","organizations_url":"https://api.github.com/users/chavdar/orgs","repos_url":"https://api.github.com/users/chavdar/repos","events_url":"https://api.github.com/users/chavdar/events{/privacy}","received_events_url":"https://api.github.com/users/chavdar/received_events","type":"User","site_admin":false},"body":"minor: The scope prefix in the next three cases should be declared in this class not the factories as a different class may want to use a different scope.","created_at":"2016-03-29T21:43:32Z","updated_at":"2016-03-29T21:43:32Z","html_url":"https://github.com/linkedin/gobblin/pull/869#discussion_r57806183","pull_request_url":"https://api.github.com/repos/linkedin/gobblin/pulls/869","_links":{"self":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/comments/57806183"},"html":{"href":"https://github.com/linkedin/gobblin/pull/869#discussion_r57806183"},"pull_request":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/869"}}},"pull_request":{"url":"https://api.github.com/repos/linkedin/gobblin/pulls/869","id":63446510,"html_url":"https://github.com/linkedin/gobblin/pull/869","diff_url":"https://github.com/linkedin/gobblin/pull/869.diff","patch_url":"https://github.com/linkedin/gobblin/pull/869.patch","issue_url":"https://api.github.com/repos/linkedin/gobblin/issues/869","number":869,"state":"open","locked":false,"title":"Base interfaces for value auditing","user":{"login":"pcadabam","id":1166372,"avatar_url":"https://avatars.githubusercontent.com/u/1166372?v=3","gravatar_id":"","url":"https://api.github.com/users/pcadabam","html_url":"https://github.com/pcadabam","followers_url":"https://api.github.com/users/pcadabam/followers","following_url":"https://api.github.com/users/pcadabam/following{/other_user}","gists_url":"https://api.github.com/users/pcadabam/gists{/gist_id}","starred_url":"https://api.github.com/users/pcadabam/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pcadabam/subscriptions","organizations_url":"https://api.github.com/users/pcadabam/orgs","repos_url":"https://api.github.com/users/pcadabam/repos","events_url":"https://api.github.com/users/pcadabam/events{/privacy}","received_events_url":"https://api.github.com/users/pcadabam/received_events","type":"User","site_admin":false},"body":"Added base interfaces for value auditing. The pr mainly has interfaces for \r\n- `RowSelectionPolicy` - An interface to decide if a row needs to be audited\r\n- `ColomnProjectionPolicy` - An interface that projects certain columns/fields of an input {@link GenericRecord} to generate a new {@link GenericRecord} that can be audited.\r\n- `AuditSink` - An interface for persisting value audits\r\n- `ValueAuditGenerator` - The class that implements value based auditing. The class captures the values of certain columns from the rows in the dataset using the {@link ColumnProjectionPolicy}. This is done for every row or for a sample of the rows as defined by the {@link RowSelectionPolicy}\r\n- The module that drives auditing will create an instance of `ValueAuditGenerator` for each avro file being written and called `ValueAuditGenerator.audit(GenericRecord)`\r\n\r\n**The complete javadoc has been generated and hosted through github pages [here] (http://pcadabam.github.io/gobblin-audit-docs/) to easily navigate through the detailed documentation**\r\n\r\nI am adding some unit tests while this is being reviewed. \r\n@chavdar  can you review?","created_at":"2016-03-18T22:49:38Z","updated_at":"2016-03-29T21:43:32Z","closed_at":null,"merged_at":null,"merge_commit_sha":"439acc1a77208675df06d2f02b201d4383fff369","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/linkedin/gobblin/pulls/869/commits","review_comments_url":"https://api.github.com/repos/linkedin/gobblin/pulls/869/comments","review_comment_url":"https://api.github.com/repos/linkedin/gobblin/pulls/comments{/number}","comments_url":"https://api.github.com/repos/linkedin/gobblin/issues/869/comments","statuses_url":"https://api.github.com/repos/linkedin/gobblin/statuses/4de6982a26277c7db5f10d87c646bc4a590871c6","head":{"label":"pcadabam:auditor","ref":"auditor","sha":"4de6982a26277c7db5f10d87c646bc4a590871c6","user":{"login":"pcadabam","id":1166372,"avatar_url":"https://avatars.githubusercontent.com/u/1166372?v=3","gravatar_id":"","url":"https://api.github.com/users/pcadabam","html_url":"https://github.com/pcadabam","followers_url":"https://api.github.com/users/pcadabam/followers","following_url":"https://api.github.com/users/pcadabam/following{/other_user}","gists_url":"https://api.github.com/users/pcadabam/gists{/gist_id}","starred_url":"https://api.github.com/users/pcadabam/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pcadabam/subscriptions","organizations_url":"https://api.github.com/users/pcadabam/orgs","repos_url":"https://api.github.com/users/pcadabam/repos","events_url":"https://api.github.com/users/pcadabam/events{/privacy}","received_events_url":"https://api.github.com/users/pcadabam/received_events","type":"User","site_admin":false},"repo":{"id":36956709,"name":"gobblin","full_name":"pcadabam/gobblin","owner":{"login":"pcadabam","id":1166372,"avatar_url":"https://avatars.githubusercontent.com/u/1166372?v=3","gravatar_id":"","url":"https://api.github.com/users/pcadabam","html_url":"https://github.com/pcadabam","followers_url":"https://api.github.com/users/pcadabam/followers","following_url":"https://api.github.com/users/pcadabam/following{/other_user}","gists_url":"https://api.github.com/users/pcadabam/gists{/gist_id}","starred_url":"https://api.github.com/users/pcadabam/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pcadabam/subscriptions","organizations_url":"https://api.github.com/users/pcadabam/orgs","repos_url":"https://api.github.com/users/pcadabam/repos","events_url":"https://api.github.com/users/pcadabam/events{/privacy}","received_events_url":"https://api.github.com/users/pcadabam/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/pcadabam/gobblin","description":"","fork":true,"url":"https://api.github.com/repos/pcadabam/gobblin","forks_url":"https://api.github.com/repos/pcadabam/gobblin/forks","keys_url":"https://api.github.com/repos/pcadabam/gobblin/keys{/key_id}","collaborators_url":"https://api.github.com/repos/pcadabam/gobblin/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/pcadabam/gobblin/teams","hooks_url":"https://api.github.com/repos/pcadabam/gobblin/hooks","issue_events_url":"https://api.github.com/repos/pcadabam/gobblin/issues/events{/number}","events_url":"https://api.github.com/repos/pcadabam/gobblin/events","assignees_url":"https://api.github.com/repos/pcadabam/gobblin/assignees{/user}","branches_url":"https://api.github.com/repos/pcadabam/gobblin/branches{/branch}","tags_url":"https://api.github.com/repos/pcadabam/gobblin/tags","blobs_url":"https://api.github.com/repos/pcadabam/gobblin/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/pcadabam/gobblin/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/pcadabam/gobblin/git/refs{/sha}","trees_url":"https://api.github.com/repos/pcadabam/gobblin/git/trees{/sha}","statuses_url":"https://api.github.com/repos/pcadabam/gobblin/statuses/{sha}","languages_url":"https://api.github.com/repos/pcadabam/gobblin/languages","stargazers_url":"https://api.github.com/repos/pcadabam/gobblin/stargazers","contributors_url":"https://api.github.com/repos/pcadabam/gobblin/contributors","subscribers_url":"https://api.github.com/repos/pcadabam/gobblin/subscribers","subscription_url":"https://api.github.com/repos/pcadabam/gobblin/subscription","commits_url":"https://api.github.com/repos/pcadabam/gobblin/commits{/sha}","git_commits_url":"https://api.github.com/repos/pcadabam/gobblin/git/commits{/sha}","comments_url":"https://api.github.com/repos/pcadabam/gobblin/comments{/number}","issue_comment_url":"https://api.github.com/repos/pcadabam/gobblin/issues/comments{/number}","contents_url":"https://api.github.com/repos/pcadabam/gobblin/contents/{+path}","compare_url":"https://api.github.com/repos/pcadabam/gobblin/compare/{base}...{head}","merges_url":"https://api.github.com/repos/pcadabam/gobblin/merges","archive_url":"https://api.github.com/repos/pcadabam/gobblin/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/pcadabam/gobblin/downloads","issues_url":"https://api.github.com/repos/pcadabam/gobblin/issues{/number}","pulls_url":"https://api.github.com/repos/pcadabam/gobblin/pulls{/number}","milestones_url":"https://api.github.com/repos/pcadabam/gobblin/milestones{/number}","notifications_url":"https://api.github.com/repos/pcadabam/gobblin/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/pcadabam/gobblin/labels{/name}","releases_url":"https://api.github.com/repos/pcadabam/gobblin/releases{/id}","deployments_url":"https://api.github.com/repos/pcadabam/gobblin/deployments","created_at":"2015-06-05T21:39:40Z","updated_at":"2016-01-28T20:06:08Z","pushed_at":"2016-03-29T20:02:46Z","git_url":"git://github.com/pcadabam/gobblin.git","ssh_url":"git@github.com:pcadabam/gobblin.git","clone_url":"https://github.com/pcadabam/gobblin.git","svn_url":"https://github.com/pcadabam/gobblin","homepage":null,"size":85162,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"linkedin:master","ref":"master","sha":"040e4ce10ee903ec0c349f35ae2d603f35bb18b6","user":{"login":"linkedin","id":357098,"avatar_url":"https://avatars.githubusercontent.com/u/357098?v=3","gravatar_id":"","url":"https://api.github.com/users/linkedin","html_url":"https://github.com/linkedin","followers_url":"https://api.github.com/users/linkedin/followers","following_url":"https://api.github.com/users/linkedin/following{/other_user}","gists_url":"https://api.github.com/users/linkedin/gists{/gist_id}","starred_url":"https://api.github.com/users/linkedin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/linkedin/subscriptions","organizations_url":"https://api.github.com/users/linkedin/orgs","repos_url":"https://api.github.com/users/linkedin/repos","events_url":"https://api.github.com/users/linkedin/events{/privacy}","received_events_url":"https://api.github.com/users/linkedin/received_events","type":"Organization","site_admin":false},"repo":{"id":27392543,"name":"gobblin","full_name":"linkedin/gobblin","owner":{"login":"linkedin","id":357098,"avatar_url":"https://avatars.githubusercontent.com/u/357098?v=3","gravatar_id":"","url":"https://api.github.com/users/linkedin","html_url":"https://github.com/linkedin","followers_url":"https://api.github.com/users/linkedin/followers","following_url":"https://api.github.com/users/linkedin/following{/other_user}","gists_url":"https://api.github.com/users/linkedin/gists{/gist_id}","starred_url":"https://api.github.com/users/linkedin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/linkedin/subscriptions","organizations_url":"https://api.github.com/users/linkedin/orgs","repos_url":"https://api.github.com/users/linkedin/repos","events_url":"https://api.github.com/users/linkedin/events{/privacy}","received_events_url":"https://api.github.com/users/linkedin/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/linkedin/gobblin","description":"Universal data ingestion framework for Hadoop.","fork":false,"url":"https://api.github.com/repos/linkedin/gobblin","forks_url":"https://api.github.com/repos/linkedin/gobblin/forks","keys_url":"https://api.github.com/repos/linkedin/gobblin/keys{/key_id}","collaborators_url":"https://api.github.com/repos/linkedin/gobblin/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/linkedin/gobblin/teams","hooks_url":"https://api.github.com/repos/linkedin/gobblin/hooks","issue_events_url":"https://api.github.com/repos/linkedin/gobblin/issues/events{/number}","events_url":"https://api.github.com/repos/linkedin/gobblin/events","assignees_url":"https://api.github.com/repos/linkedin/gobblin/assignees{/user}","branches_url":"https://api.github.com/repos/linkedin/gobblin/branches{/branch}","tags_url":"https://api.github.com/repos/linkedin/gobblin/tags","blobs_url":"https://api.github.com/repos/linkedin/gobblin/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/linkedin/gobblin/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/linkedin/gobblin/git/refs{/sha}","trees_url":"https://api.github.com/repos/linkedin/gobblin/git/trees{/sha}","statuses_url":"https://api.github.com/repos/linkedin/gobblin/statuses/{sha}","languages_url":"https://api.github.com/repos/linkedin/gobblin/languages","stargazers_url":"https://api.github.com/repos/linkedin/gobblin/stargazers","contributors_url":"https://api.github.com/repos/linkedin/gobblin/contributors","subscribers_url":"https://api.github.com/repos/linkedin/gobblin/subscribers","subscription_url":"https://api.github.com/repos/linkedin/gobblin/subscription","commits_url":"https://api.github.com/repos/linkedin/gobblin/commits{/sha}","git_commits_url":"https://api.github.com/repos/linkedin/gobblin/git/commits{/sha}","comments_url":"https://api.github.com/repos/linkedin/gobblin/comments{/number}","issue_comment_url":"https://api.github.com/repos/linkedin/gobblin/issues/comments{/number}","contents_url":"https://api.github.com/repos/linkedin/gobblin/contents/{+path}","compare_url":"https://api.github.com/repos/linkedin/gobblin/compare/{base}...{head}","merges_url":"https://api.github.com/repos/linkedin/gobblin/merges","archive_url":"https://api.github.com/repos/linkedin/gobblin/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/linkedin/gobblin/downloads","issues_url":"https://api.github.com/repos/linkedin/gobblin/issues{/number}","pulls_url":"https://api.github.com/repos/linkedin/gobblin/pulls{/number}","milestones_url":"https://api.github.com/repos/linkedin/gobblin/milestones{/number}","notifications_url":"https://api.github.com/repos/linkedin/gobblin/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/linkedin/gobblin/labels{/name}","releases_url":"https://api.github.com/repos/linkedin/gobblin/releases{/id}","deployments_url":"https://api.github.com/repos/linkedin/gobblin/deployments","created_at":"2014-12-01T18:10:50Z","updated_at":"2016-03-29T06:19:04Z","pushed_at":"2016-03-29T21:39:35Z","git_url":"git://github.com/linkedin/gobblin.git","ssh_url":"git@github.com:linkedin/gobblin.git","clone_url":"https://github.com/linkedin/gobblin.git","svn_url":"https://github.com/linkedin/gobblin","homepage":"https://github.com/linkedin/gobblin/wiki","size":165118,"stargazers_count":490,"watchers_count":490,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":193,"mirror_url":null,"open_issues_count":72,"forks":193,"open_issues":72,"watchers":490,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/869"},"html":{"href":"https://github.com/linkedin/gobblin/pull/869"},"issue":{"href":"https://api.github.com/repos/linkedin/gobblin/issues/869"},"comments":{"href":"https://api.github.com/repos/linkedin/gobblin/issues/869/comments"},"review_comments":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/869/comments"},"review_comment":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/linkedin/gobblin/pulls/869/commits"},"statuses":{"href":"https://api.github.com/repos/linkedin/gobblin/statuses/4de6982a26277c7db5f10d87c646bc4a590871c6"}}}}