{"action":"created","comment":{"url":"https://api.github.com/repos/broadinstitute/gatk/pulls/comments/67382301","id":67382301,"diff_hunk":"@@ -0,0 +1,116 @@\n+package org.broadinstitute.hellbender.tools.spark.pipelines.metrics;\n+\n+import htsjdk.samtools.SAMFileHeader;\n+\n+import htsjdk.samtools.metrics.Header;\n+import org.apache.spark.api.java.JavaRDD;\n+import org.broadinstitute.hellbender.engine.AuthHolder;\n+import org.broadinstitute.hellbender.engine.filters.ReadFilter;\n+import org.broadinstitute.hellbender.engine.filters.ReadFilterLibrary;\n+import org.broadinstitute.hellbender.metrics.MetricsArgs;\n+import org.broadinstitute.hellbender.utils.read.GATKRead;\n+\n+import java.io.Serializable;\n+import java.util.List;\n+\n+/**\n+ * Interface implemented by Spark metrics collectors to allow them\n+ * to run as either a standalone Spark tool, or under the control of\n+ * CollectMultipleMetricsSpark, which reuses the same input RDD to run\n+ * multiple collectors.\n+ *\n+ * Spark collectors should be factored into the following implementation\n+ * classes:\n+ *\n+ * <li>\n+ * A class derived from MetricsArgs that defines the input arguments for\n+ * the collector in a form that is suitable as a command line argument\n+ * collection.\n+ * A class that implements this interface, parameterized with the\n+ * MetricsArgs-derived class.\n+ * </li>\n+ *\n+ * These two classes can then be used either in the standalone collector tool (see\n+ * CollectInsertSizeMetricsSpark as an example), or from CollectMultipleMetricsSpark.\n+ *\n+ * The general lifecyle of a Spark collector looks like this:\n+ *\n+ *     CollectorType collector = new CollectorType<CollectorArgType>()\n+ *     CollectorArgType args = // may call collector.gatherInputArguments();\n+ *\n+ *     // pass the input arguments back to the collector for initialization\n+ *     collector.initializeCollector(args);\n+ *\n+ *     ReadFilter filter == collector.getReadFilter();\n+ *     collector.collectMetrics(\n+ *         getReads().filter(filter),\n+ *         getHeaderForReads(),\n+ *         getReadSourceName(),\n+ *         getAuthHolder()\n+ *     );\n+ *     collector.finishCollection();\n+ *\n+ */\n+public interface MetricsCollectorSpark<T extends MetricsArgs> extends Serializable\n+{\n+    static final long serialVersionUID = 1l;\n+\n+    /**\n+     * Advertise whether or not this collector requires a reference.\n+     * @return true if this collector requires that a reference be provided.\n+     */\n+    default boolean requiresReference() { return false;}","path":"src/main/java/org/broadinstitute/hellbender/tools/spark/pipelines/metrics/MetricsCollectorSpark.java","position":62,"original_position":62,"commit_id":"0ac3e1f749522be8f9c644932658bc765df03ea5","original_commit_id":"0ac3e1f749522be8f9c644932658bc765df03ea5","user":{"login":"cmnbroad","id":10062863,"avatar_url":"https://avatars.githubusercontent.com/u/10062863?v=3","gravatar_id":"","url":"https://api.github.com/users/cmnbroad","html_url":"https://github.com/cmnbroad","followers_url":"https://api.github.com/users/cmnbroad/followers","following_url":"https://api.github.com/users/cmnbroad/following{/other_user}","gists_url":"https://api.github.com/users/cmnbroad/gists{/gist_id}","starred_url":"https://api.github.com/users/cmnbroad/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cmnbroad/subscriptions","organizations_url":"https://api.github.com/users/cmnbroad/orgs","repos_url":"https://api.github.com/users/cmnbroad/repos","events_url":"https://api.github.com/users/cmnbroad/events{/privacy}","received_events_url":"https://api.github.com/users/cmnbroad/received_events","type":"User","site_admin":false},"body":"Done.","created_at":"2016-06-16T16:52:56Z","updated_at":"2016-06-16T16:52:56Z","html_url":"https://github.com/broadinstitute/gatk/pull/1827#discussion_r67382301","pull_request_url":"https://api.github.com/repos/broadinstitute/gatk/pulls/1827","_links":{"self":{"href":"https://api.github.com/repos/broadinstitute/gatk/pulls/comments/67382301"},"html":{"href":"https://github.com/broadinstitute/gatk/pull/1827#discussion_r67382301"},"pull_request":{"href":"https://api.github.com/repos/broadinstitute/gatk/pulls/1827"}}},"pull_request":{"url":"https://api.github.com/repos/broadinstitute/gatk/pulls/1827","id":70353768,"html_url":"https://github.com/broadinstitute/gatk/pull/1827","diff_url":"https://github.com/broadinstitute/gatk/pull/1827.diff","patch_url":"https://github.com/broadinstitute/gatk/pull/1827.patch","issue_url":"https://api.github.com/repos/broadinstitute/gatk/issues/1827","number":1827,"state":"open","locked":false,"title":"Spark metrics collector refactoring checkpoint.","user":{"login":"cmnbroad","id":10062863,"avatar_url":"https://avatars.githubusercontent.com/u/10062863?v=3","gravatar_id":"","url":"https://api.github.com/users/cmnbroad","html_url":"https://github.com/cmnbroad","followers_url":"https://api.github.com/users/cmnbroad/followers","following_url":"https://api.github.com/users/cmnbroad/following{/other_user}","gists_url":"https://api.github.com/users/cmnbroad/gists{/gist_id}","starred_url":"https://api.github.com/users/cmnbroad/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cmnbroad/subscriptions","organizations_url":"https://api.github.com/users/cmnbroad/orgs","repos_url":"https://api.github.com/users/cmnbroad/repos","events_url":"https://api.github.com/users/cmnbroad/events{/privacy}","received_events_url":"https://api.github.com/users/cmnbroad/received_events","type":"User","site_admin":false},"body":"This is a checkpoint PR for https://github.com/broadinstitute/gatk/issues/1237 and https://github.com/broadinstitute/gatk/issues/1643. This is the first step in refactoring metrics collectors so they can be pipelined in Spark and reuse RDDs, but still share metrics computation code between walker and Spark versions. The next step will be to extend MultilevelCollector to be able to merge its own instances in order to support efficient map and reduce phases for multi level collectors.\r\n\r\nSuggested review order:\r\n\r\n-MetricsCollectorSpark: interface to be implemented by all Spark collectors\r\n-MetricsArgs:base class for all collector argument sets\r\n-MetricsCollectorToolSpark: base class for all Spark metrics collector tools\r\n-CollectQualityYieldMetrics: Spark version of QualityYieldMetrics using these new interfaces\r\n-CollectInsertSizeMetricsSpark: existing Spark version of InsertSizeMetrics collector ported\r\nto these interfaces\r\n-CollectMultipleMetricsSpark: Spark version of CollectMultipleMetrics; currently only works\r\non QualityYieldMetrics and InsertSizeMetrics\r\n\r\nThe rest of the PR is refactoring existing to get QualityYieldMetrics and InsertSizeMetrics to conform to these interfaces (moving CollectInsertSizeMetrics out of the sv package and Program Groups, etc.). Note that the existing InsertSizeMetrics Spark collector doesn’t really share code with the walker\r\nversion (and their command line param sets are way out of sync) but this should be fixed separately from these changes as the interfaces evolve.","created_at":"2016-05-17T13:10:10Z","updated_at":"2016-06-16T16:52:56Z","closed_at":null,"merged_at":null,"merge_commit_sha":"c80da684d65da5c00440e91274d95b9612b4dde5","assignee":{"login":"cmnbroad","id":10062863,"avatar_url":"https://avatars.githubusercontent.com/u/10062863?v=3","gravatar_id":"","url":"https://api.github.com/users/cmnbroad","html_url":"https://github.com/cmnbroad","followers_url":"https://api.github.com/users/cmnbroad/followers","following_url":"https://api.github.com/users/cmnbroad/following{/other_user}","gists_url":"https://api.github.com/users/cmnbroad/gists{/gist_id}","starred_url":"https://api.github.com/users/cmnbroad/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cmnbroad/subscriptions","organizations_url":"https://api.github.com/users/cmnbroad/orgs","repos_url":"https://api.github.com/users/cmnbroad/repos","events_url":"https://api.github.com/users/cmnbroad/events{/privacy}","received_events_url":"https://api.github.com/users/cmnbroad/received_events","type":"User","site_admin":false},"milestone":null,"commits_url":"https://api.github.com/repos/broadinstitute/gatk/pulls/1827/commits","review_comments_url":"https://api.github.com/repos/broadinstitute/gatk/pulls/1827/comments","review_comment_url":"https://api.github.com/repos/broadinstitute/gatk/pulls/comments{/number}","comments_url":"https://api.github.com/repos/broadinstitute/gatk/issues/1827/comments","statuses_url":"https://api.github.com/repos/broadinstitute/gatk/statuses/0ac3e1f749522be8f9c644932658bc765df03ea5","head":{"label":"broadinstitute:cn_spark_metrics","ref":"cn_spark_metrics","sha":"0ac3e1f749522be8f9c644932658bc765df03ea5","user":{"login":"broadinstitute","id":393552,"avatar_url":"https://avatars.githubusercontent.com/u/393552?v=3","gravatar_id":"","url":"https://api.github.com/users/broadinstitute","html_url":"https://github.com/broadinstitute","followers_url":"https://api.github.com/users/broadinstitute/followers","following_url":"https://api.github.com/users/broadinstitute/following{/other_user}","gists_url":"https://api.github.com/users/broadinstitute/gists{/gist_id}","starred_url":"https://api.github.com/users/broadinstitute/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/broadinstitute/subscriptions","organizations_url":"https://api.github.com/users/broadinstitute/orgs","repos_url":"https://api.github.com/users/broadinstitute/repos","events_url":"https://api.github.com/users/broadinstitute/events{/privacy}","received_events_url":"https://api.github.com/users/broadinstitute/received_events","type":"Organization","site_admin":false},"repo":{"id":27452807,"name":"gatk","full_name":"broadinstitute/gatk","owner":{"login":"broadinstitute","id":393552,"avatar_url":"https://avatars.githubusercontent.com/u/393552?v=3","gravatar_id":"","url":"https://api.github.com/users/broadinstitute","html_url":"https://github.com/broadinstitute","followers_url":"https://api.github.com/users/broadinstitute/followers","following_url":"https://api.github.com/users/broadinstitute/following{/other_user}","gists_url":"https://api.github.com/users/broadinstitute/gists{/gist_id}","starred_url":"https://api.github.com/users/broadinstitute/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/broadinstitute/subscriptions","organizations_url":"https://api.github.com/users/broadinstitute/orgs","repos_url":"https://api.github.com/users/broadinstitute/repos","events_url":"https://api.github.com/users/broadinstitute/events{/privacy}","received_events_url":"https://api.github.com/users/broadinstitute/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/broadinstitute/gatk","description":"GATK4 development","fork":false,"url":"https://api.github.com/repos/broadinstitute/gatk","forks_url":"https://api.github.com/repos/broadinstitute/gatk/forks","keys_url":"https://api.github.com/repos/broadinstitute/gatk/keys{/key_id}","collaborators_url":"https://api.github.com/repos/broadinstitute/gatk/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/broadinstitute/gatk/teams","hooks_url":"https://api.github.com/repos/broadinstitute/gatk/hooks","issue_events_url":"https://api.github.com/repos/broadinstitute/gatk/issues/events{/number}","events_url":"https://api.github.com/repos/broadinstitute/gatk/events","assignees_url":"https://api.github.com/repos/broadinstitute/gatk/assignees{/user}","branches_url":"https://api.github.com/repos/broadinstitute/gatk/branches{/branch}","tags_url":"https://api.github.com/repos/broadinstitute/gatk/tags","blobs_url":"https://api.github.com/repos/broadinstitute/gatk/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/broadinstitute/gatk/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/broadinstitute/gatk/git/refs{/sha}","trees_url":"https://api.github.com/repos/broadinstitute/gatk/git/trees{/sha}","statuses_url":"https://api.github.com/repos/broadinstitute/gatk/statuses/{sha}","languages_url":"https://api.github.com/repos/broadinstitute/gatk/languages","stargazers_url":"https://api.github.com/repos/broadinstitute/gatk/stargazers","contributors_url":"https://api.github.com/repos/broadinstitute/gatk/contributors","subscribers_url":"https://api.github.com/repos/broadinstitute/gatk/subscribers","subscription_url":"https://api.github.com/repos/broadinstitute/gatk/subscription","commits_url":"https://api.github.com/repos/broadinstitute/gatk/commits{/sha}","git_commits_url":"https://api.github.com/repos/broadinstitute/gatk/git/commits{/sha}","comments_url":"https://api.github.com/repos/broadinstitute/gatk/comments{/number}","issue_comment_url":"https://api.github.com/repos/broadinstitute/gatk/issues/comments{/number}","contents_url":"https://api.github.com/repos/broadinstitute/gatk/contents/{+path}","compare_url":"https://api.github.com/repos/broadinstitute/gatk/compare/{base}...{head}","merges_url":"https://api.github.com/repos/broadinstitute/gatk/merges","archive_url":"https://api.github.com/repos/broadinstitute/gatk/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/broadinstitute/gatk/downloads","issues_url":"https://api.github.com/repos/broadinstitute/gatk/issues{/number}","pulls_url":"https://api.github.com/repos/broadinstitute/gatk/pulls{/number}","milestones_url":"https://api.github.com/repos/broadinstitute/gatk/milestones{/number}","notifications_url":"https://api.github.com/repos/broadinstitute/gatk/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/broadinstitute/gatk/labels{/name}","releases_url":"https://api.github.com/repos/broadinstitute/gatk/releases{/id}","deployments_url":"https://api.github.com/repos/broadinstitute/gatk/deployments","created_at":"2014-12-02T20:50:36Z","updated_at":"2016-06-15T22:59:52Z","pushed_at":"2016-06-16T16:40:51Z","git_url":"git://github.com/broadinstitute/gatk.git","ssh_url":"git@github.com:broadinstitute/gatk.git","clone_url":"https://github.com/broadinstitute/gatk.git","svn_url":"https://github.com/broadinstitute/gatk","homepage":"","size":113075,"stargazers_count":72,"watchers_count":72,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":28,"mirror_url":null,"open_issues_count":375,"forks":28,"open_issues":375,"watchers":72,"default_branch":"master"}},"base":{"label":"broadinstitute:master","ref":"master","sha":"7734d1172b60237e6a138202a3d6910ff7dbb7cb","user":{"login":"broadinstitute","id":393552,"avatar_url":"https://avatars.githubusercontent.com/u/393552?v=3","gravatar_id":"","url":"https://api.github.com/users/broadinstitute","html_url":"https://github.com/broadinstitute","followers_url":"https://api.github.com/users/broadinstitute/followers","following_url":"https://api.github.com/users/broadinstitute/following{/other_user}","gists_url":"https://api.github.com/users/broadinstitute/gists{/gist_id}","starred_url":"https://api.github.com/users/broadinstitute/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/broadinstitute/subscriptions","organizations_url":"https://api.github.com/users/broadinstitute/orgs","repos_url":"https://api.github.com/users/broadinstitute/repos","events_url":"https://api.github.com/users/broadinstitute/events{/privacy}","received_events_url":"https://api.github.com/users/broadinstitute/received_events","type":"Organization","site_admin":false},"repo":{"id":27452807,"name":"gatk","full_name":"broadinstitute/gatk","owner":{"login":"broadinstitute","id":393552,"avatar_url":"https://avatars.githubusercontent.com/u/393552?v=3","gravatar_id":"","url":"https://api.github.com/users/broadinstitute","html_url":"https://github.com/broadinstitute","followers_url":"https://api.github.com/users/broadinstitute/followers","following_url":"https://api.github.com/users/broadinstitute/following{/other_user}","gists_url":"https://api.github.com/users/broadinstitute/gists{/gist_id}","starred_url":"https://api.github.com/users/broadinstitute/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/broadinstitute/subscriptions","organizations_url":"https://api.github.com/users/broadinstitute/orgs","repos_url":"https://api.github.com/users/broadinstitute/repos","events_url":"https://api.github.com/users/broadinstitute/events{/privacy}","received_events_url":"https://api.github.com/users/broadinstitute/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/broadinstitute/gatk","description":"GATK4 development","fork":false,"url":"https://api.github.com/repos/broadinstitute/gatk","forks_url":"https://api.github.com/repos/broadinstitute/gatk/forks","keys_url":"https://api.github.com/repos/broadinstitute/gatk/keys{/key_id}","collaborators_url":"https://api.github.com/repos/broadinstitute/gatk/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/broadinstitute/gatk/teams","hooks_url":"https://api.github.com/repos/broadinstitute/gatk/hooks","issue_events_url":"https://api.github.com/repos/broadinstitute/gatk/issues/events{/number}","events_url":"https://api.github.com/repos/broadinstitute/gatk/events","assignees_url":"https://api.github.com/repos/broadinstitute/gatk/assignees{/user}","branches_url":"https://api.github.com/repos/broadinstitute/gatk/branches{/branch}","tags_url":"https://api.github.com/repos/broadinstitute/gatk/tags","blobs_url":"https://api.github.com/repos/broadinstitute/gatk/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/broadinstitute/gatk/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/broadinstitute/gatk/git/refs{/sha}","trees_url":"https://api.github.com/repos/broadinstitute/gatk/git/trees{/sha}","statuses_url":"https://api.github.com/repos/broadinstitute/gatk/statuses/{sha}","languages_url":"https://api.github.com/repos/broadinstitute/gatk/languages","stargazers_url":"https://api.github.com/repos/broadinstitute/gatk/stargazers","contributors_url":"https://api.github.com/repos/broadinstitute/gatk/contributors","subscribers_url":"https://api.github.com/repos/broadinstitute/gatk/subscribers","subscription_url":"https://api.github.com/repos/broadinstitute/gatk/subscription","commits_url":"https://api.github.com/repos/broadinstitute/gatk/commits{/sha}","git_commits_url":"https://api.github.com/repos/broadinstitute/gatk/git/commits{/sha}","comments_url":"https://api.github.com/repos/broadinstitute/gatk/comments{/number}","issue_comment_url":"https://api.github.com/repos/broadinstitute/gatk/issues/comments{/number}","contents_url":"https://api.github.com/repos/broadinstitute/gatk/contents/{+path}","compare_url":"https://api.github.com/repos/broadinstitute/gatk/compare/{base}...{head}","merges_url":"https://api.github.com/repos/broadinstitute/gatk/merges","archive_url":"https://api.github.com/repos/broadinstitute/gatk/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/broadinstitute/gatk/downloads","issues_url":"https://api.github.com/repos/broadinstitute/gatk/issues{/number}","pulls_url":"https://api.github.com/repos/broadinstitute/gatk/pulls{/number}","milestones_url":"https://api.github.com/repos/broadinstitute/gatk/milestones{/number}","notifications_url":"https://api.github.com/repos/broadinstitute/gatk/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/broadinstitute/gatk/labels{/name}","releases_url":"https://api.github.com/repos/broadinstitute/gatk/releases{/id}","deployments_url":"https://api.github.com/repos/broadinstitute/gatk/deployments","created_at":"2014-12-02T20:50:36Z","updated_at":"2016-06-15T22:59:52Z","pushed_at":"2016-06-16T16:40:51Z","git_url":"git://github.com/broadinstitute/gatk.git","ssh_url":"git@github.com:broadinstitute/gatk.git","clone_url":"https://github.com/broadinstitute/gatk.git","svn_url":"https://github.com/broadinstitute/gatk","homepage":"","size":113075,"stargazers_count":72,"watchers_count":72,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":28,"mirror_url":null,"open_issues_count":375,"forks":28,"open_issues":375,"watchers":72,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/broadinstitute/gatk/pulls/1827"},"html":{"href":"https://github.com/broadinstitute/gatk/pull/1827"},"issue":{"href":"https://api.github.com/repos/broadinstitute/gatk/issues/1827"},"comments":{"href":"https://api.github.com/repos/broadinstitute/gatk/issues/1827/comments"},"review_comments":{"href":"https://api.github.com/repos/broadinstitute/gatk/pulls/1827/comments"},"review_comment":{"href":"https://api.github.com/repos/broadinstitute/gatk/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/broadinstitute/gatk/pulls/1827/commits"},"statuses":{"href":"https://api.github.com/repos/broadinstitute/gatk/statuses/0ac3e1f749522be8f9c644932658bc765df03ea5"}}}}
{"actor":{"display_login":"cmnbroad"}}