{"action":"created","comment":{"url":"https://api.github.com/repos/broadinstitute/gatk/pulls/comments/49913299","id":49913299,"diff_hunk":"@@ -0,0 +1,135 @@\n+package org.broadinstitute.hellbender.tools.spark.sv;\n+\n+import htsjdk.samtools.SAMFileHeader;\n+import htsjdk.samtools.SAMSequenceDictionary;\n+import htsjdk.samtools.SAMSequenceRecord;\n+import org.apache.spark.api.java.JavaRDD;\n+import org.apache.spark.api.java.JavaSparkContext;\n+import org.broadinstitute.hellbender.cmdline.Argument;\n+import org.broadinstitute.hellbender.cmdline.CommandLineProgramProperties;\n+import org.broadinstitute.hellbender.cmdline.StandardArgumentDefinitions;\n+import org.broadinstitute.hellbender.cmdline.programgroups.SparkProgramGroup;\n+import org.broadinstitute.hellbender.engine.datasources.ReferenceMultiSource;\n+import org.broadinstitute.hellbender.engine.spark.GATKSparkTool;\n+import org.broadinstitute.hellbender.exceptions.GATKException;\n+import org.broadinstitute.hellbender.utils.SimpleInterval;\n+import scala.Tuple2;\n+\n+import java.io.*;\n+import java.util.*;\n+\n+/**\n+ * SparkTool to identify 63-mers in the reference that occur more than 3 times.\n+ *\n+ * Created by tsharpe on 12/4/15.\n+ */\n+@CommandLineProgramProperties(summary=\"Use spark to find the set of high copy number kmers in a reference.\",\n+        oneLineSummary=\"find ref kmers with high copy number\",\n+        programGroup = SparkProgramGroup.class)\n+public final class FindBadGenomicKmers extends GATKSparkTool\n+{\n+    private static final long serialVersionUID = 1L;\n+    private static final Long MAX_KMER_FREQ = 3L;\n+    private static final int KLEN = 63;\n+    private static final int REF_REC_LEN = 10000;\n+    private static final long CHUNK_SIZE = 3000000;\n+\n+    @Argument(doc = \"file for ubiquitous kmer output\", shortName = StandardArgumentDefinitions.OUTPUT_SHORT_NAME,\n+            fullName = StandardArgumentDefinitions.OUTPUT_LONG_NAME, optional = false)\n+    private String outputFile;\n+\n+    @Override\n+    public boolean requiresReference()\n+    {\n+        return true;\n+    }\n+\n+    // Get the list of high copy number kmers in the reference, and write them to a file.\n+    @Override\n+    protected void runTool( final JavaSparkContext ctx )\n+    {\n+        SAMFileHeader hdr = getHeaderForReads();\n+        SAMSequenceDictionary dict = null;\n+        if ( hdr != null ) dict = hdr.getSequenceDictionary();\n+        List<Kmer> killList = findBadGenomicKmers(ctx,getReference(),dict);\n+        final File killFile = new File(outputFile);\n+        try\n+        {\n+            final Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(killFile)));\n+\n+            for ( Kmer kmer : killList )\n+            {\n+                writer.write(kmer.toString(KLEN));\n+                writer.write('\\n');\n+            }\n+            writer.close();\n+        }\n+        catch ( IOException e )\n+        { throw new GATKException(\"Unable to write ubiquitous kmer kill file.\", e); }\n+    }\n+\n+    // Find high copy number kmers in the reference sequence\n+    public static List<Kmer> findBadGenomicKmers( final JavaSparkContext ctx,\n+                                                  final ReferenceMultiSource ref,\n+                                                  final SAMSequenceDictionary readsDict )\n+    {\n+        // Write all the reference sequences to a text file as a series of 10,000-base lines.\n+        // For each contig, records are written with a K-1 base overlap\n+        // (i.e., the last 62 bases in line n match the first 62 bases in line n+1) so that we don't miss any kmers\n+        // due to line breaks -- we can just kmerize each record.\n+        File refSequenceFile;\n+        try\n+        {\n+            refSequenceFile = File.createTempFile(\"genomic_sequence\",\"txt\");\n+            refSequenceFile.deleteOnExit();\n+            final OutputStream os = new BufferedOutputStream(new FileOutputStream(refSequenceFile));\n+\n+            final SAMSequenceDictionary dict = ref.getReferenceSequenceDictionary(readsDict);\n+            if ( dict == null )\n+                throw new GATKException(\"No reference dictionary available.\");\n+\n+            final int effectiveRecLen = REF_REC_LEN - KLEN + 1;\n+            for ( final SAMSequenceRecord rec : dict.getSequences() )\n+            {\n+                final String seqName = rec.getSequenceName();\n+                final int seqLen = rec.getSequenceLength();\n+                final byte[] bases = ref.getReferenceBases(null,new SimpleInterval(seqName,1,seqLen)).getBases();\n+                for ( int start = 0; start < seqLen; start += effectiveRecLen )\n+                {\n+                    os.write(bases,start,Math.min(REF_REC_LEN,seqLen-start));\n+                    os.write('\\n');\n+                }\n+            }\n+\n+            os.close();\n+        }\n+        catch ( IOException e )\n+        { throw new GATKException(\"Unable to write reference text file for kmerizing.\", e); }\n+\n+        // Turn that text file into an RDD, and do a classic map/reduce:\n+        // Kmerize, mapping to a pair <kmer,1>, reduce by summing values by key, filter out <kmer,N> where N < 3,\n+        // and collect the high frequency kmers back in the driver.\n+        final int nParts = (int)(refSequenceFile.length()/CHUNK_SIZE + 1);\n+        final JavaRDD<String> genomicBases = ctx.textFile(refSequenceFile.getPath(),nParts);","path":"src/main/java/org/broadinstitute/hellbender/tools/spark/sv/FindBadGenomicKmers.java","position":113,"original_position":113,"commit_id":"43f1272da7e281f185b510742adecb20f6ba4ebd","original_commit_id":"43f1272da7e281f185b510742adecb20f6ba4ebd","user":{"login":"lbergelson","id":4700332,"avatar_url":"https://avatars.githubusercontent.com/u/4700332?v=3","gravatar_id":"","url":"https://api.github.com/users/lbergelson","html_url":"https://github.com/lbergelson","followers_url":"https://api.github.com/users/lbergelson/followers","following_url":"https://api.github.com/users/lbergelson/following{/other_user}","gists_url":"https://api.github.com/users/lbergelson/gists{/gist_id}","starred_url":"https://api.github.com/users/lbergelson/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/lbergelson/subscriptions","organizations_url":"https://api.github.com/users/lbergelson/orgs","repos_url":"https://api.github.com/users/lbergelson/repos","events_url":"https://api.github.com/users/lbergelson/events{/privacy}","received_events_url":"https://api.github.com/users/lbergelson/received_events","type":"User","site_admin":false},"body":"I think it's likely that reading the entire reference, then writing it, then reading it again is a performance killer.  Instead of generating your initial RDD this way, you might be better off by coming up a with a scheme to load the reference directly into an rdd without an intermediate file.   If you can fit the whole thing in memory you could parallelize it directly.  We have a \"2bit\" reference representation that is much smaller in memory, but requires preprocessing your reference file.  \r\n\r\nThis is an area we need work on in the rest of hellbender as wel, we don't have a good way of spreading the reference to many nodes.  ","created_at":"2016-01-15T22:33:19Z","updated_at":"2016-01-15T22:33:19Z","html_url":"https://github.com/broadinstitute/gatk/pull/1435#discussion_r49913299","pull_request_url":"https://api.github.com/repos/broadinstitute/gatk/pulls/1435","_links":{"self":{"href":"https://api.github.com/repos/broadinstitute/gatk/pulls/comments/49913299"},"html":{"href":"https://github.com/broadinstitute/gatk/pull/1435#discussion_r49913299"},"pull_request":{"href":"https://api.github.com/repos/broadinstitute/gatk/pulls/1435"}}},"pull_request":{"url":"https://api.github.com/repos/broadinstitute/gatk/pulls/1435","id":56209740,"html_url":"https://github.com/broadinstitute/gatk/pull/1435","diff_url":"https://github.com/broadinstitute/gatk/pull/1435.diff","patch_url":"https://github.com/broadinstitute/gatk/pull/1435.patch","issue_url":"https://api.github.com/repos/broadinstitute/gatk/issues/1435","number":1435,"state":"open","locked":false,"title":"New Spark tools to identify putative SV breakpoints…","user":{"login":"tedsharpe","id":9284156,"avatar_url":"https://avatars.githubusercontent.com/u/9284156?v=3","gravatar_id":"","url":"https://api.github.com/users/tedsharpe","html_url":"https://github.com/tedsharpe","followers_url":"https://api.github.com/users/tedsharpe/followers","following_url":"https://api.github.com/users/tedsharpe/following{/other_user}","gists_url":"https://api.github.com/users/tedsharpe/gists{/gist_id}","starred_url":"https://api.github.com/users/tedsharpe/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tedsharpe/subscriptions","organizations_url":"https://api.github.com/users/tedsharpe/orgs","repos_url":"https://api.github.com/users/tedsharpe/repos","events_url":"https://api.github.com/users/tedsharpe/events{/privacy}","received_events_url":"https://api.github.com/users/tedsharpe/received_events","type":"User","site_admin":false},"body":" and emit data for local assemblies in these regions.","created_at":"2016-01-15T19:19:17Z","updated_at":"2016-01-15T22:33:19Z","closed_at":null,"merged_at":null,"merge_commit_sha":"828865e23ba0f3cabab4d455071aff4d1a79ae98","assignee":{"login":"lbergelson","id":4700332,"avatar_url":"https://avatars.githubusercontent.com/u/4700332?v=3","gravatar_id":"","url":"https://api.github.com/users/lbergelson","html_url":"https://github.com/lbergelson","followers_url":"https://api.github.com/users/lbergelson/followers","following_url":"https://api.github.com/users/lbergelson/following{/other_user}","gists_url":"https://api.github.com/users/lbergelson/gists{/gist_id}","starred_url":"https://api.github.com/users/lbergelson/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/lbergelson/subscriptions","organizations_url":"https://api.github.com/users/lbergelson/orgs","repos_url":"https://api.github.com/users/lbergelson/repos","events_url":"https://api.github.com/users/lbergelson/events{/privacy}","received_events_url":"https://api.github.com/users/lbergelson/received_events","type":"User","site_admin":false},"milestone":null,"commits_url":"https://api.github.com/repos/broadinstitute/gatk/pulls/1435/commits","review_comments_url":"https://api.github.com/repos/broadinstitute/gatk/pulls/1435/comments","review_comment_url":"https://api.github.com/repos/broadinstitute/gatk/pulls/comments{/number}","comments_url":"https://api.github.com/repos/broadinstitute/gatk/issues/1435/comments","statuses_url":"https://api.github.com/repos/broadinstitute/gatk/statuses/43f1272da7e281f185b510742adecb20f6ba4ebd","head":{"label":"broadinstitute:sv_master","ref":"sv_master","sha":"43f1272da7e281f185b510742adecb20f6ba4ebd","user":{"login":"broadinstitute","id":393552,"avatar_url":"https://avatars.githubusercontent.com/u/393552?v=3","gravatar_id":"","url":"https://api.github.com/users/broadinstitute","html_url":"https://github.com/broadinstitute","followers_url":"https://api.github.com/users/broadinstitute/followers","following_url":"https://api.github.com/users/broadinstitute/following{/other_user}","gists_url":"https://api.github.com/users/broadinstitute/gists{/gist_id}","starred_url":"https://api.github.com/users/broadinstitute/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/broadinstitute/subscriptions","organizations_url":"https://api.github.com/users/broadinstitute/orgs","repos_url":"https://api.github.com/users/broadinstitute/repos","events_url":"https://api.github.com/users/broadinstitute/events{/privacy}","received_events_url":"https://api.github.com/users/broadinstitute/received_events","type":"Organization","site_admin":false},"repo":{"id":27452807,"name":"gatk","full_name":"broadinstitute/gatk","owner":{"login":"broadinstitute","id":393552,"avatar_url":"https://avatars.githubusercontent.com/u/393552?v=3","gravatar_id":"","url":"https://api.github.com/users/broadinstitute","html_url":"https://github.com/broadinstitute","followers_url":"https://api.github.com/users/broadinstitute/followers","following_url":"https://api.github.com/users/broadinstitute/following{/other_user}","gists_url":"https://api.github.com/users/broadinstitute/gists{/gist_id}","starred_url":"https://api.github.com/users/broadinstitute/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/broadinstitute/subscriptions","organizations_url":"https://api.github.com/users/broadinstitute/orgs","repos_url":"https://api.github.com/users/broadinstitute/repos","events_url":"https://api.github.com/users/broadinstitute/events{/privacy}","received_events_url":"https://api.github.com/users/broadinstitute/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/broadinstitute/gatk","description":"GATK4 development","fork":false,"url":"https://api.github.com/repos/broadinstitute/gatk","forks_url":"https://api.github.com/repos/broadinstitute/gatk/forks","keys_url":"https://api.github.com/repos/broadinstitute/gatk/keys{/key_id}","collaborators_url":"https://api.github.com/repos/broadinstitute/gatk/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/broadinstitute/gatk/teams","hooks_url":"https://api.github.com/repos/broadinstitute/gatk/hooks","issue_events_url":"https://api.github.com/repos/broadinstitute/gatk/issues/events{/number}","events_url":"https://api.github.com/repos/broadinstitute/gatk/events","assignees_url":"https://api.github.com/repos/broadinstitute/gatk/assignees{/user}","branches_url":"https://api.github.com/repos/broadinstitute/gatk/branches{/branch}","tags_url":"https://api.github.com/repos/broadinstitute/gatk/tags","blobs_url":"https://api.github.com/repos/broadinstitute/gatk/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/broadinstitute/gatk/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/broadinstitute/gatk/git/refs{/sha}","trees_url":"https://api.github.com/repos/broadinstitute/gatk/git/trees{/sha}","statuses_url":"https://api.github.com/repos/broadinstitute/gatk/statuses/{sha}","languages_url":"https://api.github.com/repos/broadinstitute/gatk/languages","stargazers_url":"https://api.github.com/repos/broadinstitute/gatk/stargazers","contributors_url":"https://api.github.com/repos/broadinstitute/gatk/contributors","subscribers_url":"https://api.github.com/repos/broadinstitute/gatk/subscribers","subscription_url":"https://api.github.com/repos/broadinstitute/gatk/subscription","commits_url":"https://api.github.com/repos/broadinstitute/gatk/commits{/sha}","git_commits_url":"https://api.github.com/repos/broadinstitute/gatk/git/commits{/sha}","comments_url":"https://api.github.com/repos/broadinstitute/gatk/comments{/number}","issue_comment_url":"https://api.github.com/repos/broadinstitute/gatk/issues/comments{/number}","contents_url":"https://api.github.com/repos/broadinstitute/gatk/contents/{+path}","compare_url":"https://api.github.com/repos/broadinstitute/gatk/compare/{base}...{head}","merges_url":"https://api.github.com/repos/broadinstitute/gatk/merges","archive_url":"https://api.github.com/repos/broadinstitute/gatk/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/broadinstitute/gatk/downloads","issues_url":"https://api.github.com/repos/broadinstitute/gatk/issues{/number}","pulls_url":"https://api.github.com/repos/broadinstitute/gatk/pulls{/number}","milestones_url":"https://api.github.com/repos/broadinstitute/gatk/milestones{/number}","notifications_url":"https://api.github.com/repos/broadinstitute/gatk/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/broadinstitute/gatk/labels{/name}","releases_url":"https://api.github.com/repos/broadinstitute/gatk/releases{/id}","created_at":"2014-12-02T20:50:36Z","updated_at":"2016-01-11T21:08:29Z","pushed_at":"2016-01-15T21:30:32Z","git_url":"git://github.com/broadinstitute/gatk.git","ssh_url":"git@github.com:broadinstitute/gatk.git","clone_url":"https://github.com/broadinstitute/gatk.git","svn_url":"https://github.com/broadinstitute/gatk","homepage":"","size":57027,"stargazers_count":43,"watchers_count":43,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":18,"mirror_url":null,"open_issues_count":276,"forks":18,"open_issues":276,"watchers":43,"default_branch":"master"}},"base":{"label":"broadinstitute:master","ref":"master","sha":"c127ca2886985a33efa19bee4a0e8a95115c46b7","user":{"login":"broadinstitute","id":393552,"avatar_url":"https://avatars.githubusercontent.com/u/393552?v=3","gravatar_id":"","url":"https://api.github.com/users/broadinstitute","html_url":"https://github.com/broadinstitute","followers_url":"https://api.github.com/users/broadinstitute/followers","following_url":"https://api.github.com/users/broadinstitute/following{/other_user}","gists_url":"https://api.github.com/users/broadinstitute/gists{/gist_id}","starred_url":"https://api.github.com/users/broadinstitute/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/broadinstitute/subscriptions","organizations_url":"https://api.github.com/users/broadinstitute/orgs","repos_url":"https://api.github.com/users/broadinstitute/repos","events_url":"https://api.github.com/users/broadinstitute/events{/privacy}","received_events_url":"https://api.github.com/users/broadinstitute/received_events","type":"Organization","site_admin":false},"repo":{"id":27452807,"name":"gatk","full_name":"broadinstitute/gatk","owner":{"login":"broadinstitute","id":393552,"avatar_url":"https://avatars.githubusercontent.com/u/393552?v=3","gravatar_id":"","url":"https://api.github.com/users/broadinstitute","html_url":"https://github.com/broadinstitute","followers_url":"https://api.github.com/users/broadinstitute/followers","following_url":"https://api.github.com/users/broadinstitute/following{/other_user}","gists_url":"https://api.github.com/users/broadinstitute/gists{/gist_id}","starred_url":"https://api.github.com/users/broadinstitute/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/broadinstitute/subscriptions","organizations_url":"https://api.github.com/users/broadinstitute/orgs","repos_url":"https://api.github.com/users/broadinstitute/repos","events_url":"https://api.github.com/users/broadinstitute/events{/privacy}","received_events_url":"https://api.github.com/users/broadinstitute/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/broadinstitute/gatk","description":"GATK4 development","fork":false,"url":"https://api.github.com/repos/broadinstitute/gatk","forks_url":"https://api.github.com/repos/broadinstitute/gatk/forks","keys_url":"https://api.github.com/repos/broadinstitute/gatk/keys{/key_id}","collaborators_url":"https://api.github.com/repos/broadinstitute/gatk/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/broadinstitute/gatk/teams","hooks_url":"https://api.github.com/repos/broadinstitute/gatk/hooks","issue_events_url":"https://api.github.com/repos/broadinstitute/gatk/issues/events{/number}","events_url":"https://api.github.com/repos/broadinstitute/gatk/events","assignees_url":"https://api.github.com/repos/broadinstitute/gatk/assignees{/user}","branches_url":"https://api.github.com/repos/broadinstitute/gatk/branches{/branch}","tags_url":"https://api.github.com/repos/broadinstitute/gatk/tags","blobs_url":"https://api.github.com/repos/broadinstitute/gatk/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/broadinstitute/gatk/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/broadinstitute/gatk/git/refs{/sha}","trees_url":"https://api.github.com/repos/broadinstitute/gatk/git/trees{/sha}","statuses_url":"https://api.github.com/repos/broadinstitute/gatk/statuses/{sha}","languages_url":"https://api.github.com/repos/broadinstitute/gatk/languages","stargazers_url":"https://api.github.com/repos/broadinstitute/gatk/stargazers","contributors_url":"https://api.github.com/repos/broadinstitute/gatk/contributors","subscribers_url":"https://api.github.com/repos/broadinstitute/gatk/subscribers","subscription_url":"https://api.github.com/repos/broadinstitute/gatk/subscription","commits_url":"https://api.github.com/repos/broadinstitute/gatk/commits{/sha}","git_commits_url":"https://api.github.com/repos/broadinstitute/gatk/git/commits{/sha}","comments_url":"https://api.github.com/repos/broadinstitute/gatk/comments{/number}","issue_comment_url":"https://api.github.com/repos/broadinstitute/gatk/issues/comments{/number}","contents_url":"https://api.github.com/repos/broadinstitute/gatk/contents/{+path}","compare_url":"https://api.github.com/repos/broadinstitute/gatk/compare/{base}...{head}","merges_url":"https://api.github.com/repos/broadinstitute/gatk/merges","archive_url":"https://api.github.com/repos/broadinstitute/gatk/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/broadinstitute/gatk/downloads","issues_url":"https://api.github.com/repos/broadinstitute/gatk/issues{/number}","pulls_url":"https://api.github.com/repos/broadinstitute/gatk/pulls{/number}","milestones_url":"https://api.github.com/repos/broadinstitute/gatk/milestones{/number}","notifications_url":"https://api.github.com/repos/broadinstitute/gatk/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/broadinstitute/gatk/labels{/name}","releases_url":"https://api.github.com/repos/broadinstitute/gatk/releases{/id}","created_at":"2014-12-02T20:50:36Z","updated_at":"2016-01-11T21:08:29Z","pushed_at":"2016-01-15T21:30:32Z","git_url":"git://github.com/broadinstitute/gatk.git","ssh_url":"git@github.com:broadinstitute/gatk.git","clone_url":"https://github.com/broadinstitute/gatk.git","svn_url":"https://github.com/broadinstitute/gatk","homepage":"","size":57027,"stargazers_count":43,"watchers_count":43,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":18,"mirror_url":null,"open_issues_count":276,"forks":18,"open_issues":276,"watchers":43,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/broadinstitute/gatk/pulls/1435"},"html":{"href":"https://github.com/broadinstitute/gatk/pull/1435"},"issue":{"href":"https://api.github.com/repos/broadinstitute/gatk/issues/1435"},"comments":{"href":"https://api.github.com/repos/broadinstitute/gatk/issues/1435/comments"},"review_comments":{"href":"https://api.github.com/repos/broadinstitute/gatk/pulls/1435/comments"},"review_comment":{"href":"https://api.github.com/repos/broadinstitute/gatk/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/broadinstitute/gatk/pulls/1435/commits"},"statuses":{"href":"https://api.github.com/repos/broadinstitute/gatk/statuses/43f1272da7e281f185b510742adecb20f6ba4ebd"}}}}