{"action":"created","comment":{"url":"https://api.github.com/repos/googlegenomics/dataflow-java/pulls/comments/52691257","id":52691257,"diff_hunk":"@@ -0,0 +1,190 @@\n+package htsjdk.samtools;\n+\n+import htsjdk.samtools.util.Log;\n+\n+import java.io.File;\n+import java.io.OutputStream;\n+\n+public class BAMShardIndexer {\n+  // output written as binary, or (for debugging) as text\n+  private final BinaryBAMShardIndexWriter outputWriter;\n+\n+  // content is built up from the input bam file using this\n+  private final BAMIndexBuilder indexBuilder;\n+  \n+  // Index of the reference for which the index is being written\n+  int reference;\n+\n+  public BAMShardIndexer(OutputStream output, SAMFileHeader header, int reference) {\n+    indexBuilder = new BAMIndexBuilder(header.getSequenceDictionary(), reference);\n+    final boolean isFirstIndexShard = reference == 0;\n+    final int numReferencesToWriteInTheHeader = isFirstIndexShard ? \n+        header.getSequenceDictionary().size() : 0;\n+    outputWriter = new BinaryBAMShardIndexWriter(numReferencesToWriteInTheHeader, output);\n+    this.reference = reference;\n+  }\n+  \n+  public void processAlignment(final SAMRecord rec) {\n+      try {\n+          indexBuilder.processAlignment(rec);\n+      } catch (final Exception e) {\n+          throw new SAMException(\"Exception creating BAM index for record \" + rec, e);\n+      }\n+  }\n+\n+  /**\n+   * Finalizes writing and closes the file.\n+   * @return count of records with no coordinates.\n+   */\n+  public long finish() {\n+    final BAMIndexContent content = indexBuilder.processReference(reference);\n+    outputWriter.writeReference(content);\n+    outputWriter.close();\n+    return indexBuilder.getNoCoordinateRecordCount();\n+  }\n+\n+  /**\n+   * Class for constructing BAM index files.\n+   * One instance is used to construct an entire index.\n+   * processAlignment is called for each alignment until a new reference is encountered, then\n+   * processReference is called when all records for the reference have been processed.\n+   */\n+  private class BAMIndexBuilder {\n+\n+      private final SAMSequenceDictionary sequenceDictionary;\n+\n+      private BinningIndexBuilder binningIndexBuilder;\n+\n+      private int currentReference = -1;\n+\n+      // information in meta data\n+      private final BAMIndexMetaData indexStats = new BAMIndexMetaData();\n+\n+      BAMIndexBuilder(final SAMSequenceDictionary sequenceDictionary, int reference) {\n+          this.sequenceDictionary = sequenceDictionary;\n+          if (!sequenceDictionary.isEmpty()) startNewReference(reference);\n+      }\n+\n+      /**\n+       * Record any index information for a given BAM record\n+       *\n+       * @param rec The BAM record. Requires rec.getFileSource() is non-null.\n+       */\n+      public void processAlignment(final SAMRecord rec) {\n+\n+          // metadata\n+          indexStats.recordMetaData(rec);\n+\n+          if (rec.getAlignmentStart() == SAMRecord.NO_ALIGNMENT_START) {\n+              return; // do nothing for records without coordinates, but count them\n+          }\n+\n+          // various checks\n+          final int reference = rec.getReferenceIndex();\n+          if (reference != currentReference) {\n+              throw new SAMException(\"Unexpected reference \" + reference +\n+                      \" when constructing index for \" + currentReference + \" for record \" + rec);\n+          }\n+\n+          binningIndexBuilder.processFeature(new BinningIndexBuilder.FeatureToBeIndexed() {\n+              @Override\n+              public int getStart() {\n+                  return rec.getAlignmentStart();\n+              }\n+\n+              @Override\n+              public int getEnd() {\n+                  return rec.getAlignmentEnd();\n+              }\n+\n+              @Override\n+              public Integer getIndexingBin() {\n+                  final Integer binNumber = rec.getIndexingBin();\n+                  return (binNumber == null ? rec.computeIndexingBin() : binNumber);\n+\n+              }\n+\n+              @Override\n+              public Chunk getChunk() {\n+                  final SAMFileSource source = rec.getFileSource();\n+                  if (source == null) {\n+                      throw new SAMException(\"No source (virtual file offsets); needed for indexing on BAM Record \" + rec);\n+                  }\n+                  return ((BAMFileSpan) source.getFilePointer()).getSingleChunk();\n+              }\n+          });\n+\n+      }\n+\n+      /**\n+       * Creates the BAMIndexContent for this reference.\n+       * Requires all alignments of the reference have already been processed.\n+       *\n+       * @return Null if there are no features for this reference.\n+       */\n+      public BAMIndexContent processReference(final int reference) {\n+\n+          if (reference != currentReference) {\n+              throw new SAMException(\"Unexpected reference \" + reference + \" when constructing index for \" + currentReference);\n+          }\n+\n+          final BinningIndexContent indexContent = binningIndexBuilder.generateIndexContent();\n+          if (indexContent == null) return null;\n+          return new BAMIndexContent(indexContent.getReferenceSequence(), indexContent.getBins(),\n+                  indexStats, indexContent.getLinearIndex());\n+\n+      }\n+\n+      /**\n+       * @return the count of records with no coordinate positions\n+       */\n+      public long getNoCoordinateRecordCount() {\n+          return indexStats.getNoCoordinateRecordCount();\n+      }\n+\n+      /**\n+       * reinitialize all data structures when the reference changes","path":"src/main/java/htsjdk/samtools/BAMShardIndexer.java","position":146,"original_position":146,"commit_id":"9fa7583736bd2422551a2ab0c4c0d016f22cfeba","original_commit_id":"9fa7583736bd2422551a2ab0c4c0d016f22cfeba","user":{"login":"pgrosu","id":6555937,"avatar_url":"https://avatars.githubusercontent.com/u/6555937?v=3","gravatar_id":"","url":"https://api.github.com/users/pgrosu","html_url":"https://github.com/pgrosu","followers_url":"https://api.github.com/users/pgrosu/followers","following_url":"https://api.github.com/users/pgrosu/following{/other_user}","gists_url":"https://api.github.com/users/pgrosu/gists{/gist_id}","starred_url":"https://api.github.com/users/pgrosu/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pgrosu/subscriptions","organizations_url":"https://api.github.com/users/pgrosu/orgs","repos_url":"https://api.github.com/users/pgrosu/repos","events_url":"https://api.github.com/users/pgrosu/events{/privacy}","received_events_url":"https://api.github.com/users/pgrosu/received_events","type":"User","site_admin":false},"body":"s/reinitialize/Reinitialize ","created_at":"2016-02-12T00:34:07Z","updated_at":"2016-02-12T00:34:07Z","html_url":"https://github.com/googlegenomics/dataflow-java/pull/165#discussion_r52691257","pull_request_url":"https://api.github.com/repos/googlegenomics/dataflow-java/pulls/165","_links":{"self":{"href":"https://api.github.com/repos/googlegenomics/dataflow-java/pulls/comments/52691257"},"html":{"href":"https://github.com/googlegenomics/dataflow-java/pull/165#discussion_r52691257"},"pull_request":{"href":"https://api.github.com/repos/googlegenomics/dataflow-java/pulls/165"}}},"pull_request":{"url":"https://api.github.com/repos/googlegenomics/dataflow-java/pulls/165","id":58298007,"html_url":"https://github.com/googlegenomics/dataflow-java/pull/165","diff_url":"https://github.com/googlegenomics/dataflow-java/pull/165.diff","patch_url":"https://github.com/googlegenomics/dataflow-java/pull/165.patch","issue_url":"https://api.github.com/repos/googlegenomics/dataflow-java/issues/165","number":165,"state":"open","locked":false,"title":"Update to handle API reading and optimize sharded writing and indexing","user":{"login":"iliat","id":4410767,"avatar_url":"https://avatars.githubusercontent.com/u/4410767?v=3","gravatar_id":"","url":"https://api.github.com/users/iliat","html_url":"https://github.com/iliat","followers_url":"https://api.github.com/users/iliat/followers","following_url":"https://api.github.com/users/iliat/following{/other_user}","gists_url":"https://api.github.com/users/iliat/gists{/gist_id}","starred_url":"https://api.github.com/users/iliat/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/iliat/subscriptions","organizations_url":"https://api.github.com/users/iliat/orgs","repos_url":"https://api.github.com/users/iliat/repos","events_url":"https://api.github.com/users/iliat/events{/privacy}","received_events_url":"https://api.github.com/users/iliat/received_events","type":"User","site_admin":false},"body":"It implements sharded index writing and removes GBK stage in writing, relying on the read side sharding.\r\n\r\nIt exports a full genome BAM file of ~60GB in ~25min.","created_at":"2016-02-04T15:00:43Z","updated_at":"2016-02-12T00:34:07Z","closed_at":null,"merged_at":null,"merge_commit_sha":"6d10becd945fc0d4439a488b0864d01a66272951","assignee":{"login":"deflaux","id":1395500,"avatar_url":"https://avatars.githubusercontent.com/u/1395500?v=3","gravatar_id":"","url":"https://api.github.com/users/deflaux","html_url":"https://github.com/deflaux","followers_url":"https://api.github.com/users/deflaux/followers","following_url":"https://api.github.com/users/deflaux/following{/other_user}","gists_url":"https://api.github.com/users/deflaux/gists{/gist_id}","starred_url":"https://api.github.com/users/deflaux/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/deflaux/subscriptions","organizations_url":"https://api.github.com/users/deflaux/orgs","repos_url":"https://api.github.com/users/deflaux/repos","events_url":"https://api.github.com/users/deflaux/events{/privacy}","received_events_url":"https://api.github.com/users/deflaux/received_events","type":"User","site_admin":false},"milestone":null,"commits_url":"https://api.github.com/repos/googlegenomics/dataflow-java/pulls/165/commits","review_comments_url":"https://api.github.com/repos/googlegenomics/dataflow-java/pulls/165/comments","review_comment_url":"https://api.github.com/repos/googlegenomics/dataflow-java/pulls/comments{/number}","comments_url":"https://api.github.com/repos/googlegenomics/dataflow-java/issues/165/comments","statuses_url":"https://api.github.com/repos/googlegenomics/dataflow-java/statuses/9fa7583736bd2422551a2ab0c4c0d016f22cfeba","head":{"label":"iliat:sharded-export","ref":"sharded-export","sha":"9fa7583736bd2422551a2ab0c4c0d016f22cfeba","user":{"login":"iliat","id":4410767,"avatar_url":"https://avatars.githubusercontent.com/u/4410767?v=3","gravatar_id":"","url":"https://api.github.com/users/iliat","html_url":"https://github.com/iliat","followers_url":"https://api.github.com/users/iliat/followers","following_url":"https://api.github.com/users/iliat/following{/other_user}","gists_url":"https://api.github.com/users/iliat/gists{/gist_id}","starred_url":"https://api.github.com/users/iliat/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/iliat/subscriptions","organizations_url":"https://api.github.com/users/iliat/orgs","repos_url":"https://api.github.com/users/iliat/repos","events_url":"https://api.github.com/users/iliat/events{/privacy}","received_events_url":"https://api.github.com/users/iliat/received_events","type":"User","site_admin":false},"repo":{"id":30418867,"name":"dataflow-java","full_name":"iliat/dataflow-java","owner":{"login":"iliat","id":4410767,"avatar_url":"https://avatars.githubusercontent.com/u/4410767?v=3","gravatar_id":"","url":"https://api.github.com/users/iliat","html_url":"https://github.com/iliat","followers_url":"https://api.github.com/users/iliat/followers","following_url":"https://api.github.com/users/iliat/following{/other_user}","gists_url":"https://api.github.com/users/iliat/gists{/gist_id}","starred_url":"https://api.github.com/users/iliat/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/iliat/subscriptions","organizations_url":"https://api.github.com/users/iliat/orgs","repos_url":"https://api.github.com/users/iliat/repos","events_url":"https://api.github.com/users/iliat/events{/privacy}","received_events_url":"https://api.github.com/users/iliat/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/iliat/dataflow-java","description":"Provides several Google Cloud Dataflow pipeline examples as well as useful utility classes.","fork":true,"url":"https://api.github.com/repos/iliat/dataflow-java","forks_url":"https://api.github.com/repos/iliat/dataflow-java/forks","keys_url":"https://api.github.com/repos/iliat/dataflow-java/keys{/key_id}","collaborators_url":"https://api.github.com/repos/iliat/dataflow-java/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/iliat/dataflow-java/teams","hooks_url":"https://api.github.com/repos/iliat/dataflow-java/hooks","issue_events_url":"https://api.github.com/repos/iliat/dataflow-java/issues/events{/number}","events_url":"https://api.github.com/repos/iliat/dataflow-java/events","assignees_url":"https://api.github.com/repos/iliat/dataflow-java/assignees{/user}","branches_url":"https://api.github.com/repos/iliat/dataflow-java/branches{/branch}","tags_url":"https://api.github.com/repos/iliat/dataflow-java/tags","blobs_url":"https://api.github.com/repos/iliat/dataflow-java/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/iliat/dataflow-java/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/iliat/dataflow-java/git/refs{/sha}","trees_url":"https://api.github.com/repos/iliat/dataflow-java/git/trees{/sha}","statuses_url":"https://api.github.com/repos/iliat/dataflow-java/statuses/{sha}","languages_url":"https://api.github.com/repos/iliat/dataflow-java/languages","stargazers_url":"https://api.github.com/repos/iliat/dataflow-java/stargazers","contributors_url":"https://api.github.com/repos/iliat/dataflow-java/contributors","subscribers_url":"https://api.github.com/repos/iliat/dataflow-java/subscribers","subscription_url":"https://api.github.com/repos/iliat/dataflow-java/subscription","commits_url":"https://api.github.com/repos/iliat/dataflow-java/commits{/sha}","git_commits_url":"https://api.github.com/repos/iliat/dataflow-java/git/commits{/sha}","comments_url":"https://api.github.com/repos/iliat/dataflow-java/comments{/number}","issue_comment_url":"https://api.github.com/repos/iliat/dataflow-java/issues/comments{/number}","contents_url":"https://api.github.com/repos/iliat/dataflow-java/contents/{+path}","compare_url":"https://api.github.com/repos/iliat/dataflow-java/compare/{base}...{head}","merges_url":"https://api.github.com/repos/iliat/dataflow-java/merges","archive_url":"https://api.github.com/repos/iliat/dataflow-java/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/iliat/dataflow-java/downloads","issues_url":"https://api.github.com/repos/iliat/dataflow-java/issues{/number}","pulls_url":"https://api.github.com/repos/iliat/dataflow-java/pulls{/number}","milestones_url":"https://api.github.com/repos/iliat/dataflow-java/milestones{/number}","notifications_url":"https://api.github.com/repos/iliat/dataflow-java/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/iliat/dataflow-java/labels{/name}","releases_url":"https://api.github.com/repos/iliat/dataflow-java/releases{/id}","deployments_url":"https://api.github.com/repos/iliat/dataflow-java/deployments","created_at":"2015-02-06T15:32:16Z","updated_at":"2016-02-05T16:17:55Z","pushed_at":"2016-02-05T16:18:46Z","git_url":"git://github.com/iliat/dataflow-java.git","ssh_url":"git@github.com:iliat/dataflow-java.git","clone_url":"https://github.com/iliat/dataflow-java.git","svn_url":"https://github.com/iliat/dataflow-java","homepage":"","size":142432,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"googlegenomics:master","ref":"master","sha":"de9258adc9bd8523d35058f43446b52e60332a27","user":{"login":"googlegenomics","id":7434120,"avatar_url":"https://avatars.githubusercontent.com/u/7434120?v=3","gravatar_id":"","url":"https://api.github.com/users/googlegenomics","html_url":"https://github.com/googlegenomics","followers_url":"https://api.github.com/users/googlegenomics/followers","following_url":"https://api.github.com/users/googlegenomics/following{/other_user}","gists_url":"https://api.github.com/users/googlegenomics/gists{/gist_id}","starred_url":"https://api.github.com/users/googlegenomics/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/googlegenomics/subscriptions","organizations_url":"https://api.github.com/users/googlegenomics/orgs","repos_url":"https://api.github.com/users/googlegenomics/repos","events_url":"https://api.github.com/users/googlegenomics/events{/privacy}","received_events_url":"https://api.github.com/users/googlegenomics/received_events","type":"Organization","site_admin":false},"repo":{"id":20532587,"name":"dataflow-java","full_name":"googlegenomics/dataflow-java","owner":{"login":"googlegenomics","id":7434120,"avatar_url":"https://avatars.githubusercontent.com/u/7434120?v=3","gravatar_id":"","url":"https://api.github.com/users/googlegenomics","html_url":"https://github.com/googlegenomics","followers_url":"https://api.github.com/users/googlegenomics/followers","following_url":"https://api.github.com/users/googlegenomics/following{/other_user}","gists_url":"https://api.github.com/users/googlegenomics/gists{/gist_id}","starred_url":"https://api.github.com/users/googlegenomics/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/googlegenomics/subscriptions","organizations_url":"https://api.github.com/users/googlegenomics/orgs","repos_url":"https://api.github.com/users/googlegenomics/repos","events_url":"https://api.github.com/users/googlegenomics/events{/privacy}","received_events_url":"https://api.github.com/users/googlegenomics/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/googlegenomics/dataflow-java","description":"Google Cloud Dataflow pipelines such as Identity-By-State as well as useful utility classes.","fork":false,"url":"https://api.github.com/repos/googlegenomics/dataflow-java","forks_url":"https://api.github.com/repos/googlegenomics/dataflow-java/forks","keys_url":"https://api.github.com/repos/googlegenomics/dataflow-java/keys{/key_id}","collaborators_url":"https://api.github.com/repos/googlegenomics/dataflow-java/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/googlegenomics/dataflow-java/teams","hooks_url":"https://api.github.com/repos/googlegenomics/dataflow-java/hooks","issue_events_url":"https://api.github.com/repos/googlegenomics/dataflow-java/issues/events{/number}","events_url":"https://api.github.com/repos/googlegenomics/dataflow-java/events","assignees_url":"https://api.github.com/repos/googlegenomics/dataflow-java/assignees{/user}","branches_url":"https://api.github.com/repos/googlegenomics/dataflow-java/branches{/branch}","tags_url":"https://api.github.com/repos/googlegenomics/dataflow-java/tags","blobs_url":"https://api.github.com/repos/googlegenomics/dataflow-java/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/googlegenomics/dataflow-java/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/googlegenomics/dataflow-java/git/refs{/sha}","trees_url":"https://api.github.com/repos/googlegenomics/dataflow-java/git/trees{/sha}","statuses_url":"https://api.github.com/repos/googlegenomics/dataflow-java/statuses/{sha}","languages_url":"https://api.github.com/repos/googlegenomics/dataflow-java/languages","stargazers_url":"https://api.github.com/repos/googlegenomics/dataflow-java/stargazers","contributors_url":"https://api.github.com/repos/googlegenomics/dataflow-java/contributors","subscribers_url":"https://api.github.com/repos/googlegenomics/dataflow-java/subscribers","subscription_url":"https://api.github.com/repos/googlegenomics/dataflow-java/subscription","commits_url":"https://api.github.com/repos/googlegenomics/dataflow-java/commits{/sha}","git_commits_url":"https://api.github.com/repos/googlegenomics/dataflow-java/git/commits{/sha}","comments_url":"https://api.github.com/repos/googlegenomics/dataflow-java/comments{/number}","issue_comment_url":"https://api.github.com/repos/googlegenomics/dataflow-java/issues/comments{/number}","contents_url":"https://api.github.com/repos/googlegenomics/dataflow-java/contents/{+path}","compare_url":"https://api.github.com/repos/googlegenomics/dataflow-java/compare/{base}...{head}","merges_url":"https://api.github.com/repos/googlegenomics/dataflow-java/merges","archive_url":"https://api.github.com/repos/googlegenomics/dataflow-java/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/googlegenomics/dataflow-java/downloads","issues_url":"https://api.github.com/repos/googlegenomics/dataflow-java/issues{/number}","pulls_url":"https://api.github.com/repos/googlegenomics/dataflow-java/pulls{/number}","milestones_url":"https://api.github.com/repos/googlegenomics/dataflow-java/milestones{/number}","notifications_url":"https://api.github.com/repos/googlegenomics/dataflow-java/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/googlegenomics/dataflow-java/labels{/name}","releases_url":"https://api.github.com/repos/googlegenomics/dataflow-java/releases{/id}","deployments_url":"https://api.github.com/repos/googlegenomics/dataflow-java/deployments","created_at":"2014-06-05T16:27:31Z","updated_at":"2016-02-09T06:07:14Z","pushed_at":"2016-02-10T18:55:58Z","git_url":"git://github.com/googlegenomics/dataflow-java.git","ssh_url":"git@github.com:googlegenomics/dataflow-java.git","clone_url":"https://github.com/googlegenomics/dataflow-java.git","svn_url":"https://github.com/googlegenomics/dataflow-java","homepage":"","size":142861,"stargazers_count":17,"watchers_count":17,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":16,"mirror_url":null,"open_issues_count":12,"forks":16,"open_issues":12,"watchers":17,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/googlegenomics/dataflow-java/pulls/165"},"html":{"href":"https://github.com/googlegenomics/dataflow-java/pull/165"},"issue":{"href":"https://api.github.com/repos/googlegenomics/dataflow-java/issues/165"},"comments":{"href":"https://api.github.com/repos/googlegenomics/dataflow-java/issues/165/comments"},"review_comments":{"href":"https://api.github.com/repos/googlegenomics/dataflow-java/pulls/165/comments"},"review_comment":{"href":"https://api.github.com/repos/googlegenomics/dataflow-java/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/googlegenomics/dataflow-java/pulls/165/commits"},"statuses":{"href":"https://api.github.com/repos/googlegenomics/dataflow-java/statuses/9fa7583736bd2422551a2ab0c4c0d016f22cfeba"}}}}