{"action":"created","comment":{"url":"https://api.github.com/repos/qiime2/q2-demux/pulls/comments/80139865","id":80139865,"diff_hunk":"@@ -1,54 +1,123 @@\n+import os.path\n import gzip\n import yaml\n import itertools\n import collections\n \n import skbio\n+import pandas as pd\n+import seaborn as sns\n \n import qiime\n from q2_types.per_sample_sequences import (\n-    SingleLanePerSampleSingleEndFastqDirFmt, FastqManifestFormat, YamlFormat)\n+    SingleLanePerSampleSingleEndFastqDirFmt, FastqManifestFormat, YamlFormat,\n+    FastqGzFormat)\n+\n+\n+def _read_fastq_seqs(filepath):\n+    # This function is adapted from @jairideout's SO post:\n+    # http://stackoverflow.com/a/39302117/3424666\n+    fh = gzip.open(filepath, 'rt')\n+    for seq_header, seq, qual_header, qual in itertools.zip_longest(*[fh] * 4):\n+        yield (seq_header.strip(), seq.strip(), qual_header.strip(),\n+               qual.strip())\n \n \n def _extract_common_header(header):\n     return header.rsplit('/', 1)[0]\n \n \n-class BarcodeSequenceIterator(collections.Iterator):\n+class BarcodeSequenceFastqIterator(collections.Iterator):\n     def __init__(self, barcode_generator, sequence_generator):\n         self.barcode_generator = barcode_generator\n         self.sequence_generator = sequence_generator\n \n     def __iter__(self):\n         # Adapted from q2-types\n-        for barcode, sequence in itertools.zip_longest(\n+        for barcode_fields, sequence_fields in itertools.zip_longest(\n                 self.barcode_generator, self.sequence_generator):\n-            if barcode is None:\n+            if barcode_fields is None:\n                 raise ValueError('More sequences were provided than barcodes.')\n-            if sequence is None:\n+            if sequence_fields is None:\n                 raise ValueError('More barcodes were provided than sequences.')\n             # The id or description fields may end with \"/read-number\", which\n-            # will differ between the sequence and barcode reads. Confirm that\n-            # they are identical up until the last /\n-            if _extract_common_header(barcode.metadata['id']) != \\\n-               _extract_common_header(sequence.metadata['id']):\n+            # and read number will differ between the sequence and barcode\n+            # reads. Confirm that they are identical up until the last /\n+            barcode_header_fields = \\\n+                barcode_fields[0][1:].split(' ', maxsplit=1)\n+            sequence_header_fields = \\\n+                sequence_fields[0][1:].split(' ', maxsplit=1)\n+\n+            # confirm that both barcode and sequene have header lines with\n+            # an equal number of space-separated fields\n+            if len(barcode_header_fields) != len(sequence_header_fields):\n+                raise ValueError('Mismatched header lines: %s and %s' %\n+                                 (barcode_fields[0], sequence_fields[0]))\n+\n+            # confirm that the id fields are equal\n+            if _extract_common_header(barcode_header_fields[0]) != \\\n+               _extract_common_header(sequence_header_fields[0]):\n                 raise ValueError(\n                     'Mismatched sequence ids: %s and %s' %\n-                    (_extract_common_header(barcode.metadata['id']),\n-                     _extract_common_header(sequence.metadata['id'])))\n-            if _extract_common_header(barcode.metadata['description']) != \\\n-               _extract_common_header(sequence.metadata['description']):\n-                raise ValueError(\n-                    'Mismatched sequence descriptions: %s and %s' %\n-                    (_extract_common_header(barcode.metadata['description']),\n-                     _extract_common_header(sequence.metadata['description'])))\n-            yield barcode, sequence\n+                    (_extract_common_header(barcode_header_fields[0]),\n+                     _extract_common_header(sequence_header_fields[0])))\n+\n+            # if a description field is present, confirm that they're equal\n+            # note that the number of fields can only be 1 or 2, due to\n+            # maxsplit\n+            if len(barcode_header_fields) == 2:","path":"q2_demux/_demux.py","position":86,"original_position":86,"commit_id":"c8af701111f78661c857b1837d2b8b5637e40259","original_commit_id":"c8af701111f78661c857b1837d2b8b5637e40259","user":{"login":"jairideout","id":1847232,"avatar_url":"https://avatars.githubusercontent.com/u/1847232?v=3","gravatar_id":"","url":"https://api.github.com/users/jairideout","html_url":"https://github.com/jairideout","followers_url":"https://api.github.com/users/jairideout/followers","following_url":"https://api.github.com/users/jairideout/following{/other_user}","gists_url":"https://api.github.com/users/jairideout/gists{/gist_id}","starred_url":"https://api.github.com/users/jairideout/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jairideout/subscriptions","organizations_url":"https://api.github.com/users/jairideout/orgs","repos_url":"https://api.github.com/users/jairideout/repos","events_url":"https://api.github.com/users/jairideout/events{/privacy}","received_events_url":"https://api.github.com/users/jairideout/received_events","type":"User","site_admin":false},"body":"Can you also verify that the unit tests are testing these different cases? They might already be","created_at":"2016-09-22T21:23:48Z","updated_at":"2016-09-22T21:23:49Z","html_url":"https://github.com/qiime2/q2-demux/pull/4#discussion_r80139865","pull_request_url":"https://api.github.com/repos/qiime2/q2-demux/pulls/4","_links":{"self":{"href":"https://api.github.com/repos/qiime2/q2-demux/pulls/comments/80139865"},"html":{"href":"https://github.com/qiime2/q2-demux/pull/4#discussion_r80139865"},"pull_request":{"href":"https://api.github.com/repos/qiime2/q2-demux/pulls/4"}}},"pull_request":{"url":"https://api.github.com/repos/qiime2/q2-demux/pulls/4","id":86255612,"html_url":"https://github.com/qiime2/q2-demux/pull/4","diff_url":"https://github.com/qiime2/q2-demux/pull/4.diff","patch_url":"https://github.com/qiime2/q2-demux/pull/4.patch","issue_url":"https://api.github.com/repos/qiime2/q2-demux/issues/4","number":4,"state":"open","locked":false,"title":"ENH: reduces demux runtime by about 10x","user":{"login":"gregcaporaso","id":192372,"avatar_url":"https://avatars.githubusercontent.com/u/192372?v=3","gravatar_id":"","url":"https://api.github.com/users/gregcaporaso","html_url":"https://github.com/gregcaporaso","followers_url":"https://api.github.com/users/gregcaporaso/followers","following_url":"https://api.github.com/users/gregcaporaso/following{/other_user}","gists_url":"https://api.github.com/users/gregcaporaso/gists{/gist_id}","starred_url":"https://api.github.com/users/gregcaporaso/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/gregcaporaso/subscriptions","organizations_url":"https://api.github.com/users/gregcaporaso/orgs","repos_url":"https://api.github.com/users/gregcaporaso/repos","events_url":"https://api.github.com/users/gregcaporaso/events{/privacy}","received_events_url":"https://api.github.com/users/gregcaporaso/received_events","type":"User","site_admin":false},"body":"this also runs in about 2/3 of the time of QIIME 1 split_libraries_fastq.py\r\n\r\npartially addresses #3, but there may still be room for improvement.","created_at":"2016-09-21T22:53:33Z","updated_at":"2016-09-22T21:23:49Z","closed_at":null,"merged_at":null,"merge_commit_sha":"3cc805c24fa5d35672763279f923b576c9001d93","assignee":{"login":"thermokarst","id":274668,"avatar_url":"https://avatars.githubusercontent.com/u/274668?v=3","gravatar_id":"","url":"https://api.github.com/users/thermokarst","html_url":"https://github.com/thermokarst","followers_url":"https://api.github.com/users/thermokarst/followers","following_url":"https://api.github.com/users/thermokarst/following{/other_user}","gists_url":"https://api.github.com/users/thermokarst/gists{/gist_id}","starred_url":"https://api.github.com/users/thermokarst/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/thermokarst/subscriptions","organizations_url":"https://api.github.com/users/thermokarst/orgs","repos_url":"https://api.github.com/users/thermokarst/repos","events_url":"https://api.github.com/users/thermokarst/events{/privacy}","received_events_url":"https://api.github.com/users/thermokarst/received_events","type":"User","site_admin":false},"assignees":[{"login":"thermokarst","id":274668,"avatar_url":"https://avatars.githubusercontent.com/u/274668?v=3","gravatar_id":"","url":"https://api.github.com/users/thermokarst","html_url":"https://github.com/thermokarst","followers_url":"https://api.github.com/users/thermokarst/followers","following_url":"https://api.github.com/users/thermokarst/following{/other_user}","gists_url":"https://api.github.com/users/thermokarst/gists{/gist_id}","starred_url":"https://api.github.com/users/thermokarst/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/thermokarst/subscriptions","organizations_url":"https://api.github.com/users/thermokarst/orgs","repos_url":"https://api.github.com/users/thermokarst/repos","events_url":"https://api.github.com/users/thermokarst/events{/privacy}","received_events_url":"https://api.github.com/users/thermokarst/received_events","type":"User","site_admin":false},{"login":"jairideout","id":1847232,"avatar_url":"https://avatars.githubusercontent.com/u/1847232?v=3","gravatar_id":"","url":"https://api.github.com/users/jairideout","html_url":"https://github.com/jairideout","followers_url":"https://api.github.com/users/jairideout/followers","following_url":"https://api.github.com/users/jairideout/following{/other_user}","gists_url":"https://api.github.com/users/jairideout/gists{/gist_id}","starred_url":"https://api.github.com/users/jairideout/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jairideout/subscriptions","organizations_url":"https://api.github.com/users/jairideout/orgs","repos_url":"https://api.github.com/users/jairideout/repos","events_url":"https://api.github.com/users/jairideout/events{/privacy}","received_events_url":"https://api.github.com/users/jairideout/received_events","type":"User","site_admin":false}],"milestone":null,"commits_url":"https://api.github.com/repos/qiime2/q2-demux/pulls/4/commits","review_comments_url":"https://api.github.com/repos/qiime2/q2-demux/pulls/4/comments","review_comment_url":"https://api.github.com/repos/qiime2/q2-demux/pulls/comments{/number}","comments_url":"https://api.github.com/repos/qiime2/q2-demux/issues/4/comments","statuses_url":"https://api.github.com/repos/qiime2/q2-demux/statuses/c8af701111f78661c857b1837d2b8b5637e40259","head":{"label":"gregcaporaso:demux-optimization","ref":"demux-optimization","sha":"c8af701111f78661c857b1837d2b8b5637e40259","user":{"login":"gregcaporaso","id":192372,"avatar_url":"https://avatars.githubusercontent.com/u/192372?v=3","gravatar_id":"","url":"https://api.github.com/users/gregcaporaso","html_url":"https://github.com/gregcaporaso","followers_url":"https://api.github.com/users/gregcaporaso/followers","following_url":"https://api.github.com/users/gregcaporaso/following{/other_user}","gists_url":"https://api.github.com/users/gregcaporaso/gists{/gist_id}","starred_url":"https://api.github.com/users/gregcaporaso/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/gregcaporaso/subscriptions","organizations_url":"https://api.github.com/users/gregcaporaso/orgs","repos_url":"https://api.github.com/users/gregcaporaso/repos","events_url":"https://api.github.com/users/gregcaporaso/events{/privacy}","received_events_url":"https://api.github.com/users/gregcaporaso/received_events","type":"User","site_admin":false},"repo":{"id":68643442,"name":"q2-demux","full_name":"gregcaporaso/q2-demux","owner":{"login":"gregcaporaso","id":192372,"avatar_url":"https://avatars.githubusercontent.com/u/192372?v=3","gravatar_id":"","url":"https://api.github.com/users/gregcaporaso","html_url":"https://github.com/gregcaporaso","followers_url":"https://api.github.com/users/gregcaporaso/followers","following_url":"https://api.github.com/users/gregcaporaso/following{/other_user}","gists_url":"https://api.github.com/users/gregcaporaso/gists{/gist_id}","starred_url":"https://api.github.com/users/gregcaporaso/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/gregcaporaso/subscriptions","organizations_url":"https://api.github.com/users/gregcaporaso/orgs","repos_url":"https://api.github.com/users/gregcaporaso/repos","events_url":"https://api.github.com/users/gregcaporaso/events{/privacy}","received_events_url":"https://api.github.com/users/gregcaporaso/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/gregcaporaso/q2-demux","description":"Map sequence barcodes to sample identifiers. ","fork":true,"url":"https://api.github.com/repos/gregcaporaso/q2-demux","forks_url":"https://api.github.com/repos/gregcaporaso/q2-demux/forks","keys_url":"https://api.github.com/repos/gregcaporaso/q2-demux/keys{/key_id}","collaborators_url":"https://api.github.com/repos/gregcaporaso/q2-demux/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/gregcaporaso/q2-demux/teams","hooks_url":"https://api.github.com/repos/gregcaporaso/q2-demux/hooks","issue_events_url":"https://api.github.com/repos/gregcaporaso/q2-demux/issues/events{/number}","events_url":"https://api.github.com/repos/gregcaporaso/q2-demux/events","assignees_url":"https://api.github.com/repos/gregcaporaso/q2-demux/assignees{/user}","branches_url":"https://api.github.com/repos/gregcaporaso/q2-demux/branches{/branch}","tags_url":"https://api.github.com/repos/gregcaporaso/q2-demux/tags","blobs_url":"https://api.github.com/repos/gregcaporaso/q2-demux/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/gregcaporaso/q2-demux/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/gregcaporaso/q2-demux/git/refs{/sha}","trees_url":"https://api.github.com/repos/gregcaporaso/q2-demux/git/trees{/sha}","statuses_url":"https://api.github.com/repos/gregcaporaso/q2-demux/statuses/{sha}","languages_url":"https://api.github.com/repos/gregcaporaso/q2-demux/languages","stargazers_url":"https://api.github.com/repos/gregcaporaso/q2-demux/stargazers","contributors_url":"https://api.github.com/repos/gregcaporaso/q2-demux/contributors","subscribers_url":"https://api.github.com/repos/gregcaporaso/q2-demux/subscribers","subscription_url":"https://api.github.com/repos/gregcaporaso/q2-demux/subscription","commits_url":"https://api.github.com/repos/gregcaporaso/q2-demux/commits{/sha}","git_commits_url":"https://api.github.com/repos/gregcaporaso/q2-demux/git/commits{/sha}","comments_url":"https://api.github.com/repos/gregcaporaso/q2-demux/comments{/number}","issue_comment_url":"https://api.github.com/repos/gregcaporaso/q2-demux/issues/comments{/number}","contents_url":"https://api.github.com/repos/gregcaporaso/q2-demux/contents/{+path}","compare_url":"https://api.github.com/repos/gregcaporaso/q2-demux/compare/{base}...{head}","merges_url":"https://api.github.com/repos/gregcaporaso/q2-demux/merges","archive_url":"https://api.github.com/repos/gregcaporaso/q2-demux/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/gregcaporaso/q2-demux/downloads","issues_url":"https://api.github.com/repos/gregcaporaso/q2-demux/issues{/number}","pulls_url":"https://api.github.com/repos/gregcaporaso/q2-demux/pulls{/number}","milestones_url":"https://api.github.com/repos/gregcaporaso/q2-demux/milestones{/number}","notifications_url":"https://api.github.com/repos/gregcaporaso/q2-demux/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/gregcaporaso/q2-demux/labels{/name}","releases_url":"https://api.github.com/repos/gregcaporaso/q2-demux/releases{/id}","deployments_url":"https://api.github.com/repos/gregcaporaso/q2-demux/deployments","created_at":"2016-09-19T20:25:11Z","updated_at":"2016-09-20T19:32:37Z","pushed_at":"2016-09-22T21:19:30Z","git_url":"git://github.com/gregcaporaso/q2-demux.git","ssh_url":"git@github.com:gregcaporaso/q2-demux.git","clone_url":"https://github.com/gregcaporaso/q2-demux.git","svn_url":"https://github.com/gregcaporaso/q2-demux","homepage":null,"size":71,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"qiime2:master","ref":"master","sha":"cb236cbb7fb411682a6b12aae386ed19b165d92f","user":{"login":"qiime2","id":18176583,"avatar_url":"https://avatars.githubusercontent.com/u/18176583?v=3","gravatar_id":"","url":"https://api.github.com/users/qiime2","html_url":"https://github.com/qiime2","followers_url":"https://api.github.com/users/qiime2/followers","following_url":"https://api.github.com/users/qiime2/following{/other_user}","gists_url":"https://api.github.com/users/qiime2/gists{/gist_id}","starred_url":"https://api.github.com/users/qiime2/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/qiime2/subscriptions","organizations_url":"https://api.github.com/users/qiime2/orgs","repos_url":"https://api.github.com/users/qiime2/repos","events_url":"https://api.github.com/users/qiime2/events{/privacy}","received_events_url":"https://api.github.com/users/qiime2/received_events","type":"Organization","site_admin":false},"repo":{"id":68643014,"name":"q2-demux","full_name":"qiime2/q2-demux","owner":{"login":"qiime2","id":18176583,"avatar_url":"https://avatars.githubusercontent.com/u/18176583?v=3","gravatar_id":"","url":"https://api.github.com/users/qiime2","html_url":"https://github.com/qiime2","followers_url":"https://api.github.com/users/qiime2/followers","following_url":"https://api.github.com/users/qiime2/following{/other_user}","gists_url":"https://api.github.com/users/qiime2/gists{/gist_id}","starred_url":"https://api.github.com/users/qiime2/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/qiime2/subscriptions","organizations_url":"https://api.github.com/users/qiime2/orgs","repos_url":"https://api.github.com/users/qiime2/repos","events_url":"https://api.github.com/users/qiime2/events{/privacy}","received_events_url":"https://api.github.com/users/qiime2/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/qiime2/q2-demux","description":"Map sequence barcodes to sample identifiers. ","fork":false,"url":"https://api.github.com/repos/qiime2/q2-demux","forks_url":"https://api.github.com/repos/qiime2/q2-demux/forks","keys_url":"https://api.github.com/repos/qiime2/q2-demux/keys{/key_id}","collaborators_url":"https://api.github.com/repos/qiime2/q2-demux/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/qiime2/q2-demux/teams","hooks_url":"https://api.github.com/repos/qiime2/q2-demux/hooks","issue_events_url":"https://api.github.com/repos/qiime2/q2-demux/issues/events{/number}","events_url":"https://api.github.com/repos/qiime2/q2-demux/events","assignees_url":"https://api.github.com/repos/qiime2/q2-demux/assignees{/user}","branches_url":"https://api.github.com/repos/qiime2/q2-demux/branches{/branch}","tags_url":"https://api.github.com/repos/qiime2/q2-demux/tags","blobs_url":"https://api.github.com/repos/qiime2/q2-demux/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/qiime2/q2-demux/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/qiime2/q2-demux/git/refs{/sha}","trees_url":"https://api.github.com/repos/qiime2/q2-demux/git/trees{/sha}","statuses_url":"https://api.github.com/repos/qiime2/q2-demux/statuses/{sha}","languages_url":"https://api.github.com/repos/qiime2/q2-demux/languages","stargazers_url":"https://api.github.com/repos/qiime2/q2-demux/stargazers","contributors_url":"https://api.github.com/repos/qiime2/q2-demux/contributors","subscribers_url":"https://api.github.com/repos/qiime2/q2-demux/subscribers","subscription_url":"https://api.github.com/repos/qiime2/q2-demux/subscription","commits_url":"https://api.github.com/repos/qiime2/q2-demux/commits{/sha}","git_commits_url":"https://api.github.com/repos/qiime2/q2-demux/git/commits{/sha}","comments_url":"https://api.github.com/repos/qiime2/q2-demux/comments{/number}","issue_comment_url":"https://api.github.com/repos/qiime2/q2-demux/issues/comments{/number}","contents_url":"https://api.github.com/repos/qiime2/q2-demux/contents/{+path}","compare_url":"https://api.github.com/repos/qiime2/q2-demux/compare/{base}...{head}","merges_url":"https://api.github.com/repos/qiime2/q2-demux/merges","archive_url":"https://api.github.com/repos/qiime2/q2-demux/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/qiime2/q2-demux/downloads","issues_url":"https://api.github.com/repos/qiime2/q2-demux/issues{/number}","pulls_url":"https://api.github.com/repos/qiime2/q2-demux/pulls{/number}","milestones_url":"https://api.github.com/repos/qiime2/q2-demux/milestones{/number}","notifications_url":"https://api.github.com/repos/qiime2/q2-demux/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/qiime2/q2-demux/labels{/name}","releases_url":"https://api.github.com/repos/qiime2/q2-demux/releases{/id}","deployments_url":"https://api.github.com/repos/qiime2/q2-demux/deployments","created_at":"2016-09-19T20:19:21Z","updated_at":"2016-09-22T17:23:36Z","pushed_at":"2016-09-22T21:19:30Z","git_url":"git://github.com/qiime2/q2-demux.git","ssh_url":"git@github.com:qiime2/q2-demux.git","clone_url":"https://github.com/qiime2/q2-demux.git","svn_url":"https://github.com/qiime2/q2-demux","homepage":null,"size":59,"stargazers_count":1,"watchers_count":1,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":3,"forks":1,"open_issues":3,"watchers":1,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/qiime2/q2-demux/pulls/4"},"html":{"href":"https://github.com/qiime2/q2-demux/pull/4"},"issue":{"href":"https://api.github.com/repos/qiime2/q2-demux/issues/4"},"comments":{"href":"https://api.github.com/repos/qiime2/q2-demux/issues/4/comments"},"review_comments":{"href":"https://api.github.com/repos/qiime2/q2-demux/pulls/4/comments"},"review_comment":{"href":"https://api.github.com/repos/qiime2/q2-demux/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/qiime2/q2-demux/pulls/4/commits"},"statuses":{"href":"https://api.github.com/repos/qiime2/q2-demux/statuses/c8af701111f78661c857b1837d2b8b5637e40259"}}}}
{"actor":{"display_login":"jairideout"}}