{"action":"created","comment":{"url":"https://api.github.com/repos/BD2KGenomics/toil-lib/pulls/comments/78221578","id":78221578,"diff_hunk":"@@ -0,0 +1,335 @@\n+import os\n+import subprocess\n+\n+from toil_lib.programs import docker_call\n+\n+\n+def gatk_select_variants(job, mode, vcf_id, config):\n+    \"\"\"\n+    Isolates a particular variant type from a VCF file using GATK SelectVariants.\n+\n+    :param JobFunctionWrappingJob job: passed automatically by Toil\n+    :param str mode: variant type (i.e. SNP or INDEL)\n+    :param str vcf_id: VCF FileStoreID\n+    :param Namespace config: Configuration and shared FileStoreIDs\n+    :return: VCF FileStoreID\n+    :rtype: str\n+    \"\"\"\n+    job.fileStore.logToMaster('GATK SelectVariants: %s' % mode)\n+\n+    inputs = {'genome.fa': config.genome_fasta,\n+              'genome.fa.fai': config.genome_fai,\n+              'genome.dict': config.genome_dict,\n+              'input.vcf': vcf_id}\n+\n+    work_dir = job.fileStore.getLocalTempDir()\n+    for name, file_store_id in inputs.iteritems():\n+        job.fileStore.readGlobalFile(file_store_id, os.path.join(work_dir, name))\n+\n+    command = ['-T', 'SelectVariants',\n+               '-R', 'genome.fa',\n+               '-V', 'input.vcf',\n+               '-o', 'output.vcf',\n+               '-selectType', mode]\n+\n+    docker_call(work_dir=work_dir,\n+                env={'JAVA_OPTS': '-Djava.io.tmpdir=/data/ -Xmx{}'.format(job.memory)},\n+                parameters=command,\n+                tool='quay.io/ucsc_cgl/gatk:3.5--dba6dae49156168a909c43330350c6161dc7ecc2',\n+                inputs=inputs.keys(),\n+                outputs={'output.vcf': None})\n+\n+    return job.fileStore.writeGlobalFile(os.path.join(work_dir, 'output.vcf'))\n+\n+\n+def gatk_variant_filtration(job, mode, vcf_id, config):\n+    \"\"\"\n+    Filters a VCF file using GATK VariantFiltration and recommended JEXL expressions. The filters\n+    are based on GATK variant annotations and are specific to SNPs or INDELs. Therefore, this\n+    function must be called separately on SNPs and INDELs, and the input VCF should just have one\n+    of these variant types.\n+\n+    SNP Filter Expression:\n+    QD < 2.0 || FS > 60.0 || MQ < 40.0 || MQRankSum < -12.5 || ReadPosRankSum < -8.0\n+\n+    INDEL Filter Expression:\n+    QD < 2.0 || FS > 200.0 || ReadPosRankSum < -20.0\n+\n+    :param JobFunctionWrappingJob job: passed automatically by Toil\n+    :param str mode: variant type (SNP or INDEL)\n+    :param str vcf_id: VCF FileStoreID\n+    :param Namespace config: Configuration parameters and shared FileStoreIDs\n+    :return: Filtered VCF FileStoreID\n+    :rtype: str\n+    \"\"\"\n+    mode = mode.upper()\n+    job.fileStore.logToMaster('Applied %s Hard Filter' % mode)\n+\n+    inputs = {'genome.fa': config.genome_fasta,\n+              'genome.fa.fai': config.genome_fai,\n+              'genome.dict': config.genome_dict,\n+              'input.vcf': vcf_id}\n+\n+    work_dir = job.fileStore.getLocalTempDir()\n+    for name, file_store_id in inputs.iteritems():\n+        job.fileStore.readGlobalFile(file_store_id, os.path.join(work_dir, name))\n+\n+    # GATK hard filters:\n+    # https://software.broadinstitute.org/gatk/documentation/article?id=2806\n+    # Filters are different for SNPs and INDELs. Please refer to GATK documentation.\n+    if mode == 'SNP':\n+        expression = '\"QD < 2.0 || FS > 60.0 || MQ < 40.0 || ' \\\n+                     'MQRankSum < -12.5 || ReadPosRankSum < -8.0\"'\n+\n+    elif mode == 'INDEL':\n+        expression = '\"QD < 2.0 || FS > 200.0 || ReadPosRankSum < -20.0\"'\n+\n+    else:\n+        raise ValueError('Variant filter modes can be SNP or INDEL, got %s' % mode)\n+\n+    command = ['-T', 'VariantFiltration',\n+               '-R', 'genome.fa',\n+               '-V', 'input.vcf',\n+               '--filterExpression', expression,\n+               '--filterName', 'GATK_Germline_Hard_Filter_%s' % mode,   # Documents filter in header\n+               '-o', 'filtered_variants.vcf']\n+\n+    docker_call(work_dir=work_dir,\n+                env={'JAVA_OPTS': '-Djava.io.tmpdir=/data/ -Xmx{}'.format(job.memory)},\n+                parameters=command,\n+                tool='quay.io/ucsc_cgl/gatk:3.5--dba6dae49156168a909c43330350c6161dc7ecc2',\n+                inputs=inputs.keys(),\n+                outputs={'filtered_variants.vcf': None})\n+\n+    # Fix extra quotation marks in FILTER line\n+    output = os.path.join(work_dir, 'filtered_variants.vcf')\n+    sed_cmd = 's/\"{}\"/{}/'.format(expression, expression)\n+    subprocess.call(['sed', '-i', sed_cmd, output])\n+\n+    return job.fileStore.writeGlobalFile(output)\n+\n+\n+def gatk_variant_recalibrator(job, mode, vcf_id, config):\n+    \"\"\"\n+    Variant quality score recalibration for SNP or INDEL variants. This function is configured\n+    to run either SNP or INDEL VQSR, so run this function twice for the two variant types. It is\n+    not necessary to split the input VCF by variant type to run this function.\n+\n+    :param JobFunctionWrappingJob job: Job instance\n+    :param str mode: Determines variant recalibration mode (SNP or INDEL)\n+    :param str vcf_id: VCF FileStoreID\n+    :param Namespace config: Input parameters and shared FileStoreIDs","path":"src/toil_lib/tools/variant_manipulation.py","position":121,"original_position":121,"commit_id":"a62a51fd7dad40576ca8192c8fce885ed8ad4496","original_commit_id":"a62a51fd7dad40576ca8192c8fce885ed8ad4496","user":{"login":"jvivian","id":7576256,"avatar_url":"https://avatars.githubusercontent.com/u/7576256?v=3","gravatar_id":"","url":"https://api.github.com/users/jvivian","html_url":"https://github.com/jvivian","followers_url":"https://api.github.com/users/jvivian/followers","following_url":"https://api.github.com/users/jvivian/following{/other_user}","gists_url":"https://api.github.com/users/jvivian/gists{/gist_id}","starred_url":"https://api.github.com/users/jvivian/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jvivian/subscriptions","organizations_url":"https://api.github.com/users/jvivian/orgs","repos_url":"https://api.github.com/users/jvivian/repos","events_url":"https://api.github.com/users/jvivian/events{/privacy}","received_events_url":"https://api.github.com/users/jvivian/received_events","type":"User","site_admin":false},"body":"`config` and ID issue. ","created_at":"2016-09-09T17:53:43Z","updated_at":"2016-09-09T17:53:43Z","html_url":"https://github.com/BD2KGenomics/toil-lib/pull/10#discussion_r78221578","pull_request_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/pulls/10","_links":{"self":{"href":"https://api.github.com/repos/BD2KGenomics/toil-lib/pulls/comments/78221578"},"html":{"href":"https://github.com/BD2KGenomics/toil-lib/pull/10#discussion_r78221578"},"pull_request":{"href":"https://api.github.com/repos/BD2KGenomics/toil-lib/pulls/10"}}},"pull_request":{"url":"https://api.github.com/repos/BD2KGenomics/toil-lib/pulls/10","id":84344151,"html_url":"https://github.com/BD2KGenomics/toil-lib/pull/10","diff_url":"https://github.com/BD2KGenomics/toil-lib/pull/10.diff","patch_url":"https://github.com/BD2KGenomics/toil-lib/pull/10.patch","issue_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/issues/10","number":10,"state":"open","locked":false,"title":"Add vcf manipulation tools (resolves #9)","user":{"login":"jpfeil","id":12463956,"avatar_url":"https://avatars.githubusercontent.com/u/12463956?v=3","gravatar_id":"","url":"https://api.github.com/users/jpfeil","html_url":"https://github.com/jpfeil","followers_url":"https://api.github.com/users/jpfeil/followers","following_url":"https://api.github.com/users/jpfeil/following{/other_user}","gists_url":"https://api.github.com/users/jpfeil/gists{/gist_id}","starred_url":"https://api.github.com/users/jpfeil/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jpfeil/subscriptions","organizations_url":"https://api.github.com/users/jpfeil/orgs","repos_url":"https://api.github.com/users/jpfeil/repos","events_url":"https://api.github.com/users/jpfeil/events{/privacy}","received_events_url":"https://api.github.com/users/jpfeil/received_events","type":"User","site_admin":false},"body":"resolves #9 ","created_at":"2016-09-07T17:46:29Z","updated_at":"2016-09-09T17:53:43Z","closed_at":null,"merged_at":null,"merge_commit_sha":"b7791792374b911bf0f9f059b474dc75f270bbce","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/pulls/10/commits","review_comments_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/pulls/10/comments","review_comment_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/pulls/comments{/number}","comments_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/issues/10/comments","statuses_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/statuses/a62a51fd7dad40576ca8192c8fce885ed8ad4496","head":{"label":"BD2KGenomics:issues/9-add-vcf-manipulation-tools","ref":"issues/9-add-vcf-manipulation-tools","sha":"a62a51fd7dad40576ca8192c8fce885ed8ad4496","user":{"login":"BD2KGenomics","id":8701293,"avatar_url":"https://avatars.githubusercontent.com/u/8701293?v=3","gravatar_id":"","url":"https://api.github.com/users/BD2KGenomics","html_url":"https://github.com/BD2KGenomics","followers_url":"https://api.github.com/users/BD2KGenomics/followers","following_url":"https://api.github.com/users/BD2KGenomics/following{/other_user}","gists_url":"https://api.github.com/users/BD2KGenomics/gists{/gist_id}","starred_url":"https://api.github.com/users/BD2KGenomics/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/BD2KGenomics/subscriptions","organizations_url":"https://api.github.com/users/BD2KGenomics/orgs","repos_url":"https://api.github.com/users/BD2KGenomics/repos","events_url":"https://api.github.com/users/BD2KGenomics/events{/privacy}","received_events_url":"https://api.github.com/users/BD2KGenomics/received_events","type":"Organization","site_admin":false},"repo":{"id":66871728,"name":"toil-lib","full_name":"BD2KGenomics/toil-lib","owner":{"login":"BD2KGenomics","id":8701293,"avatar_url":"https://avatars.githubusercontent.com/u/8701293?v=3","gravatar_id":"","url":"https://api.github.com/users/BD2KGenomics","html_url":"https://github.com/BD2KGenomics","followers_url":"https://api.github.com/users/BD2KGenomics/followers","following_url":"https://api.github.com/users/BD2KGenomics/following{/other_user}","gists_url":"https://api.github.com/users/BD2KGenomics/gists{/gist_id}","starred_url":"https://api.github.com/users/BD2KGenomics/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/BD2KGenomics/subscriptions","organizations_url":"https://api.github.com/users/BD2KGenomics/orgs","repos_url":"https://api.github.com/users/BD2KGenomics/repos","events_url":"https://api.github.com/users/BD2KGenomics/events{/privacy}","received_events_url":"https://api.github.com/users/BD2KGenomics/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/BD2KGenomics/toil-lib","description":"A common library for functions and tools used in toil-based pipelines","fork":false,"url":"https://api.github.com/repos/BD2KGenomics/toil-lib","forks_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/forks","keys_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/keys{/key_id}","collaborators_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/teams","hooks_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/hooks","issue_events_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/issues/events{/number}","events_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/events","assignees_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/assignees{/user}","branches_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/branches{/branch}","tags_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/tags","blobs_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/git/refs{/sha}","trees_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/git/trees{/sha}","statuses_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/statuses/{sha}","languages_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/languages","stargazers_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/stargazers","contributors_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/contributors","subscribers_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/subscribers","subscription_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/subscription","commits_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/commits{/sha}","git_commits_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/git/commits{/sha}","comments_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/comments{/number}","issue_comment_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/issues/comments{/number}","contents_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/contents/{+path}","compare_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/compare/{base}...{head}","merges_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/merges","archive_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/downloads","issues_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/issues{/number}","pulls_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/pulls{/number}","milestones_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/milestones{/number}","notifications_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/labels{/name}","releases_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/releases{/id}","deployments_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/deployments","created_at":"2016-08-29T18:58:58Z","updated_at":"2016-09-07T17:42:20Z","pushed_at":"2016-09-09T16:56:28Z","git_url":"git://github.com/BD2KGenomics/toil-lib.git","ssh_url":"git@github.com:BD2KGenomics/toil-lib.git","clone_url":"https://github.com/BD2KGenomics/toil-lib.git","svn_url":"https://github.com/BD2KGenomics/toil-lib","homepage":null,"size":58,"stargazers_count":1,"watchers_count":1,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":10,"forks":1,"open_issues":10,"watchers":1,"default_branch":"master"}},"base":{"label":"BD2KGenomics:master","ref":"master","sha":"ae5ff1839845caea58d34e0b269cc5da2f498a71","user":{"login":"BD2KGenomics","id":8701293,"avatar_url":"https://avatars.githubusercontent.com/u/8701293?v=3","gravatar_id":"","url":"https://api.github.com/users/BD2KGenomics","html_url":"https://github.com/BD2KGenomics","followers_url":"https://api.github.com/users/BD2KGenomics/followers","following_url":"https://api.github.com/users/BD2KGenomics/following{/other_user}","gists_url":"https://api.github.com/users/BD2KGenomics/gists{/gist_id}","starred_url":"https://api.github.com/users/BD2KGenomics/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/BD2KGenomics/subscriptions","organizations_url":"https://api.github.com/users/BD2KGenomics/orgs","repos_url":"https://api.github.com/users/BD2KGenomics/repos","events_url":"https://api.github.com/users/BD2KGenomics/events{/privacy}","received_events_url":"https://api.github.com/users/BD2KGenomics/received_events","type":"Organization","site_admin":false},"repo":{"id":66871728,"name":"toil-lib","full_name":"BD2KGenomics/toil-lib","owner":{"login":"BD2KGenomics","id":8701293,"avatar_url":"https://avatars.githubusercontent.com/u/8701293?v=3","gravatar_id":"","url":"https://api.github.com/users/BD2KGenomics","html_url":"https://github.com/BD2KGenomics","followers_url":"https://api.github.com/users/BD2KGenomics/followers","following_url":"https://api.github.com/users/BD2KGenomics/following{/other_user}","gists_url":"https://api.github.com/users/BD2KGenomics/gists{/gist_id}","starred_url":"https://api.github.com/users/BD2KGenomics/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/BD2KGenomics/subscriptions","organizations_url":"https://api.github.com/users/BD2KGenomics/orgs","repos_url":"https://api.github.com/users/BD2KGenomics/repos","events_url":"https://api.github.com/users/BD2KGenomics/events{/privacy}","received_events_url":"https://api.github.com/users/BD2KGenomics/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/BD2KGenomics/toil-lib","description":"A common library for functions and tools used in toil-based pipelines","fork":false,"url":"https://api.github.com/repos/BD2KGenomics/toil-lib","forks_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/forks","keys_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/keys{/key_id}","collaborators_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/teams","hooks_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/hooks","issue_events_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/issues/events{/number}","events_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/events","assignees_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/assignees{/user}","branches_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/branches{/branch}","tags_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/tags","blobs_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/git/refs{/sha}","trees_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/git/trees{/sha}","statuses_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/statuses/{sha}","languages_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/languages","stargazers_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/stargazers","contributors_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/contributors","subscribers_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/subscribers","subscription_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/subscription","commits_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/commits{/sha}","git_commits_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/git/commits{/sha}","comments_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/comments{/number}","issue_comment_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/issues/comments{/number}","contents_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/contents/{+path}","compare_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/compare/{base}...{head}","merges_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/merges","archive_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/downloads","issues_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/issues{/number}","pulls_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/pulls{/number}","milestones_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/milestones{/number}","notifications_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/labels{/name}","releases_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/releases{/id}","deployments_url":"https://api.github.com/repos/BD2KGenomics/toil-lib/deployments","created_at":"2016-08-29T18:58:58Z","updated_at":"2016-09-07T17:42:20Z","pushed_at":"2016-09-09T16:56:28Z","git_url":"git://github.com/BD2KGenomics/toil-lib.git","ssh_url":"git@github.com:BD2KGenomics/toil-lib.git","clone_url":"https://github.com/BD2KGenomics/toil-lib.git","svn_url":"https://github.com/BD2KGenomics/toil-lib","homepage":null,"size":58,"stargazers_count":1,"watchers_count":1,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":10,"forks":1,"open_issues":10,"watchers":1,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/BD2KGenomics/toil-lib/pulls/10"},"html":{"href":"https://github.com/BD2KGenomics/toil-lib/pull/10"},"issue":{"href":"https://api.github.com/repos/BD2KGenomics/toil-lib/issues/10"},"comments":{"href":"https://api.github.com/repos/BD2KGenomics/toil-lib/issues/10/comments"},"review_comments":{"href":"https://api.github.com/repos/BD2KGenomics/toil-lib/pulls/10/comments"},"review_comment":{"href":"https://api.github.com/repos/BD2KGenomics/toil-lib/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/BD2KGenomics/toil-lib/pulls/10/commits"},"statuses":{"href":"https://api.github.com/repos/BD2KGenomics/toil-lib/statuses/a62a51fd7dad40576ca8192c8fce885ed8ad4496"}}}}
{"actor":{"display_login":"jvivian"}}