{"action":"created","comment":{"url":"https://api.github.com/repos/galaxyproject/tools-devteam/pulls/comments/75307345","id":75307345,"diff_hunk":"@@ -175,155 +200,163 @@ def _sort_fasta_custom( fasta_filename, params ):\n     else:\n         _write_sorted_fasta( sorted_names, fasta_offsets, fasta_filename, unsorted_filename )\n \n-def download_from_ucsc( data_manager_dict, params, target_directory, dbkey, dbkey_name, sequence_id, sequence_name ):\n+\n+def _download_file(start, fh):\n+    tmp = tempfile.NamedTemporaryFile()\n+    tmp.write(start)\n+    tmp.write(fh.read())\n+    tmp.flush()\n+    tmp.seek(0)\n+    return tmp\n+\n+\n+def get_stream_reader(fh, tmp_dir):\n+    \"\"\"\n+    Check if file is compressed and return correct stream reader.\n+    If file has to be downloaded, do it now.\n+    \"\"\"\n+    magic_dict = {\n+        b\"\\x1f\\x8b\\x08\": _get_stream_readers_for_gzip,\n+        b\"\\x42\\x5a\\x68\": _get_stream_readers_for_bz2,\n+        b\"\\x50\\x4b\\x03\\x04\": _get_stream_readers_for_zip,\n+    }\n+    start_of_file = fh.read(CHUNK_SIZE)\n+    try:\n+        fh.seek(0)\n+    except UnsupportedOperation:  # This is if fh has been created by urlopen\n+        fh = _download_file(start_of_file, fh)\n+    for k,v in magic_dict.items():\n+        if start_of_file.startswith(k):\n+            return v(fh, tmp_dir)\n+    try:  # Check if file is tar file\n+        if tarfile.open(fileobj=StringIO(start_of_file)):\n+            return _get_stream_readers_for_tar(fh, tmp_dir)\n+    except tarfile.ReadError:\n+        pass\n+    return fh\n+\n+\n+def _get_ucsc_download_address(params, dbkey):\n+    \"\"\"\n+    Check if we can find the correct file for the supplied dbkey on UCSC's FTP server\n+    \"\"\"\n     UCSC_FTP_SERVER = 'hgdownload.cse.ucsc.edu'\n     UCSC_DOWNLOAD_PATH = '/goldenPath/%s/bigZips/'\n-    COMPRESSED_EXTENSIONS = [ ( '.tar.gz', _get_stream_readers_for_tar ), ( '.tar.bz2', _get_stream_readers_for_tar ), ( '.zip', _get_stream_readers_for_zip ), ( '.fa.gz', _get_stream_readers_for_gzip ), ( '.fa.bz2', _get_stream_readers_for_bz2 ) ]\n-    \n+    COMPRESSED_EXTENSIONS = ['.tar.gz', '.tar.bz2', '.zip', '.fa.gz', '.fa.bz2']\n+\n     email = params['param_dict']['__user_email__']\n     if not email:\n         email = 'anonymous@example.com'\n \n     ucsc_dbkey = params['param_dict']['reference_source']['requested_dbkey'] or dbkey\n-    UCSC_CHROM_FA_FILENAMES = [ '%s.chromFa' % ucsc_dbkey, 'chromFa', ucsc_dbkey ]\n-    \n-    ftp = FTP( UCSC_FTP_SERVER )\n-    ftp.login( 'anonymous', email )\n-    \n+    UCSC_CHROM_FA_FILENAMES = ['%s.chromFa' % ucsc_dbkey, 'chromFa', ucsc_dbkey]\n+\n+    ftp = FTP(UCSC_FTP_SERVER)\n+    ftp.login('anonymous', email)\n+\n     ucsc_path = UCSC_DOWNLOAD_PATH % ucsc_dbkey\n-    path_contents = _get_files_in_ftp_path( ftp, ucsc_path )\n-    \n-    ucsc_file_name = None\n-    get_stream_reader = None\n-    ext = None\n-    ucsc_chrom_fa_filename = None\n+    path_contents = _get_files_in_ftp_path(ftp, ucsc_path)\n+    ftp.close()","path":"data_managers/data_manager_fetch_genome_dbkeys_all_fasta/data_manager/data_manager_fetch_genome_all_fasta_dbkeys.py","position":205,"original_position":205,"commit_id":"58b1ecd0d99b20548956b08039b1274e44a1f866","original_commit_id":"58b1ecd0d99b20548956b08039b1274e44a1f866","user":{"login":"nsoranzo","id":4924623,"avatar_url":"https://avatars.githubusercontent.com/u/4924623?v=3","gravatar_id":"","url":"https://api.github.com/users/nsoranzo","html_url":"https://github.com/nsoranzo","followers_url":"https://api.github.com/users/nsoranzo/followers","following_url":"https://api.github.com/users/nsoranzo/following{/other_user}","gists_url":"https://api.github.com/users/nsoranzo/gists{/gist_id}","starred_url":"https://api.github.com/users/nsoranzo/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nsoranzo/subscriptions","organizations_url":"https://api.github.com/users/nsoranzo/orgs","repos_url":"https://api.github.com/users/nsoranzo/repos","events_url":"https://api.github.com/users/nsoranzo/events{/privacy}","received_events_url":"https://api.github.com/users/nsoranzo/received_events","type":"User","site_admin":false},"body":"It seems from Python doc and Google search, that `ftp.quit()` should be the proper way to close the connection, but not really sure.","created_at":"2016-08-18T13:29:43Z","updated_at":"2016-08-18T13:29:43Z","html_url":"https://github.com/galaxyproject/tools-devteam/pull/384#discussion_r75307345","pull_request_url":"https://api.github.com/repos/galaxyproject/tools-devteam/pulls/384","_links":{"self":{"href":"https://api.github.com/repos/galaxyproject/tools-devteam/pulls/comments/75307345"},"html":{"href":"https://github.com/galaxyproject/tools-devteam/pull/384#discussion_r75307345"},"pull_request":{"href":"https://api.github.com/repos/galaxyproject/tools-devteam/pulls/384"}}},"pull_request":{"url":"https://api.github.com/repos/galaxyproject/tools-devteam/pulls/384","id":81801133,"html_url":"https://github.com/galaxyproject/tools-devteam/pull/384","diff_url":"https://github.com/galaxyproject/tools-devteam/pull/384.diff","patch_url":"https://github.com/galaxyproject/tools-devteam/pull/384.patch","issue_url":"https://api.github.com/repos/galaxyproject/tools-devteam/issues/384","number":384,"state":"open","locked":false,"title":"Support compressed files in data_manager_fetch_genome_all_fasta_dbkeys","user":{"login":"mvdbeek","id":6804901,"avatar_url":"https://avatars.githubusercontent.com/u/6804901?v=3","gravatar_id":"","url":"https://api.github.com/users/mvdbeek","html_url":"https://github.com/mvdbeek","followers_url":"https://api.github.com/users/mvdbeek/followers","following_url":"https://api.github.com/users/mvdbeek/following{/other_user}","gists_url":"https://api.github.com/users/mvdbeek/gists{/gist_id}","starred_url":"https://api.github.com/users/mvdbeek/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mvdbeek/subscriptions","organizations_url":"https://api.github.com/users/mvdbeek/orgs","repos_url":"https://api.github.com/users/mvdbeek/repos","events_url":"https://api.github.com/users/mvdbeek/events{/privacy}","received_events_url":"https://api.github.com/users/mvdbeek/received_events","type":"User","site_admin":false},"body":"It's hard these days to find uncompressed fasta downloads anywhere ... so this PR\r\nwill check if input files (download or history/directory, except for symlinks for obvious reasons) look like they are compressed and uncompress them before writing them out.\r\n\r\nAlso minor tweaks to the help text, use of CDATA tag, avoid `<command interpreter=\"python\">`\r\nand py2/py3 compatibility.\r\n\r\nI have uploaded the manager under my name to the TTS if someone would like to test https://testtoolshed.g2.bx.psu.edu/view/mvdbeek/data_manager_fetch_genome_dbkeys_all_fasta_compressed/0738889ff4d2 \r\n\r\nI couldn't quite figure out if/how I can use planemo to test a data manager, but I have uploaded a workflow [here](https://gist.github.com/mvdbeek/9cee730483a68b36d9bf23c3eb3897af) that tests most of the functionality.","created_at":"2016-08-18T11:40:21Z","updated_at":"2016-08-18T13:29:43Z","closed_at":null,"merged_at":null,"merge_commit_sha":"d54c04556f990f99ad3d815a2501122c027d6493","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/galaxyproject/tools-devteam/pulls/384/commits","review_comments_url":"https://api.github.com/repos/galaxyproject/tools-devteam/pulls/384/comments","review_comment_url":"https://api.github.com/repos/galaxyproject/tools-devteam/pulls/comments{/number}","comments_url":"https://api.github.com/repos/galaxyproject/tools-devteam/issues/384/comments","statuses_url":"https://api.github.com/repos/galaxyproject/tools-devteam/statuses/58b1ecd0d99b20548956b08039b1274e44a1f866","head":{"label":"mvdbeek:data_manager","ref":"data_manager","sha":"58b1ecd0d99b20548956b08039b1274e44a1f866","user":{"login":"mvdbeek","id":6804901,"avatar_url":"https://avatars.githubusercontent.com/u/6804901?v=3","gravatar_id":"","url":"https://api.github.com/users/mvdbeek","html_url":"https://github.com/mvdbeek","followers_url":"https://api.github.com/users/mvdbeek/followers","following_url":"https://api.github.com/users/mvdbeek/following{/other_user}","gists_url":"https://api.github.com/users/mvdbeek/gists{/gist_id}","starred_url":"https://api.github.com/users/mvdbeek/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mvdbeek/subscriptions","organizations_url":"https://api.github.com/users/mvdbeek/orgs","repos_url":"https://api.github.com/users/mvdbeek/repos","events_url":"https://api.github.com/users/mvdbeek/events{/privacy}","received_events_url":"https://api.github.com/users/mvdbeek/received_events","type":"User","site_admin":false},"repo":{"id":38748548,"name":"tools-devteam","full_name":"mvdbeek/tools-devteam","owner":{"login":"mvdbeek","id":6804901,"avatar_url":"https://avatars.githubusercontent.com/u/6804901?v=3","gravatar_id":"","url":"https://api.github.com/users/mvdbeek","html_url":"https://github.com/mvdbeek","followers_url":"https://api.github.com/users/mvdbeek/followers","following_url":"https://api.github.com/users/mvdbeek/following{/other_user}","gists_url":"https://api.github.com/users/mvdbeek/gists{/gist_id}","starred_url":"https://api.github.com/users/mvdbeek/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mvdbeek/subscriptions","organizations_url":"https://api.github.com/users/mvdbeek/orgs","repos_url":"https://api.github.com/users/mvdbeek/repos","events_url":"https://api.github.com/users/mvdbeek/events{/privacy}","received_events_url":"https://api.github.com/users/mvdbeek/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/mvdbeek/tools-devteam","description":"Contains a set of Galaxy Tools mostly written by the Galaxy Team.","fork":true,"url":"https://api.github.com/repos/mvdbeek/tools-devteam","forks_url":"https://api.github.com/repos/mvdbeek/tools-devteam/forks","keys_url":"https://api.github.com/repos/mvdbeek/tools-devteam/keys{/key_id}","collaborators_url":"https://api.github.com/repos/mvdbeek/tools-devteam/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/mvdbeek/tools-devteam/teams","hooks_url":"https://api.github.com/repos/mvdbeek/tools-devteam/hooks","issue_events_url":"https://api.github.com/repos/mvdbeek/tools-devteam/issues/events{/number}","events_url":"https://api.github.com/repos/mvdbeek/tools-devteam/events","assignees_url":"https://api.github.com/repos/mvdbeek/tools-devteam/assignees{/user}","branches_url":"https://api.github.com/repos/mvdbeek/tools-devteam/branches{/branch}","tags_url":"https://api.github.com/repos/mvdbeek/tools-devteam/tags","blobs_url":"https://api.github.com/repos/mvdbeek/tools-devteam/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/mvdbeek/tools-devteam/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/mvdbeek/tools-devteam/git/refs{/sha}","trees_url":"https://api.github.com/repos/mvdbeek/tools-devteam/git/trees{/sha}","statuses_url":"https://api.github.com/repos/mvdbeek/tools-devteam/statuses/{sha}","languages_url":"https://api.github.com/repos/mvdbeek/tools-devteam/languages","stargazers_url":"https://api.github.com/repos/mvdbeek/tools-devteam/stargazers","contributors_url":"https://api.github.com/repos/mvdbeek/tools-devteam/contributors","subscribers_url":"https://api.github.com/repos/mvdbeek/tools-devteam/subscribers","subscription_url":"https://api.github.com/repos/mvdbeek/tools-devteam/subscription","commits_url":"https://api.github.com/repos/mvdbeek/tools-devteam/commits{/sha}","git_commits_url":"https://api.github.com/repos/mvdbeek/tools-devteam/git/commits{/sha}","comments_url":"https://api.github.com/repos/mvdbeek/tools-devteam/comments{/number}","issue_comment_url":"https://api.github.com/repos/mvdbeek/tools-devteam/issues/comments{/number}","contents_url":"https://api.github.com/repos/mvdbeek/tools-devteam/contents/{+path}","compare_url":"https://api.github.com/repos/mvdbeek/tools-devteam/compare/{base}...{head}","merges_url":"https://api.github.com/repos/mvdbeek/tools-devteam/merges","archive_url":"https://api.github.com/repos/mvdbeek/tools-devteam/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/mvdbeek/tools-devteam/downloads","issues_url":"https://api.github.com/repos/mvdbeek/tools-devteam/issues{/number}","pulls_url":"https://api.github.com/repos/mvdbeek/tools-devteam/pulls{/number}","milestones_url":"https://api.github.com/repos/mvdbeek/tools-devteam/milestones{/number}","notifications_url":"https://api.github.com/repos/mvdbeek/tools-devteam/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/mvdbeek/tools-devteam/labels{/name}","releases_url":"https://api.github.com/repos/mvdbeek/tools-devteam/releases{/id}","deployments_url":"https://api.github.com/repos/mvdbeek/tools-devteam/deployments","created_at":"2015-07-08T10:52:56Z","updated_at":"2015-07-08T10:53:01Z","pushed_at":"2016-08-18T11:34:10Z","git_url":"git://github.com/mvdbeek/tools-devteam.git","ssh_url":"git@github.com:mvdbeek/tools-devteam.git","clone_url":"https://github.com/mvdbeek/tools-devteam.git","svn_url":"https://github.com/mvdbeek/tools-devteam","homepage":"","size":56237,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"galaxyproject:master","ref":"master","sha":"fc158bfe5f5927dc199321a2cf43310373cbc8ba","user":{"login":"galaxyproject","id":7937847,"avatar_url":"https://avatars.githubusercontent.com/u/7937847?v=3","gravatar_id":"","url":"https://api.github.com/users/galaxyproject","html_url":"https://github.com/galaxyproject","followers_url":"https://api.github.com/users/galaxyproject/followers","following_url":"https://api.github.com/users/galaxyproject/following{/other_user}","gists_url":"https://api.github.com/users/galaxyproject/gists{/gist_id}","starred_url":"https://api.github.com/users/galaxyproject/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/galaxyproject/subscriptions","organizations_url":"https://api.github.com/users/galaxyproject/orgs","repos_url":"https://api.github.com/users/galaxyproject/repos","events_url":"https://api.github.com/users/galaxyproject/events{/privacy}","received_events_url":"https://api.github.com/users/galaxyproject/received_events","type":"Organization","site_admin":false},"repo":{"id":24609401,"name":"tools-devteam","full_name":"galaxyproject/tools-devteam","owner":{"login":"galaxyproject","id":7937847,"avatar_url":"https://avatars.githubusercontent.com/u/7937847?v=3","gravatar_id":"","url":"https://api.github.com/users/galaxyproject","html_url":"https://github.com/galaxyproject","followers_url":"https://api.github.com/users/galaxyproject/followers","following_url":"https://api.github.com/users/galaxyproject/following{/other_user}","gists_url":"https://api.github.com/users/galaxyproject/gists{/gist_id}","starred_url":"https://api.github.com/users/galaxyproject/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/galaxyproject/subscriptions","organizations_url":"https://api.github.com/users/galaxyproject/orgs","repos_url":"https://api.github.com/users/galaxyproject/repos","events_url":"https://api.github.com/users/galaxyproject/events{/privacy}","received_events_url":"https://api.github.com/users/galaxyproject/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/galaxyproject/tools-devteam","description":"Contains a set of Galaxy Tools mostly written by the Galaxy Team.","fork":false,"url":"https://api.github.com/repos/galaxyproject/tools-devteam","forks_url":"https://api.github.com/repos/galaxyproject/tools-devteam/forks","keys_url":"https://api.github.com/repos/galaxyproject/tools-devteam/keys{/key_id}","collaborators_url":"https://api.github.com/repos/galaxyproject/tools-devteam/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/galaxyproject/tools-devteam/teams","hooks_url":"https://api.github.com/repos/galaxyproject/tools-devteam/hooks","issue_events_url":"https://api.github.com/repos/galaxyproject/tools-devteam/issues/events{/number}","events_url":"https://api.github.com/repos/galaxyproject/tools-devteam/events","assignees_url":"https://api.github.com/repos/galaxyproject/tools-devteam/assignees{/user}","branches_url":"https://api.github.com/repos/galaxyproject/tools-devteam/branches{/branch}","tags_url":"https://api.github.com/repos/galaxyproject/tools-devteam/tags","blobs_url":"https://api.github.com/repos/galaxyproject/tools-devteam/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/galaxyproject/tools-devteam/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/galaxyproject/tools-devteam/git/refs{/sha}","trees_url":"https://api.github.com/repos/galaxyproject/tools-devteam/git/trees{/sha}","statuses_url":"https://api.github.com/repos/galaxyproject/tools-devteam/statuses/{sha}","languages_url":"https://api.github.com/repos/galaxyproject/tools-devteam/languages","stargazers_url":"https://api.github.com/repos/galaxyproject/tools-devteam/stargazers","contributors_url":"https://api.github.com/repos/galaxyproject/tools-devteam/contributors","subscribers_url":"https://api.github.com/repos/galaxyproject/tools-devteam/subscribers","subscription_url":"https://api.github.com/repos/galaxyproject/tools-devteam/subscription","commits_url":"https://api.github.com/repos/galaxyproject/tools-devteam/commits{/sha}","git_commits_url":"https://api.github.com/repos/galaxyproject/tools-devteam/git/commits{/sha}","comments_url":"https://api.github.com/repos/galaxyproject/tools-devteam/comments{/number}","issue_comment_url":"https://api.github.com/repos/galaxyproject/tools-devteam/issues/comments{/number}","contents_url":"https://api.github.com/repos/galaxyproject/tools-devteam/contents/{+path}","compare_url":"https://api.github.com/repos/galaxyproject/tools-devteam/compare/{base}...{head}","merges_url":"https://api.github.com/repos/galaxyproject/tools-devteam/merges","archive_url":"https://api.github.com/repos/galaxyproject/tools-devteam/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/galaxyproject/tools-devteam/downloads","issues_url":"https://api.github.com/repos/galaxyproject/tools-devteam/issues{/number}","pulls_url":"https://api.github.com/repos/galaxyproject/tools-devteam/pulls{/number}","milestones_url":"https://api.github.com/repos/galaxyproject/tools-devteam/milestones{/number}","notifications_url":"https://api.github.com/repos/galaxyproject/tools-devteam/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/galaxyproject/tools-devteam/labels{/name}","releases_url":"https://api.github.com/repos/galaxyproject/tools-devteam/releases{/id}","deployments_url":"https://api.github.com/repos/galaxyproject/tools-devteam/deployments","created_at":"2014-09-29T19:22:30Z","updated_at":"2016-07-27T18:11:33Z","pushed_at":"2016-08-18T11:40:22Z","git_url":"git://github.com/galaxyproject/tools-devteam.git","ssh_url":"git@github.com:galaxyproject/tools-devteam.git","clone_url":"https://github.com/galaxyproject/tools-devteam.git","svn_url":"https://github.com/galaxyproject/tools-devteam","homepage":"","size":57121,"stargazers_count":28,"watchers_count":28,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":56,"mirror_url":null,"open_issues_count":81,"forks":56,"open_issues":81,"watchers":28,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/galaxyproject/tools-devteam/pulls/384"},"html":{"href":"https://github.com/galaxyproject/tools-devteam/pull/384"},"issue":{"href":"https://api.github.com/repos/galaxyproject/tools-devteam/issues/384"},"comments":{"href":"https://api.github.com/repos/galaxyproject/tools-devteam/issues/384/comments"},"review_comments":{"href":"https://api.github.com/repos/galaxyproject/tools-devteam/pulls/384/comments"},"review_comment":{"href":"https://api.github.com/repos/galaxyproject/tools-devteam/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/galaxyproject/tools-devteam/pulls/384/commits"},"statuses":{"href":"https://api.github.com/repos/galaxyproject/tools-devteam/statuses/58b1ecd0d99b20548956b08039b1274e44a1f866"}}}}
{"actor":{"display_login":"nsoranzo"}}