{"action":"created","comment":{"url":"https://api.github.com/repos/astropy/astroquery/pulls/comments/166738497","pull_request_review_id":94848687,"id":166738497,"diff_hunk":"@@ -1304,7 +1339,69 @@ def _download_files(self, products, base_dir, cache=True):\n             url = None\n \n             try:\n-                self._download_file(dataUrl, localPath, cache=cache)\n+                s3_fetch_ok = False\n+                if boto3 is not None and dataUri.startswith(\"mast:HST/product\"):\n+                    # The following is a mishmash of BaseQuery._download_file and s3 access through boto\n+\n+                    try:\n+                        obs_id = obs_id.lower()\n+\n+                        # magic associations logic per Brian - 0-9/a-e we convert to 0\n+                        magicValues = \"123456789abcde\"\n+                        if obs_id[-1] in magicValues:\n+                            print(\"WARNING: IPPPSSOOT Magic in %s\" % dataUri)\n+                            new_obs_id = obs_id[:-1] + \"0\"\n+                            dataUri = dataUri.replace(obs_id, new_obs_id)\n+                            obs_id = new_obs_id\n+                            print(\"WARNING: New URI %s\" % dataUri)\n+\n+                        bucketPath = \"hst/public/\" + obs_id[:4] + dataUri.replace(\"mast:HST/product\", \"\")\n+\n+                        # CAOM LIES!\n+                        # We can't use the reported file size\n+                        #length = dataProduct[\"size\"]\n+                        info_lookup = s3_client.head_object(Bucket='stpubdata', Key=bucketPath, RequestPayer='requester')\n+                        length = info_lookup[\"ContentLength\"]\n+\n+                        if cache and os.path.exists(localPath):\n+                            if length is not None:\n+                                statinfo = os.stat(localPath)\n+                                if statinfo.st_size != length:\n+                                    log.warning(\"Found cached file {0} with size {1} that is \"\n+                                                \"different from expected size {2}\"\n+                                                .format(localPath,\n+                                                        statinfo.st_size,\n+                                                    length))\n+                                else:\n+                                    log.info(\"Found cached file {0} with expected size {1}.\"\n+                                             .format(localPath, statinfo.st_size))\n+                                    s3_fetch_ok = True\n+\n+                        if not s3_fetch_ok:\n+                            with ProgressBarOrSpinner(length, ('Downloading URL s3://stpubdata/{0} to {1} ...'.format(bucketPath, localPath))) as pb:\n+                                global bytes_read\n+                                bytes_read = 0\n+\n+                                progress_lock = threading.Lock()\n+\n+                                def progress_callback(numbytes):\n+                                    global bytes_read\n+\n+                                    # This callback can be called in multiple threads\n+                                    # Access to updating the console needs to be locked\n+                                    with progress_lock:\n+                                        bytes_read += numbytes\n+                                        pb.update(bytes_read)\n+\n+                                bkt.download_file(bucketPath, localPath, ExtraArgs={\"RequestPayer\": \"requester\"}, Callback=progress_callback)\n+\n+                            s3_fetch_ok = True\n+                    except Exception as ex:\n+                        print(\"Error pulling from S3 bucket: %s\" % ex)\n+                        print(\"Falling back to mast download...\")\n+\n+                if not s3_fetch_ok:\n+                    self._download_file(dataUrl, localPath, cache=cache)","path":"astroquery/mast/core.py","position":148,"original_position":144,"commit_id":"892e6c790c1438792ac8cda40cbe1f087f2d154a","original_commit_id":"ff18ca7abcf9ea72ce086b8906a464efdc097eea","user":{"login":"keflavich","id":143715,"avatar_url":"https://avatars1.githubusercontent.com/u/143715?v=4","gravatar_id":"","url":"https://api.github.com/users/keflavich","html_url":"https://github.com/keflavich","followers_url":"https://api.github.com/users/keflavich/followers","following_url":"https://api.github.com/users/keflavich/following{/other_user}","gists_url":"https://api.github.com/users/keflavich/gists{/gist_id}","starred_url":"https://api.github.com/users/keflavich/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/keflavich/subscriptions","organizations_url":"https://api.github.com/users/keflavich/orgs","repos_url":"https://api.github.com/users/keflavich/repos","events_url":"https://api.github.com/users/keflavich/events{/privacy}","received_events_url":"https://api.github.com/users/keflavich/received_events","type":"User","site_admin":false},"body":"while this approach is fine (i.e., `s3_fetch_ok = False`, then `if not s3_fetch_ok:`), but why not just use a try/except block for this whole thing?","created_at":"2018-02-07T20:02:03Z","updated_at":"2018-02-07T20:02:15Z","html_url":"https://github.com/astropy/astroquery/pull/1040#discussion_r166738497","pull_request_url":"https://api.github.com/repos/astropy/astroquery/pulls/1040","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/astropy/astroquery/pulls/comments/166738497"},"html":{"href":"https://github.com/astropy/astroquery/pull/1040#discussion_r166738497"},"pull_request":{"href":"https://api.github.com/repos/astropy/astroquery/pulls/1040"}}},"pull_request":{"url":"https://api.github.com/repos/astropy/astroquery/pulls/1040","id":167789707,"html_url":"https://github.com/astropy/astroquery/pull/1040","diff_url":"https://github.com/astropy/astroquery/pull/1040.diff","patch_url":"https://github.com/astropy/astroquery/pull/1040.patch","issue_url":"https://api.github.com/repos/astropy/astroquery/issues/1040","number":1040,"state":"open","locked":false,"title":"STScIDSI-31: Enable downloading MAST HST data from S3","user":{"login":"cam72cam","id":892136,"avatar_url":"https://avatars3.githubusercontent.com/u/892136?v=4","gravatar_id":"","url":"https://api.github.com/users/cam72cam","html_url":"https://github.com/cam72cam","followers_url":"https://api.github.com/users/cam72cam/followers","following_url":"https://api.github.com/users/cam72cam/following{/other_user}","gists_url":"https://api.github.com/users/cam72cam/gists{/gist_id}","starred_url":"https://api.github.com/users/cam72cam/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cam72cam/subscriptions","organizations_url":"https://api.github.com/users/cam72cam/orgs","repos_url":"https://api.github.com/users/cam72cam/repos","events_url":"https://api.github.com/users/cam72cam/events{/privacy}","received_events_url":"https://api.github.com/users/cam72cam/received_events","type":"User","site_admin":false},"body":"Users who are running jobs on AWS can now retrieve public HST data directly from a S3 bucket through AWS's public dataset program.  https://aws.amazon.com/public-datasets/\r\n\r\nThis functionality is **disabled** by default and is opt in due to the nature of a required small service charge for pulling the data from S3.\r\n\r\n\r\nExample:\r\n```\r\nfrom astroquery.mast import Observations\r\nobsTable = Observations.query_object(\"M101\",radius=0.02)\r\nhstObs = obsTable[obsTable[\"obs_collection\"]==\"HST\"]\r\nproducts = Observations.get_product_list(hstObs[:4])\r\nObservations.enable_s3_hst_dataset()\r\nmanifest = Observations.download_products(products)\r\n```","created_at":"2018-02-07T19:42:25Z","updated_at":"2018-02-07T20:02:15Z","closed_at":null,"merged_at":null,"merge_commit_sha":"509ffb3f4189d1994d68055e3b435a9be78a23ec","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"milestone":null,"commits_url":"https://api.github.com/repos/astropy/astroquery/pulls/1040/commits","review_comments_url":"https://api.github.com/repos/astropy/astroquery/pulls/1040/comments","review_comment_url":"https://api.github.com/repos/astropy/astroquery/pulls/comments{/number}","comments_url":"https://api.github.com/repos/astropy/astroquery/issues/1040/comments","statuses_url":"https://api.github.com/repos/astropy/astroquery/statuses/892e6c790c1438792ac8cda40cbe1f087f2d154a","head":{"label":"cam72cam:master","ref":"master","sha":"892e6c790c1438792ac8cda40cbe1f087f2d154a","user":{"login":"cam72cam","id":892136,"avatar_url":"https://avatars3.githubusercontent.com/u/892136?v=4","gravatar_id":"","url":"https://api.github.com/users/cam72cam","html_url":"https://github.com/cam72cam","followers_url":"https://api.github.com/users/cam72cam/followers","following_url":"https://api.github.com/users/cam72cam/following{/other_user}","gists_url":"https://api.github.com/users/cam72cam/gists{/gist_id}","starred_url":"https://api.github.com/users/cam72cam/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cam72cam/subscriptions","organizations_url":"https://api.github.com/users/cam72cam/orgs","repos_url":"https://api.github.com/users/cam72cam/repos","events_url":"https://api.github.com/users/cam72cam/events{/privacy}","received_events_url":"https://api.github.com/users/cam72cam/received_events","type":"User","site_admin":false},"repo":{"id":120653427,"name":"astroquery","full_name":"cam72cam/astroquery","owner":{"login":"cam72cam","id":892136,"avatar_url":"https://avatars3.githubusercontent.com/u/892136?v=4","gravatar_id":"","url":"https://api.github.com/users/cam72cam","html_url":"https://github.com/cam72cam","followers_url":"https://api.github.com/users/cam72cam/followers","following_url":"https://api.github.com/users/cam72cam/following{/other_user}","gists_url":"https://api.github.com/users/cam72cam/gists{/gist_id}","starred_url":"https://api.github.com/users/cam72cam/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cam72cam/subscriptions","organizations_url":"https://api.github.com/users/cam72cam/orgs","repos_url":"https://api.github.com/users/cam72cam/repos","events_url":"https://api.github.com/users/cam72cam/events{/privacy}","received_events_url":"https://api.github.com/users/cam72cam/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/cam72cam/astroquery","description":"Functions and classes to access online data resources. Maintainer: @keflavich and @bsipocz","fork":true,"url":"https://api.github.com/repos/cam72cam/astroquery","forks_url":"https://api.github.com/repos/cam72cam/astroquery/forks","keys_url":"https://api.github.com/repos/cam72cam/astroquery/keys{/key_id}","collaborators_url":"https://api.github.com/repos/cam72cam/astroquery/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/cam72cam/astroquery/teams","hooks_url":"https://api.github.com/repos/cam72cam/astroquery/hooks","issue_events_url":"https://api.github.com/repos/cam72cam/astroquery/issues/events{/number}","events_url":"https://api.github.com/repos/cam72cam/astroquery/events","assignees_url":"https://api.github.com/repos/cam72cam/astroquery/assignees{/user}","branches_url":"https://api.github.com/repos/cam72cam/astroquery/branches{/branch}","tags_url":"https://api.github.com/repos/cam72cam/astroquery/tags","blobs_url":"https://api.github.com/repos/cam72cam/astroquery/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/cam72cam/astroquery/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/cam72cam/astroquery/git/refs{/sha}","trees_url":"https://api.github.com/repos/cam72cam/astroquery/git/trees{/sha}","statuses_url":"https://api.github.com/repos/cam72cam/astroquery/statuses/{sha}","languages_url":"https://api.github.com/repos/cam72cam/astroquery/languages","stargazers_url":"https://api.github.com/repos/cam72cam/astroquery/stargazers","contributors_url":"https://api.github.com/repos/cam72cam/astroquery/contributors","subscribers_url":"https://api.github.com/repos/cam72cam/astroquery/subscribers","subscription_url":"https://api.github.com/repos/cam72cam/astroquery/subscription","commits_url":"https://api.github.com/repos/cam72cam/astroquery/commits{/sha}","git_commits_url":"https://api.github.com/repos/cam72cam/astroquery/git/commits{/sha}","comments_url":"https://api.github.com/repos/cam72cam/astroquery/comments{/number}","issue_comment_url":"https://api.github.com/repos/cam72cam/astroquery/issues/comments{/number}","contents_url":"https://api.github.com/repos/cam72cam/astroquery/contents/{+path}","compare_url":"https://api.github.com/repos/cam72cam/astroquery/compare/{base}...{head}","merges_url":"https://api.github.com/repos/cam72cam/astroquery/merges","archive_url":"https://api.github.com/repos/cam72cam/astroquery/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/cam72cam/astroquery/downloads","issues_url":"https://api.github.com/repos/cam72cam/astroquery/issues{/number}","pulls_url":"https://api.github.com/repos/cam72cam/astroquery/pulls{/number}","milestones_url":"https://api.github.com/repos/cam72cam/astroquery/milestones{/number}","notifications_url":"https://api.github.com/repos/cam72cam/astroquery/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/cam72cam/astroquery/labels{/name}","releases_url":"https://api.github.com/repos/cam72cam/astroquery/releases{/id}","deployments_url":"https://api.github.com/repos/cam72cam/astroquery/deployments","created_at":"2018-02-07T18:20:55Z","updated_at":"2018-02-07T18:20:58Z","pushed_at":"2018-02-07T19:58:20Z","git_url":"git://github.com/cam72cam/astroquery.git","ssh_url":"git@github.com:cam72cam/astroquery.git","clone_url":"https://github.com/cam72cam/astroquery.git","svn_url":"https://github.com/cam72cam/astroquery","homepage":"http://astroquery.readthedocs.org/en/latest/","size":57167,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":null,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"astropy:master","ref":"master","sha":"7ff83bf645e51ca7cf37a7c7118f82594e2e5277","user":{"login":"astropy","id":847984,"avatar_url":"https://avatars0.githubusercontent.com/u/847984?v=4","gravatar_id":"","url":"https://api.github.com/users/astropy","html_url":"https://github.com/astropy","followers_url":"https://api.github.com/users/astropy/followers","following_url":"https://api.github.com/users/astropy/following{/other_user}","gists_url":"https://api.github.com/users/astropy/gists{/gist_id}","starred_url":"https://api.github.com/users/astropy/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/astropy/subscriptions","organizations_url":"https://api.github.com/users/astropy/orgs","repos_url":"https://api.github.com/users/astropy/repos","events_url":"https://api.github.com/users/astropy/events{/privacy}","received_events_url":"https://api.github.com/users/astropy/received_events","type":"Organization","site_admin":false},"repo":{"id":4787269,"name":"astroquery","full_name":"astropy/astroquery","owner":{"login":"astropy","id":847984,"avatar_url":"https://avatars0.githubusercontent.com/u/847984?v=4","gravatar_id":"","url":"https://api.github.com/users/astropy","html_url":"https://github.com/astropy","followers_url":"https://api.github.com/users/astropy/followers","following_url":"https://api.github.com/users/astropy/following{/other_user}","gists_url":"https://api.github.com/users/astropy/gists{/gist_id}","starred_url":"https://api.github.com/users/astropy/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/astropy/subscriptions","organizations_url":"https://api.github.com/users/astropy/orgs","repos_url":"https://api.github.com/users/astropy/repos","events_url":"https://api.github.com/users/astropy/events{/privacy}","received_events_url":"https://api.github.com/users/astropy/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/astropy/astroquery","description":"Functions and classes to access online data resources. Maintainer: @keflavich and @bsipocz","fork":false,"url":"https://api.github.com/repos/astropy/astroquery","forks_url":"https://api.github.com/repos/astropy/astroquery/forks","keys_url":"https://api.github.com/repos/astropy/astroquery/keys{/key_id}","collaborators_url":"https://api.github.com/repos/astropy/astroquery/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/astropy/astroquery/teams","hooks_url":"https://api.github.com/repos/astropy/astroquery/hooks","issue_events_url":"https://api.github.com/repos/astropy/astroquery/issues/events{/number}","events_url":"https://api.github.com/repos/astropy/astroquery/events","assignees_url":"https://api.github.com/repos/astropy/astroquery/assignees{/user}","branches_url":"https://api.github.com/repos/astropy/astroquery/branches{/branch}","tags_url":"https://api.github.com/repos/astropy/astroquery/tags","blobs_url":"https://api.github.com/repos/astropy/astroquery/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/astropy/astroquery/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/astropy/astroquery/git/refs{/sha}","trees_url":"https://api.github.com/repos/astropy/astroquery/git/trees{/sha}","statuses_url":"https://api.github.com/repos/astropy/astroquery/statuses/{sha}","languages_url":"https://api.github.com/repos/astropy/astroquery/languages","stargazers_url":"https://api.github.com/repos/astropy/astroquery/stargazers","contributors_url":"https://api.github.com/repos/astropy/astroquery/contributors","subscribers_url":"https://api.github.com/repos/astropy/astroquery/subscribers","subscription_url":"https://api.github.com/repos/astropy/astroquery/subscription","commits_url":"https://api.github.com/repos/astropy/astroquery/commits{/sha}","git_commits_url":"https://api.github.com/repos/astropy/astroquery/git/commits{/sha}","comments_url":"https://api.github.com/repos/astropy/astroquery/comments{/number}","issue_comment_url":"https://api.github.com/repos/astropy/astroquery/issues/comments{/number}","contents_url":"https://api.github.com/repos/astropy/astroquery/contents/{+path}","compare_url":"https://api.github.com/repos/astropy/astroquery/compare/{base}...{head}","merges_url":"https://api.github.com/repos/astropy/astroquery/merges","archive_url":"https://api.github.com/repos/astropy/astroquery/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/astropy/astroquery/downloads","issues_url":"https://api.github.com/repos/astropy/astroquery/issues{/number}","pulls_url":"https://api.github.com/repos/astropy/astroquery/pulls{/number}","milestones_url":"https://api.github.com/repos/astropy/astroquery/milestones{/number}","notifications_url":"https://api.github.com/repos/astropy/astroquery/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/astropy/astroquery/labels{/name}","releases_url":"https://api.github.com/repos/astropy/astroquery/releases{/id}","deployments_url":"https://api.github.com/repos/astropy/astroquery/deployments","created_at":"2012-06-25T20:52:07Z","updated_at":"2018-01-29T20:51:55Z","pushed_at":"2018-02-07T19:58:22Z","git_url":"git://github.com/astropy/astroquery.git","ssh_url":"git@github.com:astropy/astroquery.git","clone_url":"https://github.com/astropy/astroquery.git","svn_url":"https://github.com/astropy/astroquery","homepage":"http://astroquery.readthedocs.org/en/latest/","size":57159,"stargazers_count":136,"watchers_count":136,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":128,"mirror_url":null,"archived":false,"open_issues_count":165,"license":{"key":"bsd-3-clause","name":"BSD 3-Clause \"New\" or \"Revised\" License","spdx_id":"BSD-3-Clause","url":"https://api.github.com/licenses/bsd-3-clause"},"forks":128,"open_issues":165,"watchers":136,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/astropy/astroquery/pulls/1040"},"html":{"href":"https://github.com/astropy/astroquery/pull/1040"},"issue":{"href":"https://api.github.com/repos/astropy/astroquery/issues/1040"},"comments":{"href":"https://api.github.com/repos/astropy/astroquery/issues/1040/comments"},"review_comments":{"href":"https://api.github.com/repos/astropy/astroquery/pulls/1040/comments"},"review_comment":{"href":"https://api.github.com/repos/astropy/astroquery/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/astropy/astroquery/pulls/1040/commits"},"statuses":{"href":"https://api.github.com/repos/astropy/astroquery/statuses/892e6c790c1438792ac8cda40cbe1f087f2d154a"}},"author_association":"NONE"}}
{"actor":{"display_login":"keflavich"}}