{"action":"created","comment":{"url":"https://api.github.com/repos/SilviaTerra/samsts/pulls/comments/225012029","pull_request_review_id":164518053,"id":225012029,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDIyNTAxMjAyOQ==","diff_hunk":"@@ -0,0 +1,115 @@\n+import os\n+import logging\n+\n+import parse\n+import boto3\n+import rasterio\n+import numpy as np\n+\n+from landsat import HIGH_CONF_CLOUD, HIGH_CONF_CLOUD_SHADOW, MEDIUM_CONF_CLOUD\n+\n+ospj = os.path.join\n+ospb = os.path.basename\n+\n+PUBLIC_LANDSAT_AWS_BUCKET = 'landsat-pds'\n+C1_KEY = 'c1/L8/{path}/{row}/'\n+\n+# controls what QA band categories we want to SAMSTS fill\n+CUT_PIX_VALS = HIGH_CONF_CLOUD + HIGH_CONF_CLOUD_SHADOW + MEDIUM_CONF_CLOUD\n+\n+\n+def download(path, row, download_dir, max_cloud_cover=0.6,\n+             max_stack_depth=25, min_stack_depth=14):\n+    \"\"\"\n+    Download all bands for generated stack of landsat dates\n+    pulled from AWS archive, filtering by max_cloud_cover\n+\n+    returns list of band tif file paths\n+    \"\"\"\n+    # Get top-level col-1 scenes list\n+    prefix = C1_KEY.format(path=str(path).zfill(3),\n+                           row=str(row).zfill(3))\n+\n+    s3_client = boto3.client('s3', region_name='us-west-2')\n+\n+    objects = s3_client.list_objects_v2(\n+        Bucket=PUBLIC_LANDSAT_AWS_BUCKET,\n+        Prefix=prefix,\n+        Delimiter='/')\n+\n+    # exclude the Real Time (unprocessed) or Tier 2 (low quality) imagery\n+    col_1_prefixes = [o.get('Prefix') for o in objects.get('CommonPrefixes')\n+                      if o.get('Prefix').endswith('_T1/')]\n+\n+    logging.info(\"Found {num_scenes} candidate col_1 scenes\".format(\n+        num_scenes=len(col_1_prefixes)))\n+\n+    # Download all scenes over max_cloud_cover\n+    downloaded_tifs = []\n+    for prefix in col_1_prefixes:\n+\n+        vals = parse.parse(C1_KEY + '{product_id}/', prefix)\n+        assert vals, \"Parse error: {}\".format(prefix)\n+\n+        # make product_id download subdir\n+        sub_dir = ospj(download_dir, vals['product_id'])\n+        if not os.path.exists(sub_dir):\n+            os.makedirs(sub_dir)\n+\n+        tif_keys = get_tif_keys(prefix, s3_client)\n+        qa_key = [k for k in tif_keys if k.endswith('T1_BQA.TIF')]\n+        assert len(qa_key) == 1, \\\n+            \"Should only have 1 qa key, instead got {}\".format(qa_key)\n+        qa_key = qa_key[0]\n+\n+        # Check cloud cover\n+        qa_band_fn = ospj(download_dir, ospb(qa_key))\n+        s3_client.download_file(PUBLIC_LANDSAT_AWS_BUCKET, qa_key, qa_band_fn)\n+        cloud_cover = get_cloud_cover(qa_band_fn)\n+        logging.info(\"{product_id} is {cloud} cloud cover\".format(\n+            product_id=vals['product_id'], cloud=cloud_cover))\n+\n+        if max_cloud_cover < cloud_cover:\n+            logging.info(\"Dropping\")\n+            continue\n+\n+        for key in tif_keys:\n+            # download into product_id subfolder\n+            tif_fn = ospj(download_dir, vals['product_id'], ospb(key))\n+            s3_client.download_file(PUBLIC_LANDSAT_AWS_BUCKET, key, tif_fn)\n+            downloaded_tifs.append(tif_fn)\n+\n+    return downloaded_tifs\n+\n+\n+def get_tif_keys(prefix, s3_client):\n+    \"\"\"\n+    return list of all .TIF keys in given prefix\n+    \"\"\"\n+    objects = s3_client.list_objects_v2(\n+        Bucket=PUBLIC_LANDSAT_AWS_BUCKET,\n+        Prefix=prefix)\n+\n+    return [o['Key'] for o in objects['Contents']\n+            if o['Key'].endswith('.TIF')]\n+\n+\n+def get_cloud_cover(qa_fn):\n+    \"\"\"\n+    for given qa band, determine cloud cover\n+\n+    return decimal of cloud cover (e.g. 0.5 for half cloud)\n+    \"\"\"\n+    with rasterio.open(qa_fn) as src:\n+        qa_band = src.read(1)\n+        pix_vals, pix_ct = np.unique(qa_band, return_counts=True)\n+\n+        total_pix = sum([c for v, c in zip(pix_vals, pix_ct)\n+                         if v is not 1])","path":"samsts/stack_prep.py","position":108,"original_position":108,"commit_id":"d683d28357f0538c155425770c8a44cce544a994","original_commit_id":"d683d28357f0538c155425770c8a44cce544a994","user":{"login":"max-nova","id":960231,"node_id":"MDQ6VXNlcjk2MDIzMQ==","avatar_url":"https://avatars0.githubusercontent.com/u/960231?v=4","gravatar_id":"","url":"https://api.github.com/users/max-nova","html_url":"https://github.com/max-nova","followers_url":"https://api.github.com/users/max-nova/followers","following_url":"https://api.github.com/users/max-nova/following{/other_user}","gists_url":"https://api.github.com/users/max-nova/gists{/gist_id}","starred_url":"https://api.github.com/users/max-nova/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/max-nova/subscriptions","organizations_url":"https://api.github.com/users/max-nova/orgs","repos_url":"https://api.github.com/users/max-nova/repos","events_url":"https://api.github.com/users/max-nova/events{/privacy}","received_events_url":"https://api.github.com/users/max-nova/received_events","type":"User","site_admin":false},"body":"1 is a magic number here!  what does it represent?","created_at":"2018-10-14T22:37:08Z","updated_at":"2018-10-14T22:37:08Z","html_url":"https://github.com/SilviaTerra/samsts/pull/4#discussion_r225012029","pull_request_url":"https://api.github.com/repos/SilviaTerra/samsts/pulls/4","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/SilviaTerra/samsts/pulls/comments/225012029"},"html":{"href":"https://github.com/SilviaTerra/samsts/pull/4#discussion_r225012029"},"pull_request":{"href":"https://api.github.com/repos/SilviaTerra/samsts/pulls/4"}}},"pull_request":{"url":"https://api.github.com/repos/SilviaTerra/samsts/pulls/4","id":222689679,"node_id":"MDExOlB1bGxSZXF1ZXN0MjIyNjg5Njc5","html_url":"https://github.com/SilviaTerra/samsts/pull/4","diff_url":"https://github.com/SilviaTerra/samsts/pull/4.diff","patch_url":"https://github.com/SilviaTerra/samsts/pull/4.patch","issue_url":"https://api.github.com/repos/SilviaTerra/samsts/issues/4","number":4,"state":"open","locked":false,"title":"SAMSTS Plumbing","user":{"login":"mholkesvik","id":1395356,"node_id":"MDQ6VXNlcjEzOTUzNTY=","avatar_url":"https://avatars1.githubusercontent.com/u/1395356?v=4","gravatar_id":"","url":"https://api.github.com/users/mholkesvik","html_url":"https://github.com/mholkesvik","followers_url":"https://api.github.com/users/mholkesvik/followers","following_url":"https://api.github.com/users/mholkesvik/following{/other_user}","gists_url":"https://api.github.com/users/mholkesvik/gists{/gist_id}","starred_url":"https://api.github.com/users/mholkesvik/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mholkesvik/subscriptions","organizations_url":"https://api.github.com/users/mholkesvik/orgs","repos_url":"https://api.github.com/users/mholkesvik/repos","events_url":"https://api.github.com/users/mholkesvik/events{/privacy}","received_events_url":"https://api.github.com/users/mholkesvik/received_events","type":"User","site_admin":false},"body":"https://app.asana.com/0/835770503717173/862719561458979","created_at":"2018-10-14T03:01:45Z","updated_at":"2018-10-14T22:37:08Z","closed_at":null,"merged_at":null,"merge_commit_sha":"918c0107960076965f40458dcd83f842225d6d27","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/SilviaTerra/samsts/pulls/4/commits","review_comments_url":"https://api.github.com/repos/SilviaTerra/samsts/pulls/4/comments","review_comment_url":"https://api.github.com/repos/SilviaTerra/samsts/pulls/comments{/number}","comments_url":"https://api.github.com/repos/SilviaTerra/samsts/issues/4/comments","statuses_url":"https://api.github.com/repos/SilviaTerra/samsts/statuses/d683d28357f0538c155425770c8a44cce544a994","head":{"label":"SilviaTerra:samsts_plumbing","ref":"samsts_plumbing","sha":"d683d28357f0538c155425770c8a44cce544a994","user":{"login":"SilviaTerra","id":3497011,"node_id":"MDEyOk9yZ2FuaXphdGlvbjM0OTcwMTE=","avatar_url":"https://avatars3.githubusercontent.com/u/3497011?v=4","gravatar_id":"","url":"https://api.github.com/users/SilviaTerra","html_url":"https://github.com/SilviaTerra","followers_url":"https://api.github.com/users/SilviaTerra/followers","following_url":"https://api.github.com/users/SilviaTerra/following{/other_user}","gists_url":"https://api.github.com/users/SilviaTerra/gists{/gist_id}","starred_url":"https://api.github.com/users/SilviaTerra/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/SilviaTerra/subscriptions","organizations_url":"https://api.github.com/users/SilviaTerra/orgs","repos_url":"https://api.github.com/users/SilviaTerra/repos","events_url":"https://api.github.com/users/SilviaTerra/events{/privacy}","received_events_url":"https://api.github.com/users/SilviaTerra/received_events","type":"Organization","site_admin":false},"repo":{"id":146045850,"node_id":"MDEwOlJlcG9zaXRvcnkxNDYwNDU4NTA=","name":"samsts","full_name":"SilviaTerra/samsts","private":false,"owner":{"login":"SilviaTerra","id":3497011,"node_id":"MDEyOk9yZ2FuaXphdGlvbjM0OTcwMTE=","avatar_url":"https://avatars3.githubusercontent.com/u/3497011?v=4","gravatar_id":"","url":"https://api.github.com/users/SilviaTerra","html_url":"https://github.com/SilviaTerra","followers_url":"https://api.github.com/users/SilviaTerra/followers","following_url":"https://api.github.com/users/SilviaTerra/following{/other_user}","gists_url":"https://api.github.com/users/SilviaTerra/gists{/gist_id}","starred_url":"https://api.github.com/users/SilviaTerra/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/SilviaTerra/subscriptions","organizations_url":"https://api.github.com/users/SilviaTerra/orgs","repos_url":"https://api.github.com/users/SilviaTerra/repos","events_url":"https://api.github.com/users/SilviaTerra/events{/privacy}","received_events_url":"https://api.github.com/users/SilviaTerra/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/SilviaTerra/samsts","description":"Python implementation of the SAMSTS algorithm for cloud-filling landsat","fork":false,"url":"https://api.github.com/repos/SilviaTerra/samsts","forks_url":"https://api.github.com/repos/SilviaTerra/samsts/forks","keys_url":"https://api.github.com/repos/SilviaTerra/samsts/keys{/key_id}","collaborators_url":"https://api.github.com/repos/SilviaTerra/samsts/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/SilviaTerra/samsts/teams","hooks_url":"https://api.github.com/repos/SilviaTerra/samsts/hooks","issue_events_url":"https://api.github.com/repos/SilviaTerra/samsts/issues/events{/number}","events_url":"https://api.github.com/repos/SilviaTerra/samsts/events","assignees_url":"https://api.github.com/repos/SilviaTerra/samsts/assignees{/user}","branches_url":"https://api.github.com/repos/SilviaTerra/samsts/branches{/branch}","tags_url":"https://api.github.com/repos/SilviaTerra/samsts/tags","blobs_url":"https://api.github.com/repos/SilviaTerra/samsts/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/SilviaTerra/samsts/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/SilviaTerra/samsts/git/refs{/sha}","trees_url":"https://api.github.com/repos/SilviaTerra/samsts/git/trees{/sha}","statuses_url":"https://api.github.com/repos/SilviaTerra/samsts/statuses/{sha}","languages_url":"https://api.github.com/repos/SilviaTerra/samsts/languages","stargazers_url":"https://api.github.com/repos/SilviaTerra/samsts/stargazers","contributors_url":"https://api.github.com/repos/SilviaTerra/samsts/contributors","subscribers_url":"https://api.github.com/repos/SilviaTerra/samsts/subscribers","subscription_url":"https://api.github.com/repos/SilviaTerra/samsts/subscription","commits_url":"https://api.github.com/repos/SilviaTerra/samsts/commits{/sha}","git_commits_url":"https://api.github.com/repos/SilviaTerra/samsts/git/commits{/sha}","comments_url":"https://api.github.com/repos/SilviaTerra/samsts/comments{/number}","issue_comment_url":"https://api.github.com/repos/SilviaTerra/samsts/issues/comments{/number}","contents_url":"https://api.github.com/repos/SilviaTerra/samsts/contents/{+path}","compare_url":"https://api.github.com/repos/SilviaTerra/samsts/compare/{base}...{head}","merges_url":"https://api.github.com/repos/SilviaTerra/samsts/merges","archive_url":"https://api.github.com/repos/SilviaTerra/samsts/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/SilviaTerra/samsts/downloads","issues_url":"https://api.github.com/repos/SilviaTerra/samsts/issues{/number}","pulls_url":"https://api.github.com/repos/SilviaTerra/samsts/pulls{/number}","milestones_url":"https://api.github.com/repos/SilviaTerra/samsts/milestones{/number}","notifications_url":"https://api.github.com/repos/SilviaTerra/samsts/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/SilviaTerra/samsts/labels{/name}","releases_url":"https://api.github.com/repos/SilviaTerra/samsts/releases{/id}","deployments_url":"https://api.github.com/repos/SilviaTerra/samsts/deployments","created_at":"2018-08-24T22:36:39Z","updated_at":"2018-10-14T02:59:47Z","pushed_at":"2018-10-14T03:01:45Z","git_url":"git://github.com/SilviaTerra/samsts.git","ssh_url":"git@github.com:SilviaTerra/samsts.git","clone_url":"https://github.com/SilviaTerra/samsts.git","svn_url":"https://github.com/SilviaTerra/samsts","homepage":"http://www.mdpi.com/2072-4292/10/4/609","size":147,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":1,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit","node_id":"MDc6TGljZW5zZTEz"},"forks":0,"open_issues":1,"watchers":0,"default_branch":"master"}},"base":{"label":"SilviaTerra:master","ref":"master","sha":"8896551d27e9f04eba9a0e1b792b43c1ec3fdeae","user":{"login":"SilviaTerra","id":3497011,"node_id":"MDEyOk9yZ2FuaXphdGlvbjM0OTcwMTE=","avatar_url":"https://avatars3.githubusercontent.com/u/3497011?v=4","gravatar_id":"","url":"https://api.github.com/users/SilviaTerra","html_url":"https://github.com/SilviaTerra","followers_url":"https://api.github.com/users/SilviaTerra/followers","following_url":"https://api.github.com/users/SilviaTerra/following{/other_user}","gists_url":"https://api.github.com/users/SilviaTerra/gists{/gist_id}","starred_url":"https://api.github.com/users/SilviaTerra/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/SilviaTerra/subscriptions","organizations_url":"https://api.github.com/users/SilviaTerra/orgs","repos_url":"https://api.github.com/users/SilviaTerra/repos","events_url":"https://api.github.com/users/SilviaTerra/events{/privacy}","received_events_url":"https://api.github.com/users/SilviaTerra/received_events","type":"Organization","site_admin":false},"repo":{"id":146045850,"node_id":"MDEwOlJlcG9zaXRvcnkxNDYwNDU4NTA=","name":"samsts","full_name":"SilviaTerra/samsts","private":false,"owner":{"login":"SilviaTerra","id":3497011,"node_id":"MDEyOk9yZ2FuaXphdGlvbjM0OTcwMTE=","avatar_url":"https://avatars3.githubusercontent.com/u/3497011?v=4","gravatar_id":"","url":"https://api.github.com/users/SilviaTerra","html_url":"https://github.com/SilviaTerra","followers_url":"https://api.github.com/users/SilviaTerra/followers","following_url":"https://api.github.com/users/SilviaTerra/following{/other_user}","gists_url":"https://api.github.com/users/SilviaTerra/gists{/gist_id}","starred_url":"https://api.github.com/users/SilviaTerra/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/SilviaTerra/subscriptions","organizations_url":"https://api.github.com/users/SilviaTerra/orgs","repos_url":"https://api.github.com/users/SilviaTerra/repos","events_url":"https://api.github.com/users/SilviaTerra/events{/privacy}","received_events_url":"https://api.github.com/users/SilviaTerra/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/SilviaTerra/samsts","description":"Python implementation of the SAMSTS algorithm for cloud-filling landsat","fork":false,"url":"https://api.github.com/repos/SilviaTerra/samsts","forks_url":"https://api.github.com/repos/SilviaTerra/samsts/forks","keys_url":"https://api.github.com/repos/SilviaTerra/samsts/keys{/key_id}","collaborators_url":"https://api.github.com/repos/SilviaTerra/samsts/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/SilviaTerra/samsts/teams","hooks_url":"https://api.github.com/repos/SilviaTerra/samsts/hooks","issue_events_url":"https://api.github.com/repos/SilviaTerra/samsts/issues/events{/number}","events_url":"https://api.github.com/repos/SilviaTerra/samsts/events","assignees_url":"https://api.github.com/repos/SilviaTerra/samsts/assignees{/user}","branches_url":"https://api.github.com/repos/SilviaTerra/samsts/branches{/branch}","tags_url":"https://api.github.com/repos/SilviaTerra/samsts/tags","blobs_url":"https://api.github.com/repos/SilviaTerra/samsts/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/SilviaTerra/samsts/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/SilviaTerra/samsts/git/refs{/sha}","trees_url":"https://api.github.com/repos/SilviaTerra/samsts/git/trees{/sha}","statuses_url":"https://api.github.com/repos/SilviaTerra/samsts/statuses/{sha}","languages_url":"https://api.github.com/repos/SilviaTerra/samsts/languages","stargazers_url":"https://api.github.com/repos/SilviaTerra/samsts/stargazers","contributors_url":"https://api.github.com/repos/SilviaTerra/samsts/contributors","subscribers_url":"https://api.github.com/repos/SilviaTerra/samsts/subscribers","subscription_url":"https://api.github.com/repos/SilviaTerra/samsts/subscription","commits_url":"https://api.github.com/repos/SilviaTerra/samsts/commits{/sha}","git_commits_url":"https://api.github.com/repos/SilviaTerra/samsts/git/commits{/sha}","comments_url":"https://api.github.com/repos/SilviaTerra/samsts/comments{/number}","issue_comment_url":"https://api.github.com/repos/SilviaTerra/samsts/issues/comments{/number}","contents_url":"https://api.github.com/repos/SilviaTerra/samsts/contents/{+path}","compare_url":"https://api.github.com/repos/SilviaTerra/samsts/compare/{base}...{head}","merges_url":"https://api.github.com/repos/SilviaTerra/samsts/merges","archive_url":"https://api.github.com/repos/SilviaTerra/samsts/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/SilviaTerra/samsts/downloads","issues_url":"https://api.github.com/repos/SilviaTerra/samsts/issues{/number}","pulls_url":"https://api.github.com/repos/SilviaTerra/samsts/pulls{/number}","milestones_url":"https://api.github.com/repos/SilviaTerra/samsts/milestones{/number}","notifications_url":"https://api.github.com/repos/SilviaTerra/samsts/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/SilviaTerra/samsts/labels{/name}","releases_url":"https://api.github.com/repos/SilviaTerra/samsts/releases{/id}","deployments_url":"https://api.github.com/repos/SilviaTerra/samsts/deployments","created_at":"2018-08-24T22:36:39Z","updated_at":"2018-10-14T02:59:47Z","pushed_at":"2018-10-14T03:01:45Z","git_url":"git://github.com/SilviaTerra/samsts.git","ssh_url":"git@github.com:SilviaTerra/samsts.git","clone_url":"https://github.com/SilviaTerra/samsts.git","svn_url":"https://github.com/SilviaTerra/samsts","homepage":"http://www.mdpi.com/2072-4292/10/4/609","size":147,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":1,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit","node_id":"MDc6TGljZW5zZTEz"},"forks":0,"open_issues":1,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/SilviaTerra/samsts/pulls/4"},"html":{"href":"https://github.com/SilviaTerra/samsts/pull/4"},"issue":{"href":"https://api.github.com/repos/SilviaTerra/samsts/issues/4"},"comments":{"href":"https://api.github.com/repos/SilviaTerra/samsts/issues/4/comments"},"review_comments":{"href":"https://api.github.com/repos/SilviaTerra/samsts/pulls/4/comments"},"review_comment":{"href":"https://api.github.com/repos/SilviaTerra/samsts/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/SilviaTerra/samsts/pulls/4/commits"},"statuses":{"href":"https://api.github.com/repos/SilviaTerra/samsts/statuses/d683d28357f0538c155425770c8a44cce544a994"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"max-nova"}}