{"action":"created","comment":{"url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/pulls/comments/226728747","pull_request_review_id":166628229,"id":226728747,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDIyNjcyODc0Nw==","diff_hunk":"@@ -104,3 +134,20 @@ create_format_task_makefile <- function(task_plan, makefile, ...) {\n     include = \"1_format.yml\",\n     ...)\n }\n+\n+separate_obs <- function(site_id, out_ind, all_obs_file_ind) {\n+  #assuming data is already fully clean\n+  site_obs <- read_feather(as_data_file(all_obs_file_ind)) %>% filter(nhd_id == site_id) %>%\n+    rename(DateTime = date) %>% as.data.frame() #need these col names for glmtools::resample_to_field\n+  saveRDS(object = site_obs, file = as_data_file(out_ind))\n+  sc_indicate(ind_file = out_ind, data_file = as_data_file(out_ind))\n+}\n+\n+syncr_indicate <- function(out_ind, glm_yeti_path) {\n+  user <- Sys.info()[['user']] #assumes this is your yeti login\n+  out_data_file <- as_data_file(out_ind)\n+  src <- sprintf('%s@yeti.cr.usgs.gov:%s/%s', user, glm_yeti_path, basename(out_data_file))\n+  dest <- sprintf('1_format/in/glm_preds/%s_output.nc', site_id)\n+  syncr(src, dest)","path":"1_format/src/format_tasks.R","position":139,"original_position":139,"commit_id":"8c6588ee6fceedbf2c20c054279dfd8669e3d643","original_commit_id":"f3855eb34f574b8debdd20587334fa01f4f13b57","user":{"login":"aappling-usgs","id":12039957,"node_id":"MDQ6VXNlcjEyMDM5OTU3","avatar_url":"https://avatars0.githubusercontent.com/u/12039957?v=4","gravatar_id":"","url":"https://api.github.com/users/aappling-usgs","html_url":"https://github.com/aappling-usgs","followers_url":"https://api.github.com/users/aappling-usgs/followers","following_url":"https://api.github.com/users/aappling-usgs/following{/other_user}","gists_url":"https://api.github.com/users/aappling-usgs/gists{/gist_id}","starred_url":"https://api.github.com/users/aappling-usgs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/aappling-usgs/subscriptions","organizations_url":"https://api.github.com/users/aappling-usgs/orgs","repos_url":"https://api.github.com/users/aappling-usgs/repos","events_url":"https://api.github.com/users/aappling-usgs/events{/privacy}","received_events_url":"https://api.github.com/users/aappling-usgs/received_events","type":"User","site_admin":false},"body":"great, glad it works with windows. i'm fine with using a github package; let's just get it into the remake system (`packages` section) and assume that all of us are comfortable using `install_github()`","created_at":"2018-10-19T17:44:17Z","updated_at":"2018-10-19T17:44:17Z","html_url":"https://github.com/USGS-R/lake-temperature-neural-networks/pull/16#discussion_r226728747","pull_request_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/pulls/16","author_association":"MEMBER","_links":{"self":{"href":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/pulls/comments/226728747"},"html":{"href":"https://github.com/USGS-R/lake-temperature-neural-networks/pull/16#discussion_r226728747"},"pull_request":{"href":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/pulls/16"}},"in_reply_to_id":226483643},"pull_request":{"url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/pulls/16","id":219028122,"node_id":"MDExOlB1bGxSZXF1ZXN0MjE5MDI4MTIy","html_url":"https://github.com/USGS-R/lake-temperature-neural-networks/pull/16","diff_url":"https://github.com/USGS-R/lake-temperature-neural-networks/pull/16.diff","patch_url":"https://github.com/USGS-R/lake-temperature-neural-networks/pull/16.patch","issue_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/issues/16","number":16,"state":"open","locked":false,"title":"Fully connecting to pipeline 1 and 2","user":{"login":"wdwatkins","id":13987349,"node_id":"MDQ6VXNlcjEzOTg3MzQ5","avatar_url":"https://avatars3.githubusercontent.com/u/13987349?v=4","gravatar_id":"","url":"https://api.github.com/users/wdwatkins","html_url":"https://github.com/wdwatkins","followers_url":"https://api.github.com/users/wdwatkins/followers","following_url":"https://api.github.com/users/wdwatkins/following{/other_user}","gists_url":"https://api.github.com/users/wdwatkins/gists{/gist_id}","starred_url":"https://api.github.com/users/wdwatkins/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/wdwatkins/subscriptions","organizations_url":"https://api.github.com/users/wdwatkins/orgs","repos_url":"https://api.github.com/users/wdwatkins/repos","events_url":"https://api.github.com/users/wdwatkins/events{/privacy}","received_events_url":"https://api.github.com/users/wdwatkins/received_events","type":"User","site_admin":false},"body":"This involves a bit less manual pulling than we previously planned:\r\n-  Only the GLM driver files are pulled manually, since they are only a few GB total.  I downloaded all of them from Yeti manually\r\n- the merged temperature observations are pulled down from google drive based on the indicator file from pipeline 1\r\n- the NLDAS crosswalk with NHD IDs, lake names, and corresponding NLDAS tiles are pulled from Google drive based on the indicator files form pipeline 2\r\n- The GLM output is rsynced from Yeti as a pipeline step.  This assumes that you have an SSH key set up to Yeti, otherwise you will need to enter your password every time.  I didn't want to pull all the GLM output, since that will be probably be on the order of at least 100 GB for 10k lakes with maximum compression and dropping unused variables.  It would be clunky to have to determine which lakes need output, and then run a manual step.  We will need to know to manually delete all the GLM output files if they change on Yeti.\r\n\r\nI have built in a switch in the `combine_priorities` function to only output Mille Lacs and Mendota for now, until we have all the GLM output.  \r\n\r\n","created_at":"2018-09-28T16:54:31Z","updated_at":"2018-10-19T17:44:17Z","closed_at":null,"merged_at":null,"merge_commit_sha":"1581bf05e94cd6c0ef4528207fa2f2bf5ee8896e","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/pulls/16/commits","review_comments_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/pulls/16/comments","review_comment_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/pulls/comments{/number}","comments_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/issues/16/comments","statuses_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/statuses/8c6588ee6fceedbf2c20c054279dfd8669e3d643","head":{"label":"wdwatkins:master","ref":"master","sha":"8c6588ee6fceedbf2c20c054279dfd8669e3d643","user":{"login":"wdwatkins","id":13987349,"node_id":"MDQ6VXNlcjEzOTg3MzQ5","avatar_url":"https://avatars3.githubusercontent.com/u/13987349?v=4","gravatar_id":"","url":"https://api.github.com/users/wdwatkins","html_url":"https://github.com/wdwatkins","followers_url":"https://api.github.com/users/wdwatkins/followers","following_url":"https://api.github.com/users/wdwatkins/following{/other_user}","gists_url":"https://api.github.com/users/wdwatkins/gists{/gist_id}","starred_url":"https://api.github.com/users/wdwatkins/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/wdwatkins/subscriptions","organizations_url":"https://api.github.com/users/wdwatkins/orgs","repos_url":"https://api.github.com/users/wdwatkins/repos","events_url":"https://api.github.com/users/wdwatkins/events{/privacy}","received_events_url":"https://api.github.com/users/wdwatkins/received_events","type":"User","site_admin":false},"repo":{"id":143456068,"node_id":"MDEwOlJlcG9zaXRvcnkxNDM0NTYwNjg=","name":"lake-temperature-neural-networks","full_name":"wdwatkins/lake-temperature-neural-networks","private":false,"owner":{"login":"wdwatkins","id":13987349,"node_id":"MDQ6VXNlcjEzOTg3MzQ5","avatar_url":"https://avatars3.githubusercontent.com/u/13987349?v=4","gravatar_id":"","url":"https://api.github.com/users/wdwatkins","html_url":"https://github.com/wdwatkins","followers_url":"https://api.github.com/users/wdwatkins/followers","following_url":"https://api.github.com/users/wdwatkins/following{/other_user}","gists_url":"https://api.github.com/users/wdwatkins/gists{/gist_id}","starred_url":"https://api.github.com/users/wdwatkins/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/wdwatkins/subscriptions","organizations_url":"https://api.github.com/users/wdwatkins/orgs","repos_url":"https://api.github.com/users/wdwatkins/repos","events_url":"https://api.github.com/users/wdwatkins/events{/privacy}","received_events_url":"https://api.github.com/users/wdwatkins/received_events","type":"User","site_admin":false},"html_url":"https://github.com/wdwatkins/lake-temperature-neural-networks","description":"Pipeline #3 ","fork":true,"url":"https://api.github.com/repos/wdwatkins/lake-temperature-neural-networks","forks_url":"https://api.github.com/repos/wdwatkins/lake-temperature-neural-networks/forks","keys_url":"https://api.github.com/repos/wdwatkins/lake-temperature-neural-networks/keys{/key_id}","collaborators_url":"https://api.github.com/repos/wdwatkins/lake-temperature-neural-networks/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/wdwatkins/lake-temperature-neural-networks/teams","hooks_url":"https://api.github.com/repos/wdwatkins/lake-temperature-neural-networks/hooks","issue_events_url":"https://api.github.com/repos/wdwatkins/lake-temperature-neural-networks/issues/events{/number}","events_url":"https://api.github.com/repos/wdwatkins/lake-temperature-neural-networks/events","assignees_url":"https://api.github.com/repos/wdwatkins/lake-temperature-neural-networks/assignees{/user}","branches_url":"https://api.github.com/repos/wdwatkins/lake-temperature-neural-networks/branches{/branch}","tags_url":"https://api.github.com/repos/wdwatkins/lake-temperature-neural-networks/tags","blobs_url":"https://api.github.com/repos/wdwatkins/lake-temperature-neural-networks/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/wdwatkins/lake-temperature-neural-networks/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/wdwatkins/lake-temperature-neural-networks/git/refs{/sha}","trees_url":"https://api.github.com/repos/wdwatkins/lake-temperature-neural-networks/git/trees{/sha}","statuses_url":"https://api.github.com/repos/wdwatkins/lake-temperature-neural-networks/statuses/{sha}","languages_url":"https://api.github.com/repos/wdwatkins/lake-temperature-neural-networks/languages","stargazers_url":"https://api.github.com/repos/wdwatkins/lake-temperature-neural-networks/stargazers","contributors_url":"https://api.github.com/repos/wdwatkins/lake-temperature-neural-networks/contributors","subscribers_url":"https://api.github.com/repos/wdwatkins/lake-temperature-neural-networks/subscribers","subscription_url":"https://api.github.com/repos/wdwatkins/lake-temperature-neural-networks/subscription","commits_url":"https://api.github.com/repos/wdwatkins/lake-temperature-neural-networks/commits{/sha}","git_commits_url":"https://api.github.com/repos/wdwatkins/lake-temperature-neural-networks/git/commits{/sha}","comments_url":"https://api.github.com/repos/wdwatkins/lake-temperature-neural-networks/comments{/number}","issue_comment_url":"https://api.github.com/repos/wdwatkins/lake-temperature-neural-networks/issues/comments{/number}","contents_url":"https://api.github.com/repos/wdwatkins/lake-temperature-neural-networks/contents/{+path}","compare_url":"https://api.github.com/repos/wdwatkins/lake-temperature-neural-networks/compare/{base}...{head}","merges_url":"https://api.github.com/repos/wdwatkins/lake-temperature-neural-networks/merges","archive_url":"https://api.github.com/repos/wdwatkins/lake-temperature-neural-networks/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/wdwatkins/lake-temperature-neural-networks/downloads","issues_url":"https://api.github.com/repos/wdwatkins/lake-temperature-neural-networks/issues{/number}","pulls_url":"https://api.github.com/repos/wdwatkins/lake-temperature-neural-networks/pulls{/number}","milestones_url":"https://api.github.com/repos/wdwatkins/lake-temperature-neural-networks/milestones{/number}","notifications_url":"https://api.github.com/repos/wdwatkins/lake-temperature-neural-networks/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/wdwatkins/lake-temperature-neural-networks/labels{/name}","releases_url":"https://api.github.com/repos/wdwatkins/lake-temperature-neural-networks/releases{/id}","deployments_url":"https://api.github.com/repos/wdwatkins/lake-temperature-neural-networks/deployments","created_at":"2018-08-03T17:37:01Z","updated_at":"2018-10-19T16:13:19Z","pushed_at":"2018-10-19T16:13:18Z","git_url":"git://github.com/wdwatkins/lake-temperature-neural-networks.git","ssh_url":"git@github.com:wdwatkins/lake-temperature-neural-networks.git","clone_url":"https://github.com/wdwatkins/lake-temperature-neural-networks.git","svn_url":"https://github.com/wdwatkins/lake-temperature-neural-networks","homepage":null,"size":7936,"stargazers_count":0,"watchers_count":0,"language":"R","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":null,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"USGS-R:master","ref":"master","sha":"0a48d027a71ef8c83cc2adde2ae90c9ba6ba99f8","user":{"login":"USGS-R","id":3188813,"node_id":"MDEyOk9yZ2FuaXphdGlvbjMxODg4MTM=","avatar_url":"https://avatars3.githubusercontent.com/u/3188813?v=4","gravatar_id":"","url":"https://api.github.com/users/USGS-R","html_url":"https://github.com/USGS-R","followers_url":"https://api.github.com/users/USGS-R/followers","following_url":"https://api.github.com/users/USGS-R/following{/other_user}","gists_url":"https://api.github.com/users/USGS-R/gists{/gist_id}","starred_url":"https://api.github.com/users/USGS-R/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/USGS-R/subscriptions","organizations_url":"https://api.github.com/users/USGS-R/orgs","repos_url":"https://api.github.com/users/USGS-R/repos","events_url":"https://api.github.com/users/USGS-R/events{/privacy}","received_events_url":"https://api.github.com/users/USGS-R/received_events","type":"Organization","site_admin":false},"repo":{"id":143456054,"node_id":"MDEwOlJlcG9zaXRvcnkxNDM0NTYwNTQ=","name":"lake-temperature-neural-networks","full_name":"USGS-R/lake-temperature-neural-networks","private":false,"owner":{"login":"USGS-R","id":3188813,"node_id":"MDEyOk9yZ2FuaXphdGlvbjMxODg4MTM=","avatar_url":"https://avatars3.githubusercontent.com/u/3188813?v=4","gravatar_id":"","url":"https://api.github.com/users/USGS-R","html_url":"https://github.com/USGS-R","followers_url":"https://api.github.com/users/USGS-R/followers","following_url":"https://api.github.com/users/USGS-R/following{/other_user}","gists_url":"https://api.github.com/users/USGS-R/gists{/gist_id}","starred_url":"https://api.github.com/users/USGS-R/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/USGS-R/subscriptions","organizations_url":"https://api.github.com/users/USGS-R/orgs","repos_url":"https://api.github.com/users/USGS-R/repos","events_url":"https://api.github.com/users/USGS-R/events{/privacy}","received_events_url":"https://api.github.com/users/USGS-R/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/USGS-R/lake-temperature-neural-networks","description":"Pipeline #3 ","fork":false,"url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks","forks_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/forks","keys_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/keys{/key_id}","collaborators_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/teams","hooks_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/hooks","issue_events_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/issues/events{/number}","events_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/events","assignees_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/assignees{/user}","branches_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/branches{/branch}","tags_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/tags","blobs_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/git/refs{/sha}","trees_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/git/trees{/sha}","statuses_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/statuses/{sha}","languages_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/languages","stargazers_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/stargazers","contributors_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/contributors","subscribers_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/subscribers","subscription_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/subscription","commits_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/commits{/sha}","git_commits_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/git/commits{/sha}","comments_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/comments{/number}","issue_comment_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/issues/comments{/number}","contents_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/contents/{+path}","compare_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/compare/{base}...{head}","merges_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/merges","archive_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/downloads","issues_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/issues{/number}","pulls_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/pulls{/number}","milestones_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/milestones{/number}","notifications_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/labels{/name}","releases_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/releases{/id}","deployments_url":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/deployments","created_at":"2018-08-03T17:36:52Z","updated_at":"2018-09-27T18:46:18Z","pushed_at":"2018-10-19T16:13:19Z","git_url":"git://github.com/USGS-R/lake-temperature-neural-networks.git","ssh_url":"git@github.com:USGS-R/lake-temperature-neural-networks.git","clone_url":"https://github.com/USGS-R/lake-temperature-neural-networks.git","svn_url":"https://github.com/USGS-R/lake-temperature-neural-networks","homepage":null,"size":7905,"stargazers_count":0,"watchers_count":0,"language":"R","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":5,"mirror_url":null,"archived":false,"open_issues_count":5,"license":null,"forks":5,"open_issues":5,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/pulls/16"},"html":{"href":"https://github.com/USGS-R/lake-temperature-neural-networks/pull/16"},"issue":{"href":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/issues/16"},"comments":{"href":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/issues/16/comments"},"review_comments":{"href":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/pulls/16/comments"},"review_comment":{"href":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/pulls/16/commits"},"statuses":{"href":"https://api.github.com/repos/USGS-R/lake-temperature-neural-networks/statuses/8c6588ee6fceedbf2c20c054279dfd8669e3d643"}},"author_association":"COLLABORATOR"}}
{"actor":{"display_login":"aappling-usgs"}}