{"action":"created","comment":{"url":"https://api.github.com/repos/DataBiosphere/leonardo/pulls/comments/166838568","pull_request_review_id":94963381,"id":166838568,"diff_hunk":"@@ -0,0 +1,417 @@\n+package org.broadinstitute.dsde.workbench.leonardo.dao.google\n+\n+import java.time.Instant\n+import java.util.UUID\n+\n+import akka.actor.ActorSystem\n+import akka.http.scaladsl.model.StatusCodes\n+import akka.http.scaladsl.model.headers.OAuth2BearerToken\n+import cats.data.OptionT\n+import cats.implicits._\n+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport\n+import com.google.api.client.googleapis.json.GoogleJsonResponseException\n+import com.google.api.client.http.HttpResponseException\n+import com.google.api.client.json.jackson2.JacksonFactory\n+import com.google.api.services.bigquery.BigqueryScopes\n+import com.google.api.services.cloudresourcemanager.CloudResourceManager\n+import com.google.api.services.compute.model.Firewall.Allowed\n+import com.google.api.services.compute.model.{Firewall, Instance}\n+import com.google.api.services.compute.{Compute, ComputeScopes}\n+import com.google.api.services.dataproc.Dataproc\n+import com.google.api.services.dataproc.model.{Cluster => DataprocCluster, ClusterConfig => DataprocClusterConfig, ClusterStatus => DataprocClusterStatus, Operation => DataprocOperation, _}\n+import com.google.api.services.oauth2.{Oauth2, Oauth2Scopes}\n+import org.broadinstitute.dsde.workbench.google.GoogleCredentialModes.GoogleCredentialMode\n+import org.broadinstitute.dsde.workbench.google.AbstractHttpGoogleDAO\n+import org.broadinstitute.dsde.workbench.leonardo.model.google.ClusterStatus.ClusterStatus\n+import org.broadinstitute.dsde.workbench.leonardo.model.google._\n+import org.broadinstitute.dsde.workbench.metrics.GoogleInstrumentedService\n+import org.broadinstitute.dsde.workbench.metrics.GoogleInstrumentedService.GoogleInstrumentedService\n+import org.broadinstitute.dsde.workbench.model.google.{GcsBucketName, GcsPath, GoogleProject}\n+import org.broadinstitute.dsde.workbench.model.{UserInfo, WorkbenchEmail, WorkbenchUserId}\n+\n+import scala.collection.JavaConverters._\n+import scala.concurrent.{ExecutionContext, Future}\n+import scala.util.Try\n+\n+class HttpGoogleDataprocDAO(appName: String,\n+                            googleCredentialMode: GoogleCredentialMode,\n+                            override val workbenchMetricBaseName: String,\n+                            defaultNetworkTag: NetworkTag,\n+                            defaultRegion: String)\n+                           (implicit override val system: ActorSystem, override val executionContext: ExecutionContext)\n+  extends AbstractHttpGoogleDAO(appName, googleCredentialMode, workbenchMetricBaseName) with GoogleDataprocDAO {\n+\n+  override implicit val service: GoogleInstrumentedService = GoogleInstrumentedService.Dataproc\n+\n+  override val scopes: Seq[String] = Seq(ComputeScopes.CLOUD_PLATFORM)\n+\n+  private lazy val oauth2Scopes = List(Oauth2Scopes.USERINFO_EMAIL, Oauth2Scopes.USERINFO_PROFILE)\n+  private lazy val bigqueryScopes = List(BigqueryScopes.BIGQUERY)\n+\n+  private val httpTransport = GoogleNetHttpTransport.newTrustedTransport\n+  private val jsonFactory = JacksonFactory.getDefaultInstance\n+\n+  private lazy val dataproc = {\n+    new Dataproc.Builder(httpTransport, jsonFactory, googleCredential)\n+      .setApplicationName(appName).build()\n+  }\n+\n+  private lazy val compute = {\n+    new Compute.Builder(httpTransport, jsonFactory, googleCredential)\n+      .setApplicationName(appName).build()\n+  }\n+\n+  // TODO move out of this DAO\n+  private lazy val oauth2 =","path":"src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/google/HttpGoogleDataprocDAO.scala","position":65,"original_position":65,"commit_id":"b0f629574041b97ae2420371e449c0a0b9244a29","original_commit_id":"b0f629574041b97ae2420371e449c0a0b9244a29","user":{"login":"rtitle","id":5368863,"avatar_url":"https://avatars3.githubusercontent.com/u/5368863?v=4","gravatar_id":"","url":"https://api.github.com/users/rtitle","html_url":"https://github.com/rtitle","followers_url":"https://api.github.com/users/rtitle/followers","following_url":"https://api.github.com/users/rtitle/following{/other_user}","gists_url":"https://api.github.com/users/rtitle/gists{/gist_id}","starred_url":"https://api.github.com/users/rtitle/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rtitle/subscriptions","organizations_url":"https://api.github.com/users/rtitle/orgs","repos_url":"https://api.github.com/users/rtitle/repos","events_url":"https://api.github.com/users/rtitle/events{/privacy}","received_events_url":"https://api.github.com/users/rtitle/received_events","type":"User","site_admin":false},"body":"I'd like to move oauth and project-y things out of this DAO as well, but it's just a couple things not likely to change -- can do this later.","created_at":"2018-02-08T05:55:02Z","updated_at":"2018-02-08T05:55:02Z","html_url":"https://github.com/DataBiosphere/leonardo/pull/182#discussion_r166838568","pull_request_url":"https://api.github.com/repos/DataBiosphere/leonardo/pulls/182","author_association":"COLLABORATOR","_links":{"self":{"href":"https://api.github.com/repos/DataBiosphere/leonardo/pulls/comments/166838568"},"html":{"href":"https://github.com/DataBiosphere/leonardo/pull/182#discussion_r166838568"},"pull_request":{"href":"https://api.github.com/repos/DataBiosphere/leonardo/pulls/182"}}},"pull_request":{"url":"https://api.github.com/repos/DataBiosphere/leonardo/pulls/182","id":167882549,"html_url":"https://github.com/DataBiosphere/leonardo/pull/182","diff_url":"https://github.com/DataBiosphere/leonardo/pull/182.diff","patch_url":"https://github.com/DataBiosphere/leonardo/pull/182.patch","issue_url":"https://api.github.com/repos/DataBiosphere/leonardo/issues/182","number":182,"state":"open","locked":false,"title":"Use StorageDAO in Leo, plus other refactoring","user":{"login":"rtitle","id":5368863,"avatar_url":"https://avatars3.githubusercontent.com/u/5368863?v=4","gravatar_id":"","url":"https://api.github.com/users/rtitle","html_url":"https://github.com/rtitle","followers_url":"https://api.github.com/users/rtitle/followers","following_url":"https://api.github.com/users/rtitle/following{/other_user}","gists_url":"https://api.github.com/users/rtitle/gists{/gist_id}","starred_url":"https://api.github.com/users/rtitle/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rtitle/subscriptions","organizations_url":"https://api.github.com/users/rtitle/orgs","repos_url":"https://api.github.com/users/rtitle/repos","events_url":"https://api.github.com/users/rtitle/events{/privacy}","received_events_url":"https://api.github.com/users/rtitle/received_events","type":"User","site_admin":false},"body":"I know this is a big PR, and I'm sorry. Given the scope of the changes, it was hard to keep it small. It's going to be impossible for me to keep this rebased so I'd like to merge it fairly soon. Unit and automation tests are passing. I will do some manual regression testing too to make sure everything is OK.\r\n\r\nFriends with https://github.com/broadinstitute/firecloud-develop/pull/965\r\n\r\nThis does a number of good things:\r\n\r\n- Removed all storage functionality from `GoogleDataprocDAO`, and use workbench-libs `StorageDAO` instead (https://github.com/DataBiosphere/leonardo/issues/146)\r\n- Renamed `DataprocDAO`/`GoogleDataprocDAO` to `GoogleDataprocDAO`/`HttpGoogleDataprocDAO` (this is consistent with workbench-libs)\r\n- Removed Leo `StringValueClass` in favor of workbench-libs `ValueObject`\r\n- Split out some things in `LeonardoModel` to `GoogleModel`. The idea is that the Google DAO depends on the Google model, and is not dependent on any Leo functionality.\r\n- Split out bucket-related things from `LeonardoService` to `BucketService`. This makes it a little easier to distinguish between the various buckets (init bucket, staging bucket, user buckets, ...).\r\n- Use `SwaggerSamClient` for pets instead of `HttpSamDAO` (https://github.com/DataBiosphere/leonardo/issues/164). The _only_ thing `HttpSamDAO` is used for now is status checks.\r\n- Refactored the providers a bit so common Sam client code is shared in a trait\r\n\r\nI will add inline comments for anything else. I hope that this will make the code more maintainable going forward.\r\n\r\n\r\nHave you read [CONTRIBUTING.md](../CONTRIBUTING.md) lately? If not, do that first.\r\n\r\nI, the developer opening this PR, do solemnly pinky swear that:\r\n\r\n- [ ] I've documented my API changes in Swagger\r\n\r\nIn all cases:\r\n\r\n- [ ] Get two thumbsworth of review and PO signoff if necessary\r\n- [ ] Verify all tests go green\r\n- [ ] Squash and merge; you can delete your branch after this\r\n- [ ] Test this change deployed correctly and works on dev environment after deployment\r\n","created_at":"2018-02-08T05:42:43Z","updated_at":"2018-02-08T05:55:02Z","closed_at":null,"merged_at":null,"merge_commit_sha":"a4593feee183a2a51f5cb8ba8e67cdd95a6bee47","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"milestone":null,"commits_url":"https://api.github.com/repos/DataBiosphere/leonardo/pulls/182/commits","review_comments_url":"https://api.github.com/repos/DataBiosphere/leonardo/pulls/182/comments","review_comment_url":"https://api.github.com/repos/DataBiosphere/leonardo/pulls/comments{/number}","comments_url":"https://api.github.com/repos/DataBiosphere/leonardo/issues/182/comments","statuses_url":"https://api.github.com/repos/DataBiosphere/leonardo/statuses/b0f629574041b97ae2420371e449c0a0b9244a29","head":{"label":"DataBiosphere:rt-storage-dao-yak2","ref":"rt-storage-dao-yak2","sha":"b0f629574041b97ae2420371e449c0a0b9244a29","user":{"login":"DataBiosphere","id":32805087,"avatar_url":"https://avatars0.githubusercontent.com/u/32805087?v=4","gravatar_id":"","url":"https://api.github.com/users/DataBiosphere","html_url":"https://github.com/DataBiosphere","followers_url":"https://api.github.com/users/DataBiosphere/followers","following_url":"https://api.github.com/users/DataBiosphere/following{/other_user}","gists_url":"https://api.github.com/users/DataBiosphere/gists{/gist_id}","starred_url":"https://api.github.com/users/DataBiosphere/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/DataBiosphere/subscriptions","organizations_url":"https://api.github.com/users/DataBiosphere/orgs","repos_url":"https://api.github.com/users/DataBiosphere/repos","events_url":"https://api.github.com/users/DataBiosphere/events{/privacy}","received_events_url":"https://api.github.com/users/DataBiosphere/received_events","type":"Organization","site_admin":false},"repo":{"id":96942484,"name":"leonardo","full_name":"DataBiosphere/leonardo","owner":{"login":"DataBiosphere","id":32805087,"avatar_url":"https://avatars0.githubusercontent.com/u/32805087?v=4","gravatar_id":"","url":"https://api.github.com/users/DataBiosphere","html_url":"https://github.com/DataBiosphere","followers_url":"https://api.github.com/users/DataBiosphere/followers","following_url":"https://api.github.com/users/DataBiosphere/following{/other_user}","gists_url":"https://api.github.com/users/DataBiosphere/gists{/gist_id}","starred_url":"https://api.github.com/users/DataBiosphere/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/DataBiosphere/subscriptions","organizations_url":"https://api.github.com/users/DataBiosphere/orgs","repos_url":"https://api.github.com/users/DataBiosphere/repos","events_url":"https://api.github.com/users/DataBiosphere/events{/privacy}","received_events_url":"https://api.github.com/users/DataBiosphere/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/DataBiosphere/leonardo","description":"Notebook service","fork":false,"url":"https://api.github.com/repos/DataBiosphere/leonardo","forks_url":"https://api.github.com/repos/DataBiosphere/leonardo/forks","keys_url":"https://api.github.com/repos/DataBiosphere/leonardo/keys{/key_id}","collaborators_url":"https://api.github.com/repos/DataBiosphere/leonardo/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/DataBiosphere/leonardo/teams","hooks_url":"https://api.github.com/repos/DataBiosphere/leonardo/hooks","issue_events_url":"https://api.github.com/repos/DataBiosphere/leonardo/issues/events{/number}","events_url":"https://api.github.com/repos/DataBiosphere/leonardo/events","assignees_url":"https://api.github.com/repos/DataBiosphere/leonardo/assignees{/user}","branches_url":"https://api.github.com/repos/DataBiosphere/leonardo/branches{/branch}","tags_url":"https://api.github.com/repos/DataBiosphere/leonardo/tags","blobs_url":"https://api.github.com/repos/DataBiosphere/leonardo/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/DataBiosphere/leonardo/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/DataBiosphere/leonardo/git/refs{/sha}","trees_url":"https://api.github.com/repos/DataBiosphere/leonardo/git/trees{/sha}","statuses_url":"https://api.github.com/repos/DataBiosphere/leonardo/statuses/{sha}","languages_url":"https://api.github.com/repos/DataBiosphere/leonardo/languages","stargazers_url":"https://api.github.com/repos/DataBiosphere/leonardo/stargazers","contributors_url":"https://api.github.com/repos/DataBiosphere/leonardo/contributors","subscribers_url":"https://api.github.com/repos/DataBiosphere/leonardo/subscribers","subscription_url":"https://api.github.com/repos/DataBiosphere/leonardo/subscription","commits_url":"https://api.github.com/repos/DataBiosphere/leonardo/commits{/sha}","git_commits_url":"https://api.github.com/repos/DataBiosphere/leonardo/git/commits{/sha}","comments_url":"https://api.github.com/repos/DataBiosphere/leonardo/comments{/number}","issue_comment_url":"https://api.github.com/repos/DataBiosphere/leonardo/issues/comments{/number}","contents_url":"https://api.github.com/repos/DataBiosphere/leonardo/contents/{+path}","compare_url":"https://api.github.com/repos/DataBiosphere/leonardo/compare/{base}...{head}","merges_url":"https://api.github.com/repos/DataBiosphere/leonardo/merges","archive_url":"https://api.github.com/repos/DataBiosphere/leonardo/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/DataBiosphere/leonardo/downloads","issues_url":"https://api.github.com/repos/DataBiosphere/leonardo/issues{/number}","pulls_url":"https://api.github.com/repos/DataBiosphere/leonardo/pulls{/number}","milestones_url":"https://api.github.com/repos/DataBiosphere/leonardo/milestones{/number}","notifications_url":"https://api.github.com/repos/DataBiosphere/leonardo/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/DataBiosphere/leonardo/labels{/name}","releases_url":"https://api.github.com/repos/DataBiosphere/leonardo/releases{/id}","deployments_url":"https://api.github.com/repos/DataBiosphere/leonardo/deployments","created_at":"2017-07-11T22:29:16Z","updated_at":"2018-01-05T02:47:00Z","pushed_at":"2018-02-08T05:42:44Z","git_url":"git://github.com/DataBiosphere/leonardo.git","ssh_url":"git@github.com:DataBiosphere/leonardo.git","clone_url":"https://github.com/DataBiosphere/leonardo.git","svn_url":"https://github.com/DataBiosphere/leonardo","homepage":null,"size":1466,"stargazers_count":2,"watchers_count":2,"language":"Scala","has_issues":true,"has_projects":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":3,"mirror_url":null,"archived":false,"open_issues_count":46,"license":null,"forks":3,"open_issues":46,"watchers":2,"default_branch":"develop"}},"base":{"label":"DataBiosphere:develop","ref":"develop","sha":"c683156d1b0dc4fc74cacef0c5587b5d94a766b8","user":{"login":"DataBiosphere","id":32805087,"avatar_url":"https://avatars0.githubusercontent.com/u/32805087?v=4","gravatar_id":"","url":"https://api.github.com/users/DataBiosphere","html_url":"https://github.com/DataBiosphere","followers_url":"https://api.github.com/users/DataBiosphere/followers","following_url":"https://api.github.com/users/DataBiosphere/following{/other_user}","gists_url":"https://api.github.com/users/DataBiosphere/gists{/gist_id}","starred_url":"https://api.github.com/users/DataBiosphere/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/DataBiosphere/subscriptions","organizations_url":"https://api.github.com/users/DataBiosphere/orgs","repos_url":"https://api.github.com/users/DataBiosphere/repos","events_url":"https://api.github.com/users/DataBiosphere/events{/privacy}","received_events_url":"https://api.github.com/users/DataBiosphere/received_events","type":"Organization","site_admin":false},"repo":{"id":96942484,"name":"leonardo","full_name":"DataBiosphere/leonardo","owner":{"login":"DataBiosphere","id":32805087,"avatar_url":"https://avatars0.githubusercontent.com/u/32805087?v=4","gravatar_id":"","url":"https://api.github.com/users/DataBiosphere","html_url":"https://github.com/DataBiosphere","followers_url":"https://api.github.com/users/DataBiosphere/followers","following_url":"https://api.github.com/users/DataBiosphere/following{/other_user}","gists_url":"https://api.github.com/users/DataBiosphere/gists{/gist_id}","starred_url":"https://api.github.com/users/DataBiosphere/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/DataBiosphere/subscriptions","organizations_url":"https://api.github.com/users/DataBiosphere/orgs","repos_url":"https://api.github.com/users/DataBiosphere/repos","events_url":"https://api.github.com/users/DataBiosphere/events{/privacy}","received_events_url":"https://api.github.com/users/DataBiosphere/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/DataBiosphere/leonardo","description":"Notebook service","fork":false,"url":"https://api.github.com/repos/DataBiosphere/leonardo","forks_url":"https://api.github.com/repos/DataBiosphere/leonardo/forks","keys_url":"https://api.github.com/repos/DataBiosphere/leonardo/keys{/key_id}","collaborators_url":"https://api.github.com/repos/DataBiosphere/leonardo/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/DataBiosphere/leonardo/teams","hooks_url":"https://api.github.com/repos/DataBiosphere/leonardo/hooks","issue_events_url":"https://api.github.com/repos/DataBiosphere/leonardo/issues/events{/number}","events_url":"https://api.github.com/repos/DataBiosphere/leonardo/events","assignees_url":"https://api.github.com/repos/DataBiosphere/leonardo/assignees{/user}","branches_url":"https://api.github.com/repos/DataBiosphere/leonardo/branches{/branch}","tags_url":"https://api.github.com/repos/DataBiosphere/leonardo/tags","blobs_url":"https://api.github.com/repos/DataBiosphere/leonardo/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/DataBiosphere/leonardo/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/DataBiosphere/leonardo/git/refs{/sha}","trees_url":"https://api.github.com/repos/DataBiosphere/leonardo/git/trees{/sha}","statuses_url":"https://api.github.com/repos/DataBiosphere/leonardo/statuses/{sha}","languages_url":"https://api.github.com/repos/DataBiosphere/leonardo/languages","stargazers_url":"https://api.github.com/repos/DataBiosphere/leonardo/stargazers","contributors_url":"https://api.github.com/repos/DataBiosphere/leonardo/contributors","subscribers_url":"https://api.github.com/repos/DataBiosphere/leonardo/subscribers","subscription_url":"https://api.github.com/repos/DataBiosphere/leonardo/subscription","commits_url":"https://api.github.com/repos/DataBiosphere/leonardo/commits{/sha}","git_commits_url":"https://api.github.com/repos/DataBiosphere/leonardo/git/commits{/sha}","comments_url":"https://api.github.com/repos/DataBiosphere/leonardo/comments{/number}","issue_comment_url":"https://api.github.com/repos/DataBiosphere/leonardo/issues/comments{/number}","contents_url":"https://api.github.com/repos/DataBiosphere/leonardo/contents/{+path}","compare_url":"https://api.github.com/repos/DataBiosphere/leonardo/compare/{base}...{head}","merges_url":"https://api.github.com/repos/DataBiosphere/leonardo/merges","archive_url":"https://api.github.com/repos/DataBiosphere/leonardo/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/DataBiosphere/leonardo/downloads","issues_url":"https://api.github.com/repos/DataBiosphere/leonardo/issues{/number}","pulls_url":"https://api.github.com/repos/DataBiosphere/leonardo/pulls{/number}","milestones_url":"https://api.github.com/repos/DataBiosphere/leonardo/milestones{/number}","notifications_url":"https://api.github.com/repos/DataBiosphere/leonardo/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/DataBiosphere/leonardo/labels{/name}","releases_url":"https://api.github.com/repos/DataBiosphere/leonardo/releases{/id}","deployments_url":"https://api.github.com/repos/DataBiosphere/leonardo/deployments","created_at":"2017-07-11T22:29:16Z","updated_at":"2018-01-05T02:47:00Z","pushed_at":"2018-02-08T05:42:44Z","git_url":"git://github.com/DataBiosphere/leonardo.git","ssh_url":"git@github.com:DataBiosphere/leonardo.git","clone_url":"https://github.com/DataBiosphere/leonardo.git","svn_url":"https://github.com/DataBiosphere/leonardo","homepage":null,"size":1466,"stargazers_count":2,"watchers_count":2,"language":"Scala","has_issues":true,"has_projects":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":3,"mirror_url":null,"archived":false,"open_issues_count":46,"license":null,"forks":3,"open_issues":46,"watchers":2,"default_branch":"develop"}},"_links":{"self":{"href":"https://api.github.com/repos/DataBiosphere/leonardo/pulls/182"},"html":{"href":"https://github.com/DataBiosphere/leonardo/pull/182"},"issue":{"href":"https://api.github.com/repos/DataBiosphere/leonardo/issues/182"},"comments":{"href":"https://api.github.com/repos/DataBiosphere/leonardo/issues/182/comments"},"review_comments":{"href":"https://api.github.com/repos/DataBiosphere/leonardo/pulls/182/comments"},"review_comment":{"href":"https://api.github.com/repos/DataBiosphere/leonardo/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/DataBiosphere/leonardo/pulls/182/commits"},"statuses":{"href":"https://api.github.com/repos/DataBiosphere/leonardo/statuses/b0f629574041b97ae2420371e449c0a0b9244a29"}},"author_association":"COLLABORATOR"}}
{"actor":{"display_login":"rtitle"}}