{"action":"created","comment":{"url":"https://api.github.com/repos/connexta/ion-store/pulls/comments/350469265","pull_request_review_id":322616409,"id":350469265,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDM1MDQ2OTI2NQ==","diff_hunk":"@@ -0,0 +1,281 @@\n+/*\n+ * Copyright (c) 2019 Connexta, LLC\n+ *\n+ * Released under the GNU Lesser General Public License version 3; see\n+ * https://www.gnu.org/licenses/lgpl-3.0.html\n+ */\n+package com.connexta.store.poller;\n+\n+import com.connexta.store.config.TransformConfiguration;\n+import com.connexta.store.controllers.StoreController;\n+import com.connexta.store.rest.models.AddMetadataRequest;\n+import com.connexta.store.rest.models.MetadataInfo;\n+import com.connexta.store.rest.models.QuarantineRequest;\n+import com.connexta.transformation.rest.models.MetadataInformation;\n+import com.connexta.transformation.rest.models.Status;\n+import com.connexta.transformation.rest.models.TransformationPollResponse;\n+import com.google.common.annotations.VisibleForTesting;\n+import java.util.List;\n+import java.util.concurrent.BlockingQueue;\n+import java.util.concurrent.TimeUnit;\n+import lombok.extern.slf4j.Slf4j;\n+import org.springframework.http.HttpStatus;\n+import org.springframework.http.ResponseEntity;\n+import org.springframework.web.reactive.function.BodyInserters;\n+import org.springframework.web.reactive.function.client.ClientResponse;\n+import org.springframework.web.reactive.function.client.WebClient;\n+import org.springframework.web.util.UriComponentsBuilder;\n+\n+/** Class capable of polling the transformation poll endpoint. */\n+@Slf4j\n+public class TransformStatusPoller extends Thread {\n+\n+  private final WebClient storeWebClient;\n+\n+  private final WebClient transformWebClient;\n+\n+  private final BlockingQueue<TransformStatusTask> transformStatusQueue;\n+\n+  /** Creates a new TransformStatusPoller. */\n+  public TransformStatusPoller(\n+      BlockingQueue<TransformStatusTask> transformStatusQueue,\n+      WebClient transformWebClient,\n+      WebClient storeWebClient) {\n+    this.transformStatusQueue = transformStatusQueue;\n+    this.transformWebClient = transformWebClient;\n+    this.storeWebClient = storeWebClient;\n+  }\n+\n+  // TODO 11/20/2019 PeterHuffer: This class should be managed and be able to be properly destroyed\n+  @SuppressWarnings(\"squid:S2189\" /* Run forever by design */)\n+  @Override\n+  public void run() {\n+    while (true) {\n+      try {\n+        doRun();\n+      } catch (InterruptedException e) {\n+        Thread.currentThread().interrupt();\n+        return;\n+      }\n+    }\n+  }\n+\n+  @VisibleForTesting\n+  void doRun() throws InterruptedException {\n+    TransformStatusTask transformStatusTask = transformStatusQueue.take();\n+    boolean shouldRetry = handleTask(transformStatusTask);\n+    if (shouldRetry) {\n+      boolean resubmitted = transformStatusQueue.offer(transformStatusTask, 10, TimeUnit.SECONDS);\n+      if (resubmitted) {\n+        log.debug(\n+            \"Transform status {} will be tried again for dataset {}.\",\n+            transformStatusTask.getTransformStatusUrl(),\n+            transformStatusTask.getDatasetId());\n+      } else {\n+        log.debug(\n+            \"Timed out trying to resubmit transform status {} for dataset {}. The file corresponding to this dataset should be reingsted.\",\n+            transformStatusTask.getTransformStatusUrl(),\n+            transformStatusTask.getDatasetId());\n+      }\n+    }\n+  }\n+\n+  /** @return {@code true} if the task should be retried, otherwise {@code false} */\n+  private boolean handleTask(TransformStatusTask transformStatusTask) {\n+    final String transformPollUrl = transformStatusTask.getTransformStatusUrl().toString();\n+    final String datasetId = transformStatusTask.getDatasetId();\n+\n+    // this could be an ErrorMessage too, need to account for that\n+    ClientResponse response = transformWebClient.get().uri(transformPollUrl).exchange().block();\n+\n+    if (response != null) {\n+      final HttpStatus httpStatus = response.statusCode();\n+\n+      if (HttpStatus.OK.equals(httpStatus)) {\n+        ResponseEntity<TransformationPollResponse> transformationPollResponse =\n+            response.toEntity(TransformationPollResponse.class).block();\n+        if (transformationPollResponse != null) {\n+          if (transformationPollResponse.getBody() == null) {\n+            log.debug(\n+                \"No body available in the successful transform status response for status {}. This URL will be retried.\",\n+                transformPollUrl);\n+            return true;\n+          }\n+\n+          return handlePollResponse(datasetId, transformationPollResponse.getBody());\n+        } else {\n+          log.debug(\n+              \"No body available in the successful transform status response for status {}. This URL will be retried.\",\n+              transformPollUrl);\n+          return true;\n+        }\n+      } else if (HttpStatus.NOT_FOUND.equals(httpStatus)) {\n+        log.debug(\n+            \"Transform URL status for {} does not exist anymore, removing from task queue.\",\n+            transformPollUrl);","path":"src/main/java/com/connexta/store/poller/TransformStatusPoller.java","position":115,"original_position":115,"commit_id":"6b391d33a4a78b886ccb78e1bf6b0cf7bed9cd4d","original_commit_id":"6b391d33a4a78b886ccb78e1bf6b0cf7bed9cd4d","user":{"login":"mcalcote","id":11355332,"node_id":"MDQ6VXNlcjExMzU1MzMy","avatar_url":"https://avatars0.githubusercontent.com/u/11355332?v=4","gravatar_id":"","url":"https://api.github.com/users/mcalcote","html_url":"https://github.com/mcalcote","followers_url":"https://api.github.com/users/mcalcote/followers","following_url":"https://api.github.com/users/mcalcote/following{/other_user}","gists_url":"https://api.github.com/users/mcalcote/gists{/gist_id}","starred_url":"https://api.github.com/users/mcalcote/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mcalcote/subscriptions","organizations_url":"https://api.github.com/users/mcalcote/orgs","repos_url":"https://api.github.com/users/mcalcote/repos","events_url":"https://api.github.com/users/mcalcote/events{/privacy}","received_events_url":"https://api.github.com/users/mcalcote/received_events","type":"User","site_admin":false},"body":"❓ When would we run into this scenario? Would we want to log something more severe if we were trying to poll something not there?","created_at":"2019-11-25T23:06:08Z","updated_at":"2019-11-25T23:30:18Z","html_url":"https://github.com/connexta/ion-store/pull/235#discussion_r350469265","pull_request_url":"https://api.github.com/repos/connexta/ion-store/pulls/235","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/connexta/ion-store/pulls/comments/350469265"},"html":{"href":"https://github.com/connexta/ion-store/pull/235#discussion_r350469265"},"pull_request":{"href":"https://api.github.com/repos/connexta/ion-store/pulls/235"}}},"pull_request":{"url":"https://api.github.com/repos/connexta/ion-store/pulls/235","id":344116773,"node_id":"MDExOlB1bGxSZXF1ZXN0MzQ0MTE2Nzcz","html_url":"https://github.com/connexta/ion-store/pull/235","diff_url":"https://github.com/connexta/ion-store/pull/235.diff","patch_url":"https://github.com/connexta/ion-store/pull/235.patch","issue_url":"https://api.github.com/repos/connexta/ion-store/issues/235","number":235,"state":"open","locked":false,"title":"[WIP] Add support for Transform Status polling","user":{"login":"peterhuffer","id":17572782,"node_id":"MDQ6VXNlcjE3NTcyNzgy","avatar_url":"https://avatars3.githubusercontent.com/u/17572782?v=4","gravatar_id":"","url":"https://api.github.com/users/peterhuffer","html_url":"https://github.com/peterhuffer","followers_url":"https://api.github.com/users/peterhuffer/followers","following_url":"https://api.github.com/users/peterhuffer/following{/other_user}","gists_url":"https://api.github.com/users/peterhuffer/gists{/gist_id}","starred_url":"https://api.github.com/users/peterhuffer/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/peterhuffer/subscriptions","organizations_url":"https://api.github.com/users/peterhuffer/orgs","repos_url":"https://api.github.com/users/peterhuffer/repos","events_url":"https://api.github.com/users/peterhuffer/events{/privacy}","received_events_url":"https://api.github.com/users/peterhuffer/received_events","type":"User","site_admin":false},"body":"[WIP] - Putting this up to get eyes on it. Going to start testing it to see if actually works!\r\n\r\n- Upgrades transform version and updates transform request and adds transform status poller\r\n- Modifies controller and StoreService APIs to handle multiple Datas\r\n- Update code to deal with 36 length UUIDs\r\n\r\nCorrect error handling is currently difficult because I don't know if Datasets or Data exist or not based on the current StorageAdapter API.","created_at":"2019-11-21T16:20:19Z","updated_at":"2019-11-25T23:30:18Z","closed_at":null,"merged_at":null,"merge_commit_sha":"dd159737c44fbc60553b66a9905ec5e8f8a4d0df","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/connexta/ion-store/pulls/235/commits","review_comments_url":"https://api.github.com/repos/connexta/ion-store/pulls/235/comments","review_comment_url":"https://api.github.com/repos/connexta/ion-store/pulls/comments{/number}","comments_url":"https://api.github.com/repos/connexta/ion-store/issues/235/comments","statuses_url":"https://api.github.com/repos/connexta/ion-store/statuses/6b391d33a4a78b886ccb78e1bf6b0cf7bed9cd4d","head":{"label":"peterhuffer:poller","ref":"poller","sha":"6b391d33a4a78b886ccb78e1bf6b0cf7bed9cd4d","user":{"login":"peterhuffer","id":17572782,"node_id":"MDQ6VXNlcjE3NTcyNzgy","avatar_url":"https://avatars3.githubusercontent.com/u/17572782?v=4","gravatar_id":"","url":"https://api.github.com/users/peterhuffer","html_url":"https://github.com/peterhuffer","followers_url":"https://api.github.com/users/peterhuffer/followers","following_url":"https://api.github.com/users/peterhuffer/following{/other_user}","gists_url":"https://api.github.com/users/peterhuffer/gists{/gist_id}","starred_url":"https://api.github.com/users/peterhuffer/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/peterhuffer/subscriptions","organizations_url":"https://api.github.com/users/peterhuffer/orgs","repos_url":"https://api.github.com/users/peterhuffer/repos","events_url":"https://api.github.com/users/peterhuffer/events{/privacy}","received_events_url":"https://api.github.com/users/peterhuffer/received_events","type":"User","site_admin":false},"repo":{"id":211895887,"node_id":"MDEwOlJlcG9zaXRvcnkyMTE4OTU4ODc=","name":"ion-store","full_name":"peterhuffer/ion-store","private":false,"owner":{"login":"peterhuffer","id":17572782,"node_id":"MDQ6VXNlcjE3NTcyNzgy","avatar_url":"https://avatars3.githubusercontent.com/u/17572782?v=4","gravatar_id":"","url":"https://api.github.com/users/peterhuffer","html_url":"https://github.com/peterhuffer","followers_url":"https://api.github.com/users/peterhuffer/followers","following_url":"https://api.github.com/users/peterhuffer/following{/other_user}","gists_url":"https://api.github.com/users/peterhuffer/gists{/gist_id}","starred_url":"https://api.github.com/users/peterhuffer/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/peterhuffer/subscriptions","organizations_url":"https://api.github.com/users/peterhuffer/orgs","repos_url":"https://api.github.com/users/peterhuffer/repos","events_url":"https://api.github.com/users/peterhuffer/events{/privacy}","received_events_url":"https://api.github.com/users/peterhuffer/received_events","type":"User","site_admin":false},"html_url":"https://github.com/peterhuffer/ion-store","description":null,"fork":true,"url":"https://api.github.com/repos/peterhuffer/ion-store","forks_url":"https://api.github.com/repos/peterhuffer/ion-store/forks","keys_url":"https://api.github.com/repos/peterhuffer/ion-store/keys{/key_id}","collaborators_url":"https://api.github.com/repos/peterhuffer/ion-store/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/peterhuffer/ion-store/teams","hooks_url":"https://api.github.com/repos/peterhuffer/ion-store/hooks","issue_events_url":"https://api.github.com/repos/peterhuffer/ion-store/issues/events{/number}","events_url":"https://api.github.com/repos/peterhuffer/ion-store/events","assignees_url":"https://api.github.com/repos/peterhuffer/ion-store/assignees{/user}","branches_url":"https://api.github.com/repos/peterhuffer/ion-store/branches{/branch}","tags_url":"https://api.github.com/repos/peterhuffer/ion-store/tags","blobs_url":"https://api.github.com/repos/peterhuffer/ion-store/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/peterhuffer/ion-store/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/peterhuffer/ion-store/git/refs{/sha}","trees_url":"https://api.github.com/repos/peterhuffer/ion-store/git/trees{/sha}","statuses_url":"https://api.github.com/repos/peterhuffer/ion-store/statuses/{sha}","languages_url":"https://api.github.com/repos/peterhuffer/ion-store/languages","stargazers_url":"https://api.github.com/repos/peterhuffer/ion-store/stargazers","contributors_url":"https://api.github.com/repos/peterhuffer/ion-store/contributors","subscribers_url":"https://api.github.com/repos/peterhuffer/ion-store/subscribers","subscription_url":"https://api.github.com/repos/peterhuffer/ion-store/subscription","commits_url":"https://api.github.com/repos/peterhuffer/ion-store/commits{/sha}","git_commits_url":"https://api.github.com/repos/peterhuffer/ion-store/git/commits{/sha}","comments_url":"https://api.github.com/repos/peterhuffer/ion-store/comments{/number}","issue_comment_url":"https://api.github.com/repos/peterhuffer/ion-store/issues/comments{/number}","contents_url":"https://api.github.com/repos/peterhuffer/ion-store/contents/{+path}","compare_url":"https://api.github.com/repos/peterhuffer/ion-store/compare/{base}...{head}","merges_url":"https://api.github.com/repos/peterhuffer/ion-store/merges","archive_url":"https://api.github.com/repos/peterhuffer/ion-store/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/peterhuffer/ion-store/downloads","issues_url":"https://api.github.com/repos/peterhuffer/ion-store/issues{/number}","pulls_url":"https://api.github.com/repos/peterhuffer/ion-store/pulls{/number}","milestones_url":"https://api.github.com/repos/peterhuffer/ion-store/milestones{/number}","notifications_url":"https://api.github.com/repos/peterhuffer/ion-store/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/peterhuffer/ion-store/labels{/name}","releases_url":"https://api.github.com/repos/peterhuffer/ion-store/releases{/id}","deployments_url":"https://api.github.com/repos/peterhuffer/ion-store/deployments","created_at":"2019-09-30T15:45:17Z","updated_at":"2019-09-30T21:37:40Z","pushed_at":"2019-11-25T22:35:03Z","git_url":"git://github.com/peterhuffer/ion-store.git","ssh_url":"git@github.com:peterhuffer/ion-store.git","clone_url":"https://github.com/peterhuffer/ion-store.git","svn_url":"https://github.com/peterhuffer/ion-store","homepage":null,"size":710,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"disabled":false,"open_issues_count":0,"license":{"key":"other","name":"Other","spdx_id":"NOASSERTION","url":null,"node_id":"MDc6TGljZW5zZTA="},"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"connexta:master","ref":"master","sha":"e007a56b48aa6476955357372496ba9061fa6b18","user":{"login":"connexta","id":6706726,"node_id":"MDEyOk9yZ2FuaXphdGlvbjY3MDY3MjY=","avatar_url":"https://avatars3.githubusercontent.com/u/6706726?v=4","gravatar_id":"","url":"https://api.github.com/users/connexta","html_url":"https://github.com/connexta","followers_url":"https://api.github.com/users/connexta/followers","following_url":"https://api.github.com/users/connexta/following{/other_user}","gists_url":"https://api.github.com/users/connexta/gists{/gist_id}","starred_url":"https://api.github.com/users/connexta/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/connexta/subscriptions","organizations_url":"https://api.github.com/users/connexta/orgs","repos_url":"https://api.github.com/users/connexta/repos","events_url":"https://api.github.com/users/connexta/events{/privacy}","received_events_url":"https://api.github.com/users/connexta/received_events","type":"Organization","site_admin":false},"repo":{"id":175548670,"node_id":"MDEwOlJlcG9zaXRvcnkxNzU1NDg2NzA=","name":"ion-store","full_name":"connexta/ion-store","private":false,"owner":{"login":"connexta","id":6706726,"node_id":"MDEyOk9yZ2FuaXphdGlvbjY3MDY3MjY=","avatar_url":"https://avatars3.githubusercontent.com/u/6706726?v=4","gravatar_id":"","url":"https://api.github.com/users/connexta","html_url":"https://github.com/connexta","followers_url":"https://api.github.com/users/connexta/followers","following_url":"https://api.github.com/users/connexta/following{/other_user}","gists_url":"https://api.github.com/users/connexta/gists{/gist_id}","starred_url":"https://api.github.com/users/connexta/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/connexta/subscriptions","organizations_url":"https://api.github.com/users/connexta/orgs","repos_url":"https://api.github.com/users/connexta/repos","events_url":"https://api.github.com/users/connexta/events{/privacy}","received_events_url":"https://api.github.com/users/connexta/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/connexta/ion-store","description":null,"fork":false,"url":"https://api.github.com/repos/connexta/ion-store","forks_url":"https://api.github.com/repos/connexta/ion-store/forks","keys_url":"https://api.github.com/repos/connexta/ion-store/keys{/key_id}","collaborators_url":"https://api.github.com/repos/connexta/ion-store/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/connexta/ion-store/teams","hooks_url":"https://api.github.com/repos/connexta/ion-store/hooks","issue_events_url":"https://api.github.com/repos/connexta/ion-store/issues/events{/number}","events_url":"https://api.github.com/repos/connexta/ion-store/events","assignees_url":"https://api.github.com/repos/connexta/ion-store/assignees{/user}","branches_url":"https://api.github.com/repos/connexta/ion-store/branches{/branch}","tags_url":"https://api.github.com/repos/connexta/ion-store/tags","blobs_url":"https://api.github.com/repos/connexta/ion-store/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/connexta/ion-store/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/connexta/ion-store/git/refs{/sha}","trees_url":"https://api.github.com/repos/connexta/ion-store/git/trees{/sha}","statuses_url":"https://api.github.com/repos/connexta/ion-store/statuses/{sha}","languages_url":"https://api.github.com/repos/connexta/ion-store/languages","stargazers_url":"https://api.github.com/repos/connexta/ion-store/stargazers","contributors_url":"https://api.github.com/repos/connexta/ion-store/contributors","subscribers_url":"https://api.github.com/repos/connexta/ion-store/subscribers","subscription_url":"https://api.github.com/repos/connexta/ion-store/subscription","commits_url":"https://api.github.com/repos/connexta/ion-store/commits{/sha}","git_commits_url":"https://api.github.com/repos/connexta/ion-store/git/commits{/sha}","comments_url":"https://api.github.com/repos/connexta/ion-store/comments{/number}","issue_comment_url":"https://api.github.com/repos/connexta/ion-store/issues/comments{/number}","contents_url":"https://api.github.com/repos/connexta/ion-store/contents/{+path}","compare_url":"https://api.github.com/repos/connexta/ion-store/compare/{base}...{head}","merges_url":"https://api.github.com/repos/connexta/ion-store/merges","archive_url":"https://api.github.com/repos/connexta/ion-store/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/connexta/ion-store/downloads","issues_url":"https://api.github.com/repos/connexta/ion-store/issues{/number}","pulls_url":"https://api.github.com/repos/connexta/ion-store/pulls{/number}","milestones_url":"https://api.github.com/repos/connexta/ion-store/milestones{/number}","notifications_url":"https://api.github.com/repos/connexta/ion-store/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/connexta/ion-store/labels{/name}","releases_url":"https://api.github.com/repos/connexta/ion-store/releases{/id}","deployments_url":"https://api.github.com/repos/connexta/ion-store/deployments","created_at":"2019-03-14T04:34:22Z","updated_at":"2019-11-22T19:19:22Z","pushed_at":"2019-11-25T22:35:05Z","git_url":"git://github.com/connexta/ion-store.git","ssh_url":"git@github.com:connexta/ion-store.git","clone_url":"https://github.com/connexta/ion-store.git","svn_url":"https://github.com/connexta/ion-store","homepage":null,"size":737,"stargazers_count":2,"watchers_count":2,"language":"Java","has_issues":true,"has_projects":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":10,"mirror_url":null,"archived":false,"disabled":false,"open_issues_count":5,"license":{"key":"other","name":"Other","spdx_id":"NOASSERTION","url":null,"node_id":"MDc6TGljZW5zZTA="},"forks":10,"open_issues":5,"watchers":2,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/connexta/ion-store/pulls/235"},"html":{"href":"https://github.com/connexta/ion-store/pull/235"},"issue":{"href":"https://api.github.com/repos/connexta/ion-store/issues/235"},"comments":{"href":"https://api.github.com/repos/connexta/ion-store/issues/235/comments"},"review_comments":{"href":"https://api.github.com/repos/connexta/ion-store/pulls/235/comments"},"review_comment":{"href":"https://api.github.com/repos/connexta/ion-store/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/connexta/ion-store/pulls/235/commits"},"statuses":{"href":"https://api.github.com/repos/connexta/ion-store/statuses/6b391d33a4a78b886ccb78e1bf6b0cf7bed9cd4d"}},"author_association":"MEMBER"}}
{"actor":{"display_login":"mcalcote"}}