{"action":"created","comment":{"url":"https://api.github.com/repos/treasure-data/digdag/pulls/comments/68474656","id":68474656,"diff_hunk":"@@ -78,46 +85,126 @@ public Operator newTaskExecutor(Path workspacePath, TaskRequest request)\n     private class TdOperator\n             extends BaseOperator\n     {\n+        private final Config params;\n+        private final String query;\n+        private final Optional<TableParam> insertInto;\n+        private final Optional<TableParam> createTable;\n+        private final int priority;\n+        private final Optional<String> resultUrl;\n+        private final int jobRetry;\n+        private final String engine;\n+        private final Optional<String> downloadFile;\n+        private final boolean storeLastResults;\n+        private final boolean preview;\n+        private final Config state;\n+        private final Optional<String> existingJobId;\n+        private final Optional<String> existingDomainKey;\n+\n         public TdOperator(Path workspacePath, TaskRequest request)\n         {\n             super(workspacePath, request);\n-        }\n \n-        @Override\n-        public TaskResult runTask()\n-        {\n-            Config params = request.getConfig().mergeDefault(\n+            this.params = request.getConfig().mergeDefault(\n                     request.getConfig().getNestedOrGetEmpty(\"td\"));\n \n-            String query = templateEngine.templateCommand(workspacePath, params, \"query\", UTF_8);\n+            this.query = templateEngine.templateCommand(workspacePath, params, \"query\", UTF_8);\n \n-            Optional<TableParam> insertInto = params.getOptional(\"insert_into\", TableParam.class);\n-            Optional<TableParam> createTable = params.getOptional(\"create_table\", TableParam.class);\n+            this.insertInto = params.getOptional(\"insert_into\", TableParam.class);\n+            this.createTable = params.getOptional(\"create_table\", TableParam.class);\n             if (insertInto.isPresent() && createTable.isPresent()) {\n                 throw new ConfigException(\"Setting both insert_into and create_table is invalid\");\n             }\n \n-            int priority = params.get(\"priority\", int.class, 0);  // TODO this should accept string (VERY_LOW, LOW, NORMAL, HIGH VERY_HIGH)\n-            Optional<String> resultUrl = params.getOptional(\"result_url\", String.class);\n+            this.priority = params.get(\"priority\", int.class, 0);  // TODO this should accept string (VERY_LOW, LOW, NORMAL, HIGH VERY_HIGH)\n+            this.resultUrl = params.getOptional(\"result_url\", String.class);\n \n-            int jobRetry = params.get(\"job_retry\", int.class, 0);\n+            this.jobRetry = params.get(\"job_retry\", int.class, 0);\n \n-            String engine = params.get(\"engine\", String.class, \"presto\");\n+            this.engine = params.get(\"engine\", String.class, \"presto\");\n \n-            Optional<String> downloadFile = params.getOptional(\"download_file\", String.class);\n+            this.downloadFile = params.getOptional(\"download_file\", String.class);\n             if (downloadFile.isPresent() && (insertInto.isPresent() || createTable.isPresent())) {\n                 // query results become empty if INSERT INTO or CREATE TABLE query runs\n                 throw new ConfigException(\"download_file is invalid if insert_into or create_table is set\");\n             }\n \n-            boolean storeLastResults = params.get(\"store_last_results\", boolean.class, false);\n+            this.storeLastResults = params.get(\"store_last_results\", boolean.class, false);\n+\n+            this.preview = params.get(\"preview\", boolean.class, false);\n+\n+            this.state = request.getLastStateParams().deepCopy();\n+\n+            this.existingJobId = state.getOptional(JOB_ID, String.class);\n+            this.existingDomainKey = state.getOptional(DOMAIN_KEY, String.class);\n+        }\n+\n+        @Override\n+        public TaskResult runTask()\n+        {\n+            try (TDOperator op = TDOperator.fromConfigWithRetries(params)) {\n+\n+                // Generate and store domain key before starting the job\n+                if (!existingDomainKey.isPresent()) {\n+                    String domainKey = UUID.randomUUID().toString();\n+                    state.set(DOMAIN_KEY, domainKey);\n+                    throw TaskExecutionException.ofNextPolling(0, ConfigElement.copyOf(state.deepCopy()));\n+                }\n+\n+                // Start the job\n+                if (!existingJobId.isPresent()) {\n+                    String jobId = startJob(op);\n+                    state.set(JOB_ID, jobId);\n+                    state.set(POLL_INTERVAL, INITIAL_POLL_INTERVAL);\n+                    throw TaskExecutionException.ofNextPolling(INITIAL_POLL_INTERVAL, ConfigElement.copyOf(state.deepCopy()));\n+                }\n+\n+                // Check if the job is done\n+                String jobId = existingJobId.get();\n+                TDJobOperator job = op.newJobOperator(jobId);\n+                TDJobSummary status = checkJobStatus(job);\n+                boolean done = status.getStatus().isFinished();\n+                if (!done) {\n+                    int prevPollInterval = state.get(POLL_INTERVAL, int.class, INITIAL_POLL_INTERVAL);\n+                    int nextPollInterval = Math.min(MAX_POLL_INTERVAL, prevPollInterval * 2);\n+                    state.set(POLL_INTERVAL, nextPollInterval);","path":"digdag-standards/src/main/java/io/digdag/standards/operator/td/TdOperatorFactory.java","position":139,"original_position":139,"commit_id":"a8812972ee8b67f74b4345128f72b419dfe180b5","original_commit_id":"a8812972ee8b67f74b4345128f72b419dfe180b5","user":{"login":"frsyuki","id":40720,"avatar_url":"https://avatars.githubusercontent.com/u/40720?v=3","gravatar_id":"","url":"https://api.github.com/users/frsyuki","html_url":"https://github.com/frsyuki","followers_url":"https://api.github.com/users/frsyuki/followers","following_url":"https://api.github.com/users/frsyuki/following{/other_user}","gists_url":"https://api.github.com/users/frsyuki/gists{/gist_id}","starred_url":"https://api.github.com/users/frsyuki/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/frsyuki/subscriptions","organizations_url":"https://api.github.com/users/frsyuki/orgs","repos_url":"https://api.github.com/users/frsyuki/repos","events_url":"https://api.github.com/users/frsyuki/events{/privacy}","received_events_url":"https://api.github.com/users/frsyuki/received_events","type":"User","site_admin":false},"body":"I think it's easier to include number of retries in stateParams.\r\nPolling interval can be calculated using `min(initialInterval * pow(2, retryCount), maxInterval)`.","created_at":"2016-06-24T23:05:04Z","updated_at":"2016-06-24T23:05:04Z","html_url":"https://github.com/treasure-data/digdag/pull/127#discussion_r68474656","pull_request_url":"https://api.github.com/repos/treasure-data/digdag/pulls/127","_links":{"self":{"href":"https://api.github.com/repos/treasure-data/digdag/pulls/comments/68474656"},"html":{"href":"https://github.com/treasure-data/digdag/pull/127#discussion_r68474656"},"pull_request":{"href":"https://api.github.com/repos/treasure-data/digdag/pulls/127"}}},"pull_request":{"url":"https://api.github.com/repos/treasure-data/digdag/pulls/127","id":74208113,"html_url":"https://github.com/treasure-data/digdag/pull/127","diff_url":"https://github.com/treasure-data/digdag/pull/127.diff","patch_url":"https://github.com/treasure-data/digdag/pull/127.patch","issue_url":"https://api.github.com/repos/treasure-data/digdag/issues/127","number":127,"state":"open","locked":false,"title":"wip: make td operator polling instead of blocking","user":{"login":"danielnorberg","id":393289,"avatar_url":"https://avatars.githubusercontent.com/u/393289?v=3","gravatar_id":"","url":"https://api.github.com/users/danielnorberg","html_url":"https://github.com/danielnorberg","followers_url":"https://api.github.com/users/danielnorberg/followers","following_url":"https://api.github.com/users/danielnorberg/following{/other_user}","gists_url":"https://api.github.com/users/danielnorberg/gists{/gist_id}","starred_url":"https://api.github.com/users/danielnorberg/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/danielnorberg/subscriptions","organizations_url":"https://api.github.com/users/danielnorberg/orgs","repos_url":"https://api.github.com/users/danielnorberg/repos","events_url":"https://api.github.com/users/danielnorberg/events{/privacy}","received_events_url":"https://api.github.com/users/danielnorberg/received_events","type":"User","site_admin":false},"body":"This is a first step towards making all the operators that start td jobs polling instead of blocking. Most of the polling logic should be possible to implement in a manner that it can be reused across all\r\ntd job operators.","created_at":"2016-06-17T08:47:04Z","updated_at":"2016-06-24T23:05:04Z","closed_at":null,"merged_at":null,"merge_commit_sha":"d331159f40307c22525d3791c22f57adf3e38ba5","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/treasure-data/digdag/pulls/127/commits","review_comments_url":"https://api.github.com/repos/treasure-data/digdag/pulls/127/comments","review_comment_url":"https://api.github.com/repos/treasure-data/digdag/pulls/comments{/number}","comments_url":"https://api.github.com/repos/treasure-data/digdag/issues/127/comments","statuses_url":"https://api.github.com/repos/treasure-data/digdag/statuses/a8812972ee8b67f74b4345128f72b419dfe180b5","head":{"label":"treasure-data:dano/td-job-polling","ref":"dano/td-job-polling","sha":"a8812972ee8b67f74b4345128f72b419dfe180b5","user":{"login":"treasure-data","id":747746,"avatar_url":"https://avatars.githubusercontent.com/u/747746?v=3","gravatar_id":"","url":"https://api.github.com/users/treasure-data","html_url":"https://github.com/treasure-data","followers_url":"https://api.github.com/users/treasure-data/followers","following_url":"https://api.github.com/users/treasure-data/following{/other_user}","gists_url":"https://api.github.com/users/treasure-data/gists{/gist_id}","starred_url":"https://api.github.com/users/treasure-data/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/treasure-data/subscriptions","organizations_url":"https://api.github.com/users/treasure-data/orgs","repos_url":"https://api.github.com/users/treasure-data/repos","events_url":"https://api.github.com/users/treasure-data/events{/privacy}","received_events_url":"https://api.github.com/users/treasure-data/received_events","type":"Organization","site_admin":false},"repo":{"id":42751387,"name":"digdag","full_name":"treasure-data/digdag","owner":{"login":"treasure-data","id":747746,"avatar_url":"https://avatars.githubusercontent.com/u/747746?v=3","gravatar_id":"","url":"https://api.github.com/users/treasure-data","html_url":"https://github.com/treasure-data","followers_url":"https://api.github.com/users/treasure-data/followers","following_url":"https://api.github.com/users/treasure-data/following{/other_user}","gists_url":"https://api.github.com/users/treasure-data/gists{/gist_id}","starred_url":"https://api.github.com/users/treasure-data/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/treasure-data/subscriptions","organizations_url":"https://api.github.com/users/treasure-data/orgs","repos_url":"https://api.github.com/users/treasure-data/repos","events_url":"https://api.github.com/users/treasure-data/events{/privacy}","received_events_url":"https://api.github.com/users/treasure-data/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/treasure-data/digdag","description":"Workload Automation System","fork":false,"url":"https://api.github.com/repos/treasure-data/digdag","forks_url":"https://api.github.com/repos/treasure-data/digdag/forks","keys_url":"https://api.github.com/repos/treasure-data/digdag/keys{/key_id}","collaborators_url":"https://api.github.com/repos/treasure-data/digdag/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/treasure-data/digdag/teams","hooks_url":"https://api.github.com/repos/treasure-data/digdag/hooks","issue_events_url":"https://api.github.com/repos/treasure-data/digdag/issues/events{/number}","events_url":"https://api.github.com/repos/treasure-data/digdag/events","assignees_url":"https://api.github.com/repos/treasure-data/digdag/assignees{/user}","branches_url":"https://api.github.com/repos/treasure-data/digdag/branches{/branch}","tags_url":"https://api.github.com/repos/treasure-data/digdag/tags","blobs_url":"https://api.github.com/repos/treasure-data/digdag/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/treasure-data/digdag/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/treasure-data/digdag/git/refs{/sha}","trees_url":"https://api.github.com/repos/treasure-data/digdag/git/trees{/sha}","statuses_url":"https://api.github.com/repos/treasure-data/digdag/statuses/{sha}","languages_url":"https://api.github.com/repos/treasure-data/digdag/languages","stargazers_url":"https://api.github.com/repos/treasure-data/digdag/stargazers","contributors_url":"https://api.github.com/repos/treasure-data/digdag/contributors","subscribers_url":"https://api.github.com/repos/treasure-data/digdag/subscribers","subscription_url":"https://api.github.com/repos/treasure-data/digdag/subscription","commits_url":"https://api.github.com/repos/treasure-data/digdag/commits{/sha}","git_commits_url":"https://api.github.com/repos/treasure-data/digdag/git/commits{/sha}","comments_url":"https://api.github.com/repos/treasure-data/digdag/comments{/number}","issue_comment_url":"https://api.github.com/repos/treasure-data/digdag/issues/comments{/number}","contents_url":"https://api.github.com/repos/treasure-data/digdag/contents/{+path}","compare_url":"https://api.github.com/repos/treasure-data/digdag/compare/{base}...{head}","merges_url":"https://api.github.com/repos/treasure-data/digdag/merges","archive_url":"https://api.github.com/repos/treasure-data/digdag/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/treasure-data/digdag/downloads","issues_url":"https://api.github.com/repos/treasure-data/digdag/issues{/number}","pulls_url":"https://api.github.com/repos/treasure-data/digdag/pulls{/number}","milestones_url":"https://api.github.com/repos/treasure-data/digdag/milestones{/number}","notifications_url":"https://api.github.com/repos/treasure-data/digdag/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/treasure-data/digdag/labels{/name}","releases_url":"https://api.github.com/repos/treasure-data/digdag/releases{/id}","deployments_url":"https://api.github.com/repos/treasure-data/digdag/deployments","created_at":"2015-09-18T23:18:08Z","updated_at":"2016-06-24T20:52:52Z","pushed_at":"2016-06-24T21:18:39Z","git_url":"git://github.com/treasure-data/digdag.git","ssh_url":"git@github.com:treasure-data/digdag.git","clone_url":"https://github.com/treasure-data/digdag.git","svn_url":"https://github.com/treasure-data/digdag","homepage":"http://www.digdag.io/","size":2780,"stargazers_count":129,"watchers_count":129,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":11,"mirror_url":null,"open_issues_count":26,"forks":11,"open_issues":26,"watchers":129,"default_branch":"master"}},"base":{"label":"treasure-data:master","ref":"master","sha":"94b55038ce091a9205989d206c612441768ba5f8","user":{"login":"treasure-data","id":747746,"avatar_url":"https://avatars.githubusercontent.com/u/747746?v=3","gravatar_id":"","url":"https://api.github.com/users/treasure-data","html_url":"https://github.com/treasure-data","followers_url":"https://api.github.com/users/treasure-data/followers","following_url":"https://api.github.com/users/treasure-data/following{/other_user}","gists_url":"https://api.github.com/users/treasure-data/gists{/gist_id}","starred_url":"https://api.github.com/users/treasure-data/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/treasure-data/subscriptions","organizations_url":"https://api.github.com/users/treasure-data/orgs","repos_url":"https://api.github.com/users/treasure-data/repos","events_url":"https://api.github.com/users/treasure-data/events{/privacy}","received_events_url":"https://api.github.com/users/treasure-data/received_events","type":"Organization","site_admin":false},"repo":{"id":42751387,"name":"digdag","full_name":"treasure-data/digdag","owner":{"login":"treasure-data","id":747746,"avatar_url":"https://avatars.githubusercontent.com/u/747746?v=3","gravatar_id":"","url":"https://api.github.com/users/treasure-data","html_url":"https://github.com/treasure-data","followers_url":"https://api.github.com/users/treasure-data/followers","following_url":"https://api.github.com/users/treasure-data/following{/other_user}","gists_url":"https://api.github.com/users/treasure-data/gists{/gist_id}","starred_url":"https://api.github.com/users/treasure-data/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/treasure-data/subscriptions","organizations_url":"https://api.github.com/users/treasure-data/orgs","repos_url":"https://api.github.com/users/treasure-data/repos","events_url":"https://api.github.com/users/treasure-data/events{/privacy}","received_events_url":"https://api.github.com/users/treasure-data/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/treasure-data/digdag","description":"Workload Automation System","fork":false,"url":"https://api.github.com/repos/treasure-data/digdag","forks_url":"https://api.github.com/repos/treasure-data/digdag/forks","keys_url":"https://api.github.com/repos/treasure-data/digdag/keys{/key_id}","collaborators_url":"https://api.github.com/repos/treasure-data/digdag/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/treasure-data/digdag/teams","hooks_url":"https://api.github.com/repos/treasure-data/digdag/hooks","issue_events_url":"https://api.github.com/repos/treasure-data/digdag/issues/events{/number}","events_url":"https://api.github.com/repos/treasure-data/digdag/events","assignees_url":"https://api.github.com/repos/treasure-data/digdag/assignees{/user}","branches_url":"https://api.github.com/repos/treasure-data/digdag/branches{/branch}","tags_url":"https://api.github.com/repos/treasure-data/digdag/tags","blobs_url":"https://api.github.com/repos/treasure-data/digdag/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/treasure-data/digdag/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/treasure-data/digdag/git/refs{/sha}","trees_url":"https://api.github.com/repos/treasure-data/digdag/git/trees{/sha}","statuses_url":"https://api.github.com/repos/treasure-data/digdag/statuses/{sha}","languages_url":"https://api.github.com/repos/treasure-data/digdag/languages","stargazers_url":"https://api.github.com/repos/treasure-data/digdag/stargazers","contributors_url":"https://api.github.com/repos/treasure-data/digdag/contributors","subscribers_url":"https://api.github.com/repos/treasure-data/digdag/subscribers","subscription_url":"https://api.github.com/repos/treasure-data/digdag/subscription","commits_url":"https://api.github.com/repos/treasure-data/digdag/commits{/sha}","git_commits_url":"https://api.github.com/repos/treasure-data/digdag/git/commits{/sha}","comments_url":"https://api.github.com/repos/treasure-data/digdag/comments{/number}","issue_comment_url":"https://api.github.com/repos/treasure-data/digdag/issues/comments{/number}","contents_url":"https://api.github.com/repos/treasure-data/digdag/contents/{+path}","compare_url":"https://api.github.com/repos/treasure-data/digdag/compare/{base}...{head}","merges_url":"https://api.github.com/repos/treasure-data/digdag/merges","archive_url":"https://api.github.com/repos/treasure-data/digdag/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/treasure-data/digdag/downloads","issues_url":"https://api.github.com/repos/treasure-data/digdag/issues{/number}","pulls_url":"https://api.github.com/repos/treasure-data/digdag/pulls{/number}","milestones_url":"https://api.github.com/repos/treasure-data/digdag/milestones{/number}","notifications_url":"https://api.github.com/repos/treasure-data/digdag/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/treasure-data/digdag/labels{/name}","releases_url":"https://api.github.com/repos/treasure-data/digdag/releases{/id}","deployments_url":"https://api.github.com/repos/treasure-data/digdag/deployments","created_at":"2015-09-18T23:18:08Z","updated_at":"2016-06-24T20:52:52Z","pushed_at":"2016-06-24T21:18:39Z","git_url":"git://github.com/treasure-data/digdag.git","ssh_url":"git@github.com:treasure-data/digdag.git","clone_url":"https://github.com/treasure-data/digdag.git","svn_url":"https://github.com/treasure-data/digdag","homepage":"http://www.digdag.io/","size":2780,"stargazers_count":129,"watchers_count":129,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":11,"mirror_url":null,"open_issues_count":26,"forks":11,"open_issues":26,"watchers":129,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/treasure-data/digdag/pulls/127"},"html":{"href":"https://github.com/treasure-data/digdag/pull/127"},"issue":{"href":"https://api.github.com/repos/treasure-data/digdag/issues/127"},"comments":{"href":"https://api.github.com/repos/treasure-data/digdag/issues/127/comments"},"review_comments":{"href":"https://api.github.com/repos/treasure-data/digdag/pulls/127/comments"},"review_comment":{"href":"https://api.github.com/repos/treasure-data/digdag/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/treasure-data/digdag/pulls/127/commits"},"statuses":{"href":"https://api.github.com/repos/treasure-data/digdag/statuses/a8812972ee8b67f74b4345128f72b419dfe180b5"}}}}
{"actor":{"display_login":"frsyuki"}}