{"action":"created","comment":{"url":"https://api.github.com/repos/elastic/elasticsearch/pulls/comments/71593769","id":71593769,"diff_hunk":"@@ -0,0 +1,363 @@\n+/*\n+ * Licensed to Elasticsearch under one or more contributor\n+ * license agreements. See the NOTICE file distributed with\n+ * this work for additional information regarding copyright\n+ * ownership. Elasticsearch licenses this file to you under\n+ * the Apache License, Version 2.0 (the \"License\"); you may\n+ * not use this file except in compliance with the License.\n+ * You may obtain a copy of the License at\n+ *\n+ *    http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing,\n+ * software distributed under the License is distributed on an\n+ * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n+ * KIND, either express or implied.  See the License for the\n+ * specific language governing permissions and limitations\n+ * under the License.\n+ */\n+package org.elasticsearch.cockroach;\n+\n+import org.elasticsearch.action.ActionListener;\n+import org.elasticsearch.cluster.ClusterChangedEvent;\n+import org.elasticsearch.cluster.ClusterStateListener;\n+import org.elasticsearch.cockroach.CockroachActionRegistry.CockroachActionHolder;\n+import org.elasticsearch.cockroach.CockroachTasksInProgress.CockroachTaskInProgress;\n+import org.elasticsearch.common.Nullable;\n+import org.elasticsearch.common.Strings;\n+import org.elasticsearch.common.component.AbstractComponent;\n+import org.elasticsearch.common.inject.Provider;\n+import org.elasticsearch.common.io.stream.StreamInput;\n+import org.elasticsearch.common.io.stream.StreamOutput;\n+import org.elasticsearch.common.settings.Settings;\n+import org.elasticsearch.common.util.concurrent.AbstractRunnable;\n+import org.elasticsearch.common.xcontent.XContentBuilder;\n+import org.elasticsearch.tasks.Task;\n+import org.elasticsearch.tasks.TaskManager;\n+import org.elasticsearch.threadpool.ThreadPool;\n+import org.elasticsearch.transport.TransportResponse.Empty;\n+\n+import java.io.IOException;\n+import java.util.HashMap;\n+import java.util.HashSet;\n+import java.util.Map;\n+import java.util.Objects;\n+import java.util.Set;\n+import java.util.concurrent.atomic.AtomicReference;\n+\n+import static java.util.Objects.requireNonNull;\n+\n+/**\n+ * This component is responsible for execution of cockroach actions on individual nodes. It runs on all\n+ * non-transport client nodes in the cluster and monitors cluster state changes to detect started commands.\n+ */\n+public class CockroachActionExecutor extends AbstractComponent implements ClusterStateListener {\n+    private final Map<String, RunningCockroachTask> runningTasks = new HashMap<>();\n+    private final CockroachActionRegistry cockroachActionRegistry;\n+    private final TaskManager taskManager;\n+    private final ThreadPool threadPool;\n+    private final CockroachTransportService cockroachTransportService;\n+\n+\n+    public CockroachActionExecutor(final Settings settings, final CockroachActionRegistry cockroachActionRegistry,\n+                                   final TaskManager taskManager, final CockroachTransportService cockroachTransportService,\n+                                   final ThreadPool threadPool) {\n+        super(settings);\n+        this.cockroachActionRegistry = cockroachActionRegistry;\n+        this.taskManager = taskManager;\n+        this.cockroachTransportService = cockroachTransportService;\n+        this.threadPool = threadPool;\n+    }\n+\n+    @Override\n+    public void clusterChanged(ClusterChangedEvent event) {\n+        CockroachTasksInProgress tasks = event.state().custom(CockroachTasksInProgress.TYPE);\n+        CockroachTasksInProgress previousTasks = event.previousState().custom(CockroachTasksInProgress.TYPE);\n+\n+        if (Objects.equals(tasks, previousTasks) == false || event.nodesChanged()) {\n+            // We have some changes let's check if they are related to our node\n+            String localNodeId = event.state().getNodes().getLocalNodeId();\n+            Set<String> notVisitedTasks = new HashSet<>(runningTasks.keySet());\n+            if (tasks != null) {\n+                for (CockroachTaskInProgress taskInProgress : tasks.entries()) {\n+                    if (localNodeId.equals(taskInProgress.getExecutorNode())) {\n+                        RunningCockroachTask cockroachTask = runningTasks.get(taskInProgress.getUuid());\n+                        if (cockroachTask == null) {\n+                            // New task - let's start it\n+                            startTask(taskInProgress);\n+                        } else {\n+                            // The task is still running\n+                            notVisitedTasks.remove(taskInProgress.getUuid());\n+                            if (taskInProgress.isFinished() == false && cockroachTask.getState() == State.FAILED_NOTIFICATION) {\n+                                // We tried to notify the master about this task before but the notification failed and\n+                                // the master doesn't seem to know about it - retry notification\n+                                restartNotification(cockroachTask);\n+                            }\n+                        }\n+                    }\n+                }\n+            }\n+\n+            for (String s : notVisitedTasks) {\n+                RunningCockroachTask task = runningTasks.get(s);\n+                if (task.getState() == State.NOTIFIED || task.getState() == State.DONE) {\n+                    // Result was sent to the caller and the caller acknowledged acceptance of the result\n+                    finishTask(s);\n+                } else if (task.getState() == State.FAILED_NOTIFICATION) {\n+                    // We tried to send result to master, but it failed and master doesn't know about this task\n+                    logger.warn(\"failed to notify master about task {}\", task.getUuid());\n+                    finishTask(s);\n+                }\n+                // TODO: task is running locally, but master doesn't know about it","path":"core/src/main/java/org/elasticsearch/cockroach/CockroachActionExecutor.java","position":111,"original_position":111,"commit_id":"50594bddb707b487c2a87aee67faab0019b43266","original_commit_id":"50594bddb707b487c2a87aee67faab0019b43266","user":{"login":"abeyad","id":1631297,"avatar_url":"https://avatars.githubusercontent.com/u/1631297?v=3","gravatar_id":"","url":"https://api.github.com/users/abeyad","html_url":"https://github.com/abeyad","followers_url":"https://api.github.com/users/abeyad/followers","following_url":"https://api.github.com/users/abeyad/following{/other_user}","gists_url":"https://api.github.com/users/abeyad/gists{/gist_id}","starred_url":"https://api.github.com/users/abeyad/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/abeyad/subscriptions","organizations_url":"https://api.github.com/users/abeyad/orgs","repos_url":"https://api.github.com/users/abeyad/repos","events_url":"https://api.github.com/users/abeyad/events{/privacy}","received_events_url":"https://api.github.com/users/abeyad/received_events","type":"User","site_admin":false},"body":"I'm not following this comment and how the task could be running locally if master didn't have it in the cluster state?","created_at":"2016-07-20T19:47:13Z","updated_at":"2016-07-20T19:47:13Z","html_url":"https://github.com/elastic/elasticsearch/pull/19288#discussion_r71593769","pull_request_url":"https://api.github.com/repos/elastic/elasticsearch/pulls/19288","_links":{"self":{"href":"https://api.github.com/repos/elastic/elasticsearch/pulls/comments/71593769"},"html":{"href":"https://github.com/elastic/elasticsearch/pull/19288#discussion_r71593769"},"pull_request":{"href":"https://api.github.com/repos/elastic/elasticsearch/pulls/19288"}}},"pull_request":{"url":"https://api.github.com/repos/elastic/elasticsearch/pulls/19288","id":76470069,"html_url":"https://github.com/elastic/elasticsearch/pull/19288","diff_url":"https://github.com/elastic/elasticsearch/pull/19288.diff","patch_url":"https://github.com/elastic/elasticsearch/pull/19288.patch","issue_url":"https://api.github.com/repos/elastic/elasticsearch/issues/19288","number":19288,"state":"open","locked":false,"title":"Adds support for cockroach actions","user":{"login":"imotov","id":655851,"avatar_url":"https://avatars.githubusercontent.com/u/655851?v=3","gravatar_id":"","url":"https://api.github.com/users/imotov","html_url":"https://github.com/imotov","followers_url":"https://api.github.com/users/imotov/followers","following_url":"https://api.github.com/users/imotov/following{/other_user}","gists_url":"https://api.github.com/users/imotov/gists{/gist_id}","starred_url":"https://api.github.com/users/imotov/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/imotov/subscriptions","organizations_url":"https://api.github.com/users/imotov/orgs","repos_url":"https://api.github.com/users/imotov/repos","events_url":"https://api.github.com/users/imotov/events{/privacy}","received_events_url":"https://api.github.com/users/imotov/received_events","type":"User","site_admin":false},"body":"A cockroach action is a transport-like action that is using the cluster state instead of transport to start tasks and send tasks responses. This allows cockroach tasks to survive restart of coordinating and executing nodes. A cockroach action can be implemented by extending TransportCockroachAction. TransportCockroachAction will start the task by using CockroachService, which controls cockroach tasks lifecycle.  See TestCockroachActionPlugin for an example implementing a cockroach action.\r\n\r\nThis PR just adds an infrastructure for creating cockroach tasks and tests. The first use of the tasks will be to migrate snapshot/restore functionality to it. Since it's a big change, it probably makes more sense to target 6.0.0 with it.","created_at":"2016-07-06T17:28:24Z","updated_at":"2016-07-20T19:47:13Z","closed_at":null,"merged_at":null,"merge_commit_sha":"9c5653547ccc24a02ad55ede414175bad1c98b98","assignee":{"login":"abeyad","id":1631297,"avatar_url":"https://avatars.githubusercontent.com/u/1631297?v=3","gravatar_id":"","url":"https://api.github.com/users/abeyad","html_url":"https://github.com/abeyad","followers_url":"https://api.github.com/users/abeyad/followers","following_url":"https://api.github.com/users/abeyad/following{/other_user}","gists_url":"https://api.github.com/users/abeyad/gists{/gist_id}","starred_url":"https://api.github.com/users/abeyad/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/abeyad/subscriptions","organizations_url":"https://api.github.com/users/abeyad/orgs","repos_url":"https://api.github.com/users/abeyad/repos","events_url":"https://api.github.com/users/abeyad/events{/privacy}","received_events_url":"https://api.github.com/users/abeyad/received_events","type":"User","site_admin":false},"assignees":[{"login":"abeyad","id":1631297,"avatar_url":"https://avatars.githubusercontent.com/u/1631297?v=3","gravatar_id":"","url":"https://api.github.com/users/abeyad","html_url":"https://github.com/abeyad","followers_url":"https://api.github.com/users/abeyad/followers","following_url":"https://api.github.com/users/abeyad/following{/other_user}","gists_url":"https://api.github.com/users/abeyad/gists{/gist_id}","starred_url":"https://api.github.com/users/abeyad/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/abeyad/subscriptions","organizations_url":"https://api.github.com/users/abeyad/orgs","repos_url":"https://api.github.com/users/abeyad/repos","events_url":"https://api.github.com/users/abeyad/events{/privacy}","received_events_url":"https://api.github.com/users/abeyad/received_events","type":"User","site_admin":false}],"milestone":null,"commits_url":"https://api.github.com/repos/elastic/elasticsearch/pulls/19288/commits","review_comments_url":"https://api.github.com/repos/elastic/elasticsearch/pulls/19288/comments","review_comment_url":"https://api.github.com/repos/elastic/elasticsearch/pulls/comments{/number}","comments_url":"https://api.github.com/repos/elastic/elasticsearch/issues/19288/comments","statuses_url":"https://api.github.com/repos/elastic/elasticsearch/statuses/50594bddb707b487c2a87aee67faab0019b43266","head":{"label":"imotov:background-tasks","ref":"background-tasks","sha":"50594bddb707b487c2a87aee67faab0019b43266","user":{"login":"imotov","id":655851,"avatar_url":"https://avatars.githubusercontent.com/u/655851?v=3","gravatar_id":"","url":"https://api.github.com/users/imotov","html_url":"https://github.com/imotov","followers_url":"https://api.github.com/users/imotov/followers","following_url":"https://api.github.com/users/imotov/following{/other_user}","gists_url":"https://api.github.com/users/imotov/gists{/gist_id}","starred_url":"https://api.github.com/users/imotov/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/imotov/subscriptions","organizations_url":"https://api.github.com/users/imotov/orgs","repos_url":"https://api.github.com/users/imotov/repos","events_url":"https://api.github.com/users/imotov/events{/privacy}","received_events_url":"https://api.github.com/users/imotov/received_events","type":"User","site_admin":false},"repo":{"id":1484646,"name":"elasticsearch","full_name":"imotov/elasticsearch","owner":{"login":"imotov","id":655851,"avatar_url":"https://avatars.githubusercontent.com/u/655851?v=3","gravatar_id":"","url":"https://api.github.com/users/imotov","html_url":"https://github.com/imotov","followers_url":"https://api.github.com/users/imotov/followers","following_url":"https://api.github.com/users/imotov/following{/other_user}","gists_url":"https://api.github.com/users/imotov/gists{/gist_id}","starred_url":"https://api.github.com/users/imotov/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/imotov/subscriptions","organizations_url":"https://api.github.com/users/imotov/orgs","repos_url":"https://api.github.com/users/imotov/repos","events_url":"https://api.github.com/users/imotov/events{/privacy}","received_events_url":"https://api.github.com/users/imotov/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/imotov/elasticsearch","description":"Open Source, Distributed, RESTful Search Engine","fork":true,"url":"https://api.github.com/repos/imotov/elasticsearch","forks_url":"https://api.github.com/repos/imotov/elasticsearch/forks","keys_url":"https://api.github.com/repos/imotov/elasticsearch/keys{/key_id}","collaborators_url":"https://api.github.com/repos/imotov/elasticsearch/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/imotov/elasticsearch/teams","hooks_url":"https://api.github.com/repos/imotov/elasticsearch/hooks","issue_events_url":"https://api.github.com/repos/imotov/elasticsearch/issues/events{/number}","events_url":"https://api.github.com/repos/imotov/elasticsearch/events","assignees_url":"https://api.github.com/repos/imotov/elasticsearch/assignees{/user}","branches_url":"https://api.github.com/repos/imotov/elasticsearch/branches{/branch}","tags_url":"https://api.github.com/repos/imotov/elasticsearch/tags","blobs_url":"https://api.github.com/repos/imotov/elasticsearch/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/imotov/elasticsearch/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/imotov/elasticsearch/git/refs{/sha}","trees_url":"https://api.github.com/repos/imotov/elasticsearch/git/trees{/sha}","statuses_url":"https://api.github.com/repos/imotov/elasticsearch/statuses/{sha}","languages_url":"https://api.github.com/repos/imotov/elasticsearch/languages","stargazers_url":"https://api.github.com/repos/imotov/elasticsearch/stargazers","contributors_url":"https://api.github.com/repos/imotov/elasticsearch/contributors","subscribers_url":"https://api.github.com/repos/imotov/elasticsearch/subscribers","subscription_url":"https://api.github.com/repos/imotov/elasticsearch/subscription","commits_url":"https://api.github.com/repos/imotov/elasticsearch/commits{/sha}","git_commits_url":"https://api.github.com/repos/imotov/elasticsearch/git/commits{/sha}","comments_url":"https://api.github.com/repos/imotov/elasticsearch/comments{/number}","issue_comment_url":"https://api.github.com/repos/imotov/elasticsearch/issues/comments{/number}","contents_url":"https://api.github.com/repos/imotov/elasticsearch/contents/{+path}","compare_url":"https://api.github.com/repos/imotov/elasticsearch/compare/{base}...{head}","merges_url":"https://api.github.com/repos/imotov/elasticsearch/merges","archive_url":"https://api.github.com/repos/imotov/elasticsearch/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/imotov/elasticsearch/downloads","issues_url":"https://api.github.com/repos/imotov/elasticsearch/issues{/number}","pulls_url":"https://api.github.com/repos/imotov/elasticsearch/pulls{/number}","milestones_url":"https://api.github.com/repos/imotov/elasticsearch/milestones{/number}","notifications_url":"https://api.github.com/repos/imotov/elasticsearch/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/imotov/elasticsearch/labels{/name}","releases_url":"https://api.github.com/repos/imotov/elasticsearch/releases{/id}","deployments_url":"https://api.github.com/repos/imotov/elasticsearch/deployments","created_at":"2011-03-15T21:12:02Z","updated_at":"2015-12-21T11:41:57Z","pushed_at":"2016-07-18T15:11:38Z","git_url":"git://github.com/imotov/elasticsearch.git","ssh_url":"git@github.com:imotov/elasticsearch.git","clone_url":"https://github.com/imotov/elasticsearch.git","svn_url":"https://github.com/imotov/elasticsearch","homepage":"http://elasticsearch.org","size":219497,"stargazers_count":2,"watchers_count":2,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":0,"forks":1,"open_issues":0,"watchers":2,"default_branch":"master"}},"base":{"label":"elastic:master","ref":"master","sha":"76057e6b0532959332d516aff5d7d215fab97b23","user":{"login":"elastic","id":6764390,"avatar_url":"https://avatars.githubusercontent.com/u/6764390?v=3","gravatar_id":"","url":"https://api.github.com/users/elastic","html_url":"https://github.com/elastic","followers_url":"https://api.github.com/users/elastic/followers","following_url":"https://api.github.com/users/elastic/following{/other_user}","gists_url":"https://api.github.com/users/elastic/gists{/gist_id}","starred_url":"https://api.github.com/users/elastic/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/elastic/subscriptions","organizations_url":"https://api.github.com/users/elastic/orgs","repos_url":"https://api.github.com/users/elastic/repos","events_url":"https://api.github.com/users/elastic/events{/privacy}","received_events_url":"https://api.github.com/users/elastic/received_events","type":"Organization","site_admin":false},"repo":{"id":507775,"name":"elasticsearch","full_name":"elastic/elasticsearch","owner":{"login":"elastic","id":6764390,"avatar_url":"https://avatars.githubusercontent.com/u/6764390?v=3","gravatar_id":"","url":"https://api.github.com/users/elastic","html_url":"https://github.com/elastic","followers_url":"https://api.github.com/users/elastic/followers","following_url":"https://api.github.com/users/elastic/following{/other_user}","gists_url":"https://api.github.com/users/elastic/gists{/gist_id}","starred_url":"https://api.github.com/users/elastic/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/elastic/subscriptions","organizations_url":"https://api.github.com/users/elastic/orgs","repos_url":"https://api.github.com/users/elastic/repos","events_url":"https://api.github.com/users/elastic/events{/privacy}","received_events_url":"https://api.github.com/users/elastic/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/elastic/elasticsearch","description":"Open Source, Distributed, RESTful Search Engine","fork":false,"url":"https://api.github.com/repos/elastic/elasticsearch","forks_url":"https://api.github.com/repos/elastic/elasticsearch/forks","keys_url":"https://api.github.com/repos/elastic/elasticsearch/keys{/key_id}","collaborators_url":"https://api.github.com/repos/elastic/elasticsearch/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/elastic/elasticsearch/teams","hooks_url":"https://api.github.com/repos/elastic/elasticsearch/hooks","issue_events_url":"https://api.github.com/repos/elastic/elasticsearch/issues/events{/number}","events_url":"https://api.github.com/repos/elastic/elasticsearch/events","assignees_url":"https://api.github.com/repos/elastic/elasticsearch/assignees{/user}","branches_url":"https://api.github.com/repos/elastic/elasticsearch/branches{/branch}","tags_url":"https://api.github.com/repos/elastic/elasticsearch/tags","blobs_url":"https://api.github.com/repos/elastic/elasticsearch/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/elastic/elasticsearch/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/elastic/elasticsearch/git/refs{/sha}","trees_url":"https://api.github.com/repos/elastic/elasticsearch/git/trees{/sha}","statuses_url":"https://api.github.com/repos/elastic/elasticsearch/statuses/{sha}","languages_url":"https://api.github.com/repos/elastic/elasticsearch/languages","stargazers_url":"https://api.github.com/repos/elastic/elasticsearch/stargazers","contributors_url":"https://api.github.com/repos/elastic/elasticsearch/contributors","subscribers_url":"https://api.github.com/repos/elastic/elasticsearch/subscribers","subscription_url":"https://api.github.com/repos/elastic/elasticsearch/subscription","commits_url":"https://api.github.com/repos/elastic/elasticsearch/commits{/sha}","git_commits_url":"https://api.github.com/repos/elastic/elasticsearch/git/commits{/sha}","comments_url":"https://api.github.com/repos/elastic/elasticsearch/comments{/number}","issue_comment_url":"https://api.github.com/repos/elastic/elasticsearch/issues/comments{/number}","contents_url":"https://api.github.com/repos/elastic/elasticsearch/contents/{+path}","compare_url":"https://api.github.com/repos/elastic/elasticsearch/compare/{base}...{head}","merges_url":"https://api.github.com/repos/elastic/elasticsearch/merges","archive_url":"https://api.github.com/repos/elastic/elasticsearch/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/elastic/elasticsearch/downloads","issues_url":"https://api.github.com/repos/elastic/elasticsearch/issues{/number}","pulls_url":"https://api.github.com/repos/elastic/elasticsearch/pulls{/number}","milestones_url":"https://api.github.com/repos/elastic/elasticsearch/milestones{/number}","notifications_url":"https://api.github.com/repos/elastic/elasticsearch/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/elastic/elasticsearch/labels{/name}","releases_url":"https://api.github.com/repos/elastic/elasticsearch/releases{/id}","deployments_url":"https://api.github.com/repos/elastic/elasticsearch/deployments","created_at":"2010-02-08T13:20:56Z","updated_at":"2016-07-20T18:10:36Z","pushed_at":"2016-07-20T19:45:22Z","git_url":"git://github.com/elastic/elasticsearch.git","ssh_url":"git@github.com:elastic/elasticsearch.git","clone_url":"https://github.com/elastic/elasticsearch.git","svn_url":"https://github.com/elastic/elasticsearch","homepage":"https://www.elastic.co/products/elasticsearch","size":344459,"stargazers_count":17486,"watchers_count":17486,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":5923,"mirror_url":null,"open_issues_count":1088,"forks":5923,"open_issues":1088,"watchers":17486,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/elastic/elasticsearch/pulls/19288"},"html":{"href":"https://github.com/elastic/elasticsearch/pull/19288"},"issue":{"href":"https://api.github.com/repos/elastic/elasticsearch/issues/19288"},"comments":{"href":"https://api.github.com/repos/elastic/elasticsearch/issues/19288/comments"},"review_comments":{"href":"https://api.github.com/repos/elastic/elasticsearch/pulls/19288/comments"},"review_comment":{"href":"https://api.github.com/repos/elastic/elasticsearch/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/elastic/elasticsearch/pulls/19288/commits"},"statuses":{"href":"https://api.github.com/repos/elastic/elasticsearch/statuses/50594bddb707b487c2a87aee67faab0019b43266"}}}}
{"actor":{"display_login":"abeyad"}}