{"action":"created","comment":{"url":"https://api.github.com/repos/elastic/elasticsearch/pulls/comments/48995922","id":48995922,"diff_hunk":"@@ -0,0 +1,232 @@\n+/*\n+ * Licensed to Elasticsearch under one or more contributor\n+ * license agreements. See the NOTICE file distributed with\n+ * this work for additional information regarding copyright\n+ * ownership. Elasticsearch licenses this file to you under\n+ * the Apache License, Version 2.0 (the \"License\"); you may\n+ * not use this file except in compliance with the License.\n+ * You may obtain a copy of the License at\n+ *\n+ *    http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing,\n+ * software distributed under the License is distributed on an\n+ * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n+ * KIND, either express or implied.  See the License for the\n+ * specific language governing permissions and limitations\n+ * under the License.\n+ */\n+\n+package org.elasticsearch.plugin.reindex;\n+\n+import static java.util.Objects.requireNonNull;\n+import static org.elasticsearch.index.VersionType.INTERNAL;\n+\n+import java.util.Objects;\n+\n+import org.elasticsearch.action.ActionListener;\n+import org.elasticsearch.action.ActionRequestValidationException;\n+import org.elasticsearch.action.index.IndexRequest;\n+import org.elasticsearch.action.support.ActionFilters;\n+import org.elasticsearch.action.support.AutoCreateIndex;\n+import org.elasticsearch.action.support.HandledTransportAction;\n+import org.elasticsearch.client.Client;\n+import org.elasticsearch.cluster.ClusterService;\n+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;\n+import org.elasticsearch.common.inject.Inject;\n+import org.elasticsearch.common.lucene.uid.Versions;\n+import org.elasticsearch.common.settings.Settings;\n+import org.elasticsearch.index.mapper.internal.TTLFieldMapper;\n+import org.elasticsearch.index.mapper.internal.VersionFieldMapper;\n+import org.elasticsearch.script.ScriptService;\n+import org.elasticsearch.search.SearchHit;\n+import org.elasticsearch.threadpool.ThreadPool;\n+import org.elasticsearch.transport.TransportService;\n+\n+public class TransportReindexAction extends HandledTransportAction<ReindexRequest, ReindexResponse> {\n+    private final ClusterService clusterService;\n+    private final ScriptService scriptService;\n+    private final AutoCreateIndex autoCreateIndex;\n+    private final Client client;\n+\n+    @Inject\n+    public TransportReindexAction(Settings settings, ThreadPool threadPool, ActionFilters actionFilters,\n+            IndexNameExpressionResolver indexNameExpressionResolver, ClusterService clusterService, ScriptService scriptService,\n+            AutoCreateIndex autoCreateIndex, Client client, TransportService transportService) {\n+        super(settings, ReindexAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver,\n+                ReindexRequest::new);\n+        this.clusterService = clusterService;\n+        this.scriptService = scriptService;\n+        this.autoCreateIndex = autoCreateIndex;\n+        this.client = client;\n+    }\n+\n+    @Override\n+    protected void doExecute(ReindexRequest request, ActionListener<ReindexResponse> listener) {\n+        String target = request.destination().index();\n+        if (false == autoCreateIndex.shouldAutoCreate(target, clusterService.state())) {\n+            /*\n+             * If we're going to autocreate the index we don't need to resolve\n+             * it. This is the same sort of dance that TransportIndexRequest\n+             * uses to decide to autocreate the index.\n+             */\n+            target = indexNameExpressionResolver.concreteIndices(clusterService.state(), request.destination())[0];\n+        }\n+        for (String sourceIndex: indexNameExpressionResolver.concreteIndices(clusterService.state(), request.source())) {\n+            if (sourceIndex.equals(target)) {\n+                ActionRequestValidationException e = new ActionRequestValidationException();\n+                e.addValidationError(\"reindex cannot write into an index its reading from [\" + target + ']');\n+                throw e;\n+            }\n+        }\n+        new AsyncIndexBySearchAction(request, listener).start();\n+    }\n+\n+    /**\n+     * Simple implementation of reindex using scrolling and bulk. There are tons\n+     * of optimizations that can be done on certain types of reindex requests\n+     * but this makes no attempt to do any of them so it can be as simple\n+     * possible.\n+     */\n+    class AsyncIndexBySearchAction extends AbstractAsyncBulkIndexByScrollAction<ReindexRequest, ReindexResponse> {\n+        public AsyncIndexBySearchAction(ReindexRequest request, ActionListener<ReindexResponse> listener) {\n+            super(logger, scriptService, client, threadPool, request, request.source(), listener);\n+        }\n+\n+        @Override\n+        protected IndexRequest buildIndexRequest(SearchHit doc) {\n+            IndexRequest index = new IndexRequest(mainRequest.destination(), mainRequest);\n+\n+            // We want the index from the copied request, not the doc.\n+            index.id(doc.id());\n+            if (index.type() == null) {\n+                /*\n+                 * Default to doc's type if not specified in request so its easy\n+                 * to do a scripted update.\n+                 */\n+                index.type(doc.type());\n+            }\n+            index.source(doc.sourceRef());\n+            /*\n+             * We've already copied the versionType but we need to be careful\n+             * with internal versioning. It either represents \"create\" (version\n+             * == MATCH_DELETED) or overwrite (version == MATCH_ANY). All other\n+             * values are invalid.\n+             */\n+            if (index.versionType() == INTERNAL) {\n+                if (index.version() != Versions.MATCH_DELETED && index.version() != Versions.MATCH_ANY) {\n+                    throw new IllegalArgumentException(\"Invalid version for reindex with internal versioning [\" + index.version() + ']');","path":"plugins/reindex/src/main/java/org/elasticsearch/plugin/reindex/TransportReindexAction.java","position":null,"original_position":118,"commit_id":"f4fd2f68bdfcced9e1469bed69902c2d0c57f09b","original_commit_id":"c0bc003d4efbcf45e591de3438fc7ccd61b33ea7","user":{"login":"nik9000","id":215970,"avatar_url":"https://avatars.githubusercontent.com/u/215970?v=3","gravatar_id":"","url":"https://api.github.com/users/nik9000","html_url":"https://github.com/nik9000","followers_url":"https://api.github.com/users/nik9000/followers","following_url":"https://api.github.com/users/nik9000/following{/other_user}","gists_url":"https://api.github.com/users/nik9000/gists{/gist_id}","starred_url":"https://api.github.com/users/nik9000/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nik9000/subscriptions","organizations_url":"https://api.github.com/users/nik9000/orgs","repos_url":"https://api.github.com/users/nik9000/repos","events_url":"https://api.github.com/users/nik9000/events{/privacy}","received_events_url":"https://api.github.com/users/nik9000/received_events","type":"User","site_admin":false},"body":"Yup. Removing.","created_at":"2016-01-06T19:11:30Z","updated_at":"2016-01-06T19:11:30Z","html_url":"https://github.com/elastic/elasticsearch/pull/15125#discussion_r48995922","pull_request_url":"https://api.github.com/repos/elastic/elasticsearch/pulls/15125","_links":{"self":{"href":"https://api.github.com/repos/elastic/elasticsearch/pulls/comments/48995922"},"html":{"href":"https://github.com/elastic/elasticsearch/pull/15125#discussion_r48995922"},"pull_request":{"href":"https://api.github.com/repos/elastic/elasticsearch/pulls/15125"}}},"pull_request":{"url":"https://api.github.com/repos/elastic/elasticsearch/pulls/15125","id":52176503,"html_url":"https://github.com/elastic/elasticsearch/pull/15125","diff_url":"https://github.com/elastic/elasticsearch/pull/15125.diff","patch_url":"https://github.com/elastic/elasticsearch/pull/15125.patch","issue_url":"https://api.github.com/repos/elastic/elasticsearch/issues/15125","number":15125,"state":"open","locked":false,"title":"Basic reindex implementation","user":{"login":"nik9000","id":215970,"avatar_url":"https://avatars.githubusercontent.com/u/215970?v=3","gravatar_id":"","url":"https://api.github.com/users/nik9000","html_url":"https://github.com/nik9000","followers_url":"https://api.github.com/users/nik9000/followers","following_url":"https://api.github.com/users/nik9000/following{/other_user}","gists_url":"https://api.github.com/users/nik9000/gists{/gist_id}","starred_url":"https://api.github.com/users/nik9000/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nik9000/subscriptions","organizations_url":"https://api.github.com/users/nik9000/orgs","repos_url":"https://api.github.com/users/nik9000/repos","events_url":"https://api.github.com/users/nik9000/events{/privacy}","received_events_url":"https://api.github.com/users/nik9000/received_events","type":"User","site_admin":false},"body":"Note: this has been edited from my first comment.\r\n\r\nThis creates basic reindex and update-by-query implementations that work like delete-by-query. They scroll documents and then turn around and issue bulk requests for each.\r\n\r\nThis isn't the last word on reindex. Before we merge the feature branch down to master we'll need to fix it to retry on bulk rejection and integrate it with the task management api.\r\n\r\nFor those of you reading the issues, realize that all the discussion about APIs is discussion. The API we're going to merge is the result of those discussions. You can read about it by reading the asciidoc files in the PR.","created_at":"2015-11-30T20:43:51Z","updated_at":"2016-01-06T19:11:30Z","closed_at":null,"merged_at":null,"merge_commit_sha":"9687e2934a21e2787b260771e166f33b42b33c5c","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/elastic/elasticsearch/pulls/15125/commits","review_comments_url":"https://api.github.com/repos/elastic/elasticsearch/pulls/15125/comments","review_comment_url":"https://api.github.com/repos/elastic/elasticsearch/pulls/comments{/number}","comments_url":"https://api.github.com/repos/elastic/elasticsearch/issues/15125/comments","statuses_url":"https://api.github.com/repos/elastic/elasticsearch/statuses/f4fd2f68bdfcced9e1469bed69902c2d0c57f09b","head":{"label":"nik9000:first_reindex","ref":"first_reindex","sha":"f4fd2f68bdfcced9e1469bed69902c2d0c57f09b","user":{"login":"nik9000","id":215970,"avatar_url":"https://avatars.githubusercontent.com/u/215970?v=3","gravatar_id":"","url":"https://api.github.com/users/nik9000","html_url":"https://github.com/nik9000","followers_url":"https://api.github.com/users/nik9000/followers","following_url":"https://api.github.com/users/nik9000/following{/other_user}","gists_url":"https://api.github.com/users/nik9000/gists{/gist_id}","starred_url":"https://api.github.com/users/nik9000/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nik9000/subscriptions","organizations_url":"https://api.github.com/users/nik9000/orgs","repos_url":"https://api.github.com/users/nik9000/repos","events_url":"https://api.github.com/users/nik9000/events{/privacy}","received_events_url":"https://api.github.com/users/nik9000/received_events","type":"User","site_admin":false},"repo":{"id":11487195,"name":"elasticsearch","full_name":"nik9000/elasticsearch","owner":{"login":"nik9000","id":215970,"avatar_url":"https://avatars.githubusercontent.com/u/215970?v=3","gravatar_id":"","url":"https://api.github.com/users/nik9000","html_url":"https://github.com/nik9000","followers_url":"https://api.github.com/users/nik9000/followers","following_url":"https://api.github.com/users/nik9000/following{/other_user}","gists_url":"https://api.github.com/users/nik9000/gists{/gist_id}","starred_url":"https://api.github.com/users/nik9000/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nik9000/subscriptions","organizations_url":"https://api.github.com/users/nik9000/orgs","repos_url":"https://api.github.com/users/nik9000/repos","events_url":"https://api.github.com/users/nik9000/events{/privacy}","received_events_url":"https://api.github.com/users/nik9000/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/nik9000/elasticsearch","description":"Open Source, Distributed, RESTful Search Engine","fork":true,"url":"https://api.github.com/repos/nik9000/elasticsearch","forks_url":"https://api.github.com/repos/nik9000/elasticsearch/forks","keys_url":"https://api.github.com/repos/nik9000/elasticsearch/keys{/key_id}","collaborators_url":"https://api.github.com/repos/nik9000/elasticsearch/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/nik9000/elasticsearch/teams","hooks_url":"https://api.github.com/repos/nik9000/elasticsearch/hooks","issue_events_url":"https://api.github.com/repos/nik9000/elasticsearch/issues/events{/number}","events_url":"https://api.github.com/repos/nik9000/elasticsearch/events","assignees_url":"https://api.github.com/repos/nik9000/elasticsearch/assignees{/user}","branches_url":"https://api.github.com/repos/nik9000/elasticsearch/branches{/branch}","tags_url":"https://api.github.com/repos/nik9000/elasticsearch/tags","blobs_url":"https://api.github.com/repos/nik9000/elasticsearch/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/nik9000/elasticsearch/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/nik9000/elasticsearch/git/refs{/sha}","trees_url":"https://api.github.com/repos/nik9000/elasticsearch/git/trees{/sha}","statuses_url":"https://api.github.com/repos/nik9000/elasticsearch/statuses/{sha}","languages_url":"https://api.github.com/repos/nik9000/elasticsearch/languages","stargazers_url":"https://api.github.com/repos/nik9000/elasticsearch/stargazers","contributors_url":"https://api.github.com/repos/nik9000/elasticsearch/contributors","subscribers_url":"https://api.github.com/repos/nik9000/elasticsearch/subscribers","subscription_url":"https://api.github.com/repos/nik9000/elasticsearch/subscription","commits_url":"https://api.github.com/repos/nik9000/elasticsearch/commits{/sha}","git_commits_url":"https://api.github.com/repos/nik9000/elasticsearch/git/commits{/sha}","comments_url":"https://api.github.com/repos/nik9000/elasticsearch/comments{/number}","issue_comment_url":"https://api.github.com/repos/nik9000/elasticsearch/issues/comments{/number}","contents_url":"https://api.github.com/repos/nik9000/elasticsearch/contents/{+path}","compare_url":"https://api.github.com/repos/nik9000/elasticsearch/compare/{base}...{head}","merges_url":"https://api.github.com/repos/nik9000/elasticsearch/merges","archive_url":"https://api.github.com/repos/nik9000/elasticsearch/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/nik9000/elasticsearch/downloads","issues_url":"https://api.github.com/repos/nik9000/elasticsearch/issues{/number}","pulls_url":"https://api.github.com/repos/nik9000/elasticsearch/pulls{/number}","milestones_url":"https://api.github.com/repos/nik9000/elasticsearch/milestones{/number}","notifications_url":"https://api.github.com/repos/nik9000/elasticsearch/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/nik9000/elasticsearch/labels{/name}","releases_url":"https://api.github.com/repos/nik9000/elasticsearch/releases{/id}","created_at":"2013-07-17T20:48:13Z","updated_at":"2014-12-15T18:37:06Z","pushed_at":"2016-01-06T19:11:28Z","git_url":"git://github.com/nik9000/elasticsearch.git","ssh_url":"git@github.com:nik9000/elasticsearch.git","clone_url":"https://github.com/nik9000/elasticsearch.git","svn_url":"https://github.com/nik9000/elasticsearch","homepage":"http://elasticsearch.org","size":185750,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"elastic:feature/reindex","ref":"feature/reindex","sha":"f8026ed8fd80ece83f33dd7a5d7e66b371dc12a4","user":{"login":"elastic","id":6764390,"avatar_url":"https://avatars.githubusercontent.com/u/6764390?v=3","gravatar_id":"","url":"https://api.github.com/users/elastic","html_url":"https://github.com/elastic","followers_url":"https://api.github.com/users/elastic/followers","following_url":"https://api.github.com/users/elastic/following{/other_user}","gists_url":"https://api.github.com/users/elastic/gists{/gist_id}","starred_url":"https://api.github.com/users/elastic/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/elastic/subscriptions","organizations_url":"https://api.github.com/users/elastic/orgs","repos_url":"https://api.github.com/users/elastic/repos","events_url":"https://api.github.com/users/elastic/events{/privacy}","received_events_url":"https://api.github.com/users/elastic/received_events","type":"Organization","site_admin":false},"repo":{"id":507775,"name":"elasticsearch","full_name":"elastic/elasticsearch","owner":{"login":"elastic","id":6764390,"avatar_url":"https://avatars.githubusercontent.com/u/6764390?v=3","gravatar_id":"","url":"https://api.github.com/users/elastic","html_url":"https://github.com/elastic","followers_url":"https://api.github.com/users/elastic/followers","following_url":"https://api.github.com/users/elastic/following{/other_user}","gists_url":"https://api.github.com/users/elastic/gists{/gist_id}","starred_url":"https://api.github.com/users/elastic/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/elastic/subscriptions","organizations_url":"https://api.github.com/users/elastic/orgs","repos_url":"https://api.github.com/users/elastic/repos","events_url":"https://api.github.com/users/elastic/events{/privacy}","received_events_url":"https://api.github.com/users/elastic/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/elastic/elasticsearch","description":"Open Source, Distributed, RESTful Search Engine","fork":false,"url":"https://api.github.com/repos/elastic/elasticsearch","forks_url":"https://api.github.com/repos/elastic/elasticsearch/forks","keys_url":"https://api.github.com/repos/elastic/elasticsearch/keys{/key_id}","collaborators_url":"https://api.github.com/repos/elastic/elasticsearch/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/elastic/elasticsearch/teams","hooks_url":"https://api.github.com/repos/elastic/elasticsearch/hooks","issue_events_url":"https://api.github.com/repos/elastic/elasticsearch/issues/events{/number}","events_url":"https://api.github.com/repos/elastic/elasticsearch/events","assignees_url":"https://api.github.com/repos/elastic/elasticsearch/assignees{/user}","branches_url":"https://api.github.com/repos/elastic/elasticsearch/branches{/branch}","tags_url":"https://api.github.com/repos/elastic/elasticsearch/tags","blobs_url":"https://api.github.com/repos/elastic/elasticsearch/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/elastic/elasticsearch/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/elastic/elasticsearch/git/refs{/sha}","trees_url":"https://api.github.com/repos/elastic/elasticsearch/git/trees{/sha}","statuses_url":"https://api.github.com/repos/elastic/elasticsearch/statuses/{sha}","languages_url":"https://api.github.com/repos/elastic/elasticsearch/languages","stargazers_url":"https://api.github.com/repos/elastic/elasticsearch/stargazers","contributors_url":"https://api.github.com/repos/elastic/elasticsearch/contributors","subscribers_url":"https://api.github.com/repos/elastic/elasticsearch/subscribers","subscription_url":"https://api.github.com/repos/elastic/elasticsearch/subscription","commits_url":"https://api.github.com/repos/elastic/elasticsearch/commits{/sha}","git_commits_url":"https://api.github.com/repos/elastic/elasticsearch/git/commits{/sha}","comments_url":"https://api.github.com/repos/elastic/elasticsearch/comments{/number}","issue_comment_url":"https://api.github.com/repos/elastic/elasticsearch/issues/comments{/number}","contents_url":"https://api.github.com/repos/elastic/elasticsearch/contents/{+path}","compare_url":"https://api.github.com/repos/elastic/elasticsearch/compare/{base}...{head}","merges_url":"https://api.github.com/repos/elastic/elasticsearch/merges","archive_url":"https://api.github.com/repos/elastic/elasticsearch/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/elastic/elasticsearch/downloads","issues_url":"https://api.github.com/repos/elastic/elasticsearch/issues{/number}","pulls_url":"https://api.github.com/repos/elastic/elasticsearch/pulls{/number}","milestones_url":"https://api.github.com/repos/elastic/elasticsearch/milestones{/number}","notifications_url":"https://api.github.com/repos/elastic/elasticsearch/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/elastic/elasticsearch/labels{/name}","releases_url":"https://api.github.com/repos/elastic/elasticsearch/releases{/id}","created_at":"2010-02-08T13:20:56Z","updated_at":"2016-01-06T17:53:19Z","pushed_at":"2016-01-06T19:11:29Z","git_url":"git://github.com/elastic/elasticsearch.git","ssh_url":"git@github.com:elastic/elasticsearch.git","clone_url":"https://github.com/elastic/elasticsearch.git","svn_url":"https://github.com/elastic/elasticsearch","homepage":"https://www.elastic.co/products/elasticsearch","size":234780,"stargazers_count":14251,"watchers_count":14251,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":4748,"mirror_url":null,"open_issues_count":1257,"forks":4748,"open_issues":1257,"watchers":14251,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/elastic/elasticsearch/pulls/15125"},"html":{"href":"https://github.com/elastic/elasticsearch/pull/15125"},"issue":{"href":"https://api.github.com/repos/elastic/elasticsearch/issues/15125"},"comments":{"href":"https://api.github.com/repos/elastic/elasticsearch/issues/15125/comments"},"review_comments":{"href":"https://api.github.com/repos/elastic/elasticsearch/pulls/15125/comments"},"review_comment":{"href":"https://api.github.com/repos/elastic/elasticsearch/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/elastic/elasticsearch/pulls/15125/commits"},"statuses":{"href":"https://api.github.com/repos/elastic/elasticsearch/statuses/f4fd2f68bdfcced9e1469bed69902c2d0c57f09b"}}}}