{"action":"created","comment":{"url":"https://api.github.com/repos/elastic/elasticsearch/pulls/comments/68808990","id":68808990,"diff_hunk":"@@ -0,0 +1,171 @@\n+/*\n+ * Licensed to Elasticsearch under one or more contributor\n+ * license agreements. See the NOTICE file distributed with\n+ * this work for additional information regarding copyright\n+ * ownership. Elasticsearch licenses this file to you under\n+ * the Apache License, Version 2.0 (the \"License\"); you may\n+ * not use this file except in compliance with the License.\n+ * You may obtain a copy of the License at\n+ *\n+ *    http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing,\n+ * software distributed under the License is distributed on an\n+ * \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n+ * KIND, either express or implied.  See the License for the\n+ * specific language governing permissions and limitations\n+ * under the License.\n+ */\n+\n+package org.elasticsearch.index.reindex.remote;\n+\n+import org.apache.http.HttpEntity;\n+import org.apache.http.entity.ByteArrayEntity;\n+import org.apache.http.entity.ContentType;\n+import org.apache.http.entity.StringEntity;\n+import org.elasticsearch.Version;\n+import org.elasticsearch.action.search.SearchRequest;\n+import org.elasticsearch.common.bytes.BytesReference;\n+import org.elasticsearch.common.unit.TimeValue;\n+import org.elasticsearch.common.xcontent.XContentBuilder;\n+import org.elasticsearch.common.xcontent.XContentHelper;\n+import org.elasticsearch.common.xcontent.XContentParser;\n+import org.elasticsearch.common.xcontent.json.JsonXContent;\n+import org.elasticsearch.search.sort.FieldSortBuilder;\n+import org.elasticsearch.search.sort.SortBuilder;\n+\n+import java.io.IOException;\n+import java.util.HashMap;\n+import java.util.Map;\n+\n+import static java.util.Collections.singletonMap;\n+\n+final class RemoteRequestBuilders {\n+    private RemoteRequestBuilders() {}\n+\n+    static class InitialSearch {\n+        private InitialSearch() {}\n+\n+        static String path(SearchRequest searchRequest) {\n+            // It is nasty to build paths with StringBuilder but we'll be careful....\n+            StringBuilder path = new StringBuilder(\"/\");\n+            addIndexesOrTypes(path, \"Index\", searchRequest.indices());\n+            addIndexesOrTypes(path, \"Type\", searchRequest.types());\n+            path.append(\"_search\");\n+            return path.toString();\n+        }\n+\n+        static Map<String, String> params(SearchRequest searchRequest, Version remoteVersion) {\n+            Map<String, String> params = new HashMap<>();\n+            if (searchRequest.scroll() != null) {\n+                params.put(\"scroll\", searchRequest.scroll().keepAlive().toString());\n+            }\n+            params.put(\"size\", Integer.toString(searchRequest.source().size()));\n+            if (searchRequest.source().version() == null || searchRequest.source().version() == true) {\n+                // false is the only value that makes it false. Null defaults to true....\n+                params.put(\"version\", null);","path":"modules/reindex/src/main/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuilders.java","position":66,"original_position":66,"commit_id":"f0b0c694a7fce557494b3ede9ca6d9e1bf62b390","original_commit_id":"f0b0c694a7fce557494b3ede9ca6d9e1bf62b390","user":{"login":"dakrone","id":19060,"avatar_url":"https://avatars.githubusercontent.com/u/19060?v=3","gravatar_id":"","url":"https://api.github.com/users/dakrone","html_url":"https://github.com/dakrone","followers_url":"https://api.github.com/users/dakrone/followers","following_url":"https://api.github.com/users/dakrone/following{/other_user}","gists_url":"https://api.github.com/users/dakrone/gists{/gist_id}","starred_url":"https://api.github.com/users/dakrone/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dakrone/subscriptions","organizations_url":"https://api.github.com/users/dakrone/orgs","repos_url":"https://api.github.com/users/dakrone/repos","events_url":"https://api.github.com/users/dakrone/events{/privacy}","received_events_url":"https://api.github.com/users/dakrone/received_events","type":"User","site_admin":false},"body":"I don't really understand this, when reindexing why would we fetch the `version` field? do we attempt to copy the version over for the document?","created_at":"2016-06-28T17:58:44Z","updated_at":"2016-06-28T17:58:44Z","html_url":"https://github.com/elastic/elasticsearch/pull/18585#discussion_r68808990","pull_request_url":"https://api.github.com/repos/elastic/elasticsearch/pulls/18585","_links":{"self":{"href":"https://api.github.com/repos/elastic/elasticsearch/pulls/comments/68808990"},"html":{"href":"https://github.com/elastic/elasticsearch/pull/18585#discussion_r68808990"},"pull_request":{"href":"https://api.github.com/repos/elastic/elasticsearch/pulls/18585"}}},"pull_request":{"url":"https://api.github.com/repos/elastic/elasticsearch/pulls/18585","id":71490434,"html_url":"https://github.com/elastic/elasticsearch/pull/18585","diff_url":"https://github.com/elastic/elasticsearch/pull/18585.diff","patch_url":"https://github.com/elastic/elasticsearch/pull/18585.patch","issue_url":"https://api.github.com/repos/elastic/elasticsearch/issues/18585","number":18585,"state":"open","locked":false,"title":"Reindex from remote","user":{"login":"nik9000","id":215970,"avatar_url":"https://avatars.githubusercontent.com/u/215970?v=3","gravatar_id":"","url":"https://api.github.com/users/nik9000","html_url":"https://github.com/nik9000","followers_url":"https://api.github.com/users/nik9000/followers","following_url":"https://api.github.com/users/nik9000/following{/other_user}","gists_url":"https://api.github.com/users/nik9000/gists{/gist_id}","starred_url":"https://api.github.com/users/nik9000/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nik9000/subscriptions","organizations_url":"https://api.github.com/users/nik9000/orgs","repos_url":"https://api.github.com/users/nik9000/repos","events_url":"https://api.github.com/users/nik9000/events{/privacy}","received_events_url":"https://api.github.com/users/nik9000/received_events","type":"User","site_admin":false},"body":"This adds a `remote` option to reindex that looks like\r\n\r\n```\r\ncurl -POST 'localhost:9200/_reindex?pretty' -d'{\r\n  \"source\": {\r\n    \"remote\": {\r\n      \"host\": \"otherhost:9200\"\r\n    },\r\n    \"index\": \"target\",\r\n    \"query\": {\r\n      \"match\": {\r\n        \"foo\": \"bar\"\r\n      }\r\n    }\r\n  },\r\n  \"dest\": {\r\n    \"index\": \"target\"\r\n  }\r\n}'\r\n```\r\n\r\nThis reindex has all of the features of local reindex:\r\n* Using queries to filter what is copied\r\n* Retry on rejection\r\n* Throttle/rethottle\r\n\r\nThe big advantage of this version is that it goes over the HTTP API which can be made backwards compatible. I have yet to test it against any version of Elasticsearch other than the modern one but it should be fairly easy to retrofit it to work with anything.\r\n\r\nSome things are different:\r\n* The `query` field is sent directly to the other node rather than parsed on the coordinating node. This should allow it to support constructs that are invalid on the coordinating node for whatever reason.\r\n\r\nLots of things still need to be sorted out:\r\n* [x] A bunch of NOCOMMITS\r\n* [x] This uses httpclient directly and doesn't support tls or basic auth at all. I'd like to replace it with Elasticsearch's currently-being-built HTTP based java client.\r\n* [x] At this point there isn't a whitelist for the `host`. This doesn't turn Elasticsearch into an arbitrary `curl`ing machine because reindex will only allow three commands - the one to start the scroll, the one to pull the next scroll, and the one clear the scroll. The URLs aren't arbitrary and neither are the methods and it only does them in that order. But, still, I'm going to have to add a whitelist.\r\n* [x] I have to test it against more Elasticsearch versions.\r\n* [x] We have to decide if we're going to integration test against real elasticsearch versions or just use mocks of their returned json and call that good enough. It is obviously much more work to setup the integration tests and much slower to test. 50ms per test vs 30 seconds per test kind of slower.\r\n* [x] Building this involved a lot of refactoring. That refactoring needs to inform some unit test changes. Probably.\r\nMore?\r\n\r\nCloses https://github.com/elastic/elasticsearch/issues/17447","created_at":"2016-05-26T02:06:55Z","updated_at":"2016-06-28T17:58:44Z","closed_at":null,"merged_at":null,"merge_commit_sha":"5ec8dba69c0a9ca5a373e757eced8e2e0b77b51a","assignee":{"login":"dakrone","id":19060,"avatar_url":"https://avatars.githubusercontent.com/u/19060?v=3","gravatar_id":"","url":"https://api.github.com/users/dakrone","html_url":"https://github.com/dakrone","followers_url":"https://api.github.com/users/dakrone/followers","following_url":"https://api.github.com/users/dakrone/following{/other_user}","gists_url":"https://api.github.com/users/dakrone/gists{/gist_id}","starred_url":"https://api.github.com/users/dakrone/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dakrone/subscriptions","organizations_url":"https://api.github.com/users/dakrone/orgs","repos_url":"https://api.github.com/users/dakrone/repos","events_url":"https://api.github.com/users/dakrone/events{/privacy}","received_events_url":"https://api.github.com/users/dakrone/received_events","type":"User","site_admin":false},"assignees":[{"login":"dakrone","id":19060,"avatar_url":"https://avatars.githubusercontent.com/u/19060?v=3","gravatar_id":"","url":"https://api.github.com/users/dakrone","html_url":"https://github.com/dakrone","followers_url":"https://api.github.com/users/dakrone/followers","following_url":"https://api.github.com/users/dakrone/following{/other_user}","gists_url":"https://api.github.com/users/dakrone/gists{/gist_id}","starred_url":"https://api.github.com/users/dakrone/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dakrone/subscriptions","organizations_url":"https://api.github.com/users/dakrone/orgs","repos_url":"https://api.github.com/users/dakrone/repos","events_url":"https://api.github.com/users/dakrone/events{/privacy}","received_events_url":"https://api.github.com/users/dakrone/received_events","type":"User","site_admin":false}],"milestone":null,"commits_url":"https://api.github.com/repos/elastic/elasticsearch/pulls/18585/commits","review_comments_url":"https://api.github.com/repos/elastic/elasticsearch/pulls/18585/comments","review_comment_url":"https://api.github.com/repos/elastic/elasticsearch/pulls/comments{/number}","comments_url":"https://api.github.com/repos/elastic/elasticsearch/issues/18585/comments","statuses_url":"https://api.github.com/repos/elastic/elasticsearch/statuses/f0b0c694a7fce557494b3ede9ca6d9e1bf62b390","head":{"label":"nik9000:reindex_from_remote","ref":"reindex_from_remote","sha":"f0b0c694a7fce557494b3ede9ca6d9e1bf62b390","user":{"login":"nik9000","id":215970,"avatar_url":"https://avatars.githubusercontent.com/u/215970?v=3","gravatar_id":"","url":"https://api.github.com/users/nik9000","html_url":"https://github.com/nik9000","followers_url":"https://api.github.com/users/nik9000/followers","following_url":"https://api.github.com/users/nik9000/following{/other_user}","gists_url":"https://api.github.com/users/nik9000/gists{/gist_id}","starred_url":"https://api.github.com/users/nik9000/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nik9000/subscriptions","organizations_url":"https://api.github.com/users/nik9000/orgs","repos_url":"https://api.github.com/users/nik9000/repos","events_url":"https://api.github.com/users/nik9000/events{/privacy}","received_events_url":"https://api.github.com/users/nik9000/received_events","type":"User","site_admin":false},"repo":{"id":11487195,"name":"elasticsearch","full_name":"nik9000/elasticsearch","owner":{"login":"nik9000","id":215970,"avatar_url":"https://avatars.githubusercontent.com/u/215970?v=3","gravatar_id":"","url":"https://api.github.com/users/nik9000","html_url":"https://github.com/nik9000","followers_url":"https://api.github.com/users/nik9000/followers","following_url":"https://api.github.com/users/nik9000/following{/other_user}","gists_url":"https://api.github.com/users/nik9000/gists{/gist_id}","starred_url":"https://api.github.com/users/nik9000/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/nik9000/subscriptions","organizations_url":"https://api.github.com/users/nik9000/orgs","repos_url":"https://api.github.com/users/nik9000/repos","events_url":"https://api.github.com/users/nik9000/events{/privacy}","received_events_url":"https://api.github.com/users/nik9000/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/nik9000/elasticsearch","description":"Open Source, Distributed, RESTful Search Engine","fork":true,"url":"https://api.github.com/repos/nik9000/elasticsearch","forks_url":"https://api.github.com/repos/nik9000/elasticsearch/forks","keys_url":"https://api.github.com/repos/nik9000/elasticsearch/keys{/key_id}","collaborators_url":"https://api.github.com/repos/nik9000/elasticsearch/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/nik9000/elasticsearch/teams","hooks_url":"https://api.github.com/repos/nik9000/elasticsearch/hooks","issue_events_url":"https://api.github.com/repos/nik9000/elasticsearch/issues/events{/number}","events_url":"https://api.github.com/repos/nik9000/elasticsearch/events","assignees_url":"https://api.github.com/repos/nik9000/elasticsearch/assignees{/user}","branches_url":"https://api.github.com/repos/nik9000/elasticsearch/branches{/branch}","tags_url":"https://api.github.com/repos/nik9000/elasticsearch/tags","blobs_url":"https://api.github.com/repos/nik9000/elasticsearch/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/nik9000/elasticsearch/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/nik9000/elasticsearch/git/refs{/sha}","trees_url":"https://api.github.com/repos/nik9000/elasticsearch/git/trees{/sha}","statuses_url":"https://api.github.com/repos/nik9000/elasticsearch/statuses/{sha}","languages_url":"https://api.github.com/repos/nik9000/elasticsearch/languages","stargazers_url":"https://api.github.com/repos/nik9000/elasticsearch/stargazers","contributors_url":"https://api.github.com/repos/nik9000/elasticsearch/contributors","subscribers_url":"https://api.github.com/repos/nik9000/elasticsearch/subscribers","subscription_url":"https://api.github.com/repos/nik9000/elasticsearch/subscription","commits_url":"https://api.github.com/repos/nik9000/elasticsearch/commits{/sha}","git_commits_url":"https://api.github.com/repos/nik9000/elasticsearch/git/commits{/sha}","comments_url":"https://api.github.com/repos/nik9000/elasticsearch/comments{/number}","issue_comment_url":"https://api.github.com/repos/nik9000/elasticsearch/issues/comments{/number}","contents_url":"https://api.github.com/repos/nik9000/elasticsearch/contents/{+path}","compare_url":"https://api.github.com/repos/nik9000/elasticsearch/compare/{base}...{head}","merges_url":"https://api.github.com/repos/nik9000/elasticsearch/merges","archive_url":"https://api.github.com/repos/nik9000/elasticsearch/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/nik9000/elasticsearch/downloads","issues_url":"https://api.github.com/repos/nik9000/elasticsearch/issues{/number}","pulls_url":"https://api.github.com/repos/nik9000/elasticsearch/pulls{/number}","milestones_url":"https://api.github.com/repos/nik9000/elasticsearch/milestones{/number}","notifications_url":"https://api.github.com/repos/nik9000/elasticsearch/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/nik9000/elasticsearch/labels{/name}","releases_url":"https://api.github.com/repos/nik9000/elasticsearch/releases{/id}","deployments_url":"https://api.github.com/repos/nik9000/elasticsearch/deployments","created_at":"2013-07-17T20:48:13Z","updated_at":"2014-12-15T18:37:06Z","pushed_at":"2016-06-28T16:48:52Z","git_url":"git://github.com/nik9000/elasticsearch.git","ssh_url":"git@github.com:nik9000/elasticsearch.git","clone_url":"https://github.com/nik9000/elasticsearch.git","svn_url":"https://github.com/nik9000/elasticsearch","homepage":"http://elasticsearch.org","size":224362,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"elastic:master","ref":"master","sha":"599a5489981e3b8f58e77c0433c382096dcfd591","user":{"login":"elastic","id":6764390,"avatar_url":"https://avatars.githubusercontent.com/u/6764390?v=3","gravatar_id":"","url":"https://api.github.com/users/elastic","html_url":"https://github.com/elastic","followers_url":"https://api.github.com/users/elastic/followers","following_url":"https://api.github.com/users/elastic/following{/other_user}","gists_url":"https://api.github.com/users/elastic/gists{/gist_id}","starred_url":"https://api.github.com/users/elastic/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/elastic/subscriptions","organizations_url":"https://api.github.com/users/elastic/orgs","repos_url":"https://api.github.com/users/elastic/repos","events_url":"https://api.github.com/users/elastic/events{/privacy}","received_events_url":"https://api.github.com/users/elastic/received_events","type":"Organization","site_admin":false},"repo":{"id":507775,"name":"elasticsearch","full_name":"elastic/elasticsearch","owner":{"login":"elastic","id":6764390,"avatar_url":"https://avatars.githubusercontent.com/u/6764390?v=3","gravatar_id":"","url":"https://api.github.com/users/elastic","html_url":"https://github.com/elastic","followers_url":"https://api.github.com/users/elastic/followers","following_url":"https://api.github.com/users/elastic/following{/other_user}","gists_url":"https://api.github.com/users/elastic/gists{/gist_id}","starred_url":"https://api.github.com/users/elastic/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/elastic/subscriptions","organizations_url":"https://api.github.com/users/elastic/orgs","repos_url":"https://api.github.com/users/elastic/repos","events_url":"https://api.github.com/users/elastic/events{/privacy}","received_events_url":"https://api.github.com/users/elastic/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/elastic/elasticsearch","description":"Open Source, Distributed, RESTful Search Engine","fork":false,"url":"https://api.github.com/repos/elastic/elasticsearch","forks_url":"https://api.github.com/repos/elastic/elasticsearch/forks","keys_url":"https://api.github.com/repos/elastic/elasticsearch/keys{/key_id}","collaborators_url":"https://api.github.com/repos/elastic/elasticsearch/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/elastic/elasticsearch/teams","hooks_url":"https://api.github.com/repos/elastic/elasticsearch/hooks","issue_events_url":"https://api.github.com/repos/elastic/elasticsearch/issues/events{/number}","events_url":"https://api.github.com/repos/elastic/elasticsearch/events","assignees_url":"https://api.github.com/repos/elastic/elasticsearch/assignees{/user}","branches_url":"https://api.github.com/repos/elastic/elasticsearch/branches{/branch}","tags_url":"https://api.github.com/repos/elastic/elasticsearch/tags","blobs_url":"https://api.github.com/repos/elastic/elasticsearch/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/elastic/elasticsearch/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/elastic/elasticsearch/git/refs{/sha}","trees_url":"https://api.github.com/repos/elastic/elasticsearch/git/trees{/sha}","statuses_url":"https://api.github.com/repos/elastic/elasticsearch/statuses/{sha}","languages_url":"https://api.github.com/repos/elastic/elasticsearch/languages","stargazers_url":"https://api.github.com/repos/elastic/elasticsearch/stargazers","contributors_url":"https://api.github.com/repos/elastic/elasticsearch/contributors","subscribers_url":"https://api.github.com/repos/elastic/elasticsearch/subscribers","subscription_url":"https://api.github.com/repos/elastic/elasticsearch/subscription","commits_url":"https://api.github.com/repos/elastic/elasticsearch/commits{/sha}","git_commits_url":"https://api.github.com/repos/elastic/elasticsearch/git/commits{/sha}","comments_url":"https://api.github.com/repos/elastic/elasticsearch/comments{/number}","issue_comment_url":"https://api.github.com/repos/elastic/elasticsearch/issues/comments{/number}","contents_url":"https://api.github.com/repos/elastic/elasticsearch/contents/{+path}","compare_url":"https://api.github.com/repos/elastic/elasticsearch/compare/{base}...{head}","merges_url":"https://api.github.com/repos/elastic/elasticsearch/merges","archive_url":"https://api.github.com/repos/elastic/elasticsearch/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/elastic/elasticsearch/downloads","issues_url":"https://api.github.com/repos/elastic/elasticsearch/issues{/number}","pulls_url":"https://api.github.com/repos/elastic/elasticsearch/pulls{/number}","milestones_url":"https://api.github.com/repos/elastic/elasticsearch/milestones{/number}","notifications_url":"https://api.github.com/repos/elastic/elasticsearch/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/elastic/elasticsearch/labels{/name}","releases_url":"https://api.github.com/repos/elastic/elasticsearch/releases{/id}","deployments_url":"https://api.github.com/repos/elastic/elasticsearch/deployments","created_at":"2010-02-08T13:20:56Z","updated_at":"2016-06-28T16:06:21Z","pushed_at":"2016-06-28T17:40:26Z","git_url":"git://github.com/elastic/elasticsearch.git","ssh_url":"git@github.com:elastic/elasticsearch.git","clone_url":"https://github.com/elastic/elasticsearch.git","svn_url":"https://github.com/elastic/elasticsearch","homepage":"https://www.elastic.co/products/elasticsearch","size":335146,"stargazers_count":17113,"watchers_count":17113,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":5799,"mirror_url":null,"open_issues_count":1068,"forks":5799,"open_issues":1068,"watchers":17113,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/elastic/elasticsearch/pulls/18585"},"html":{"href":"https://github.com/elastic/elasticsearch/pull/18585"},"issue":{"href":"https://api.github.com/repos/elastic/elasticsearch/issues/18585"},"comments":{"href":"https://api.github.com/repos/elastic/elasticsearch/issues/18585/comments"},"review_comments":{"href":"https://api.github.com/repos/elastic/elasticsearch/pulls/18585/comments"},"review_comment":{"href":"https://api.github.com/repos/elastic/elasticsearch/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/elastic/elasticsearch/pulls/18585/commits"},"statuses":{"href":"https://api.github.com/repos/elastic/elasticsearch/statuses/f0b0c694a7fce557494b3ede9ca6d9e1bf62b390"}}}}
{"actor":{"display_login":"dakrone"}}