{"action":"created","comment":{"url":"https://api.github.com/repos/rmagen/unipop/pulls/comments/67620175","id":67620175,"diff_hunk":"@@ -1,65 +1,96 @@\n package org.unipop.elastic.common;\n \n+import com.google.gson.Gson;\n+import com.google.gson.JsonElement;\n+import io.searchbox.client.JestClient;\n+import io.searchbox.client.JestResult;\n+import io.searchbox.client.http.JestHttpClient;\n+import io.searchbox.core.Search;\n+import io.searchbox.core.SearchScroll;\n+import io.searchbox.params.Parameters;\n import org.apache.tinkerpop.gremlin.structure.Element;\n import org.elasticsearch.action.search.SearchRequestBuilder;\n import org.elasticsearch.action.search.SearchResponse;\n import org.elasticsearch.client.Client;\n import org.elasticsearch.common.unit.TimeValue;\n import org.elasticsearch.index.query.QueryBuilder;\n import org.elasticsearch.search.SearchHit;\n-import org.unipop.query.StepDescriptor;\n+import org.elasticsearch.search.builder.SearchSourceBuilder;\n+import org.elasticsearch.search.internal.InternalSearchHit;\n+import org.jooq.lambda.Seq;\n \n+import java.io.IOException;\n import java.util.Iterator;\n+import java.util.Map;\n \n public class QueryIterator<E extends Element> implements Iterator<E> {\n \n-    private final StepDescriptor stepDescriptor;\n-    private SearchResponse scrollResponse;\n+    private JestResult scrollResponse;\n+    private String scrollId;\n     private final Parser<E> parser;\n     private final int scrollSize;\n-    private Client client;\n-    private Iterator<SearchHit> hits;\n+    private JestClient client;\n+    private Iterator<JsonElement> hits;\n+    private final Gson gson;\n \n-    public QueryIterator(QueryBuilder query, StepDescriptor stepDescriptor, int scrollSize, int maxSize, Client client,\n-                         Parser<E> parser, String... indices) {\n-        this.stepDescriptor = stepDescriptor;\n+    public QueryIterator(QueryBuilder query, int scrollSize, int maxSize, JestClient client,\n+                         Parser<E> parser, String... indices) throws IOException {\n         this.scrollSize = scrollSize;\n         this.client = client;\n         this.parser = parser;\n \n-        SearchRequestBuilder searchRequestBuilder = client.prepareSearch(indices).setQuery(query);\n+        this.gson = new Gson();\n+\n+        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder()\n+                .query(query);\n+\n+        Search.Builder searchBuilder = new Search.Builder(searchSourceBuilder.toString())\n+                .addIndex(Seq.of(indices).toList());\n+\n         if(maxSize == -1) maxSize = 10000;\n \n         if (scrollSize > 0) {\n-            searchRequestBuilder.setScroll(new TimeValue(60000))\n-                    .setSize(maxSize < scrollSize ? maxSize : scrollSize);\n+            searchBuilder.setParameter(Parameters.SCROLL, new TimeValue(60000));\n+            searchBuilder.setParameter(Parameters.SIZE, Math.min(maxSize, scrollSize));\n+        }\n+\n+        else {\n+            searchBuilder.setParameter(Parameters.SIZE, maxSize);\n         }\n-        else searchRequestBuilder.setSize(maxSize);\n \n-        this.scrollResponse = searchRequestBuilder.execute().actionGet();\n+        Search search = searchBuilder.build();\n \n-        hits = scrollResponse.getHits().iterator();\n+        this.scrollResponse = client.execute(search);\n+        this.hits = this.scrollResponse.getJsonObject().getAsJsonObject(\"hits\").getAsJsonArray(\"hits\").iterator();\n     }\n \n     @Override\n     public boolean hasNext() {\n         if (hits.hasNext()) return true;\n \n         if (scrollSize > 0) {\n-            scrollResponse = client.prepareSearchScroll(scrollResponse.getScrollId()).setScroll(new TimeValue(600000)).execute().actionGet();\n-            hits = scrollResponse.getHits().iterator();\n+            try {\n+                SearchScroll scroll = new SearchScroll.Builder(\n+                        this.scrollResponse.getJsonObject().getAsJsonPrimitive(\"_scroll_id\").getAsString(),\n+                        new TimeValue(60000).format())\n+                        .build();\n+                this.scrollResponse = client.execute(scroll);\n+                this.hits = scrollResponse.getJsonObject().getAsJsonObject(\"hits\").getAsJsonArray(\"hits\").iterator();\n+            } catch (IOException e) {\n+                // TODO: Handle Exception","path":"unipop-elastic/src/org/unipop/elastic/common/QueryIterator.java","position":96,"original_position":96,"commit_id":"dcd025f1e8e83a2bef4500024d2070174bc9d3e7","original_commit_id":"dcd025f1e8e83a2bef4500024d2070174bc9d3e7","user":{"login":"GurRonen","id":9721883,"avatar_url":"https://avatars.githubusercontent.com/u/9721883?v=3","gravatar_id":"","url":"https://api.github.com/users/GurRonen","html_url":"https://github.com/GurRonen","followers_url":"https://api.github.com/users/GurRonen/followers","following_url":"https://api.github.com/users/GurRonen/following{/other_user}","gists_url":"https://api.github.com/users/GurRonen/gists{/gist_id}","starred_url":"https://api.github.com/users/GurRonen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/GurRonen/subscriptions","organizations_url":"https://api.github.com/users/GurRonen/orgs","repos_url":"https://api.github.com/users/GurRonen/repos","events_url":"https://api.github.com/users/GurRonen/events{/privacy}","received_events_url":"https://api.github.com/users/GurRonen/received_events","type":"User","site_admin":false},"body":"left this as a todo until we decide what to do about logging","created_at":"2016-06-19T17:03:15Z","updated_at":"2016-06-19T17:03:15Z","html_url":"https://github.com/rmagen/unipop/pull/41#discussion_r67620175","pull_request_url":"https://api.github.com/repos/rmagen/unipop/pulls/41","_links":{"self":{"href":"https://api.github.com/repos/rmagen/unipop/pulls/comments/67620175"},"html":{"href":"https://github.com/rmagen/unipop/pull/41#discussion_r67620175"},"pull_request":{"href":"https://api.github.com/repos/rmagen/unipop/pulls/41"}}},"pull_request":{"url":"https://api.github.com/repos/rmagen/unipop/pulls/41","id":73504089,"html_url":"https://github.com/rmagen/unipop/pull/41","diff_url":"https://github.com/rmagen/unipop/pull/41.diff","patch_url":"https://github.com/rmagen/unipop/pull/41.patch","issue_url":"https://api.github.com/repos/rmagen/unipop/issues/41","number":41,"state":"open","locked":false,"title":"Feature/add jest support","user":{"login":"GurRonen","id":9721883,"avatar_url":"https://avatars.githubusercontent.com/u/9721883?v=3","gravatar_id":"","url":"https://api.github.com/users/GurRonen","html_url":"https://github.com/GurRonen","followers_url":"https://api.github.com/users/GurRonen/followers","following_url":"https://api.github.com/users/GurRonen/following{/other_user}","gists_url":"https://api.github.com/users/GurRonen/gists{/gist_id}","starred_url":"https://api.github.com/users/GurRonen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/GurRonen/subscriptions","organizations_url":"https://api.github.com/users/GurRonen/orgs","repos_url":"https://api.github.com/users/GurRonen/repos","events_url":"https://api.github.com/users/GurRonen/events{/privacy}","received_events_url":"https://api.github.com/users/GurRonen/received_events","type":"User","site_admin":false},"body":"Finished feature, works for both elastic 1 and 2.\r\n\r\na few remaining issues before merge:\r\n    1. should the elastic2 module be deleted\r\n    2. process tests fail due to messup in indexing, with pre-existing index and proper configuration\r\n","created_at":"2016-06-12T19:02:46Z","updated_at":"2016-06-19T17:03:15Z","closed_at":null,"merged_at":null,"merge_commit_sha":"b920dfc24ec2051b7f1dc36431ee40a1af9dae8e","assignee":{"login":"rmagen","id":7149043,"avatar_url":"https://avatars.githubusercontent.com/u/7149043?v=3","gravatar_id":"","url":"https://api.github.com/users/rmagen","html_url":"https://github.com/rmagen","followers_url":"https://api.github.com/users/rmagen/followers","following_url":"https://api.github.com/users/rmagen/following{/other_user}","gists_url":"https://api.github.com/users/rmagen/gists{/gist_id}","starred_url":"https://api.github.com/users/rmagen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rmagen/subscriptions","organizations_url":"https://api.github.com/users/rmagen/orgs","repos_url":"https://api.github.com/users/rmagen/repos","events_url":"https://api.github.com/users/rmagen/events{/privacy}","received_events_url":"https://api.github.com/users/rmagen/received_events","type":"User","site_admin":false},"milestone":{"url":"https://api.github.com/repos/rmagen/unipop/milestones/1","html_url":"https://github.com/rmagen/unipop/milestones/Unipop%200.2","labels_url":"https://api.github.com/repos/rmagen/unipop/milestones/1/labels","id":1759891,"number":1,"title":"Unipop 0.2","description":"","creator":{"login":"rmagen","id":7149043,"avatar_url":"https://avatars.githubusercontent.com/u/7149043?v=3","gravatar_id":"","url":"https://api.github.com/users/rmagen","html_url":"https://github.com/rmagen","followers_url":"https://api.github.com/users/rmagen/followers","following_url":"https://api.github.com/users/rmagen/following{/other_user}","gists_url":"https://api.github.com/users/rmagen/gists{/gist_id}","starred_url":"https://api.github.com/users/rmagen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rmagen/subscriptions","organizations_url":"https://api.github.com/users/rmagen/orgs","repos_url":"https://api.github.com/users/rmagen/repos","events_url":"https://api.github.com/users/rmagen/events{/privacy}","received_events_url":"https://api.github.com/users/rmagen/received_events","type":"User","site_admin":false},"open_issues":11,"closed_issues":1,"state":"open","created_at":"2016-05-11T11:35:05Z","updated_at":"2016-06-18T18:44:06Z","due_on":null,"closed_at":null},"commits_url":"https://api.github.com/repos/rmagen/unipop/pulls/41/commits","review_comments_url":"https://api.github.com/repos/rmagen/unipop/pulls/41/comments","review_comment_url":"https://api.github.com/repos/rmagen/unipop/pulls/comments{/number}","comments_url":"https://api.github.com/repos/rmagen/unipop/issues/41/comments","statuses_url":"https://api.github.com/repos/rmagen/unipop/statuses/dcd025f1e8e83a2bef4500024d2070174bc9d3e7","head":{"label":"rmagen:feature/addJestSupport","ref":"feature/addJestSupport","sha":"dcd025f1e8e83a2bef4500024d2070174bc9d3e7","user":{"login":"rmagen","id":7149043,"avatar_url":"https://avatars.githubusercontent.com/u/7149043?v=3","gravatar_id":"","url":"https://api.github.com/users/rmagen","html_url":"https://github.com/rmagen","followers_url":"https://api.github.com/users/rmagen/followers","following_url":"https://api.github.com/users/rmagen/following{/other_user}","gists_url":"https://api.github.com/users/rmagen/gists{/gist_id}","starred_url":"https://api.github.com/users/rmagen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rmagen/subscriptions","organizations_url":"https://api.github.com/users/rmagen/orgs","repos_url":"https://api.github.com/users/rmagen/repos","events_url":"https://api.github.com/users/rmagen/events{/privacy}","received_events_url":"https://api.github.com/users/rmagen/received_events","type":"User","site_admin":false},"repo":{"id":43837063,"name":"unipop","full_name":"rmagen/unipop","owner":{"login":"rmagen","id":7149043,"avatar_url":"https://avatars.githubusercontent.com/u/7149043?v=3","gravatar_id":"","url":"https://api.github.com/users/rmagen","html_url":"https://github.com/rmagen","followers_url":"https://api.github.com/users/rmagen/followers","following_url":"https://api.github.com/users/rmagen/following{/other_user}","gists_url":"https://api.github.com/users/rmagen/gists{/gist_id}","starred_url":"https://api.github.com/users/rmagen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rmagen/subscriptions","organizations_url":"https://api.github.com/users/rmagen/orgs","repos_url":"https://api.github.com/users/rmagen/repos","events_url":"https://api.github.com/users/rmagen/events{/privacy}","received_events_url":"https://api.github.com/users/rmagen/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/rmagen/unipop","description":"Data Analytics and BI, powered by Apache Tinkerpop","fork":false,"url":"https://api.github.com/repos/rmagen/unipop","forks_url":"https://api.github.com/repos/rmagen/unipop/forks","keys_url":"https://api.github.com/repos/rmagen/unipop/keys{/key_id}","collaborators_url":"https://api.github.com/repos/rmagen/unipop/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/rmagen/unipop/teams","hooks_url":"https://api.github.com/repos/rmagen/unipop/hooks","issue_events_url":"https://api.github.com/repos/rmagen/unipop/issues/events{/number}","events_url":"https://api.github.com/repos/rmagen/unipop/events","assignees_url":"https://api.github.com/repos/rmagen/unipop/assignees{/user}","branches_url":"https://api.github.com/repos/rmagen/unipop/branches{/branch}","tags_url":"https://api.github.com/repos/rmagen/unipop/tags","blobs_url":"https://api.github.com/repos/rmagen/unipop/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/rmagen/unipop/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/rmagen/unipop/git/refs{/sha}","trees_url":"https://api.github.com/repos/rmagen/unipop/git/trees{/sha}","statuses_url":"https://api.github.com/repos/rmagen/unipop/statuses/{sha}","languages_url":"https://api.github.com/repos/rmagen/unipop/languages","stargazers_url":"https://api.github.com/repos/rmagen/unipop/stargazers","contributors_url":"https://api.github.com/repos/rmagen/unipop/contributors","subscribers_url":"https://api.github.com/repos/rmagen/unipop/subscribers","subscription_url":"https://api.github.com/repos/rmagen/unipop/subscription","commits_url":"https://api.github.com/repos/rmagen/unipop/commits{/sha}","git_commits_url":"https://api.github.com/repos/rmagen/unipop/git/commits{/sha}","comments_url":"https://api.github.com/repos/rmagen/unipop/comments{/number}","issue_comment_url":"https://api.github.com/repos/rmagen/unipop/issues/comments{/number}","contents_url":"https://api.github.com/repos/rmagen/unipop/contents/{+path}","compare_url":"https://api.github.com/repos/rmagen/unipop/compare/{base}...{head}","merges_url":"https://api.github.com/repos/rmagen/unipop/merges","archive_url":"https://api.github.com/repos/rmagen/unipop/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/rmagen/unipop/downloads","issues_url":"https://api.github.com/repos/rmagen/unipop/issues{/number}","pulls_url":"https://api.github.com/repos/rmagen/unipop/pulls{/number}","milestones_url":"https://api.github.com/repos/rmagen/unipop/milestones{/number}","notifications_url":"https://api.github.com/repos/rmagen/unipop/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/rmagen/unipop/labels{/name}","releases_url":"https://api.github.com/repos/rmagen/unipop/releases{/id}","deployments_url":"https://api.github.com/repos/rmagen/unipop/deployments","created_at":"2015-10-07T18:39:59Z","updated_at":"2016-06-16T09:13:08Z","pushed_at":"2016-06-14T18:51:37Z","git_url":"git://github.com/rmagen/unipop.git","ssh_url":"git@github.com:rmagen/unipop.git","clone_url":"https://github.com/rmagen/unipop.git","svn_url":"https://github.com/rmagen/unipop","homepage":"","size":31985,"stargazers_count":44,"watchers_count":44,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":11,"mirror_url":null,"open_issues_count":30,"forks":11,"open_issues":30,"watchers":44,"default_branch":"master"}},"base":{"label":"rmagen:master","ref":"master","sha":"7668ffd490f3647c344664fe0b753e88791c4ccf","user":{"login":"rmagen","id":7149043,"avatar_url":"https://avatars.githubusercontent.com/u/7149043?v=3","gravatar_id":"","url":"https://api.github.com/users/rmagen","html_url":"https://github.com/rmagen","followers_url":"https://api.github.com/users/rmagen/followers","following_url":"https://api.github.com/users/rmagen/following{/other_user}","gists_url":"https://api.github.com/users/rmagen/gists{/gist_id}","starred_url":"https://api.github.com/users/rmagen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rmagen/subscriptions","organizations_url":"https://api.github.com/users/rmagen/orgs","repos_url":"https://api.github.com/users/rmagen/repos","events_url":"https://api.github.com/users/rmagen/events{/privacy}","received_events_url":"https://api.github.com/users/rmagen/received_events","type":"User","site_admin":false},"repo":{"id":43837063,"name":"unipop","full_name":"rmagen/unipop","owner":{"login":"rmagen","id":7149043,"avatar_url":"https://avatars.githubusercontent.com/u/7149043?v=3","gravatar_id":"","url":"https://api.github.com/users/rmagen","html_url":"https://github.com/rmagen","followers_url":"https://api.github.com/users/rmagen/followers","following_url":"https://api.github.com/users/rmagen/following{/other_user}","gists_url":"https://api.github.com/users/rmagen/gists{/gist_id}","starred_url":"https://api.github.com/users/rmagen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rmagen/subscriptions","organizations_url":"https://api.github.com/users/rmagen/orgs","repos_url":"https://api.github.com/users/rmagen/repos","events_url":"https://api.github.com/users/rmagen/events{/privacy}","received_events_url":"https://api.github.com/users/rmagen/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/rmagen/unipop","description":"Data Analytics and BI, powered by Apache Tinkerpop","fork":false,"url":"https://api.github.com/repos/rmagen/unipop","forks_url":"https://api.github.com/repos/rmagen/unipop/forks","keys_url":"https://api.github.com/repos/rmagen/unipop/keys{/key_id}","collaborators_url":"https://api.github.com/repos/rmagen/unipop/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/rmagen/unipop/teams","hooks_url":"https://api.github.com/repos/rmagen/unipop/hooks","issue_events_url":"https://api.github.com/repos/rmagen/unipop/issues/events{/number}","events_url":"https://api.github.com/repos/rmagen/unipop/events","assignees_url":"https://api.github.com/repos/rmagen/unipop/assignees{/user}","branches_url":"https://api.github.com/repos/rmagen/unipop/branches{/branch}","tags_url":"https://api.github.com/repos/rmagen/unipop/tags","blobs_url":"https://api.github.com/repos/rmagen/unipop/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/rmagen/unipop/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/rmagen/unipop/git/refs{/sha}","trees_url":"https://api.github.com/repos/rmagen/unipop/git/trees{/sha}","statuses_url":"https://api.github.com/repos/rmagen/unipop/statuses/{sha}","languages_url":"https://api.github.com/repos/rmagen/unipop/languages","stargazers_url":"https://api.github.com/repos/rmagen/unipop/stargazers","contributors_url":"https://api.github.com/repos/rmagen/unipop/contributors","subscribers_url":"https://api.github.com/repos/rmagen/unipop/subscribers","subscription_url":"https://api.github.com/repos/rmagen/unipop/subscription","commits_url":"https://api.github.com/repos/rmagen/unipop/commits{/sha}","git_commits_url":"https://api.github.com/repos/rmagen/unipop/git/commits{/sha}","comments_url":"https://api.github.com/repos/rmagen/unipop/comments{/number}","issue_comment_url":"https://api.github.com/repos/rmagen/unipop/issues/comments{/number}","contents_url":"https://api.github.com/repos/rmagen/unipop/contents/{+path}","compare_url":"https://api.github.com/repos/rmagen/unipop/compare/{base}...{head}","merges_url":"https://api.github.com/repos/rmagen/unipop/merges","archive_url":"https://api.github.com/repos/rmagen/unipop/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/rmagen/unipop/downloads","issues_url":"https://api.github.com/repos/rmagen/unipop/issues{/number}","pulls_url":"https://api.github.com/repos/rmagen/unipop/pulls{/number}","milestones_url":"https://api.github.com/repos/rmagen/unipop/milestones{/number}","notifications_url":"https://api.github.com/repos/rmagen/unipop/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/rmagen/unipop/labels{/name}","releases_url":"https://api.github.com/repos/rmagen/unipop/releases{/id}","deployments_url":"https://api.github.com/repos/rmagen/unipop/deployments","created_at":"2015-10-07T18:39:59Z","updated_at":"2016-06-16T09:13:08Z","pushed_at":"2016-06-14T18:51:37Z","git_url":"git://github.com/rmagen/unipop.git","ssh_url":"git@github.com:rmagen/unipop.git","clone_url":"https://github.com/rmagen/unipop.git","svn_url":"https://github.com/rmagen/unipop","homepage":"","size":31985,"stargazers_count":44,"watchers_count":44,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":11,"mirror_url":null,"open_issues_count":30,"forks":11,"open_issues":30,"watchers":44,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/rmagen/unipop/pulls/41"},"html":{"href":"https://github.com/rmagen/unipop/pull/41"},"issue":{"href":"https://api.github.com/repos/rmagen/unipop/issues/41"},"comments":{"href":"https://api.github.com/repos/rmagen/unipop/issues/41/comments"},"review_comments":{"href":"https://api.github.com/repos/rmagen/unipop/pulls/41/comments"},"review_comment":{"href":"https://api.github.com/repos/rmagen/unipop/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/rmagen/unipop/pulls/41/commits"},"statuses":{"href":"https://api.github.com/repos/rmagen/unipop/statuses/dcd025f1e8e83a2bef4500024d2070174bc9d3e7"}}}}
{"actor":{"display_login":"GurRonen"}}