{"action":"created","comment":{"url":"https://api.github.com/repos/hazelcast/hazelcast/pulls/comments/84368562","pull_request_review_id":5151783,"id":84368562,"diff_hunk":"@@ -0,0 +1,317 @@\n+/*\n+ * Copyright (c) 2008-2016, Hazelcast, Inc. All Rights Reserved.\n+ *\n+ * Licensed under the Apache License, Version 2.0 (the \"License\");\n+ * you may not use this file except in compliance with the License.\n+ * You may obtain a copy of the License at\n+ *\n+ * http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+\n+package com.hazelcast.map.impl.query;\n+\n+import com.hazelcast.config.CacheDeserializedValues;\n+import com.hazelcast.internal.cluster.ClusterService;\n+import com.hazelcast.internal.serialization.InternalSerializationService;\n+import com.hazelcast.logging.ILogger;\n+import com.hazelcast.map.impl.LocalMapStatsProvider;\n+import com.hazelcast.map.impl.MapContainer;\n+import com.hazelcast.map.impl.MapServiceContext;\n+import com.hazelcast.map.impl.PartitionContainer;\n+import com.hazelcast.map.impl.record.Record;\n+import com.hazelcast.map.impl.record.Records;\n+import com.hazelcast.monitor.impl.LocalMapStatsImpl;\n+import com.hazelcast.nio.serialization.Data;\n+import com.hazelcast.query.PagingPredicate;\n+import com.hazelcast.query.Predicate;\n+import com.hazelcast.query.impl.CachedQueryEntry;\n+import com.hazelcast.query.impl.QueryableEntry;\n+import com.hazelcast.query.impl.getters.Extractors;\n+import com.hazelcast.query.impl.predicates.QueryOptimizer;\n+import com.hazelcast.spi.NodeEngine;\n+import com.hazelcast.spi.OperationService;\n+import com.hazelcast.spi.exception.RetryableHazelcastException;\n+import com.hazelcast.spi.partition.IPartitionService;\n+import com.hazelcast.util.Clock;\n+import com.hazelcast.util.IterationType;\n+import com.hazelcast.util.executor.ManagedExecutorService;\n+\n+import java.util.ArrayList;\n+import java.util.Collection;\n+import java.util.Iterator;\n+import java.util.LinkedList;\n+import java.util.List;\n+import java.util.Map;\n+import java.util.concurrent.ExecutionException;\n+\n+import static com.hazelcast.map.impl.query.MapQueryEngineUtils.newQueryResult;\n+import static com.hazelcast.map.impl.query.MapQueryEngineUtils.newQueryResultForSinglePartition;\n+import static com.hazelcast.query.PagingPredicateAccessor.getNearestAnchorEntry;\n+import static com.hazelcast.spi.ExecutionService.QUERY_EXECUTOR;\n+import static com.hazelcast.util.SortingUtil.compareAnchor;\n+import static com.hazelcast.util.SortingUtil.getSortedSubList;\n+\n+/**\n+ * Runs query operations in the calling thread (thus blucking it)\n+ * Query evaluation per partition is run in a sequential fashion.\n+ * <p>\n+ * Used by query operations only: QueryOperation & QueryPartitionOperation\n+ * Should not be used by proxies or any other query related objects.\n+ */\n+public class MapLocalQueryRunner {\n+\n+    protected final MapServiceContext mapServiceContext;\n+    protected final NodeEngine nodeEngine;\n+    protected final ILogger logger;\n+    protected final QueryResultSizeLimiter queryResultSizeLimiter;\n+    protected final InternalSerializationService serializationService;\n+    protected final IPartitionService partitionService;\n+    protected final QueryOptimizer queryOptimizer;\n+    protected final OperationService operationService;\n+    protected final ClusterService clusterService;\n+    protected final LocalMapStatsProvider localMapStatsProvider;\n+    protected final ManagedExecutorService executor;\n+\n+    public MapLocalQueryRunner(MapServiceContext mapServiceContext, QueryOptimizer optimizer) {\n+        this.mapServiceContext = mapServiceContext;\n+        this.nodeEngine = mapServiceContext.getNodeEngine();\n+        this.serializationService = (InternalSerializationService) nodeEngine.getSerializationService();\n+        this.partitionService = nodeEngine.getPartitionService();\n+        this.logger = nodeEngine.getLogger(getClass());\n+        this.queryResultSizeLimiter = new QueryResultSizeLimiter(mapServiceContext, logger);\n+        this.queryOptimizer = optimizer;\n+        this.operationService = nodeEngine.getOperationService();\n+        this.clusterService = nodeEngine.getClusterService();\n+        this.localMapStatsProvider = mapServiceContext.getLocalMapStatsProvider();\n+        this.executor = nodeEngine.getExecutionService().getExecutor(QUERY_EXECUTOR);\n+    }\n+\n+    // full query = index query (if possible), then partition-scan query\n+    public QueryResult runFullQuery(String mapName, Predicate predicate, IterationType iterationType)","path":"hazelcast/src/main/java/com/hazelcast/map/impl/query/MapLocalQueryRunner.java","position":96,"original_position":96,"commit_id":"3a9196ef22b37c8eaf4402fb5bd92b7ea70a6cf2","original_commit_id":"bfc7e376851ed2abe15828643bf7cd4106d4f834","user":{"login":"tombujok","id":1047469,"avatar_url":"https://avatars.githubusercontent.com/u/1047469?v=3","gravatar_id":"","url":"https://api.github.com/users/tombujok","html_url":"https://github.com/tombujok","followers_url":"https://api.github.com/users/tombujok/followers","following_url":"https://api.github.com/users/tombujok/following{/other_user}","gists_url":"https://api.github.com/users/tombujok/gists{/gist_id}","starred_url":"https://api.github.com/users/tombujok/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tombujok/subscriptions","organizations_url":"https://api.github.com/users/tombujok/orgs","repos_url":"https://api.github.com/users/tombujok/repos","events_url":"https://api.github.com/users/tombujok/events{/privacy}","received_events_url":"https://api.github.com/users/tombujok/received_events","type":"User","site_admin":false},"body":"Probably so. I'll be adding fast-aggregations and projections in the next couple of days, thus I would not like to optimize the API prematurely. Adding those will be a great validation of the API and will definitely result in its changes. Other hints appreciated.","created_at":"2016-10-20T20:26:03Z","updated_at":"2016-10-20T20:26:03Z","html_url":"https://github.com/hazelcast/hazelcast/pull/9149#discussion_r84368562","pull_request_url":"https://api.github.com/repos/hazelcast/hazelcast/pulls/9149","_links":{"self":{"href":"https://api.github.com/repos/hazelcast/hazelcast/pulls/comments/84368562"},"html":{"href":"https://github.com/hazelcast/hazelcast/pull/9149#discussion_r84368562"},"pull_request":{"href":"https://api.github.com/repos/hazelcast/hazelcast/pulls/9149"}}},"pull_request":{"url":"https://api.github.com/repos/hazelcast/hazelcast/pulls/9149","id":90161656,"html_url":"https://github.com/hazelcast/hazelcast/pull/9149","diff_url":"https://github.com/hazelcast/hazelcast/pull/9149.diff","patch_url":"https://github.com/hazelcast/hazelcast/pull/9149.patch","issue_url":"https://api.github.com/repos/hazelcast/hazelcast/issues/9149","number":9149,"state":"open","locked":false,"title":"WIP","user":{"login":"tombujok","id":1047469,"avatar_url":"https://avatars.githubusercontent.com/u/1047469?v=3","gravatar_id":"","url":"https://api.github.com/users/tombujok","html_url":"https://github.com/tombujok","followers_url":"https://api.github.com/users/tombujok/followers","following_url":"https://api.github.com/users/tombujok/following{/other_user}","gists_url":"https://api.github.com/users/tombujok/gists{/gist_id}","starred_url":"https://api.github.com/users/tombujok/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tombujok/subscriptions","organizations_url":"https://api.github.com/users/tombujok/orgs","repos_url":"https://api.github.com/users/tombujok/repos","events_url":"https://api.github.com/users/tombujok/events{/privacy}","received_events_url":"https://api.github.com/users/tombujok/received_events","type":"User","site_admin":false},"body":"\r\nThe problem:\r\nIn the previous approach the MapQueryEngine contained the following:\r\n- logic how to run a query (full-partition-scan or index query)\r\n- logic scheduling query operations \r\n- logic orchestrating queries (first query thread, later partition thread)\r\n\r\nSo the proxies used MapQueryEngine to schedule a query. Then the MapQueryEngine dispatched operations which called (surprise, surprise) the MapQueryEngine to run the query logic.\r\nThis was a mess and a lot of spaghetti.\r\nThere was also a lot of inconsistencies how difference scenarios of queries were run.\r\nFinally, there was a big mess when it comes to the method names, to name some: queryLocationPartition, queryTheLocalPartition, queryLocalPartitions.\r\n\r\nCurrently there are 3 entities:\r\n- MapQueryRunner -> (local and parallel) runs the query logic in the caller thread.\r\n- MapQueryDispatcher -> dispatches query operations (QueryOperation and QueryPartitionOperation). Does not know about the queryRunner and queryDispatcher.\r\n- MapQueryEngine -> orchestrates the query by dispatching the query operations using the MapQueryDispatcher and merges the result. (does not know about the queryRunner)\r\n\r\nIn this way there's no cycles and spaghetti bindings.\r\n\r\nFurthermore:\r\n- in the new approach there's no special path for PagingPredicate and non-paging predicate. Most of the stuff has been abstracted away.\r\n- queries run on the partitionThreads are optimized now (earlier the optimisation was not executed)\r\n\r\nImportant:\r\n\r\n- This refactoring is the first one of a series.  It's not meant to fix all the problems.\r\n- The result of this refactoring is consistent and does not limit us in any way.\r\n- It opens possibilities to refactor further. \r\n\r\nI'll be working on fast-aggregations and projections next, thus the MapQueryEngine will be refactored further. Test coverage will also significantly increase.","created_at":"2016-10-20T09:17:47Z","updated_at":"2016-10-20T20:26:03Z","closed_at":null,"merged_at":null,"merge_commit_sha":"c01f65f00051607ef6b6d1d71774e4a5cf93df00","assignee":null,"assignees":[],"milestone":null,"commits_url":"https://api.github.com/repos/hazelcast/hazelcast/pulls/9149/commits","review_comments_url":"https://api.github.com/repos/hazelcast/hazelcast/pulls/9149/comments","review_comment_url":"https://api.github.com/repos/hazelcast/hazelcast/pulls/comments{/number}","comments_url":"https://api.github.com/repos/hazelcast/hazelcast/issues/9149/comments","statuses_url":"https://api.github.com/repos/hazelcast/hazelcast/statuses/3a9196ef22b37c8eaf4402fb5bd92b7ea70a6cf2","head":{"label":"tombujok:query-engine","ref":"query-engine","sha":"3a9196ef22b37c8eaf4402fb5bd92b7ea70a6cf2","user":{"login":"tombujok","id":1047469,"avatar_url":"https://avatars.githubusercontent.com/u/1047469?v=3","gravatar_id":"","url":"https://api.github.com/users/tombujok","html_url":"https://github.com/tombujok","followers_url":"https://api.github.com/users/tombujok/followers","following_url":"https://api.github.com/users/tombujok/following{/other_user}","gists_url":"https://api.github.com/users/tombujok/gists{/gist_id}","starred_url":"https://api.github.com/users/tombujok/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tombujok/subscriptions","organizations_url":"https://api.github.com/users/tombujok/orgs","repos_url":"https://api.github.com/users/tombujok/repos","events_url":"https://api.github.com/users/tombujok/events{/privacy}","received_events_url":"https://api.github.com/users/tombujok/received_events","type":"User","site_admin":false},"repo":{"id":41409741,"name":"hazelcast","full_name":"tombujok/hazelcast","owner":{"login":"tombujok","id":1047469,"avatar_url":"https://avatars.githubusercontent.com/u/1047469?v=3","gravatar_id":"","url":"https://api.github.com/users/tombujok","html_url":"https://github.com/tombujok","followers_url":"https://api.github.com/users/tombujok/followers","following_url":"https://api.github.com/users/tombujok/following{/other_user}","gists_url":"https://api.github.com/users/tombujok/gists{/gist_id}","starred_url":"https://api.github.com/users/tombujok/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tombujok/subscriptions","organizations_url":"https://api.github.com/users/tombujok/orgs","repos_url":"https://api.github.com/users/tombujok/repos","events_url":"https://api.github.com/users/tombujok/events{/privacy}","received_events_url":"https://api.github.com/users/tombujok/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/tombujok/hazelcast","description":"Open Source In-Memory Data Grid","fork":true,"url":"https://api.github.com/repos/tombujok/hazelcast","forks_url":"https://api.github.com/repos/tombujok/hazelcast/forks","keys_url":"https://api.github.com/repos/tombujok/hazelcast/keys{/key_id}","collaborators_url":"https://api.github.com/repos/tombujok/hazelcast/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/tombujok/hazelcast/teams","hooks_url":"https://api.github.com/repos/tombujok/hazelcast/hooks","issue_events_url":"https://api.github.com/repos/tombujok/hazelcast/issues/events{/number}","events_url":"https://api.github.com/repos/tombujok/hazelcast/events","assignees_url":"https://api.github.com/repos/tombujok/hazelcast/assignees{/user}","branches_url":"https://api.github.com/repos/tombujok/hazelcast/branches{/branch}","tags_url":"https://api.github.com/repos/tombujok/hazelcast/tags","blobs_url":"https://api.github.com/repos/tombujok/hazelcast/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/tombujok/hazelcast/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/tombujok/hazelcast/git/refs{/sha}","trees_url":"https://api.github.com/repos/tombujok/hazelcast/git/trees{/sha}","statuses_url":"https://api.github.com/repos/tombujok/hazelcast/statuses/{sha}","languages_url":"https://api.github.com/repos/tombujok/hazelcast/languages","stargazers_url":"https://api.github.com/repos/tombujok/hazelcast/stargazers","contributors_url":"https://api.github.com/repos/tombujok/hazelcast/contributors","subscribers_url":"https://api.github.com/repos/tombujok/hazelcast/subscribers","subscription_url":"https://api.github.com/repos/tombujok/hazelcast/subscription","commits_url":"https://api.github.com/repos/tombujok/hazelcast/commits{/sha}","git_commits_url":"https://api.github.com/repos/tombujok/hazelcast/git/commits{/sha}","comments_url":"https://api.github.com/repos/tombujok/hazelcast/comments{/number}","issue_comment_url":"https://api.github.com/repos/tombujok/hazelcast/issues/comments{/number}","contents_url":"https://api.github.com/repos/tombujok/hazelcast/contents/{+path}","compare_url":"https://api.github.com/repos/tombujok/hazelcast/compare/{base}...{head}","merges_url":"https://api.github.com/repos/tombujok/hazelcast/merges","archive_url":"https://api.github.com/repos/tombujok/hazelcast/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/tombujok/hazelcast/downloads","issues_url":"https://api.github.com/repos/tombujok/hazelcast/issues{/number}","pulls_url":"https://api.github.com/repos/tombujok/hazelcast/pulls{/number}","milestones_url":"https://api.github.com/repos/tombujok/hazelcast/milestones{/number}","notifications_url":"https://api.github.com/repos/tombujok/hazelcast/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/tombujok/hazelcast/labels{/name}","releases_url":"https://api.github.com/repos/tombujok/hazelcast/releases{/id}","deployments_url":"https://api.github.com/repos/tombujok/hazelcast/deployments","created_at":"2015-08-26T06:54:32Z","updated_at":"2015-08-26T06:54:43Z","pushed_at":"2016-10-20T20:20:00Z","git_url":"git://github.com/tombujok/hazelcast.git","ssh_url":"git@github.com:tombujok/hazelcast.git","clone_url":"https://github.com/tombujok/hazelcast.git","svn_url":"https://github.com/tombujok/hazelcast","homepage":"www.hazelcast.com","size":100834,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"hazelcast:master","ref":"master","sha":"80d57012b7edc4c876ba0c1c2448dccf8d06afbe","user":{"login":"hazelcast","id":1453152,"avatar_url":"https://avatars.githubusercontent.com/u/1453152?v=3","gravatar_id":"","url":"https://api.github.com/users/hazelcast","html_url":"https://github.com/hazelcast","followers_url":"https://api.github.com/users/hazelcast/followers","following_url":"https://api.github.com/users/hazelcast/following{/other_user}","gists_url":"https://api.github.com/users/hazelcast/gists{/gist_id}","starred_url":"https://api.github.com/users/hazelcast/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hazelcast/subscriptions","organizations_url":"https://api.github.com/users/hazelcast/orgs","repos_url":"https://api.github.com/users/hazelcast/repos","events_url":"https://api.github.com/users/hazelcast/events{/privacy}","received_events_url":"https://api.github.com/users/hazelcast/received_events","type":"Organization","site_admin":false},"repo":{"id":3786237,"name":"hazelcast","full_name":"hazelcast/hazelcast","owner":{"login":"hazelcast","id":1453152,"avatar_url":"https://avatars.githubusercontent.com/u/1453152?v=3","gravatar_id":"","url":"https://api.github.com/users/hazelcast","html_url":"https://github.com/hazelcast","followers_url":"https://api.github.com/users/hazelcast/followers","following_url":"https://api.github.com/users/hazelcast/following{/other_user}","gists_url":"https://api.github.com/users/hazelcast/gists{/gist_id}","starred_url":"https://api.github.com/users/hazelcast/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/hazelcast/subscriptions","organizations_url":"https://api.github.com/users/hazelcast/orgs","repos_url":"https://api.github.com/users/hazelcast/repos","events_url":"https://api.github.com/users/hazelcast/events{/privacy}","received_events_url":"https://api.github.com/users/hazelcast/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/hazelcast/hazelcast","description":"Open Source In-Memory Data Grid","fork":false,"url":"https://api.github.com/repos/hazelcast/hazelcast","forks_url":"https://api.github.com/repos/hazelcast/hazelcast/forks","keys_url":"https://api.github.com/repos/hazelcast/hazelcast/keys{/key_id}","collaborators_url":"https://api.github.com/repos/hazelcast/hazelcast/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/hazelcast/hazelcast/teams","hooks_url":"https://api.github.com/repos/hazelcast/hazelcast/hooks","issue_events_url":"https://api.github.com/repos/hazelcast/hazelcast/issues/events{/number}","events_url":"https://api.github.com/repos/hazelcast/hazelcast/events","assignees_url":"https://api.github.com/repos/hazelcast/hazelcast/assignees{/user}","branches_url":"https://api.github.com/repos/hazelcast/hazelcast/branches{/branch}","tags_url":"https://api.github.com/repos/hazelcast/hazelcast/tags","blobs_url":"https://api.github.com/repos/hazelcast/hazelcast/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/hazelcast/hazelcast/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/hazelcast/hazelcast/git/refs{/sha}","trees_url":"https://api.github.com/repos/hazelcast/hazelcast/git/trees{/sha}","statuses_url":"https://api.github.com/repos/hazelcast/hazelcast/statuses/{sha}","languages_url":"https://api.github.com/repos/hazelcast/hazelcast/languages","stargazers_url":"https://api.github.com/repos/hazelcast/hazelcast/stargazers","contributors_url":"https://api.github.com/repos/hazelcast/hazelcast/contributors","subscribers_url":"https://api.github.com/repos/hazelcast/hazelcast/subscribers","subscription_url":"https://api.github.com/repos/hazelcast/hazelcast/subscription","commits_url":"https://api.github.com/repos/hazelcast/hazelcast/commits{/sha}","git_commits_url":"https://api.github.com/repos/hazelcast/hazelcast/git/commits{/sha}","comments_url":"https://api.github.com/repos/hazelcast/hazelcast/comments{/number}","issue_comment_url":"https://api.github.com/repos/hazelcast/hazelcast/issues/comments{/number}","contents_url":"https://api.github.com/repos/hazelcast/hazelcast/contents/{+path}","compare_url":"https://api.github.com/repos/hazelcast/hazelcast/compare/{base}...{head}","merges_url":"https://api.github.com/repos/hazelcast/hazelcast/merges","archive_url":"https://api.github.com/repos/hazelcast/hazelcast/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/hazelcast/hazelcast/downloads","issues_url":"https://api.github.com/repos/hazelcast/hazelcast/issues{/number}","pulls_url":"https://api.github.com/repos/hazelcast/hazelcast/pulls{/number}","milestones_url":"https://api.github.com/repos/hazelcast/hazelcast/milestones{/number}","notifications_url":"https://api.github.com/repos/hazelcast/hazelcast/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/hazelcast/hazelcast/labels{/name}","releases_url":"https://api.github.com/repos/hazelcast/hazelcast/releases{/id}","deployments_url":"https://api.github.com/repos/hazelcast/hazelcast/deployments","created_at":"2012-03-21T12:25:18Z","updated_at":"2016-10-20T14:33:21Z","pushed_at":"2016-10-20T20:20:02Z","git_url":"git://github.com/hazelcast/hazelcast.git","ssh_url":"git@github.com:hazelcast/hazelcast.git","clone_url":"https://github.com/hazelcast/hazelcast.git","svn_url":"https://github.com/hazelcast/hazelcast","homepage":"www.hazelcast.com","size":134979,"stargazers_count":1765,"watchers_count":1765,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":823,"mirror_url":null,"open_issues_count":642,"forks":823,"open_issues":642,"watchers":1765,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/hazelcast/hazelcast/pulls/9149"},"html":{"href":"https://github.com/hazelcast/hazelcast/pull/9149"},"issue":{"href":"https://api.github.com/repos/hazelcast/hazelcast/issues/9149"},"comments":{"href":"https://api.github.com/repos/hazelcast/hazelcast/issues/9149/comments"},"review_comments":{"href":"https://api.github.com/repos/hazelcast/hazelcast/pulls/9149/comments"},"review_comment":{"href":"https://api.github.com/repos/hazelcast/hazelcast/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/hazelcast/hazelcast/pulls/9149/commits"},"statuses":{"href":"https://api.github.com/repos/hazelcast/hazelcast/statuses/3a9196ef22b37c8eaf4402fb5bd92b7ea70a6cf2"}}}}
{"actor":{"display_login":"tombujok"}}