{"action":"created","comment":{"url":"https://api.github.com/repos/HuaweiBigData/carbondata/pulls/comments/67209297","id":67209297,"diff_hunk":"@@ -188,59 +182,138 @@ class CarbonStrategies(sqlContext: SQLContext) extends QueryPlanner[SparkPlan] {\n         projectList: Seq[NamedExpression],\n         predicates: Seq[Expression],\n         carbonRelation: CarbonDatasourceRelation,\n-        partialComputation: Seq[NamedExpression],\n         groupingExpressions: Seq[Expression],\n-        namedGroupingAttributes: Seq[Attribute],\n-        rewrittenAggregateExpressions: Seq[NamedExpression]):\n+        namedGroupingAttributes: Seq[NamedExpression]):\n     Seq[SparkPlan] = {\n       val (_, _, _, _, groupExprs, substitutesortExprs, limitExpr) = extractPlan(plan)\n \n       val s =\n         try {\n           carbonScan(projectList, predicates, carbonRelation.carbonRelation,\n-            Some(partialComputation), substitutesortExprs, limitExpr, groupingExpressions.nonEmpty)\n+            Some(namedGroupingAttributes), substitutesortExprs,\n+            limitExpr, groupingExpressions.nonEmpty)\n         } catch {\n           case e: Exception => null\n         }\n \n       if (s != null) {\n-        CarbonAggregate(\n-          partial = false,\n-          namedGroupingAttributes,\n-          rewrittenAggregateExpressions,\n-          CarbonAggregate(\n-            partial = true,\n-            groupingExpressions,\n-            partialComputation,\n-            s)(sqlContext))(sqlContext) :: Nil\n+        aggregatePlan(groupingExpressions, namedGroupingAttributes, s)\n \n       } else {\n         (aggPlan, true) match {\n-          case PartialAggregation(\n-          namedGroupingAttributes,\n-          rewrittenAggregateExpressions,\n+          case CarbonAggregation(\n           groupingExpressions,\n-          partialComputation,\n+          namedGroupingAttributes,\n           PhysicalOperation(projectList, predicates,\n           l@LogicalRelation(carbonRelation: CarbonDatasourceRelation, _))) =>\n             val (_, _, _, _, groupExprs, substitutesortExprs, limitExpr) = extractPlan(plan)\n \n \n             val s = carbonScan(projectList, predicates, carbonRelation.carbonRelation,\n-              Some(partialComputation), substitutesortExprs, limitExpr,\n+              Some(namedGroupingAttributes), substitutesortExprs, limitExpr,\n               groupingExpressions.nonEmpty, detailQuery = true)\n \n-            CarbonAggregate(\n-              partial = false,\n-              namedGroupingAttributes,\n-              rewrittenAggregateExpressions,\n-              CarbonAggregate(\n-                partial = true,\n-                groupingExpressions,\n-                partialComputation,\n-                s)(sqlContext))(sqlContext) :: Nil\n+            aggregatePlan(groupingExpressions, namedGroupingAttributes, s)\n+        }\n+      }\n+    }\n+\n+    // TODO: It is duplicated code from spark. Need to find a way\n+    private def aggregatePlan(groupingExpressions: Seq[Expression],","path":"integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonStrategies.scala","position":138,"original_position":138,"commit_id":"dd4310b14818258b46e852e2deafcf303d616ec2","original_commit_id":"663d6cee1adfa002d2c5c8e934d81bd12625f6e7","user":{"login":"ravipesala","id":8134086,"avatar_url":"https://avatars.githubusercontent.com/u/8134086?v=3","gravatar_id":"","url":"https://api.github.com/users/ravipesala","html_url":"https://github.com/ravipesala","followers_url":"https://api.github.com/users/ravipesala/followers","following_url":"https://api.github.com/users/ravipesala/following{/other_user}","gists_url":"https://api.github.com/users/ravipesala/gists{/gist_id}","starred_url":"https://api.github.com/users/ravipesala/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ravipesala/subscriptions","organizations_url":"https://api.github.com/users/ravipesala/orgs","repos_url":"https://api.github.com/users/ravipesala/repos","events_url":"https://api.github.com/users/ravipesala/events{/privacy}","received_events_url":"https://api.github.com/users/ravipesala/received_events","type":"User","site_admin":false},"body":"yes , It is only needed for push down path, for push up we use `CarbonRawStrategies`","created_at":"2016-06-15T17:42:29Z","updated_at":"2016-06-15T17:42:29Z","html_url":"https://github.com/HuaweiBigData/carbondata/pull/624#discussion_r67209297","pull_request_url":"https://api.github.com/repos/HuaweiBigData/carbondata/pulls/624","_links":{"self":{"href":"https://api.github.com/repos/HuaweiBigData/carbondata/pulls/comments/67209297"},"html":{"href":"https://github.com/HuaweiBigData/carbondata/pull/624#discussion_r67209297"},"pull_request":{"href":"https://api.github.com/repos/HuaweiBigData/carbondata/pulls/624"}}},"pull_request":{"url":"https://api.github.com/repos/HuaweiBigData/carbondata/pulls/624","id":73425392,"html_url":"https://github.com/HuaweiBigData/carbondata/pull/624","diff_url":"https://github.com/HuaweiBigData/carbondata/pull/624.diff","patch_url":"https://github.com/HuaweiBigData/carbondata/pull/624.patch","issue_url":"https://api.github.com/repos/HuaweiBigData/carbondata/issues/624","number":624,"state":"open","locked":false,"title":"[Issue 618]Supported Spark 1.6 in Carbondata","user":{"login":"ravipesala","id":8134086,"avatar_url":"https://avatars.githubusercontent.com/u/8134086?v=3","gravatar_id":"","url":"https://api.github.com/users/ravipesala","html_url":"https://github.com/ravipesala","followers_url":"https://api.github.com/users/ravipesala/followers","following_url":"https://api.github.com/users/ravipesala/following{/other_user}","gists_url":"https://api.github.com/users/ravipesala/gists{/gist_id}","starred_url":"https://api.github.com/users/ravipesala/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ravipesala/subscriptions","organizations_url":"https://api.github.com/users/ravipesala/orgs","repos_url":"https://api.github.com/users/ravipesala/repos","events_url":"https://api.github.com/users/ravipesala/events{/privacy}","received_events_url":"https://api.github.com/users/ravipesala/received_events","type":"User","site_admin":false},"body":"Spark 1.6 changes approach of aggregation and changed the interfaces.  So all old implementations need to be either rewritten or deleted in this PR.\r\n\r\nMeasureAggregator object is no longer supported in Spark 1.6 as UDT. As Spark 1.6 is completely based on UnSafeRow and codegen. So it try to read only the sqlType of UDT.\r\nSo approach is changed to avoid usage of  MeasureAggregator objects in Spark side aggregation.\r\n\r\nDistinct aggregation like count-distinct or sum-distinct is no longer supported to push down to spark layer. Spark changes the plan completely to support distinct count in its way, so carbon cannot get a chance to push down distinct aggregation.\r\n\r\nInterface identification and cleanup would be done another PR","created_at":"2016-06-10T19:41:56Z","updated_at":"2016-06-15T17:42:29Z","closed_at":null,"merged_at":null,"merge_commit_sha":"2a357d75f7fcff87609654030e277c3478601c60","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/HuaweiBigData/carbondata/pulls/624/commits","review_comments_url":"https://api.github.com/repos/HuaweiBigData/carbondata/pulls/624/comments","review_comment_url":"https://api.github.com/repos/HuaweiBigData/carbondata/pulls/comments{/number}","comments_url":"https://api.github.com/repos/HuaweiBigData/carbondata/issues/624/comments","statuses_url":"https://api.github.com/repos/HuaweiBigData/carbondata/statuses/dd4310b14818258b46e852e2deafcf303d616ec2","head":{"label":"ravipesala:spark1.6","ref":"spark1.6","sha":"dd4310b14818258b46e852e2deafcf303d616ec2","user":{"login":"ravipesala","id":8134086,"avatar_url":"https://avatars.githubusercontent.com/u/8134086?v=3","gravatar_id":"","url":"https://api.github.com/users/ravipesala","html_url":"https://github.com/ravipesala","followers_url":"https://api.github.com/users/ravipesala/followers","following_url":"https://api.github.com/users/ravipesala/following{/other_user}","gists_url":"https://api.github.com/users/ravipesala/gists{/gist_id}","starred_url":"https://api.github.com/users/ravipesala/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ravipesala/subscriptions","organizations_url":"https://api.github.com/users/ravipesala/orgs","repos_url":"https://api.github.com/users/ravipesala/repos","events_url":"https://api.github.com/users/ravipesala/events{/privacy}","received_events_url":"https://api.github.com/users/ravipesala/received_events","type":"User","site_admin":false},"repo":{"id":58879736,"name":"carbondata-1","full_name":"ravipesala/carbondata-1","owner":{"login":"ravipesala","id":8134086,"avatar_url":"https://avatars.githubusercontent.com/u/8134086?v=3","gravatar_id":"","url":"https://api.github.com/users/ravipesala","html_url":"https://github.com/ravipesala","followers_url":"https://api.github.com/users/ravipesala/followers","following_url":"https://api.github.com/users/ravipesala/following{/other_user}","gists_url":"https://api.github.com/users/ravipesala/gists{/gist_id}","starred_url":"https://api.github.com/users/ravipesala/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ravipesala/subscriptions","organizations_url":"https://api.github.com/users/ravipesala/orgs","repos_url":"https://api.github.com/users/ravipesala/repos","events_url":"https://api.github.com/users/ravipesala/events{/privacy}","received_events_url":"https://api.github.com/users/ravipesala/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/ravipesala/carbondata-1","description":"CarbonData is a new hadoop native file format for faster interactive query","fork":true,"url":"https://api.github.com/repos/ravipesala/carbondata-1","forks_url":"https://api.github.com/repos/ravipesala/carbondata-1/forks","keys_url":"https://api.github.com/repos/ravipesala/carbondata-1/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ravipesala/carbondata-1/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ravipesala/carbondata-1/teams","hooks_url":"https://api.github.com/repos/ravipesala/carbondata-1/hooks","issue_events_url":"https://api.github.com/repos/ravipesala/carbondata-1/issues/events{/number}","events_url":"https://api.github.com/repos/ravipesala/carbondata-1/events","assignees_url":"https://api.github.com/repos/ravipesala/carbondata-1/assignees{/user}","branches_url":"https://api.github.com/repos/ravipesala/carbondata-1/branches{/branch}","tags_url":"https://api.github.com/repos/ravipesala/carbondata-1/tags","blobs_url":"https://api.github.com/repos/ravipesala/carbondata-1/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ravipesala/carbondata-1/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ravipesala/carbondata-1/git/refs{/sha}","trees_url":"https://api.github.com/repos/ravipesala/carbondata-1/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ravipesala/carbondata-1/statuses/{sha}","languages_url":"https://api.github.com/repos/ravipesala/carbondata-1/languages","stargazers_url":"https://api.github.com/repos/ravipesala/carbondata-1/stargazers","contributors_url":"https://api.github.com/repos/ravipesala/carbondata-1/contributors","subscribers_url":"https://api.github.com/repos/ravipesala/carbondata-1/subscribers","subscription_url":"https://api.github.com/repos/ravipesala/carbondata-1/subscription","commits_url":"https://api.github.com/repos/ravipesala/carbondata-1/commits{/sha}","git_commits_url":"https://api.github.com/repos/ravipesala/carbondata-1/git/commits{/sha}","comments_url":"https://api.github.com/repos/ravipesala/carbondata-1/comments{/number}","issue_comment_url":"https://api.github.com/repos/ravipesala/carbondata-1/issues/comments{/number}","contents_url":"https://api.github.com/repos/ravipesala/carbondata-1/contents/{+path}","compare_url":"https://api.github.com/repos/ravipesala/carbondata-1/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ravipesala/carbondata-1/merges","archive_url":"https://api.github.com/repos/ravipesala/carbondata-1/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ravipesala/carbondata-1/downloads","issues_url":"https://api.github.com/repos/ravipesala/carbondata-1/issues{/number}","pulls_url":"https://api.github.com/repos/ravipesala/carbondata-1/pulls{/number}","milestones_url":"https://api.github.com/repos/ravipesala/carbondata-1/milestones{/number}","notifications_url":"https://api.github.com/repos/ravipesala/carbondata-1/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ravipesala/carbondata-1/labels{/name}","releases_url":"https://api.github.com/repos/ravipesala/carbondata-1/releases{/id}","deployments_url":"https://api.github.com/repos/ravipesala/carbondata-1/deployments","created_at":"2016-05-15T19:19:42Z","updated_at":"2016-05-15T19:19:46Z","pushed_at":"2016-06-15T17:31:49Z","git_url":"git://github.com/ravipesala/carbondata-1.git","ssh_url":"git@github.com:ravipesala/carbondata-1.git","clone_url":"https://github.com/ravipesala/carbondata-1.git","svn_url":"https://github.com/ravipesala/carbondata-1","homepage":"","size":33947,"stargazers_count":0,"watchers_count":0,"language":"Java","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"HuaweiBigData:master","ref":"master","sha":"efc41b2f43c18e5bf4176692dfc3b30cd4e7f7fa","user":{"login":"HuaweiBigData","id":13159946,"avatar_url":"https://avatars.githubusercontent.com/u/13159946?v=3","gravatar_id":"","url":"https://api.github.com/users/HuaweiBigData","html_url":"https://github.com/HuaweiBigData","followers_url":"https://api.github.com/users/HuaweiBigData/followers","following_url":"https://api.github.com/users/HuaweiBigData/following{/other_user}","gists_url":"https://api.github.com/users/HuaweiBigData/gists{/gist_id}","starred_url":"https://api.github.com/users/HuaweiBigData/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/HuaweiBigData/subscriptions","organizations_url":"https://api.github.com/users/HuaweiBigData/orgs","repos_url":"https://api.github.com/users/HuaweiBigData/repos","events_url":"https://api.github.com/users/HuaweiBigData/events{/privacy}","received_events_url":"https://api.github.com/users/HuaweiBigData/received_events","type":"Organization","site_admin":false},"repo":{"id":52943921,"name":"carbondata","full_name":"HuaweiBigData/carbondata","owner":{"login":"HuaweiBigData","id":13159946,"avatar_url":"https://avatars.githubusercontent.com/u/13159946?v=3","gravatar_id":"","url":"https://api.github.com/users/HuaweiBigData","html_url":"https://github.com/HuaweiBigData","followers_url":"https://api.github.com/users/HuaweiBigData/followers","following_url":"https://api.github.com/users/HuaweiBigData/following{/other_user}","gists_url":"https://api.github.com/users/HuaweiBigData/gists{/gist_id}","starred_url":"https://api.github.com/users/HuaweiBigData/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/HuaweiBigData/subscriptions","organizations_url":"https://api.github.com/users/HuaweiBigData/orgs","repos_url":"https://api.github.com/users/HuaweiBigData/repos","events_url":"https://api.github.com/users/HuaweiBigData/events{/privacy}","received_events_url":"https://api.github.com/users/HuaweiBigData/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/HuaweiBigData/carbondata","description":"CarbonData is a new hadoop native file format for faster interactive query","fork":false,"url":"https://api.github.com/repos/HuaweiBigData/carbondata","forks_url":"https://api.github.com/repos/HuaweiBigData/carbondata/forks","keys_url":"https://api.github.com/repos/HuaweiBigData/carbondata/keys{/key_id}","collaborators_url":"https://api.github.com/repos/HuaweiBigData/carbondata/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/HuaweiBigData/carbondata/teams","hooks_url":"https://api.github.com/repos/HuaweiBigData/carbondata/hooks","issue_events_url":"https://api.github.com/repos/HuaweiBigData/carbondata/issues/events{/number}","events_url":"https://api.github.com/repos/HuaweiBigData/carbondata/events","assignees_url":"https://api.github.com/repos/HuaweiBigData/carbondata/assignees{/user}","branches_url":"https://api.github.com/repos/HuaweiBigData/carbondata/branches{/branch}","tags_url":"https://api.github.com/repos/HuaweiBigData/carbondata/tags","blobs_url":"https://api.github.com/repos/HuaweiBigData/carbondata/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/HuaweiBigData/carbondata/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/HuaweiBigData/carbondata/git/refs{/sha}","trees_url":"https://api.github.com/repos/HuaweiBigData/carbondata/git/trees{/sha}","statuses_url":"https://api.github.com/repos/HuaweiBigData/carbondata/statuses/{sha}","languages_url":"https://api.github.com/repos/HuaweiBigData/carbondata/languages","stargazers_url":"https://api.github.com/repos/HuaweiBigData/carbondata/stargazers","contributors_url":"https://api.github.com/repos/HuaweiBigData/carbondata/contributors","subscribers_url":"https://api.github.com/repos/HuaweiBigData/carbondata/subscribers","subscription_url":"https://api.github.com/repos/HuaweiBigData/carbondata/subscription","commits_url":"https://api.github.com/repos/HuaweiBigData/carbondata/commits{/sha}","git_commits_url":"https://api.github.com/repos/HuaweiBigData/carbondata/git/commits{/sha}","comments_url":"https://api.github.com/repos/HuaweiBigData/carbondata/comments{/number}","issue_comment_url":"https://api.github.com/repos/HuaweiBigData/carbondata/issues/comments{/number}","contents_url":"https://api.github.com/repos/HuaweiBigData/carbondata/contents/{+path}","compare_url":"https://api.github.com/repos/HuaweiBigData/carbondata/compare/{base}...{head}","merges_url":"https://api.github.com/repos/HuaweiBigData/carbondata/merges","archive_url":"https://api.github.com/repos/HuaweiBigData/carbondata/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/HuaweiBigData/carbondata/downloads","issues_url":"https://api.github.com/repos/HuaweiBigData/carbondata/issues{/number}","pulls_url":"https://api.github.com/repos/HuaweiBigData/carbondata/pulls{/number}","milestones_url":"https://api.github.com/repos/HuaweiBigData/carbondata/milestones{/number}","notifications_url":"https://api.github.com/repos/HuaweiBigData/carbondata/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/HuaweiBigData/carbondata/labels{/name}","releases_url":"https://api.github.com/repos/HuaweiBigData/carbondata/releases{/id}","deployments_url":"https://api.github.com/repos/HuaweiBigData/carbondata/deployments","created_at":"2016-03-02T07:44:41Z","updated_at":"2016-06-15T02:35:26Z","pushed_at":"2016-06-15T17:31:50Z","git_url":"git://github.com/HuaweiBigData/carbondata.git","ssh_url":"git@github.com:HuaweiBigData/carbondata.git","clone_url":"https://github.com/HuaweiBigData/carbondata.git","svn_url":"https://github.com/HuaweiBigData/carbondata","homepage":"","size":63911,"stargazers_count":126,"watchers_count":126,"language":"Java","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":55,"mirror_url":null,"open_issues_count":114,"forks":55,"open_issues":114,"watchers":126,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/HuaweiBigData/carbondata/pulls/624"},"html":{"href":"https://github.com/HuaweiBigData/carbondata/pull/624"},"issue":{"href":"https://api.github.com/repos/HuaweiBigData/carbondata/issues/624"},"comments":{"href":"https://api.github.com/repos/HuaweiBigData/carbondata/issues/624/comments"},"review_comments":{"href":"https://api.github.com/repos/HuaweiBigData/carbondata/pulls/624/comments"},"review_comment":{"href":"https://api.github.com/repos/HuaweiBigData/carbondata/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/HuaweiBigData/carbondata/pulls/624/commits"},"statuses":{"href":"https://api.github.com/repos/HuaweiBigData/carbondata/statuses/dd4310b14818258b46e852e2deafcf303d616ec2"}}}}
{"actor":{"display_login":"ravipesala"}}