{"action":"created","comment":{"url":"https://api.github.com/repos/apache/carbondata/pulls/comments/185394341","pull_request_review_id":116771949,"id":185394341,"diff_hunk":"@@ -27,74 +29,124 @@\n import org.apache.carbondata.core.features.TableOperation;\n import org.apache.carbondata.core.metadata.schema.table.CarbonTable;\n import org.apache.carbondata.core.metadata.schema.table.DataMapSchema;\n+import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;\n import org.apache.carbondata.events.Event;\n \n+import org.apache.commons.lang.StringUtils;\n+\n /**\n- * Interface for datamap factory, it is responsible for creating the datamap.\n+ * Interface for datamap of index type, it is responsible for creating the datamap.\n  */\n-public interface DataMapFactory<T extends DataMap> {\n+public abstract class DataMapFactory<T extends DataMap> {\n+\n+  public static final String INDEX_COLUMNS = \"INDEX_COLUMNS\";\n+  protected CarbonTable carbonTable;\n+\n+  public DataMapFactory(CarbonTable carbonTable) {\n+    this.carbonTable = carbonTable;\n+  }\n \n   /**\n    * Initialization of Datamap factory with the carbonTable and datamap name\n    */\n-  void init(CarbonTable carbonTable, DataMapSchema dataMapSchema)\n+  public abstract void init(DataMapSchema dataMapSchema)\n       throws IOException, MalformedDataMapCommandException;\n \n   /**\n    * Return a new write for this datamap\n    */\n-  DataMapWriter createWriter(Segment segment, String writeDirectoryPath);\n+  public abstract DataMapWriter createWriter(Segment segment, String shardName)\n+      throws IOException;\n+\n+  public abstract DataMapRefresher createRefresher(Segment segment, String shardName)\n+      throws IOException;\n \n   /**\n    * Get the datamap for segmentid\n    */\n-  List<T> getDataMaps(Segment segment) throws IOException;\n+  public abstract List<T> getDataMaps(Segment segment) throws IOException;\n \n   /**\n    * Get datamaps for distributable object.\n    */\n-  List<T> getDataMaps(DataMapDistributable distributable)\n+  public abstract List<T> getDataMaps(DataMapDistributable distributable)\n       throws IOException;\n \n   /**\n    * Get all distributable objects of a segmentid\n    * @return\n    */\n-  List<DataMapDistributable> toDistributable(Segment segment);\n+  public abstract List<DataMapDistributable> toDistributable(Segment segment);\n \n   /**\n    *\n    * @param event\n    */\n-  void fireEvent(Event event);\n+  public abstract void fireEvent(Event event);\n \n   /**\n    * Clears datamap of the segment\n    */\n-  void clear(Segment segment);\n+  public abstract void clear(Segment segment);\n \n   /**\n    * Clear all datamaps from memory\n    */\n-  void clear();\n+  public abstract void clear();\n \n   /**\n    * Return metadata of this datamap\n    */\n-  DataMapMeta getMeta();\n+  public abstract DataMapMeta getMeta();\n \n   /**\n    *  Type of datamap whether it is FG or CG\n    */\n-  DataMapLevel getDataMapType();\n+  public abstract DataMapLevel getDataMapLevel();\n \n   /**\n    * delete datamap data if any\n    */\n-  void deleteDatamapData();\n+  public abstract void deleteDatamapData();\n \n   /**\n    * This function should return true is the input operation enum will make the datamap become stale\n    */\n-  boolean willBecomeStale(TableOperation operation);\n+  public abstract boolean willBecomeStale(TableOperation operation);\n+\n+  /**\n+   * Validate INDEX_COLUMNS property and return a array containing index column name\n+   * Following will be validated\n+   * 1. require INDEX_COLUMNS property\n+   * 2. INDEX_COLUMNS can't contains illegal argument(empty, blank)\n+   * 3. INDEX_COLUMNS can't contains duplicate same columns\n+   * 4. INDEX_COLUMNS should be exists in table columns\n+   */\n+  public void validateIndexedColumns(DataMapSchema dataMapSchema,\n+      CarbonTable carbonTable) throws MalformedDataMapCommandException {\n+    List<CarbonColumn> indexColumns = carbonTable.getIndexedColumns(dataMapSchema);\n+    Set<String> unique = new HashSet<>();\n+    for (CarbonColumn indexColumn : indexColumns) {\n+      unique.add(indexColumn.getColName());\n+    }\n+    if (unique.size() != indexColumns.size()) {\n+      throw new MalformedDataMapCommandException(INDEX_COLUMNS + \" has duplicate column\");\n+    }\n+  }\n+\n+  public static String[] getIndexColumns(DataMapSchema dataMapSchema)\n+      throws MalformedDataMapCommandException {\n+    String columns = dataMapSchema.getProperties().get(INDEX_COLUMNS);\n+    if (columns == null) {\n+      columns = dataMapSchema.getProperties().get(INDEX_COLUMNS.toLowerCase());\n+    }\n+    if (columns == null) {\n+      throw new MalformedDataMapCommandException(INDEX_COLUMNS + \" DMPROPERTY is required\");\n+    } else if (StringUtils.isBlank(columns)) {","path":"core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java","position":141,"original_position":141,"commit_id":"0ea75d2a5b68343b9933e8e78aabb37804d706fe","original_commit_id":"0ea75d2a5b68343b9933e8e78aabb37804d706fe","user":{"login":"jackylk","id":6500698,"avatar_url":"https://avatars3.githubusercontent.com/u/6500698?v=4","gravatar_id":"","url":"https://api.github.com/users/jackylk","html_url":"https://github.com/jackylk","followers_url":"https://api.github.com/users/jackylk/followers","following_url":"https://api.github.com/users/jackylk/following{/other_user}","gists_url":"https://api.github.com/users/jackylk/gists{/gist_id}","starred_url":"https://api.github.com/users/jackylk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jackylk/subscriptions","organizations_url":"https://api.github.com/users/jackylk/orgs","repos_url":"https://api.github.com/users/jackylk/repos","events_url":"https://api.github.com/users/jackylk/events{/privacy}","received_events_url":"https://api.github.com/users/jackylk/received_events","type":"User","site_admin":false},"body":"yes","created_at":"2018-05-02T05:07:16Z","updated_at":"2018-05-02T05:07:17Z","html_url":"https://github.com/apache/carbondata/pull/2254#discussion_r185394341","pull_request_url":"https://api.github.com/repos/apache/carbondata/pulls/2254","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/apache/carbondata/pulls/comments/185394341"},"html":{"href":"https://github.com/apache/carbondata/pull/2254#discussion_r185394341"},"pull_request":{"href":"https://api.github.com/repos/apache/carbondata/pulls/2254"}},"in_reply_to_id":185231346},"pull_request":{"url":"https://api.github.com/repos/apache/carbondata/pulls/2254","id":185130697,"html_url":"https://github.com/apache/carbondata/pull/2254","diff_url":"https://github.com/apache/carbondata/pull/2254.diff","patch_url":"https://github.com/apache/carbondata/pull/2254.patch","issue_url":"https://api.github.com/repos/apache/carbondata/issues/2254","number":2254,"state":"open","locked":false,"title":"[CARBONDATA-2415] Support Refresh DataMap command for all Index datamap","user":{"login":"jackylk","id":6500698,"avatar_url":"https://avatars3.githubusercontent.com/u/6500698?v=4","gravatar_id":"","url":"https://api.github.com/users/jackylk","html_url":"https://github.com/jackylk","followers_url":"https://api.github.com/users/jackylk/followers","following_url":"https://api.github.com/users/jackylk/following{/other_user}","gists_url":"https://api.github.com/users/jackylk/gists{/gist_id}","starred_url":"https://api.github.com/users/jackylk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jackylk/subscriptions","organizations_url":"https://api.github.com/users/jackylk/orgs","repos_url":"https://api.github.com/users/jackylk/repos","events_url":"https://api.github.com/users/jackylk/events{/privacy}","received_events_url":"https://api.github.com/users/jackylk/received_events","type":"User","site_admin":false},"body":"1. Refactor DataMapWriter interface to accept row instead of column page when adding data. This refactory need to modify a lot of test case files\r\n2. Add REFRESH DATAMAP support for all index datamap including Lucene and Bloom\r\n3. Make IndexDataMapRefreshRDD generic for all index datamap\r\n4. For all index datamap, added a unified DMPROPERTIES called *INDEX_COLUMNS* to specify the column name list for indexing\r\n\r\n - [X] Any interfaces changed?\r\n DataMapFactory developer interface has added new function\r\n\r\n - [X] Any backward compatibility impacted?\r\n No\r\n - [X] Document update required?\r\nYes\r\n - [X] Testing done\r\n        Please provide details on \r\n        - Whether new unit test cases have been added or why no new tests are required?\r\n        - How it is tested? Please attach test report.\r\n        - Is it a performance related change? Please attach the performance test report.\r\n        - Any additional information to help reviewers in testing this change.\r\ntest case added\r\n - [X] For large changes, please consider breaking it into sub-tasks under an umbrella JIRA. \r\nNA","created_at":"2018-05-01T06:56:09Z","updated_at":"2018-05-02T05:07:17Z","closed_at":null,"merged_at":null,"merge_commit_sha":"7aa8341d84b4184b1f7706de1e2883911b8ec221","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/apache/carbondata/pulls/2254/commits","review_comments_url":"https://api.github.com/repos/apache/carbondata/pulls/2254/comments","review_comment_url":"https://api.github.com/repos/apache/carbondata/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/carbondata/issues/2254/comments","statuses_url":"https://api.github.com/repos/apache/carbondata/statuses/0ea75d2a5b68343b9933e8e78aabb37804d706fe","head":{"label":"jackylk:refresh-datamap","ref":"refresh-datamap","sha":"0ea75d2a5b68343b9933e8e78aabb37804d706fe","user":{"login":"jackylk","id":6500698,"avatar_url":"https://avatars3.githubusercontent.com/u/6500698?v=4","gravatar_id":"","url":"https://api.github.com/users/jackylk","html_url":"https://github.com/jackylk","followers_url":"https://api.github.com/users/jackylk/followers","following_url":"https://api.github.com/users/jackylk/following{/other_user}","gists_url":"https://api.github.com/users/jackylk/gists{/gist_id}","starred_url":"https://api.github.com/users/jackylk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jackylk/subscriptions","organizations_url":"https://api.github.com/users/jackylk/orgs","repos_url":"https://api.github.com/users/jackylk/repos","events_url":"https://api.github.com/users/jackylk/events{/privacy}","received_events_url":"https://api.github.com/users/jackylk/received_events","type":"User","site_admin":false},"repo":{"id":62279476,"name":"incubator-carbondata","full_name":"jackylk/incubator-carbondata","owner":{"login":"jackylk","id":6500698,"avatar_url":"https://avatars3.githubusercontent.com/u/6500698?v=4","gravatar_id":"","url":"https://api.github.com/users/jackylk","html_url":"https://github.com/jackylk","followers_url":"https://api.github.com/users/jackylk/followers","following_url":"https://api.github.com/users/jackylk/following{/other_user}","gists_url":"https://api.github.com/users/jackylk/gists{/gist_id}","starred_url":"https://api.github.com/users/jackylk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jackylk/subscriptions","organizations_url":"https://api.github.com/users/jackylk/orgs","repos_url":"https://api.github.com/users/jackylk/repos","events_url":"https://api.github.com/users/jackylk/events{/privacy}","received_events_url":"https://api.github.com/users/jackylk/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/jackylk/incubator-carbondata","description":"Mirror of Apache CarbonData (Incubating)","fork":true,"url":"https://api.github.com/repos/jackylk/incubator-carbondata","forks_url":"https://api.github.com/repos/jackylk/incubator-carbondata/forks","keys_url":"https://api.github.com/repos/jackylk/incubator-carbondata/keys{/key_id}","collaborators_url":"https://api.github.com/repos/jackylk/incubator-carbondata/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/jackylk/incubator-carbondata/teams","hooks_url":"https://api.github.com/repos/jackylk/incubator-carbondata/hooks","issue_events_url":"https://api.github.com/repos/jackylk/incubator-carbondata/issues/events{/number}","events_url":"https://api.github.com/repos/jackylk/incubator-carbondata/events","assignees_url":"https://api.github.com/repos/jackylk/incubator-carbondata/assignees{/user}","branches_url":"https://api.github.com/repos/jackylk/incubator-carbondata/branches{/branch}","tags_url":"https://api.github.com/repos/jackylk/incubator-carbondata/tags","blobs_url":"https://api.github.com/repos/jackylk/incubator-carbondata/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/jackylk/incubator-carbondata/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/jackylk/incubator-carbondata/git/refs{/sha}","trees_url":"https://api.github.com/repos/jackylk/incubator-carbondata/git/trees{/sha}","statuses_url":"https://api.github.com/repos/jackylk/incubator-carbondata/statuses/{sha}","languages_url":"https://api.github.com/repos/jackylk/incubator-carbondata/languages","stargazers_url":"https://api.github.com/repos/jackylk/incubator-carbondata/stargazers","contributors_url":"https://api.github.com/repos/jackylk/incubator-carbondata/contributors","subscribers_url":"https://api.github.com/repos/jackylk/incubator-carbondata/subscribers","subscription_url":"https://api.github.com/repos/jackylk/incubator-carbondata/subscription","commits_url":"https://api.github.com/repos/jackylk/incubator-carbondata/commits{/sha}","git_commits_url":"https://api.github.com/repos/jackylk/incubator-carbondata/git/commits{/sha}","comments_url":"https://api.github.com/repos/jackylk/incubator-carbondata/comments{/number}","issue_comment_url":"https://api.github.com/repos/jackylk/incubator-carbondata/issues/comments{/number}","contents_url":"https://api.github.com/repos/jackylk/incubator-carbondata/contents/{+path}","compare_url":"https://api.github.com/repos/jackylk/incubator-carbondata/compare/{base}...{head}","merges_url":"https://api.github.com/repos/jackylk/incubator-carbondata/merges","archive_url":"https://api.github.com/repos/jackylk/incubator-carbondata/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/jackylk/incubator-carbondata/downloads","issues_url":"https://api.github.com/repos/jackylk/incubator-carbondata/issues{/number}","pulls_url":"https://api.github.com/repos/jackylk/incubator-carbondata/pulls{/number}","milestones_url":"https://api.github.com/repos/jackylk/incubator-carbondata/milestones{/number}","notifications_url":"https://api.github.com/repos/jackylk/incubator-carbondata/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/jackylk/incubator-carbondata/labels{/name}","releases_url":"https://api.github.com/repos/jackylk/incubator-carbondata/releases{/id}","deployments_url":"https://api.github.com/repos/jackylk/incubator-carbondata/deployments","created_at":"2016-06-30T04:32:26Z","updated_at":"2017-11-07T06:37:44Z","pushed_at":"2018-05-01T12:54:13Z","git_url":"git://github.com/jackylk/incubator-carbondata.git","ssh_url":"git@github.com:jackylk/incubator-carbondata.git","clone_url":"https://github.com/jackylk/incubator-carbondata.git","svn_url":"https://github.com/jackylk/incubator-carbondata","homepage":null,"size":83006,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":4,"mirror_url":null,"archived":false,"open_issues_count":2,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0"},"forks":4,"open_issues":2,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"4b98af22d327125f9b5a011c32be7bc1d48edb98","user":{"login":"apache","id":47359,"avatar_url":"https://avatars0.githubusercontent.com/u/47359?v=4","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":62117818,"name":"carbondata","full_name":"apache/carbondata","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars0.githubusercontent.com/u/47359?v=4","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/carbondata","description":"Mirror of Apache CarbonData","fork":false,"url":"https://api.github.com/repos/apache/carbondata","forks_url":"https://api.github.com/repos/apache/carbondata/forks","keys_url":"https://api.github.com/repos/apache/carbondata/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/carbondata/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/carbondata/teams","hooks_url":"https://api.github.com/repos/apache/carbondata/hooks","issue_events_url":"https://api.github.com/repos/apache/carbondata/issues/events{/number}","events_url":"https://api.github.com/repos/apache/carbondata/events","assignees_url":"https://api.github.com/repos/apache/carbondata/assignees{/user}","branches_url":"https://api.github.com/repos/apache/carbondata/branches{/branch}","tags_url":"https://api.github.com/repos/apache/carbondata/tags","blobs_url":"https://api.github.com/repos/apache/carbondata/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/carbondata/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/carbondata/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/carbondata/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/carbondata/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/carbondata/languages","stargazers_url":"https://api.github.com/repos/apache/carbondata/stargazers","contributors_url":"https://api.github.com/repos/apache/carbondata/contributors","subscribers_url":"https://api.github.com/repos/apache/carbondata/subscribers","subscription_url":"https://api.github.com/repos/apache/carbondata/subscription","commits_url":"https://api.github.com/repos/apache/carbondata/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/carbondata/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/carbondata/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/carbondata/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/carbondata/contents/{+path}","compare_url":"https://api.github.com/repos/apache/carbondata/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/carbondata/merges","archive_url":"https://api.github.com/repos/apache/carbondata/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/carbondata/downloads","issues_url":"https://api.github.com/repos/apache/carbondata/issues{/number}","pulls_url":"https://api.github.com/repos/apache/carbondata/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/carbondata/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/carbondata/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/carbondata/labels{/name}","releases_url":"https://api.github.com/repos/apache/carbondata/releases{/id}","deployments_url":"https://api.github.com/repos/apache/carbondata/deployments","created_at":"2016-06-28T07:00:06Z","updated_at":"2018-05-01T17:59:38Z","pushed_at":"2018-05-02T04:33:33Z","git_url":"git://github.com/apache/carbondata.git","ssh_url":"git@github.com:apache/carbondata.git","clone_url":"https://github.com/apache/carbondata.git","svn_url":"https://github.com/apache/carbondata","homepage":"","size":57337,"stargazers_count":653,"watchers_count":653,"language":"Scala","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":410,"mirror_url":null,"archived":false,"open_issues_count":105,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0"},"forks":410,"open_issues":105,"watchers":653,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/carbondata/pulls/2254"},"html":{"href":"https://github.com/apache/carbondata/pull/2254"},"issue":{"href":"https://api.github.com/repos/apache/carbondata/issues/2254"},"comments":{"href":"https://api.github.com/repos/apache/carbondata/issues/2254/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/carbondata/pulls/2254/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/carbondata/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/carbondata/pulls/2254/commits"},"statuses":{"href":"https://api.github.com/repos/apache/carbondata/statuses/0ea75d2a5b68343b9933e8e78aabb37804d706fe"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"jackylk"}}