{"action":"created","comment":{"url":"https://api.github.com/repos/apache/carbondata/pulls/comments/359672271","pull_request_review_id":334385713,"id":359672271,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDM1OTY3MjI3MQ==","diff_hunk":"@@ -0,0 +1,368 @@\n+/*\n+ * Licensed to the Apache Software Foundation (ASF) under one or more\n+ * contributor license agreements.  See the NOTICE file distributed with\n+ * this work for additional information regarding copyright ownership.\n+ * The ASF licenses this file to You under the Apache License, Version 2.0\n+ * (the \"License\"); you may not use this file except in compliance with\n+ * the License.  You may obtain a copy of the License at\n+ *\n+ *    http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+\n+package org.apache.spark.sql.execution.command.management\n+\n+import java.io.{InputStreamReader, IOException}\n+import java.util\n+import java.util.Collections\n+import java.util.concurrent.{Executors, ExecutorService}\n+\n+import scala.collection.JavaConverters._\n+\n+import com.google.gson.Gson\n+import org.apache.hadoop.conf.Configuration\n+import org.apache.log4j.Logger\n+import org.apache.spark.sql.{CarbonEnv, Row, SparkSession}\n+import org.apache.spark.sql.execution.command.{Checker, DataCommand}\n+import org.apache.spark.sql.util.SparkSQLUtil\n+\n+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException\n+import org.apache.carbondata.common.logging.LogServiceFactory\n+import org.apache.carbondata.core.constants.CarbonCommonConstants\n+import org.apache.carbondata.core.datastore.filesystem.CarbonFile\n+import org.apache.carbondata.core.datastore.impl.FileFactory\n+import org.apache.carbondata.core.locks.{CarbonLockFactory, CarbonLockUtil, ICarbonLock, LockUsage}\n+import org.apache.carbondata.core.metadata.SegmentFileStore\n+import org.apache.carbondata.core.metadata.schema.table.CarbonTable\n+import org.apache.carbondata.core.statusmanager.{SegmentStatus, SegmentStatusManager, StageInput}\n+import org.apache.carbondata.core.util.path.CarbonTablePath\n+import org.apache.carbondata.processing.loading.FailureCauses\n+import org.apache.carbondata.processing.loading.model.CarbonLoadModel\n+import org.apache.carbondata.processing.util.CarbonLoaderUtil\n+import org.apache.carbondata.spark.load.DataLoadProcessBuilderOnSpark\n+\n+/**\n+ * Collect stage input files and trigger a loading into carbon table.\n+ *\n+ * @param databaseNameOp database name\n+ * @param tableName table name\n+ */\n+case class CarbonIngestCommand(\n+    databaseNameOp: Option[String],\n+    tableName: String\n+) extends DataCommand {\n+\n+  @transient var LOGGER: Logger = _\n+\n+  override def processData(spark: SparkSession): Seq[Row] = {\n+    LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)\n+    Checker.validateTableExists(databaseNameOp, tableName, spark)\n+    val table = CarbonEnv.getCarbonTable(databaseNameOp, tableName)(spark)\n+    val hadoopConf = spark.sessionState.newHadoopConf()\n+    FileFactory.getConfiguration.addResource(hadoopConf)\n+    setAuditTable(table)\n+\n+    if (!table.getTableInfo.isTransactionalTable) {\n+      throw new MalformedCarbonCommandException(\"Unsupported operation on non transactional table\")\n+    }\n+\n+    val lock = acquireIngestLock(table)\n+    val tablePath = table.getTablePath\n+    val stagePath = CarbonTablePath.getStageDir(tablePath)\n+    val snapshotFilePath = CarbonTablePath.getStageSnapshotFile(tablePath)\n+\n+    try {\n+      // 1. Check whether we need to recover from previous failure\n+      // We use a snapshot file to indicate whether there was failure in previous\n+      // ingest operation. A Snapshot file will be created when an ingest operation\n+      // starts and will be deleted only after the whole ingest operation is finished,\n+      // which includes two actions:\n+      //   1) action1: changing segment status to SUCCESS and\n+      //   2) action2: deleting all involved stage files.\n+      //\n+      // If one of these two actions is failed, the snapshot file will be exist, so\n+      // that recovery is needed.\n+      //\n+      // To do the recovery, do following steps:\n+      //   1) Check if corresponding segment in table status is SUCCESS,\n+      //      means deleting stage files had failed. So need to read the stage\n+      //      file list from the snapshot file and delete them again.\n+      //   2) Check if corresponding segment in table status is INSERT_IN_PROGRESS,\n+      //      means data loading had failed. So need to read the stage file list\n+      //      from the snapshot file and load again.\n+      recoverIfRequired(snapshotFilePath, table, hadoopConf)\n+\n+      // 2. Start ingesting, steps:\n+      //   1) read all existing stage files\n+      //   2) read all stage files to collect input files for data loading\n+      //   3) add a new segment entry in table status as INSERT_IN_PROGRESS,\n+      //   4) write all existing stage file names into a new snapshot file\n+      //   5) do actual loading\n+      //   6) write segment file and update segment state to SUCCESS in table status\n+      //   7) delete stage files used for loading\n+      //   8) delete the snapshot file\n+\n+      // 1) read all existing stage files\n+      val stageFiles = listStageFiles(stagePath, hadoopConf)\n+      if (stageFiles.isEmpty) {\n+        // no stage files, so do nothing\n+        LOGGER.warn(\"files not found under stage metadata folder\")\n+        return Seq.empty\n+      }\n+\n+      // 2) read all stage files to collect input files for data loading\n+      // create a thread pool to read them\n+      val numThreads = Math.min(Math.max(stageFiles.length, 1), 10)\n+      val executorService = Executors.newFixedThreadPool(numThreads)\n+      val stageInputs = collectStageInputs(executorService, stageFiles)\n+\n+      // 3) add new segment with INSERT_IN_PROGRESS into table status\n+      val loadModel = DataLoadProcessBuilderOnSpark.createLoadModelForGlobalSort(spark, table)\n+      CarbonLoaderUtil.recordNewLoadMetadata(loadModel)\n+\n+      // 4) write all existing stage file names and segmentId into a new snapshot file\n+      // The content of snapshot file is: first line is segmentId, followed by each line is\n+      // one stage file name\n+      val content =\n+        (Seq(loadModel.getSegmentId) ++ stageFiles.map(_._1.getAbsolutePath)).mkString(\"\\n\")\n+      FileFactory.writeFile(content, snapshotFilePath)\n+\n+      // 5) perform data loading\n+      startLoading(spark, table, loadModel, stageInputs)\n+\n+      // 6) write segment file and update the segment entry to SUCCESS\n+      val segmentFileName = SegmentFileStore.writeSegmentFile(\n+        table, loadModel.getSegmentId, loadModel.getFactTimeStamp.toString)\n+      SegmentFileStore.updateTableStatusFile(\n+        table, loadModel.getSegmentId, segmentFileName,\n+        table.getCarbonTableIdentifier.getTableId,\n+        new SegmentFileStore(table.getTablePath, segmentFileName),\n+        SegmentStatus.SUCCESS)\n+\n+      // 7) delete stage files\n+      deleteStageFiles(executorService, stageFiles)\n+\n+      // 8) delete the snapshot file\n+      FileFactory.getCarbonFile(snapshotFilePath).delete()\n+    } catch {\n+      case ex: Throwable =>\n+        LOGGER.error(s\"failed to ingest ${table.getDatabaseName}.${table.getTableName}\", ex)\n+        throw ex\n+    } finally {\n+      lock.unlock()\n+    }\n+    Seq.empty\n+  }\n+\n+  /**\n+   * Check whether there was failure in previous ingest process and try to recover\n+   */\n+  private def recoverIfRequired(\n+      snapshotFilePath: String,\n+      table: CarbonTable,\n+      conf: Configuration): Unit = {\n+    if (!FileFactory.isFileExist(snapshotFilePath)) {\n+      // everything is fine\n+      return\n+    }\n+\n+    // something wrong, read the snapshot file and do recover steps\n+    // 1. check segment state in table status file\n+    // 2. If in SUCCESS state, delete all stage files read inn snapshot file\n+    // 3. If in IN_PROGRESS state, delete the entry in table status and load again\n+    LOGGER.info(s\"snapshot file found ($snapshotFilePath), start recovery process\")\n+    val lines = FileFactory.readLinesInFile(snapshotFilePath, conf)\n+    if (lines.size() < 2) {\n+      throw new RuntimeException(\"Invalid snapshot file, \" + lines.size() + \" lines\")\n+    }\n+\n+    val segmentId = lines.get(0)\n+    val stageFileNames = lines.remove(0)\n+    LOGGER.info(s\"Segment $segmentId need recovery, ${stageFileNames.length} stage files\")\n+\n+    // lock the table status\n+    var lock = CarbonLockFactory.getCarbonLockObj(\n+      table.getAbsoluteTableIdentifier, LockUsage.TABLE_STATUS_LOCK)\n+    if (!lock.lockWithRetries()) {\n+      throw new RuntimeException(s\"Failed to lock table status for \" +\n+                                 s\"${table.getDatabaseName}.${table.getTableName}\")\n+    }\n+    try {\n+      val segments = SegmentStatusManager.readTableStatusFile(\n+        CarbonTablePath.getTableStatusFilePath(table.getTablePath)\n+      )\n+      val matchedSegment = segments.filter(_.getLoadName.equals(segmentId))\n+      if (matchedSegment.length != 1) {\n+        throw new RuntimeException(\"unexpected \" + matchedSegment.length + \" segment found\")\n+      }\n+      matchedSegment(0).getSegmentStatus match {\n+        case SegmentStatus.SUCCESS =>\n+          // delete all stage files\n+          lock.unlock()\n+          lock = null\n+          LOGGER.info(s\"Segment $segmentId is in SUCCESS state, about to delete \" +\n+                      s\"${stageFileNames.length} stage files\")\n+          val numThreads = Math.min(Math.max(stageFileNames.length, 1), 10)\n+          val executorService = Executors.newFixedThreadPool(numThreads)\n+          stageFileNames.map { fileName =>\n+            executorService.submit(new Runnable {\n+              override def run(): Unit = {\n+                FileFactory.getCarbonFile(\n+                  CarbonTablePath.getStageDir(table.getTablePath) +\n+                  CarbonCommonConstants.FILE_SEPARATOR + fileName\n+                ).delete()\n+              }\n+            })\n+          }.map { future =>\n+            future.get()\n+          }\n+        case SegmentStatus.INSERT_IN_PROGRESS =>\n+          // delete entry in table status and load again\n+          LOGGER.info(s\"Segment $segmentId is in INSERT_IN_PROGRESS state, about to delete the \" +\n+                      s\"segment entry and load again\")\n+          val segmentToWrite = segments.filterNot(_.getLoadName.equals(segmentId))\n+          SegmentStatusManager.writeLoadDetailsIntoFile(\n+            CarbonTablePath.getTableStatusFilePath(table.getTablePath),\n+            segmentToWrite)\n+        case other =>\n+          throw new RuntimeException(s\"Segment $segmentId is in unexpected state: $other\")\n+      }\n+    } finally {\n+      if (lock != null) {\n+        lock.unlock()\n+      }\n+    }\n+    LOGGER.info(s\"Finish recovery, delete snapshot file: $snapshotFilePath\")\n+    FileFactory.getCarbonFile(snapshotFilePath).delete()\n+  }\n+\n+  /**\n+   * Start global sort loading\n+   */\n+  private def startLoading(\n+      spark: SparkSession,\n+      table: CarbonTable,\n+      loadModel: CarbonLoadModel,\n+      stageInput: Seq[StageInput]\n+  ): Unit = {\n+    val splits = stageInput.flatMap(_.createSplits().asScala)\n+    LOGGER.info(s\"start to load ${splits.size} files into \" +\n+                s\"${table.getDatabaseName}.${table.getTableName}\")\n+    val start = System.currentTimeMillis()\n+    val dataFrame = DataLoadProcessBuilderOnSpark.createInputDataFrame(spark, table, splits)\n+    DataLoadProcessBuilderOnSpark.loadDataUsingGlobalSort(\n+      spark,\n+      Option(dataFrame),\n+      loadModel,\n+      SparkSQLUtil.sessionState(spark).newHadoopConf()\n+    ).map { row =>\n+        (row._1, FailureCauses.NONE == row._2._2.failureCauses)\n+      }\n+\n+    LOGGER.info(s\"finish data loading, time taken ${System.currentTimeMillis() - start}ms\")\n+  }\n+\n+  /**\n+   * Read stage files and return input files\n+   */\n+  private def collectStageInputs(\n+      executorService: ExecutorService,\n+      stageFiles: Array[(CarbonFile, CarbonFile)]\n+  ): Seq[StageInput] = {\n+    val startTime = System.currentTimeMillis()\n+    val output = Collections.synchronizedList(new util.ArrayList[StageInput]())\n+    val gson = new Gson()\n+    stageFiles.map { stage =>\n+      executorService.submit(new Runnable {\n+        override def run(): Unit = {\n+          val filePath = stage._1.getAbsolutePath\n+          val stream = FileFactory.getDataInputStream(filePath, FileFactory.getFileType(filePath))\n+          val stageInput = gson.fromJson(new InputStreamReader(stream), classOf[StageInput])\n+          output.add(stageInput)\n+        }\n+      })\n+    }.map { future =>\n+      future.get()\n+    }\n+    LOGGER.info(s\"read stage files taken ${System.currentTimeMillis() - startTime}ms\")\n+    output.asScala\n+  }\n+\n+  /**\n+   * Delete stage file and success file\n+   */\n+  private def deleteStageFiles(\n+      executorService: ExecutorService,\n+      stageFiles: Array[(CarbonFile, CarbonFile)]): Unit = {\n+    val startTime = System.currentTimeMillis()\n+    stageFiles.map { files =>\n+      executorService.submit(new Runnable {\n+        override def run(): Unit = {\n+          files._1.delete()\n+          files._2.delete()","path":"integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonIngestCommand.scala","position":307,"original_position":307,"commit_id":"9a0a5bbd34977b7823393befd2838acb4338aa16","original_commit_id":"9a0a5bbd34977b7823393befd2838acb4338aa16","user":{"login":"niuge01","id":3700064,"node_id":"MDQ6VXNlcjM3MDAwNjQ=","avatar_url":"https://avatars2.githubusercontent.com/u/3700064?v=4","gravatar_id":"","url":"https://api.github.com/users/niuge01","html_url":"https://github.com/niuge01","followers_url":"https://api.github.com/users/niuge01/followers","following_url":"https://api.github.com/users/niuge01/following{/other_user}","gists_url":"https://api.github.com/users/niuge01/gists{/gist_id}","starred_url":"https://api.github.com/users/niuge01/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/niuge01/subscriptions","organizations_url":"https://api.github.com/users/niuge01/orgs","repos_url":"https://api.github.com/users/niuge01/repos","events_url":"https://api.github.com/users/niuge01/events{/privacy}","received_events_url":"https://api.github.com/users/niuge01/received_events","type":"User","site_admin":false},"body":"The second file will not be deleted when the first file delete fail, should we consider for the case?","created_at":"2019-12-19T03:28:36Z","updated_at":"2019-12-19T03:29:36Z","html_url":"https://github.com/apache/carbondata/pull/3488#discussion_r359672271","pull_request_url":"https://api.github.com/repos/apache/carbondata/pulls/3488","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/apache/carbondata/pulls/comments/359672271"},"html":{"href":"https://github.com/apache/carbondata/pull/3488#discussion_r359672271"},"pull_request":{"href":"https://api.github.com/repos/apache/carbondata/pulls/3488"}}},"pull_request":{"url":"https://api.github.com/repos/apache/carbondata/pulls/3488","id":346695990,"node_id":"MDExOlB1bGxSZXF1ZXN0MzQ2Njk1OTkw","html_url":"https://github.com/apache/carbondata/pull/3488","diff_url":"https://github.com/apache/carbondata/pull/3488.diff","patch_url":"https://github.com/apache/carbondata/pull/3488.patch","issue_url":"https://api.github.com/repos/apache/carbondata/issues/3488","number":3488,"state":"open","locked":false,"title":"[CARBONDATA-3599] Support auto ingest data written by Flink","user":{"login":"jackylk","id":6500698,"node_id":"MDQ6VXNlcjY1MDA2OTg=","avatar_url":"https://avatars3.githubusercontent.com/u/6500698?v=4","gravatar_id":"","url":"https://api.github.com/users/jackylk","html_url":"https://github.com/jackylk","followers_url":"https://api.github.com/users/jackylk/followers","following_url":"https://api.github.com/users/jackylk/following{/other_user}","gists_url":"https://api.github.com/users/jackylk/gists{/gist_id}","starred_url":"https://api.github.com/users/jackylk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jackylk/subscriptions","organizations_url":"https://api.github.com/users/jackylk/orgs","repos_url":"https://api.github.com/users/jackylk/repos","events_url":"https://api.github.com/users/jackylk/events{/privacy}","received_events_url":"https://api.github.com/users/jackylk/received_events","type":"User","site_admin":false},"body":" A new SQL command is added to auto ingest data written by Flink\r\n\r\nUsage:\r\nUse carbondata-flink module to write data to HDFS, S3, etc\r\nThen use following command to trigger loading into carbon table periodically.\r\n```\r\nINGEST dbName.tableName\r\n```\r\n\r\nFor more detail, see TestCarbonWriter.scala\r\n\r\n - [ ] Any interfaces changed?\r\n \r\n - [ ] Any backward compatibility impacted?\r\n \r\n - [ ] Document update required?\r\n\r\n - [ ] Testing done\r\n        Please provide details on \r\n        - Whether new unit test cases have been added or why no new tests are required?\r\n        - How it is tested? Please attach test report.\r\n        - Is it a performance related change? Please attach the performance test report.\r\n        - Any additional information to help reviewers in testing this change.\r\n       \r\n - [ ] For large changes, please consider breaking it into sub-tasks under an umbrella JIRA. \r\n\r\n","created_at":"2019-11-28T13:16:32Z","updated_at":"2019-12-19T03:29:36Z","closed_at":null,"merged_at":null,"merge_commit_sha":"caaae348ac9410e135bedfdc9d89c39f9d456cf1","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/apache/carbondata/pulls/3488/commits","review_comments_url":"https://api.github.com/repos/apache/carbondata/pulls/3488/comments","review_comment_url":"https://api.github.com/repos/apache/carbondata/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/carbondata/issues/3488/comments","statuses_url":"https://api.github.com/repos/apache/carbondata/statuses/9a0a5bbd34977b7823393befd2838acb4338aa16","head":{"label":"jackylk:flink-ingest","ref":"flink-ingest","sha":"9a0a5bbd34977b7823393befd2838acb4338aa16","user":{"login":"jackylk","id":6500698,"node_id":"MDQ6VXNlcjY1MDA2OTg=","avatar_url":"https://avatars3.githubusercontent.com/u/6500698?v=4","gravatar_id":"","url":"https://api.github.com/users/jackylk","html_url":"https://github.com/jackylk","followers_url":"https://api.github.com/users/jackylk/followers","following_url":"https://api.github.com/users/jackylk/following{/other_user}","gists_url":"https://api.github.com/users/jackylk/gists{/gist_id}","starred_url":"https://api.github.com/users/jackylk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jackylk/subscriptions","organizations_url":"https://api.github.com/users/jackylk/orgs","repos_url":"https://api.github.com/users/jackylk/repos","events_url":"https://api.github.com/users/jackylk/events{/privacy}","received_events_url":"https://api.github.com/users/jackylk/received_events","type":"User","site_admin":false},"repo":{"id":62279476,"node_id":"MDEwOlJlcG9zaXRvcnk2MjI3OTQ3Ng==","name":"incubator-carbondata","full_name":"jackylk/incubator-carbondata","private":false,"owner":{"login":"jackylk","id":6500698,"node_id":"MDQ6VXNlcjY1MDA2OTg=","avatar_url":"https://avatars3.githubusercontent.com/u/6500698?v=4","gravatar_id":"","url":"https://api.github.com/users/jackylk","html_url":"https://github.com/jackylk","followers_url":"https://api.github.com/users/jackylk/followers","following_url":"https://api.github.com/users/jackylk/following{/other_user}","gists_url":"https://api.github.com/users/jackylk/gists{/gist_id}","starred_url":"https://api.github.com/users/jackylk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jackylk/subscriptions","organizations_url":"https://api.github.com/users/jackylk/orgs","repos_url":"https://api.github.com/users/jackylk/repos","events_url":"https://api.github.com/users/jackylk/events{/privacy}","received_events_url":"https://api.github.com/users/jackylk/received_events","type":"User","site_admin":false},"html_url":"https://github.com/jackylk/incubator-carbondata","description":"Mirror of Apache CarbonData (Incubating)","fork":true,"url":"https://api.github.com/repos/jackylk/incubator-carbondata","forks_url":"https://api.github.com/repos/jackylk/incubator-carbondata/forks","keys_url":"https://api.github.com/repos/jackylk/incubator-carbondata/keys{/key_id}","collaborators_url":"https://api.github.com/repos/jackylk/incubator-carbondata/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/jackylk/incubator-carbondata/teams","hooks_url":"https://api.github.com/repos/jackylk/incubator-carbondata/hooks","issue_events_url":"https://api.github.com/repos/jackylk/incubator-carbondata/issues/events{/number}","events_url":"https://api.github.com/repos/jackylk/incubator-carbondata/events","assignees_url":"https://api.github.com/repos/jackylk/incubator-carbondata/assignees{/user}","branches_url":"https://api.github.com/repos/jackylk/incubator-carbondata/branches{/branch}","tags_url":"https://api.github.com/repos/jackylk/incubator-carbondata/tags","blobs_url":"https://api.github.com/repos/jackylk/incubator-carbondata/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/jackylk/incubator-carbondata/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/jackylk/incubator-carbondata/git/refs{/sha}","trees_url":"https://api.github.com/repos/jackylk/incubator-carbondata/git/trees{/sha}","statuses_url":"https://api.github.com/repos/jackylk/incubator-carbondata/statuses/{sha}","languages_url":"https://api.github.com/repos/jackylk/incubator-carbondata/languages","stargazers_url":"https://api.github.com/repos/jackylk/incubator-carbondata/stargazers","contributors_url":"https://api.github.com/repos/jackylk/incubator-carbondata/contributors","subscribers_url":"https://api.github.com/repos/jackylk/incubator-carbondata/subscribers","subscription_url":"https://api.github.com/repos/jackylk/incubator-carbondata/subscription","commits_url":"https://api.github.com/repos/jackylk/incubator-carbondata/commits{/sha}","git_commits_url":"https://api.github.com/repos/jackylk/incubator-carbondata/git/commits{/sha}","comments_url":"https://api.github.com/repos/jackylk/incubator-carbondata/comments{/number}","issue_comment_url":"https://api.github.com/repos/jackylk/incubator-carbondata/issues/comments{/number}","contents_url":"https://api.github.com/repos/jackylk/incubator-carbondata/contents/{+path}","compare_url":"https://api.github.com/repos/jackylk/incubator-carbondata/compare/{base}...{head}","merges_url":"https://api.github.com/repos/jackylk/incubator-carbondata/merges","archive_url":"https://api.github.com/repos/jackylk/incubator-carbondata/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/jackylk/incubator-carbondata/downloads","issues_url":"https://api.github.com/repos/jackylk/incubator-carbondata/issues{/number}","pulls_url":"https://api.github.com/repos/jackylk/incubator-carbondata/pulls{/number}","milestones_url":"https://api.github.com/repos/jackylk/incubator-carbondata/milestones{/number}","notifications_url":"https://api.github.com/repos/jackylk/incubator-carbondata/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/jackylk/incubator-carbondata/labels{/name}","releases_url":"https://api.github.com/repos/jackylk/incubator-carbondata/releases{/id}","deployments_url":"https://api.github.com/repos/jackylk/incubator-carbondata/deployments","created_at":"2016-06-30T04:32:26Z","updated_at":"2019-06-15T07:32:04Z","pushed_at":"2019-12-18T15:28:11Z","git_url":"git://github.com/jackylk/incubator-carbondata.git","ssh_url":"git@github.com:jackylk/incubator-carbondata.git","clone_url":"https://github.com/jackylk/incubator-carbondata.git","svn_url":"https://github.com/jackylk/incubator-carbondata","homepage":null,"size":109479,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":4,"mirror_url":null,"archived":false,"disabled":false,"open_issues_count":3,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":4,"open_issues":3,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:flink","ref":"flink","sha":"5322767b70615e65d1e6c98d56582028c1f20f96","user":{"login":"apache","id":47359,"node_id":"MDEyOk9yZ2FuaXphdGlvbjQ3MzU5","avatar_url":"https://avatars0.githubusercontent.com/u/47359?v=4","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":62117818,"node_id":"MDEwOlJlcG9zaXRvcnk2MjExNzgxOA==","name":"carbondata","full_name":"apache/carbondata","private":false,"owner":{"login":"apache","id":47359,"node_id":"MDEyOk9yZ2FuaXphdGlvbjQ3MzU5","avatar_url":"https://avatars0.githubusercontent.com/u/47359?v=4","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/apache/carbondata","description":"Mirror of Apache CarbonData","fork":false,"url":"https://api.github.com/repos/apache/carbondata","forks_url":"https://api.github.com/repos/apache/carbondata/forks","keys_url":"https://api.github.com/repos/apache/carbondata/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/carbondata/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/carbondata/teams","hooks_url":"https://api.github.com/repos/apache/carbondata/hooks","issue_events_url":"https://api.github.com/repos/apache/carbondata/issues/events{/number}","events_url":"https://api.github.com/repos/apache/carbondata/events","assignees_url":"https://api.github.com/repos/apache/carbondata/assignees{/user}","branches_url":"https://api.github.com/repos/apache/carbondata/branches{/branch}","tags_url":"https://api.github.com/repos/apache/carbondata/tags","blobs_url":"https://api.github.com/repos/apache/carbondata/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/carbondata/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/carbondata/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/carbondata/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/carbondata/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/carbondata/languages","stargazers_url":"https://api.github.com/repos/apache/carbondata/stargazers","contributors_url":"https://api.github.com/repos/apache/carbondata/contributors","subscribers_url":"https://api.github.com/repos/apache/carbondata/subscribers","subscription_url":"https://api.github.com/repos/apache/carbondata/subscription","commits_url":"https://api.github.com/repos/apache/carbondata/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/carbondata/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/carbondata/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/carbondata/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/carbondata/contents/{+path}","compare_url":"https://api.github.com/repos/apache/carbondata/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/carbondata/merges","archive_url":"https://api.github.com/repos/apache/carbondata/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/carbondata/downloads","issues_url":"https://api.github.com/repos/apache/carbondata/issues{/number}","pulls_url":"https://api.github.com/repos/apache/carbondata/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/carbondata/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/carbondata/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/carbondata/labels{/name}","releases_url":"https://api.github.com/repos/apache/carbondata/releases{/id}","deployments_url":"https://api.github.com/repos/apache/carbondata/deployments","created_at":"2016-06-28T07:00:06Z","updated_at":"2019-12-19T00:31:47Z","pushed_at":"2019-12-19T03:06:47Z","git_url":"git://github.com/apache/carbondata.git","ssh_url":"git@github.com:apache/carbondata.git","clone_url":"https://github.com/apache/carbondata.git","svn_url":"https://github.com/apache/carbondata","homepage":"","size":72242,"stargazers_count":1000,"watchers_count":1000,"language":"Scala","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":585,"mirror_url":null,"archived":false,"disabled":false,"open_issues_count":136,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0","node_id":"MDc6TGljZW5zZTI="},"forks":585,"open_issues":136,"watchers":1000,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/carbondata/pulls/3488"},"html":{"href":"https://github.com/apache/carbondata/pull/3488"},"issue":{"href":"https://api.github.com/repos/apache/carbondata/issues/3488"},"comments":{"href":"https://api.github.com/repos/apache/carbondata/issues/3488/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/carbondata/pulls/3488/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/carbondata/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/carbondata/pulls/3488/commits"},"statuses":{"href":"https://api.github.com/repos/apache/carbondata/statuses/9a0a5bbd34977b7823393befd2838acb4338aa16"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"niuge01"}}