{"action":"created","comment":{"url":"https://api.github.com/repos/apache/spark/pulls/comments/56424506","id":56424506,"diff_hunk":"@@ -0,0 +1,469 @@\n+/*\n+ * Licensed to the Apache Software Foundation (ASF) under one or more\n+ * contributor license agreements.  See the NOTICE file distributed with\n+ * this work for additional information regarding copyright ownership.\n+ * The ASF licenses this file to You under the Apache License, Version 2.0\n+ * (the \"License\"); you may not use this file except in compliance with\n+ * the License.  You may obtain a copy of the License at\n+ *\n+ *    http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+\n+package org.apache.spark.sql.catalyst.catalog\n+\n+import java.util.concurrent.ConcurrentHashMap\n+\n+import scala.collection.JavaConverters._\n+\n+import org.apache.spark.sql.AnalysisException\n+import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier}\n+import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, SubqueryAlias}\n+\n+\n+/**\n+ * An internal catalog that is used by a Spark Session. This internal catalog serves as a\n+ * proxy to the underlying metastore (e.g. Hive Metastore) and it also manages temporary\n+ * tables and functions of the Spark Session that it belongs to.\n+ */\n+class SessionCatalog(externalCatalog: ExternalCatalog) {\n+  import ExternalCatalog._\n+\n+  private[this] val tempTables = new ConcurrentHashMap[String, LogicalPlan]\n+  private[this] val tempFunctions = new ConcurrentHashMap[String, CatalogFunction]\n+\n+  // Note: we track current database here because certain operations do not explicitly\n+  // specify the database (e.g. DROP TABLE my_table). In these cases we must first\n+  // check whether the temporary table or function exists, then, if not, operate on\n+  // the corresponding item in the current database.\n+  private[this] var currentDb = \"default\"\n+\n+  // ----------------------------------------------------------------------------\n+  // Databases\n+  // ----------------------------------------------------------------------------\n+  // All methods in this category interact directly with the underlying catalog.\n+  // ----------------------------------------------------------------------------\n+\n+  def createDatabase(dbDefinition: CatalogDatabase, ignoreIfExists: Boolean): Unit = {\n+    externalCatalog.createDatabase(dbDefinition, ignoreIfExists)\n+  }\n+\n+  def dropDatabase(db: String, ignoreIfNotExists: Boolean, cascade: Boolean): Unit = {\n+    externalCatalog.dropDatabase(db, ignoreIfNotExists, cascade)\n+  }\n+\n+  def alterDatabase(dbDefinition: CatalogDatabase): Unit = {\n+    externalCatalog.alterDatabase(dbDefinition)\n+  }\n+\n+  def getDatabase(db: String): CatalogDatabase = {\n+    externalCatalog.getDatabase(db)\n+  }\n+\n+  def databaseExists(db: String): Boolean = {\n+    externalCatalog.databaseExists(db)\n+  }\n+\n+  def listDatabases(): Seq[String] = {\n+    externalCatalog.listDatabases()\n+  }\n+\n+  def listDatabases(pattern: String): Seq[String] = {\n+    externalCatalog.listDatabases(pattern)\n+  }\n+\n+  def getCurrentDatabase: String = currentDb\n+\n+  def setCurrentDatabase(db: String): Unit = {\n+    if (!databaseExists(db)) {\n+      throw new AnalysisException(s\"cannot set current database to non-existent '$db'\")\n+    }\n+    currentDb = db\n+  }\n+\n+  // ----------------------------------------------------------------------------\n+  // Tables\n+  // ----------------------------------------------------------------------------\n+  // There are two kinds of tables, temporary tables and metastore tables.\n+  // Temporary tables are isolated across sessions and do not belong to any\n+  // particular database. Metastore tables can be used across multiple\n+  // sessions as their metadata is persisted in the underlying catalog.\n+  // ----------------------------------------------------------------------------\n+\n+  // ----------------------------------------------------\n+  // | Methods that interact with metastore tables only |\n+  // ----------------------------------------------------\n+\n+  /**\n+   * Create a metastore table in the database specified in `tableDefinition`.\n+   * If no such database is specified, create it in the current database.\n+   */\n+  def createTable(tableDefinition: CatalogTable, ignoreIfExists: Boolean): Unit = {\n+    val db = tableDefinition.name.database.getOrElse(currentDb)\n+    val newTableDefinition = tableDefinition.copy(\n+      name = TableIdentifier(tableDefinition.name.table, Some(db)))\n+    externalCatalog.createTable(db, newTableDefinition, ignoreIfExists)\n+  }\n+\n+  /**\n+   * Alter the metadata of an existing metastore table identified by `tableDefinition`.\n+   *\n+   * If no database is specified in `tableDefinition`, assume the table is in the\n+   * current database.\n+   *\n+   * Note: If the underlying implementation does not support altering a certain field,\n+   * this becomes a no-op.\n+   */\n+  def alterTable(tableDefinition: CatalogTable): Unit = {\n+    val db = tableDefinition.name.database.getOrElse(currentDb)\n+    val newTableDefinition = tableDefinition.copy(\n+      name = TableIdentifier(tableDefinition.name.table, Some(db)))\n+    externalCatalog.alterTable(db, newTableDefinition)\n+  }\n+\n+  /**\n+   * Retrieve the metadata of an existing metastore table.\n+   * If no database is specified, assume the table is in the current database.\n+   */\n+  def getTable(name: TableIdentifier): CatalogTable = {\n+    val db = name.database.getOrElse(currentDb)\n+    externalCatalog.getTable(db, name.table)\n+  }\n+\n+  // -------------------------------------------------------------\n+  // | Methods that interact with temporary and metastore tables |\n+  // -------------------------------------------------------------\n+\n+  /**\n+   * Create a temporary table.\n+   */\n+  def createTempTable(\n+      name: String,\n+      tableDefinition: LogicalPlan,\n+      ignoreIfExists: Boolean): Unit = {\n+    if (tempTables.containsKey(name) && !ignoreIfExists) {\n+      throw new AnalysisException(s\"Temporary table '$name' already exists.\")\n+    }\n+    tempTables.put(name, tableDefinition)\n+  }\n+\n+  /**\n+   * Rename a table.\n+   *\n+   * If a database is specified in `oldName`, this will rename the table in that database.\n+   * If no database is specified, this will first attempt to rename a temporary table with\n+   * the same name, then, if that does not exist, rename the table in the current database.\n+   *\n+   * This assumes the database specified in `oldName` matches the one specified in `newName`.\n+   */\n+  def renameTable(oldName: TableIdentifier, newName: TableIdentifier): Unit = {\n+    if (oldName.database != newName.database) {\n+      throw new AnalysisException(\"rename does not support moving tables across databases\")\n+    }\n+    val db = oldName.database.getOrElse(currentDb)\n+    if (oldName.database.isDefined || !tempTables.containsKey(oldName.table)) {\n+      externalCatalog.renameTable(db, oldName.table, newName.table)\n+    } else {\n+      val table = tempTables.remove(oldName.table)\n+      tempTables.put(newName.table, table)\n+    }\n+  }\n+\n+  /**\n+   * Drop a table.\n+   *\n+   * If a database is specified in `name`, this will drop the table from that database.\n+   * If no database is specified, this will first attempt to drop a temporary table with\n+   * the same name, then, if that does not exist, drop the table from the current database.\n+   */\n+  def dropTable(name: TableIdentifier, ignoreIfNotExists: Boolean): Unit = {","path":"sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala","position":184,"original_position":184,"commit_id":"ad43a5ffdeeb881aaed8944971b63a27d1f4257f","original_commit_id":"ad43a5ffdeeb881aaed8944971b63a27d1f4257f","user":{"login":"yhuai","id":2072857,"avatar_url":"https://avatars.githubusercontent.com/u/2072857?v=3","gravatar_id":"","url":"https://api.github.com/users/yhuai","html_url":"https://github.com/yhuai","followers_url":"https://api.github.com/users/yhuai/followers","following_url":"https://api.github.com/users/yhuai/following{/other_user}","gists_url":"https://api.github.com/users/yhuai/gists{/gist_id}","starred_url":"https://api.github.com/users/yhuai/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/yhuai/subscriptions","organizations_url":"https://api.github.com/users/yhuai/orgs","repos_url":"https://api.github.com/users/yhuai/repos","events_url":"https://api.github.com/users/yhuai/events{/privacy}","received_events_url":"https://api.github.com/users/yhuai/received_events","type":"User","site_admin":false},"body":"This is the semantic of postgres and Hive.","created_at":"2016-03-16T22:07:47Z","updated_at":"2016-03-16T22:07:47Z","html_url":"https://github.com/apache/spark/pull/11750#discussion_r56424506","pull_request_url":"https://api.github.com/repos/apache/spark/pulls/11750","_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/56424506"},"html":{"href":"https://github.com/apache/spark/pull/11750#discussion_r56424506"},"pull_request":{"href":"https://api.github.com/repos/apache/spark/pulls/11750"}}},"pull_request":{"url":"https://api.github.com/repos/apache/spark/pulls/11750","id":63007217,"html_url":"https://github.com/apache/spark/pull/11750","diff_url":"https://github.com/apache/spark/pull/11750.diff","patch_url":"https://github.com/apache/spark/pull/11750.patch","issue_url":"https://api.github.com/repos/apache/spark/issues/11750","number":11750,"state":"open","locked":false,"title":"[SPARK-13923] [SQL] Implement SessionCatalog","user":{"login":"andrewor14","id":2133137,"avatar_url":"https://avatars.githubusercontent.com/u/2133137?v=3","gravatar_id":"","url":"https://api.github.com/users/andrewor14","html_url":"https://github.com/andrewor14","followers_url":"https://api.github.com/users/andrewor14/followers","following_url":"https://api.github.com/users/andrewor14/following{/other_user}","gists_url":"https://api.github.com/users/andrewor14/gists{/gist_id}","starred_url":"https://api.github.com/users/andrewor14/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/andrewor14/subscriptions","organizations_url":"https://api.github.com/users/andrewor14/orgs","repos_url":"https://api.github.com/users/andrewor14/repos","events_url":"https://api.github.com/users/andrewor14/events{/privacy}","received_events_url":"https://api.github.com/users/andrewor14/received_events","type":"User","site_admin":false},"body":"## What changes were proposed in this pull request?\r\n\r\nAs part of the effort to merge `SQLContext` and `HiveContext`, this patch implements an internal catalog called `SessionCatalog` that handles temporary functions and tables and delegates metastore operations to `ExternalCatalog`. Currently, this is still dead code, but in the future it will be part of `SessionState` and will replace `o.a.s.sql.catalyst.analysis.Catalog`.\r\n\r\nA recent patch #11573 parses Hive commands ourselves in Spark, but still passes the entire query text to Hive. In a future patch, we will use `SessionCatalog` to implement the parsed commands.\r\n\r\n## How was this patch tested?\r\n\r\n800+ lines of tests in `SessionCatalogSuite`.\r\n","created_at":"2016-03-16T02:50:10Z","updated_at":"2016-03-16T22:07:47Z","closed_at":null,"merged_at":null,"merge_commit_sha":"edf52b6b4196b9b6932943ad199651a7b7134919","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/apache/spark/pulls/11750/commits","review_comments_url":"https://api.github.com/repos/apache/spark/pulls/11750/comments","review_comment_url":"https://api.github.com/repos/apache/spark/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/spark/issues/11750/comments","statuses_url":"https://api.github.com/repos/apache/spark/statuses/ad43a5ffdeeb881aaed8944971b63a27d1f4257f","head":{"label":"andrewor14:temp-catalog","ref":"temp-catalog","sha":"ad43a5ffdeeb881aaed8944971b63a27d1f4257f","user":{"login":"andrewor14","id":2133137,"avatar_url":"https://avatars.githubusercontent.com/u/2133137?v=3","gravatar_id":"","url":"https://api.github.com/users/andrewor14","html_url":"https://github.com/andrewor14","followers_url":"https://api.github.com/users/andrewor14/followers","following_url":"https://api.github.com/users/andrewor14/following{/other_user}","gists_url":"https://api.github.com/users/andrewor14/gists{/gist_id}","starred_url":"https://api.github.com/users/andrewor14/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/andrewor14/subscriptions","organizations_url":"https://api.github.com/users/andrewor14/orgs","repos_url":"https://api.github.com/users/andrewor14/repos","events_url":"https://api.github.com/users/andrewor14/events{/privacy}","received_events_url":"https://api.github.com/users/andrewor14/received_events","type":"User","site_admin":false},"repo":{"id":17231517,"name":"spark","full_name":"andrewor14/spark","owner":{"login":"andrewor14","id":2133137,"avatar_url":"https://avatars.githubusercontent.com/u/2133137?v=3","gravatar_id":"","url":"https://api.github.com/users/andrewor14","html_url":"https://github.com/andrewor14","followers_url":"https://api.github.com/users/andrewor14/followers","following_url":"https://api.github.com/users/andrewor14/following{/other_user}","gists_url":"https://api.github.com/users/andrewor14/gists{/gist_id}","starred_url":"https://api.github.com/users/andrewor14/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/andrewor14/subscriptions","organizations_url":"https://api.github.com/users/andrewor14/orgs","repos_url":"https://api.github.com/users/andrewor14/repos","events_url":"https://api.github.com/users/andrewor14/events{/privacy}","received_events_url":"https://api.github.com/users/andrewor14/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/andrewor14/spark","description":"Mirror of Apache Spark","fork":true,"url":"https://api.github.com/repos/andrewor14/spark","forks_url":"https://api.github.com/repos/andrewor14/spark/forks","keys_url":"https://api.github.com/repos/andrewor14/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/andrewor14/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/andrewor14/spark/teams","hooks_url":"https://api.github.com/repos/andrewor14/spark/hooks","issue_events_url":"https://api.github.com/repos/andrewor14/spark/issues/events{/number}","events_url":"https://api.github.com/repos/andrewor14/spark/events","assignees_url":"https://api.github.com/repos/andrewor14/spark/assignees{/user}","branches_url":"https://api.github.com/repos/andrewor14/spark/branches{/branch}","tags_url":"https://api.github.com/repos/andrewor14/spark/tags","blobs_url":"https://api.github.com/repos/andrewor14/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/andrewor14/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/andrewor14/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/andrewor14/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/andrewor14/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/andrewor14/spark/languages","stargazers_url":"https://api.github.com/repos/andrewor14/spark/stargazers","contributors_url":"https://api.github.com/repos/andrewor14/spark/contributors","subscribers_url":"https://api.github.com/repos/andrewor14/spark/subscribers","subscription_url":"https://api.github.com/repos/andrewor14/spark/subscription","commits_url":"https://api.github.com/repos/andrewor14/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/andrewor14/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/andrewor14/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/andrewor14/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/andrewor14/spark/contents/{+path}","compare_url":"https://api.github.com/repos/andrewor14/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/andrewor14/spark/merges","archive_url":"https://api.github.com/repos/andrewor14/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/andrewor14/spark/downloads","issues_url":"https://api.github.com/repos/andrewor14/spark/issues{/number}","pulls_url":"https://api.github.com/repos/andrewor14/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/andrewor14/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/andrewor14/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/andrewor14/spark/labels{/name}","releases_url":"https://api.github.com/repos/andrewor14/spark/releases{/id}","deployments_url":"https://api.github.com/repos/andrewor14/spark/deployments","created_at":"2014-02-27T00:52:37Z","updated_at":"2015-05-08T06:53:12Z","pushed_at":"2016-03-16T22:03:19Z","git_url":"git://github.com/andrewor14/spark.git","ssh_url":"git@github.com:andrewor14/spark.git","clone_url":"https://github.com/andrewor14/spark.git","svn_url":"https://github.com/andrewor14/spark","homepage":null,"size":162226,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":1,"mirror_url":null,"open_issues_count":0,"forks":1,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"27e1f38851a8f28a28544b2021b3c5641d0ff3ab","user":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":17165658,"name":"spark","full_name":"apache/spark","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/spark","description":"Mirror of Apache Spark","fork":false,"url":"https://api.github.com/repos/apache/spark","forks_url":"https://api.github.com/repos/apache/spark/forks","keys_url":"https://api.github.com/repos/apache/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/spark/teams","hooks_url":"https://api.github.com/repos/apache/spark/hooks","issue_events_url":"https://api.github.com/repos/apache/spark/issues/events{/number}","events_url":"https://api.github.com/repos/apache/spark/events","assignees_url":"https://api.github.com/repos/apache/spark/assignees{/user}","branches_url":"https://api.github.com/repos/apache/spark/branches{/branch}","tags_url":"https://api.github.com/repos/apache/spark/tags","blobs_url":"https://api.github.com/repos/apache/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/spark/languages","stargazers_url":"https://api.github.com/repos/apache/spark/stargazers","contributors_url":"https://api.github.com/repos/apache/spark/contributors","subscribers_url":"https://api.github.com/repos/apache/spark/subscribers","subscription_url":"https://api.github.com/repos/apache/spark/subscription","commits_url":"https://api.github.com/repos/apache/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/spark/contents/{+path}","compare_url":"https://api.github.com/repos/apache/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/spark/merges","archive_url":"https://api.github.com/repos/apache/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/spark/downloads","issues_url":"https://api.github.com/repos/apache/spark/issues{/number}","pulls_url":"https://api.github.com/repos/apache/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/spark/labels{/name}","releases_url":"https://api.github.com/repos/apache/spark/releases{/id}","deployments_url":"https://api.github.com/repos/apache/spark/deployments","created_at":"2014-02-25T08:00:08Z","updated_at":"2016-03-16T21:38:59Z","pushed_at":"2016-03-16T22:02:55Z","git_url":"git://github.com/apache/spark.git","ssh_url":"git@github.com:apache/spark.git","clone_url":"https://github.com/apache/spark.git","svn_url":"https://github.com/apache/spark","homepage":null,"size":445639,"stargazers_count":7817,"watchers_count":7817,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":7172,"mirror_url":"git://git.apache.org/spark.git","open_issues_count":399,"forks":7172,"open_issues":399,"watchers":7817,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/11750"},"html":{"href":"https://github.com/apache/spark/pull/11750"},"issue":{"href":"https://api.github.com/repos/apache/spark/issues/11750"},"comments":{"href":"https://api.github.com/repos/apache/spark/issues/11750/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/spark/pulls/11750/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/spark/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/spark/pulls/11750/commits"},"statuses":{"href":"https://api.github.com/repos/apache/spark/statuses/ad43a5ffdeeb881aaed8944971b63a27d1f4257f"}}}}