diff --git a/common/utils/src/main/resources/error/error-conditions.json b/common/utils/src/main/resources/error/error-conditions.json index c0d15f96892d8..45f629e756e9e 100644 --- a/common/utils/src/main/resources/error/error-conditions.json +++ b/common/utils/src/main/resources/error/error-conditions.json @@ -1929,6 +1929,12 @@ ], "sqlState" : "23505" }, + "DUPLICATE_PATH_ENTRY" : { + "message" : [ + "Duplicate path entry . The session path cannot contain the same catalog.schema more than once (including after expanding shortcuts like DEFAULT_PATH or SYSTEM_PATH)." + ], + "sqlState" : "42P10" + }, "DUPLICATE_ROUTINE_PARAMETER_ASSIGNMENT" : { "message" : [ "Call to routine is invalid because it includes multiple argument assignments to the same parameter name ." @@ -7615,6 +7621,11 @@ "Cannot have VARIANT type columns in DataFrame which calls set operations (INTERSECT, EXCEPT, etc.), but the type of column is ." ] }, + "SET_PATH_VIA_SET" : { + "message" : [ + "The session path cannot be set using the SET statement. Use SET PATH = ... instead." + ] + }, "SET_PROPERTIES_AND_DBPROPERTIES" : { "message" : [ "set PROPERTIES and DBPROPERTIES at the same time." diff --git a/docs/sql-ref-ansi-compliance.md b/docs/sql-ref-ansi-compliance.md index c40d9c95d61c2..1157e47411e07 100644 --- a/docs/sql-ref-ansi-compliance.md +++ b/docs/sql-ref-ansi-compliance.md @@ -477,7 +477,10 @@ Below is a list of all the keywords in Spark SQL. |CROSS|reserved|strict-non-reserved|reserved| |CUBE|non-reserved|non-reserved|reserved| |CURRENT|non-reserved|non-reserved|reserved| +|CURRENT_DATABASE|non-reserved|non-reserved|non-reserved| |CURRENT_DATE|reserved|non-reserved|reserved| +|CURRENT_PATH|non-reserved|non-reserved|reserved| +|CURRENT_SCHEMA|non-reserved|non-reserved|non-reserved| |CURRENT_TIME|reserved|non-reserved|reserved| |CURRENT_TIMESTAMP|reserved|non-reserved|reserved| |CURRENT_USER|reserved|non-reserved|reserved| @@ -500,6 +503,7 @@ Below is a list of all the keywords in Spark SQL. |DEFAULT|non-reserved|non-reserved|non-reserved| |DEFINED|non-reserved|non-reserved|non-reserved| |DEFINER|non-reserved|non-reserved|non-reserved| +|DEFAULT_PATH|non-reserved|non-reserved|not a keyword| |DELAY|non-reserved|non-reserved|non-reserved| |DELETE|non-reserved|non-reserved|reserved| |DELIMITED|non-reserved|non-reserved|non-reserved| @@ -667,6 +671,7 @@ Below is a list of all the keywords in Spark SQL. |PARTITION|non-reserved|non-reserved|reserved| |PARTITIONED|non-reserved|non-reserved|non-reserved| |PARTITIONS|non-reserved|non-reserved|non-reserved| +|PATH|non-reserved|non-reserved|not a keyword| |PERCENT|non-reserved|non-reserved|non-reserved| |PIVOT|non-reserved|non-reserved|non-reserved| |PLACING|non-reserved|non-reserved|non-reserved| @@ -750,6 +755,7 @@ Below is a list of all the keywords in Spark SQL. |SUBSTR|non-reserved|non-reserved|non-reserved| |SUBSTRING|non-reserved|non-reserved|non-reserved| |SYNC|non-reserved|non-reserved|non-reserved| +|SYSTEM_PATH|non-reserved|non-reserved|not a keyword| |SYSTEM_TIME|non-reserved|non-reserved|non-reserved| |SYSTEM_VERSION|non-reserved|non-reserved|non-reserved| |TABLE|reserved|non-reserved|reserved| diff --git a/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseLexer.g4 b/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseLexer.g4 index 177695e38aacc..ae02485c4e6f7 100644 --- a/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseLexer.g4 +++ b/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseLexer.g4 @@ -196,7 +196,10 @@ CREATE: 'CREATE'; CROSS: 'CROSS'; CUBE: 'CUBE'; CURRENT: 'CURRENT'; +CURRENT_DATABASE: 'CURRENT_DATABASE'; CURRENT_DATE: 'CURRENT_DATE'; +CURRENT_PATH: 'CURRENT_PATH'; +CURRENT_SCHEMA: 'CURRENT_SCHEMA'; CURRENT_TIME: 'CURRENT_TIME'; CURRENT_TIMESTAMP: 'CURRENT_TIMESTAMP'; CURRENT_USER: 'CURRENT_USER'; @@ -217,6 +220,7 @@ DEC: 'DEC'; DECIMAL: 'DECIMAL'; DECLARE: 'DECLARE'; DEFAULT: 'DEFAULT'; +DEFAULT_PATH: 'DEFAULT_PATH'; DEFINED: 'DEFINED'; DEFINER: 'DEFINER'; DELAY: 'DELAY'; @@ -385,6 +389,7 @@ OVERWRITE: 'OVERWRITE'; PARTITION: 'PARTITION'; PARTITIONED: 'PARTITIONED'; PARTITIONS: 'PARTITIONS'; +PATH: 'PATH'; PERCENTLIT: 'PERCENT'; PIVOT: 'PIVOT'; PLACING: 'PLACING'; @@ -470,6 +475,7 @@ SUBSTRING: 'SUBSTRING'; SYNC: 'SYNC'; SYSTEM_TIME: 'SYSTEM_TIME'; SYSTEM_VERSION: 'SYSTEM_VERSION'; +SYSTEM_PATH: 'SYSTEM_PATH'; TABLE: 'TABLE'; TABLES: 'TABLES'; TABLESAMPLE: 'TABLESAMPLE'; diff --git a/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 b/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 index d79131b1caf98..1a64d46e32f3c 100644 --- a/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 +++ b/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 @@ -446,6 +446,7 @@ setResetStatement | SET TIME ZONE interval #setTimeZone | SET TIME ZONE timezone #setTimeZone | SET TIME ZONE .*? #setTimeZone + | SET PATH EQ pathElement (COMMA pathElement)* #setPath | SET variable assignmentList #setVariable | SET variable LEFT_PAREN multipartIdentifierList RIGHT_PAREN EQ LEFT_PAREN query RIGHT_PAREN #setVariable @@ -457,6 +458,15 @@ setResetStatement | RESET .*? #resetConfiguration ; +pathElement + : DEFAULT_PATH + | SYSTEM_PATH + | PATH + | CURRENT_DATABASE + | CURRENT_SCHEMA + | multipartIdentifier + ; + executeImmediate : EXECUTE IMMEDIATE queryParam=expression (INTO targetVariable=multipartIdentifierList)? executeImmediateUsing? ; @@ -1276,7 +1286,7 @@ datetimeUnit ; primaryExpression - : name=(CURRENT_DATE | CURRENT_TIMESTAMP | CURRENT_USER | USER | SESSION_USER | CURRENT_TIME) #currentLike + : name=(CURRENT_DATABASE | CURRENT_DATE | CURRENT_PATH | CURRENT_SCHEMA | CURRENT_TIME | CURRENT_TIMESTAMP | CURRENT_USER | USER | SESSION_USER) #currentLike | name=(TIMESTAMPADD | DATEADD | DATE_ADD) LEFT_PAREN (unit=datetimeUnit | invalidUnit=stringLit) COMMA unitsAmount=valueExpression COMMA timestamp=valueExpression RIGHT_PAREN #timestampadd | name=(TIMESTAMPDIFF | DATEDIFF | DATE_DIFF | TIMEDIFF) LEFT_PAREN (unit=datetimeUnit | invalidUnit=stringLit) COMMA startTimestamp=valueExpression COMMA endTimestamp=valueExpression RIGHT_PAREN #timestampdiff | CASE whenClause+ (ELSE elseExpression=expression)? END #searchedCase @@ -1939,6 +1949,9 @@ ansiNonReserved | CURSOR | CUBE | CURRENT + | CURRENT_DATABASE + | CURRENT_PATH + | CURRENT_SCHEMA | DATA | DATABASE | DATABASES @@ -1957,6 +1970,7 @@ ansiNonReserved | DEFAULT | DEFINED | DEFINER + | DEFAULT_PATH | DELAY | DELETE | DELIMITED @@ -2088,6 +2102,7 @@ ansiNonReserved | PARTITION | PARTITIONED | PARTITIONS + | PATH | PERCENTLIT | PIVOT | PLACING @@ -2163,6 +2178,7 @@ ansiNonReserved | SUBSTR | SUBSTRING | SYNC + | SYSTEM_PATH | SYSTEM_TIME | SYSTEM_VERSION | TABLES @@ -2316,7 +2332,10 @@ nonReserved | CUBE | CURRENT | CURSOR + | CURRENT_DATABASE | CURRENT_DATE + | CURRENT_PATH + | CURRENT_SCHEMA | CURRENT_TIME | CURRENT_TIMESTAMP | CURRENT_USER @@ -2336,6 +2355,7 @@ nonReserved | DECIMAL | DECLARE | DEFAULT + | DEFAULT_PATH | DEFINED | DEFINER | DELAY @@ -2507,6 +2527,7 @@ nonReserved | PROCEDURES | PROPERTIES | PURGE + | PATH | QUARTER | QUERY | RANGE @@ -2576,6 +2597,7 @@ nonReserved | SUBSTR | SUBSTRING | SYNC + | SYSTEM_PATH | SYSTEM_TIME | SYSTEM_VERSION | TABLE diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index faf6a5e07ed1a..b0b15a260218a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -290,13 +290,20 @@ object Analyzer { */ class Analyzer( override val catalogManager: CatalogManager, - private[sql] val sharedRelationCache: RelationCache = RelationCache.empty) + private[sql] val sharedRelationCache: RelationCache = RelationCache.empty, + private[sql] val sessionConf: Option[SQLConf] = None) extends RuleExecutor[LogicalPlan] with CheckAnalysis with AliasHelper with SQLConfHelper with ColumnResolutionHelper { + /** Conf to use for path-based resolution and error messages; uses session conf when available. */ + private[sql] def resolutionConf: SQLConf = sessionConf.getOrElse(SQLConf.get) + + override protected def confForRoutineResolution: SQLConf = resolutionConf + private val v1SessionCatalog: SessionCatalog = catalogManager.v1SessionCatalog private val relationResolution = new RelationResolution(catalogManager, sharedRelationCache) - private val functionResolution = new FunctionResolution(catalogManager, relationResolution) + private val functionResolution = new FunctionResolution(catalogManager, relationResolution, + resolutionConf) override protected def validatePlanChanges( previousPlan: LogicalPlan, @@ -317,20 +324,22 @@ class Analyzer( if (plan.analyzed) { plan } else { + def runAnalysis(): LogicalPlan = HybridAnalyzer.fromLegacyAnalyzer( + legacyAnalyzer = this, tracker = tracker).apply(plan) + def runWithConf(): LogicalPlan = sessionConf match { + case Some(c) => SQLConf.withExistingConf(c) { runAnalysis() } + case None => runAnalysis() + } if (AnalysisContext.get.isDefault) { AnalysisContext.reset() try { - AnalysisHelper.markInAnalyzer { - HybridAnalyzer.fromLegacyAnalyzer(legacyAnalyzer = this, tracker = tracker).apply(plan) - } + AnalysisHelper.markInAnalyzer { runWithConf() } } finally { AnalysisContext.reset() } } else { AnalysisContext.withNewAnalysisContext { - AnalysisHelper.markInAnalyzer { - HybridAnalyzer.fromLegacyAnalyzer(legacyAnalyzer = this, tracker = tracker).apply(plan) - } + AnalysisHelper.markInAnalyzer { runWithConf() } } } } @@ -2063,7 +2072,9 @@ class Analyzer( case FunctionType.NotFound => val catalogPath = catalogManager.currentCatalog.name +: catalogManager.currentNamespace - val searchPath = SQLConf.get.resolutionSearchPath(catalogPath.toSeq) + val pathEntries = resolutionConf.effectivePathEntries + .getOrElse(Seq(catalogPath.toSeq)) + val searchPath = resolutionConf.resolutionSearchPath(pathEntries) .map(_.quoted) throw QueryCompilationErrors.unresolvedRoutineError( nameParts, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala index d6613cebb2202..9e6b28ea16eb9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala @@ -43,6 +43,11 @@ trait CheckAnalysis extends LookupCatalog with QueryErrorsBase with PlanToString protected def isView(nameParts: Seq[String]): Boolean + protected def conf: org.apache.spark.sql.internal.SQLConf + + /** Conf for routine resolution/errors; override in Analyzer to use session conf. */ + protected def confForRoutineResolution: SQLConf = conf + import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._ /** @@ -304,7 +309,9 @@ trait CheckAnalysis extends LookupCatalog with QueryErrorsBase with PlanToString case u: UnresolvedFunctionName => val catalogPath = currentCatalog.name +: catalogManager.currentNamespace - val searchPath = SQLConf.get.resolutionSearchPath(catalogPath.toSeq) + val pathEntries = confForRoutineResolution.effectivePathEntries + .getOrElse(Seq(catalogPath.toSeq)) + val searchPath = confForRoutineResolution.resolutionSearchPath(pathEntries) .map(_.quoted) throw QueryCompilationErrors.unresolvedRoutineError( u.multipartIdentifier, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala index cadb7750c2460..ef7b25208928c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala @@ -854,6 +854,7 @@ object FunctionRegistry { expression[CurrentDatabase]("current_database"), expression[CurrentDatabase]("current_schema", true, Some("3.4.0")), expression[CurrentCatalog]("current_catalog"), + expression[CurrentPath]("current_path", true, Some("4.2.0")), expression[CurrentUser]("current_user"), expression[CurrentUser]("user", true, Some("3.4.0")), expression[CurrentUser]("session_user", true, Some("4.0.0")), diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionResolution.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionResolution.scala index 543979102e0ac..3f9b603e75566 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionResolution.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionResolution.scala @@ -56,13 +56,13 @@ import org.apache.spark.sql.connector.catalog.functions.{ UnboundFunction } import org.apache.spark.sql.errors.{DataTypeErrorsBase, QueryCompilationErrors} -import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.connector.V1Function import org.apache.spark.sql.types._ class FunctionResolution( override val catalogManager: CatalogManager, - relationResolution: RelationResolution) + relationResolution: RelationResolution, + conf: org.apache.spark.sql.internal.SQLConf) extends DataTypeErrorsBase with LookupCatalog with Logging { private val v1SessionCatalog = catalogManager.v1SessionCatalog @@ -94,12 +94,13 @@ class FunctionResolution( */ private def resolutionCandidates(nameParts: Seq[String]): Seq[Seq[String]] = { if (nameParts.size == 1) { - val searchPath = SQLConf.get.resolutionSearchPath(currentCatalogPath) + val pathEntries = conf.effectivePathEntries.getOrElse(Seq(currentCatalogPath)) + val searchPath = conf.resolutionSearchPath(pathEntries) searchPath.map(_ ++ nameParts) } else if (nameParts.size == 2 && FunctionResolution.sessionNamespaceKind(nameParts).isDefined) { val systemCandidate = CatalogManager.SYSTEM_CATALOG_NAME +: nameParts - if (SQLConf.get.prioritizeSystemCatalog) { + if (conf.prioritizeSystemCatalog) { Seq(systemCandidate, nameParts) } else { Seq(nameParts, systemCandidate) @@ -174,7 +175,8 @@ class FunctionResolution( case None => } } - val searchPath = SQLConf.get.resolutionSearchPath(currentCatalogPath) + val pathEntries = conf.effectivePathEntries.getOrElse(Seq(currentCatalogPath)) + val searchPath = conf.resolutionSearchPath(pathEntries) throw QueryCompilationErrors.unresolvedRoutineError( unresolvedFunc.nameParts, searchPath.map(toSQLId), unresolvedFunc.origin) } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/HybridAnalyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/HybridAnalyzer.scala index 3fc6438597300..94c3c24cbb1a8 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/HybridAnalyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/HybridAnalyzer.scala @@ -378,6 +378,7 @@ object HybridAnalyzer { extensions = legacyAnalyzer.singlePassResolverExtensions, metadataResolverExtensions = legacyAnalyzer.singlePassMetadataResolverExtensions, externalRelationResolution = Some(relationResolution), + conf = legacyAnalyzer.resolutionConf, extendedRewriteRules = legacyAnalyzer.singlePassPostHocResolutionRules, tracker = Some(tracker) ), diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/Resolver.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/Resolver.scala index 7b6f0441a9433..9e54164a1e7ee 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/Resolver.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/Resolver.scala @@ -54,6 +54,7 @@ import org.apache.spark.sql.catalyst.trees.CurrentOrigin import org.apache.spark.sql.catalyst.util.EvaluateUnresolvedInlineTable import org.apache.spark.sql.connector.catalog.CatalogManager import org.apache.spark.sql.errors.QueryCompilationErrors +import org.apache.spark.sql.internal.SQLConf /** * The Resolver implements a single-pass bottom-up analysis algorithm in the Catalyst. @@ -84,6 +85,7 @@ class Resolver( override val extensions: Seq[ResolverExtension] = Seq.empty, metadataResolverExtensions: Seq[ResolverExtension] = Seq.empty, externalRelationResolution: Option[RelationResolution] = None, + conf: SQLConf = SQLConf.get, extendedRewriteRules: Seq[Rule[LogicalPlan]] = Seq.empty, tracker: Option[QueryPlanningTracker] = None) extends LogicalPlanResolver @@ -102,7 +104,7 @@ class Resolver( private val relationResolution = externalRelationResolution.getOrElse { Resolver.createRelationResolution(catalogManager, sharedRelationCache) } - private val functionResolution = new FunctionResolution(catalogManager, relationResolution) + private val functionResolution = new FunctionResolution(catalogManager, relationResolution, conf) private val expressionResolver = new ExpressionResolver(this, functionResolution, planLogger) private val aggregateResolver = new AggregateResolver(this, expressionResolver) private val expressionIdAssigner = expressionResolver.getExpressionIdAssigner diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolverGuard.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolverGuard.scala index e2171b84b6eb2..6dce5c67cc76c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolverGuard.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/resolver/ResolverGuard.scala @@ -474,10 +474,10 @@ class ResolverGuard( private def checkUnresolvedFunction(unresolvedFunction: UnresolvedFunction) = { val nameParts = unresolvedFunction.nameParts val funcName = nameParts.last.toLowerCase(Locale.ROOT) - - if (nameParts.length == 1) { - // Unqualified: same as master (unsupported, non-builtin, or check children) - if (isUnsupportedFunction(funcName)) { + if (nameParts.size == 1) { + // Unqualified: reject if unsupported, else non-builtin or check children (same as master) + if (ResolverGuard.UNSUPPORTED_FUNCTION_NAMES.contains(funcName) || + isUnsupportedFunction(funcName)) { Some(s"unsupported function ${funcName}") } else if (!isBuiltinFunction(funcName)) { Some("non-builtin function") @@ -493,7 +493,7 @@ class ResolverGuard( unresolvedFunction.children.collectFirst { case CheckExpression(reason) => reason } } } else if (FunctionResolution.sessionNamespaceKind(nameParts).isDefined) { - // Session-qualified: allow through (system-first behavior) + // Session-qualified: allow through (PATH + system-first) unresolvedFunction.children.collectFirst { case CheckExpression(reason) => reason } } else { Some("multi-part function name") diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala index 6f806760b3736..c91d700e7a625 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala @@ -234,6 +234,28 @@ case class CurrentCatalog() final override val nodePatterns: Seq[TreePattern] = Seq(CURRENT_LIKE) } +/** + * Returns the current SQL path as a comma-separated list of qualified schema names + * (catalog.schema). Responsive to USE SCHEMA / USE CATALOG when PATH feature is enabled. + */ +@ExpressionDescription( + usage = "_FUNC_() - Returns the current SQL path (qualified schema names).", + examples = """ + Examples: + > SELECT _FUNC_(); + system.builtin,system.session,spark_catalog.default + """, + since = "4.2.0", + group = "misc_funcs") +case class CurrentPath() + extends LeafExpression + with DefaultStringProducingExpression + with Unevaluable { + override def nullable: Boolean = false + override def prettyName: String = "current_path" + final override val nodePatterns: Seq[TreePattern] = Seq(CURRENT_LIKE) +} + // scalastyle:off line.size.limit @ExpressionDescription( usage = """_FUNC_() - Returns an universally unique identifier (UUID) string. The value is returned as a canonical UUID 36-character string.""", diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala index ad4769ff8e31a..a565ca69ff889 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala @@ -329,7 +329,7 @@ abstract class Optimizer(catalogManager: CatalogManager) InsertMapSortInGroupingExpressions, InsertMapSortInRepartitionExpressions, ComputeCurrentTime, - ReplaceCurrentLike(catalogManager), + ReplaceCurrentLike(catalogManager, conf), SpecialDatetimeValues, RewriteAsOfJoin, EvalInlineTables, diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala index c9c26d473b982..4024fe5e8802b 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/finishAnalysis.scala @@ -148,21 +148,27 @@ object ComputeCurrentTime extends Rule[LogicalPlan] { } /** - * Replaces the expression of CurrentDatabase, CurrentCatalog, and CurrentUser + * Replaces the expression of CurrentDatabase, CurrentCatalog, CurrentPath, and CurrentUser * with the current values. */ -case class ReplaceCurrentLike(catalogManager: CatalogManager) extends Rule[LogicalPlan] { +case class ReplaceCurrentLike( + catalogManager: CatalogManager, + sqlConf: org.apache.spark.sql.internal.SQLConf) extends Rule[LogicalPlan] { def apply(plan: LogicalPlan): LogicalPlan = { import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._ lazy val currentNamespace = catalogManager.currentNamespace.quoted + lazy val currentNamespaceSeq = catalogManager.currentNamespace.toSeq lazy val currentCatalog = catalogManager.currentCatalog.name() lazy val currentUser = CurrentUserContext.getCurrentUser + lazy val currentPathStr = sqlConf.currentPathString(currentCatalog, currentNamespaceSeq) plan.transformAllExpressionsWithPruning(_.containsPattern(CURRENT_LIKE)) { case CurrentDatabase() => Literal.create(currentNamespace, StringType) case CurrentCatalog() => Literal.create(currentCatalog, StringType) + case CurrentPath() => + Literal.create(currentPathStr, StringType) case CurrentUser() => Literal.create(currentUser, StringType) } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index 1dd911e02de9d..300206e342ec9 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -3088,14 +3088,22 @@ class AstBuilder extends DataTypeAstBuilder CurrentTimestamp() case SqlBaseParser.CURRENT_TIME => CurrentTime() + case SqlBaseParser.CURRENT_PATH => + CurrentPath() + case SqlBaseParser.CURRENT_DATABASE | SqlBaseParser.CURRENT_SCHEMA => + CurrentDatabase() case SqlBaseParser.CURRENT_USER | SqlBaseParser.USER | SqlBaseParser.SESSION_USER => CurrentUser() } } else { - // If the parser is not in ansi mode, we should return `UnresolvedAttribute`, in case there - // are columns named `CURRENT_DATE` or `CURRENT_TIMESTAMP` or `CURRENT_TIME`. - // ctx.name is a token, not an identifier context. - UnresolvedAttribute.quoted(ctx.name.getText) + ctx.name.getType match { + case SqlBaseParser.CURRENT_PATH => + CurrentPath() + case SqlBaseParser.CURRENT_DATABASE | SqlBaseParser.CURRENT_SCHEMA => + CurrentDatabase() + case _ => + UnresolvedAttribute.quoted(ctx.name.getText) + } } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 71438536d1125..dcf409ba29bb5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -2386,6 +2386,57 @@ object SQLConf { .booleanConf .createWithDefault(false) + val PATH_ENABLED = + buildConf("spark.sql.path.enabled") + .version("4.2.0") + .doc("When true, enables the SQL Standard PATH feature: SET PATH, path-based routine " + + "resolution, and CURRENT_PATH(). When false, SET PATH has no effect and resolution uses " + + "the default path only.") + .withBindingPolicy(ConfigBindingPolicy.SESSION) + .booleanConf + .createWithDefault(false) + + val SESSION_PATH = + buildConf("spark.sql.session.path") + .internal() + .version("4.2.0") + .doc("Session search path for routine resolution (catalog-qualified schema list). " + + "Only settable via SET PATH statement; direct SET of this config is ignored.") + .withBindingPolicy(ConfigBindingPolicy.SESSION) + .stringConf + .createOptional + + /** + * Separator between path entries in serialized session path + * (catalog.namespace,catalog.namespace). + */ + private[sql] val SESSION_PATH_ENTRY_SEPARATOR = "," + + /** + * Parses a session path string into a list of path entries. + * Format: "catalog1.namespace1,catalog2.namespace2" + * (comma-separated, each entry catalog.namespace). + */ + private[sql] def parseSessionPath(pathStr: String): Seq[Seq[String]] = { + if (pathStr == null || pathStr.trim.isEmpty) return Seq.empty + pathStr.split(SESSION_PATH_ENTRY_SEPARATOR).map { entry => + val trimmed = entry.trim + val dot = trimmed.indexOf('.') + if (dot <= 0 || dot == trimmed.length - 1) { + Seq(trimmed) // single part, treat as namespace only (caller may qualify) + } else { + Seq(trimmed.substring(0, dot).trim, trimmed.substring(dot + 1).trim) + } + }.toSeq.filter(_.nonEmpty) + } + + /** + * Formats path entries to session path string. + * Each entry is catalog.namespace; entries separated by comma. + */ + private[sql] def formatSessionPath(pathEntries: Seq[Seq[String]]): String = + pathEntries.map(_.mkString(".")).mkString(SESSION_PATH_ENTRY_SEPARATOR) + // Whether to retain group by columns or not in GroupedData.agg. val DATAFRAME_RETAIN_GROUP_COLUMNS = buildConf("spark.sql.retainGroupColumns") .version("1.4.0") @@ -8217,26 +8268,55 @@ class SQLConf extends Serializable with Logging with SqlApiConf { */ def prioritizeSystemCatalog: Boolean = !getConf(SQLConf.PERSISTENT_CATALOG_FIRST) + def pathEnabled: Boolean = getConf(SQLConf.PATH_ENABLED) + + def sessionPath: Option[String] = getConf(SQLConf.SESSION_PATH) + + /** + * Returns the session path as path entries when PATH is enabled and set; None otherwise. + * Callers should use effectivePathEntries().getOrElse(Seq(currentCatalogPath)). + */ + def effectivePathEntries: Option[Seq[Seq[String]]] = + if (pathEnabled) sessionPath.map(SQLConf.parseSessionPath).filter(_.nonEmpty) + else None + + /** + * Returns the current path as a comma-separated string of qualified schema names + * (catalog.schema). Used by CURRENT_PATH(). When PATH is disabled, returns default path. + */ + def currentPathString(currentCatalog: String, currentNamespace: Seq[String]): String = { + val defaultEntry = + if (currentNamespace.isEmpty) Seq(currentCatalog) else currentCatalog +: currentNamespace + val pathEntries = effectivePathEntries.getOrElse(Seq(defaultEntry)) + val fullPath = resolutionSearchPath(pathEntries) + SQLConf.formatSessionPath(fullPath) + } + /** * Returns the resolution search path for error messages and resolution order. * This is the single source of truth for the search path used for functions, tables, and views. * Uses [[sessionFunctionResolutionOrder]]: "first" (session first), "second" (session second), - * "last" (session last). When catalogPath is empty, returns only system namespaces. + * "last" (session last). When pathEntries is empty, returns only system namespaces. + * + * @param pathEntries Ordered list of persistent path entries (each entry is catalog + namespace, + * e.g. Seq("system", "builtin") or Seq("spark_catalog", "default")). + * When PATH feature is disabled or not set, callers pass + * Seq(currentCatalogPath). */ - def resolutionSearchPath(catalogPath: Seq[String]): Seq[Seq[String]] = { + def resolutionSearchPath(pathEntries: Seq[Seq[String]]): Seq[Seq[String]] = { val order = sessionFunctionResolutionOrder val session = Seq("system", "session") val builtin = Seq("system", "builtin") order match { case "first" => - if (catalogPath.isEmpty) Seq(session, builtin) - else Seq(session, builtin, catalogPath) + if (pathEntries.isEmpty) Seq(session, builtin) + else Seq(session, builtin) ++ pathEntries case "last" => - if (catalogPath.isEmpty) Seq(builtin, session) - else Seq(builtin, catalogPath, session) + if (pathEntries.isEmpty) Seq(builtin, session) + else Seq(builtin) ++ pathEntries ++ Seq(session) case _ => // "second" - if (catalogPath.isEmpty) Seq(builtin, session) - else Seq(builtin, session, catalogPath) + if (pathEntries.isEmpty) Seq(builtin, session) + else Seq(builtin, session) ++ pathEntries } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/resolver/TimezoneAwareExpressionResolverSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/resolver/TimezoneAwareExpressionResolverSuite.scala index 8897d65654540..1f4b3dea81b31 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/resolver/TimezoneAwareExpressionResolverSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/resolver/TimezoneAwareExpressionResolverSuite.scala @@ -37,7 +37,10 @@ class TimezoneAwareExpressionResolverSuite extends SparkFunSuite { extends ExpressionResolver( resolver = new Resolver(catalogManager), functionResolution = - new FunctionResolution(catalogManager, Resolver.createRelationResolution(catalogManager)), + new FunctionResolution( + catalogManager, + Resolver.createRelationResolution(catalogManager), + org.apache.spark.sql.internal.SQLConf.get), planLogger = new PlanLogger ) { override def resolve(expression: Expression): Expression = resolvedExpression diff --git a/sql/connect/client/jdbc/src/test/scala/org/apache/spark/sql/connect/client/jdbc/SparkConnectDatabaseMetaDataSuite.scala b/sql/connect/client/jdbc/src/test/scala/org/apache/spark/sql/connect/client/jdbc/SparkConnectDatabaseMetaDataSuite.scala index 20ca9698c6fb2..fbaee3cf87363 100644 --- a/sql/connect/client/jdbc/src/test/scala/org/apache/spark/sql/connect/client/jdbc/SparkConnectDatabaseMetaDataSuite.scala +++ b/sql/connect/client/jdbc/src/test/scala/org/apache/spark/sql/connect/client/jdbc/SparkConnectDatabaseMetaDataSuite.scala @@ -209,7 +209,7 @@ class SparkConnectDatabaseMetaDataSuite extends ConnectFunSuite with RemoteSpark withConnection { conn => val metadata = conn.getMetaData // scalastyle:off line.size.limit - assert(metadata.getSQLKeywords === "ADD,AFTER,AGGREGATE,ALWAYS,ANALYZE,ANTI,ANY_VALUE,ARCHIVE,ASC,BINDING,BUCKET,BUCKETS,BYTE,CACHE,CASCADE,CATALOG,CATALOGS,CHANGE,CLEAR,CLUSTER,CLUSTERED,CODEGEN,COLLATION,COLLECTION,COLUMNS,COMMENT,COMPACT,COMPACTIONS,COMPENSATION,COMPUTE,CONCATENATE,CONTAINS,CONTINUE,COST,DATA,DATABASE,DATABASES,DATEADD,DATEDIFF,DATE_ADD,DATE_DIFF,DAYOFYEAR,DAYS,DBPROPERTIES,DEFINED,DEFINER,DELAY,DELIMITED,DESC,DFS,DIRECTORIES,DIRECTORY,DISTRIBUTE,DIV,DO,ELSEIF,ENFORCED,ESCAPED,EVOLUTION,EXCHANGE,EXCLUDE,EXIT,EXPLAIN,EXPORT,EXTEND,EXTENDED,FIELDS,FILEFORMAT,FIRST,FLOW,FOLLOWING,FORMAT,FORMATTED,FOUND,FUNCTIONS,GENERATED,GEOGRAPHY,GEOMETRY,HANDLER,HOURS,IDENTIFIED,IDENTIFIER,IF,IGNORE,ILIKE,IMMEDIATE,INCLUDE,INCREMENT,INDEX,INDEXES,INPATH,INPUT,INPUTFORMAT,INVOKER,ITEMS,ITERATE,JSON,KEY,KEYS,LAST,LAZY,LEAVE,LEVEL,LIMIT,LINES,LIST,LOAD,LOCATION,LOCK,LOCKS,LOGICAL,LONG,LOOP,MACRO,MAP,MATCHED,MATERIALIZED,MEASURE,METRICS,MICROSECOND,MICROSECONDS,MILLISECOND,MILLISECONDS,MINUS,MINUTES,MONTHS,MSCK,NAME,NAMESPACE,NAMESPACES,NANOSECOND,NANOSECONDS,NORELY,NULLS,OFFSET,OPTION,OPTIONS,OUTPUTFORMAT,OVERWRITE,PARTITIONED,PARTITIONS,PERCENT,PIVOT,PLACING,PRECEDING,PRINCIPALS,PROCEDURES,PROPERTIES,PURGE,QUARTER,QUERY,RECORDREADER,RECORDWRITER,RECOVER,RECURSION,REDUCE,REFRESH,RELY,RENAME,REPAIR,REPEAT,REPEATABLE,REPLACE,RESET,RESPECT,RESTRICT,ROLE,ROLES,SCHEMA,SCHEMAS,SECONDS,SECURITY,SEMI,SEPARATED,SERDE,SERDEPROPERTIES,SETS,SHORT,SHOW,SINGLE,SKEWED,SORT,SORTED,SOURCE,STATISTICS,STORED,STRATIFY,STREAM,STREAMING,STRING,STRUCT,SUBSTR,SYNC,SYSTEM_TIME,SYSTEM_VERSION,TABLES,TARGET,TBLPROPERTIES,TERMINATED,TIMEDIFF,TIMESTAMPADD,TIMESTAMPDIFF,TIMESTAMP_LTZ,TIMESTAMP_NTZ,TINYINT,TOUCH,TRANSACTION,TRANSACTIONS,TRANSFORM,TRUNCATE,TRY_CAST,TYPE,UNARCHIVE,UNBOUNDED,UNCACHE,UNLOCK,UNPIVOT,UNSET,UNTIL,USE,VAR,VARIABLE,VARIANT,VERSION,VIEW,VIEWS,VOID,WATERMARK,WEEK,WEEKS,WHILE,X,YEARS,ZONE") + assert(metadata.getSQLKeywords === "ADD,AFTER,AGGREGATE,ALWAYS,ANALYZE,ANTI,ANY_VALUE,ARCHIVE,ASC,BINDING,BUCKET,BUCKETS,BYTE,CACHE,CASCADE,CATALOG,CATALOGS,CHANGE,CLEAR,CLUSTER,CLUSTERED,CODEGEN,COLLATION,COLLECTION,COLUMNS,COMMENT,COMPACT,COMPACTIONS,COMPENSATION,COMPUTE,CONCATENATE,CONTAINS,CONTINUE,COST,CURRENT_DATABASE,CURRENT_SCHEMA,DATA,DATABASE,DATABASES,DATEADD,DATEDIFF,DATE_ADD,DATE_DIFF,DAYOFYEAR,DAYS,DBPROPERTIES,DEFAULT_PATH,DEFINED,DEFINER,DELAY,DELIMITED,DESC,DFS,DIRECTORIES,DIRECTORY,DISTRIBUTE,DIV,DO,ELSEIF,ENFORCED,ESCAPED,EVOLUTION,EXCHANGE,EXCLUDE,EXIT,EXPLAIN,EXPORT,EXTEND,EXTENDED,FIELDS,FILEFORMAT,FIRST,FLOW,FOLLOWING,FORMAT,FORMATTED,FOUND,FUNCTIONS,GENERATED,GEOGRAPHY,GEOMETRY,HANDLER,HOURS,IDENTIFIED,IDENTIFIER,IF,IGNORE,ILIKE,IMMEDIATE,INCLUDE,INCREMENT,INDEX,INDEXES,INPATH,INPUT,INPUTFORMAT,INVOKER,ITEMS,ITERATE,JSON,KEY,KEYS,LAST,LAZY,LEAVE,LEVEL,LIMIT,LINES,LIST,LOAD,LOCATION,LOCK,LOCKS,LOGICAL,LONG,LOOP,MACRO,MAP,MATCHED,MATERIALIZED,MEASURE,METRICS,MICROSECOND,MICROSECONDS,MILLISECOND,MILLISECONDS,MINUS,MINUTES,MONTHS,MSCK,NAME,NAMESPACE,NAMESPACES,NANOSECOND,NANOSECONDS,NORELY,NULLS,OFFSET,OPTION,OPTIONS,OUTPUTFORMAT,OVERWRITE,PARTITIONED,PARTITIONS,PATH,PERCENT,PIVOT,PLACING,PRECEDING,PRINCIPALS,PROCEDURES,PROPERTIES,PURGE,QUARTER,QUERY,RECORDREADER,RECORDWRITER,RECOVER,RECURSION,REDUCE,REFRESH,RELY,RENAME,REPAIR,REPEAT,REPEATABLE,REPLACE,RESET,RESPECT,RESTRICT,ROLE,ROLES,SCHEMA,SCHEMAS,SECONDS,SECURITY,SEMI,SEPARATED,SERDE,SERDEPROPERTIES,SETS,SHORT,SHOW,SINGLE,SKEWED,SORT,SORTED,SOURCE,STATISTICS,STORED,STRATIFY,STREAM,STREAMING,STRING,STRUCT,SUBSTR,SYNC,SYSTEM_PATH,SYSTEM_TIME,SYSTEM_VERSION,TABLES,TARGET,TBLPROPERTIES,TERMINATED,TIMEDIFF,TIMESTAMPADD,TIMESTAMPDIFF,TIMESTAMP_LTZ,TIMESTAMP_NTZ,TINYINT,TOUCH,TRANSACTION,TRANSACTIONS,TRANSFORM,TRUNCATE,TRY_CAST,TYPE,UNARCHIVE,UNBOUNDED,UNCACHE,UNLOCK,UNPIVOT,UNSET,UNTIL,USE,VAR,VARIABLE,VARIANT,VERSION,VIEW,VIEWS,VOID,WATERMARK,WEEK,WEEKS,WHILE,X,YEARS,ZONE") // scalastyle:on line.size.limit } } diff --git a/sql/core/src/main/scala/org/apache/spark/sql/classic/Catalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/classic/Catalog.scala index 251877842bcee..45dd7c4f77e5f 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/classic/Catalog.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/classic/Catalog.scala @@ -368,7 +368,9 @@ class Catalog(sparkSession: SparkSession) extends catalog.Catalog { val catalogPath = (Seq(currentCatalog()) ++ sparkSession.sessionState.catalogManager.currentNamespace).toSeq - val searchPath = sparkSession.sessionState.conf.resolutionSearchPath(catalogPath) + val pathEntries = sparkSession.sessionState.conf.effectivePathEntries + .getOrElse(Seq(catalogPath)) + val searchPath = sparkSession.sessionState.conf.resolutionSearchPath(pathEntries) throw QueryCompilationErrors.unresolvedRoutineError( ident, searchPath.map(_.quoted), diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala index 6c19f53d2dc42..11e6e4a03a7cc 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala @@ -326,6 +326,19 @@ class SparkSqlAstBuilder extends AstBuilder { * SET TIME ZONE INTERVAL 10 HOURS; * }}} */ + override def visitSetPath(ctx: SetPathContext): LogicalPlan = withOrigin(ctx) { + val elements = (0 until ctx.pathElement().size()).map { i => + val pe = ctx.pathElement(i) + if (pe.DEFAULT_PATH() != null) PathElement.DefaultPath + else if (pe.SYSTEM_PATH() != null) PathElement.SystemPath + else if (pe.PATH() != null) PathElement.PathRef + else if (pe.CURRENT_DATABASE() != null) PathElement.CurrentDatabase + else if (pe.CURRENT_SCHEMA() != null) PathElement.CurrentSchema + else PathElement.SchemaInPath(visitMultipartIdentifier(pe.multipartIdentifier())) + } + SetPathCommand(elements.toSeq) + } + override def visitSetTimeZone(ctx: SetTimeZoneContext): LogicalPlan = withOrigin(ctx) { val key = SQLConf.SESSION_LOCAL_TIMEZONE.key if (ctx.interval != null) { diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetCommand.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetCommand.scala index e248f0eea96de..54328d16a0baf 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetCommand.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetCommand.scala @@ -119,6 +119,11 @@ case class SetCommand(kv: Option[(String, Option[String])]) messageParameters = Map("variableName" -> toSQLId(varName))) } } + if (key == SQLConf.SESSION_PATH.key) { + throw new AnalysisException( + errorClass = "UNSUPPORTED_FEATURE.SET_PATH_VIA_SET", + messageParameters = Map.empty) + } if (sparkSession.conf.get(CATALOG_IMPLEMENTATION.key).equals("hive") && key.startsWith("hive.")) { logWarning(log"'SET ${MDC(KEY, key)}=${MDC(VALUE, value)}' might not work, since Spark " + diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetPathCommand.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetPathCommand.scala new file mode 100644 index 0000000000000..a693ab45aab0f --- /dev/null +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/SetPathCommand.scala @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.command + +import org.apache.spark.sql.{AnalysisException, Row, SparkSession} +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.connector.catalog.CatalogManager +import org.apache.spark.sql.internal.SQLConf + +/** + * Path element for SET PATH: either a well-known shortcut or a schema (optionally qualified). + * For SchemaInPath(parts), qualification with current catalog or SYSTEM is done at run time. + */ +sealed trait PathElement + +object PathElement { + case object DefaultPath extends PathElement + case object SystemPath extends PathElement + case object PathRef extends PathElement + /** Current database/schema (same in Spark); expands to current catalog + current namespace. */ + case object CurrentDatabase extends PathElement + case object CurrentSchema extends PathElement + /** Schema name parts (1 = unqualified namespace, 2+ = catalog.namespace...). Qualified at run. */ + case class SchemaInPath(parts: Seq[String]) extends PathElement +} + +/** + * Command for SET PATH = pathElement (, pathElement)* + * Expands shortcuts at run time, validates no duplicates, and sets the internal session path. + */ +case class SetPathCommand(elements: Seq[PathElement]) extends LeafRunnableCommand { + + override def output: Seq[Attribute] = Seq.empty + + override def run(sparkSession: SparkSession): Seq[Row] = { + if (!sparkSession.sessionState.conf.pathEnabled) { + // Feature disabled: no-op (or could throw; plan says reject or no-op) + return Seq.empty + } + val conf = sparkSession.sessionState.conf + val catalogManager = sparkSession.sessionState.catalogManager + val currentCatalog = catalogManager.currentCatalog.name + val currentNamespace = catalogManager.currentNamespace.toSeq + + val expanded = expandPathElements(elements, conf, currentCatalog, currentNamespace) + val seen = new scala.collection.mutable.HashSet[(String, String)] + val deduped = expanded.flatMap { entry => + val key = (entry.head.toLowerCase(java.util.Locale.ROOT), + entry.lift(1).getOrElse("").toLowerCase(java.util.Locale.ROOT)) + if (seen.contains(key)) { + throw new AnalysisException( + errorClass = "DUPLICATE_PATH_ENTRY", + messageParameters = Map("pathEntry" -> entry.mkString("."))) + } + seen.add(key) + Some(entry) + } + + if (deduped.isEmpty) { + conf.unsetConf(SQLConf.SESSION_PATH) + } else { + conf.setConfString(SQLConf.SESSION_PATH.key, SQLConf.formatSessionPath(deduped)) + } + Seq.empty + } + + private def expandPathElements( + elements: Seq[PathElement], + conf: SQLConf, + currentCatalog: String, + currentNamespace: Seq[String]): Seq[Seq[String]] = { + val systemCatalog = CatalogManager.SYSTEM_CATALOG_NAME + val builtin = CatalogManager.BUILTIN_NAMESPACE + val session = CatalogManager.SESSION_NAMESPACE + + elements.flatMap { + case PathElement.DefaultPath => + // Default path = session order (first/second/last). Clear path; use at resolution time. + Seq.empty + case PathElement.SystemPath => + Seq(Seq(systemCatalog, builtin), Seq(systemCatalog, session)) + case PathElement.CurrentDatabase | PathElement.CurrentSchema => + if (currentNamespace.isEmpty) Seq(Seq(currentCatalog)) + else Seq(currentCatalog +: currentNamespace) + case PathElement.PathRef => + conf.sessionPath match { + case Some(s) => SQLConf.parseSessionPath(s) + case None => Seq.empty + } + case PathElement.SchemaInPath(parts) => + qualifySchemaParts(parts, systemCatalog, currentCatalog) + } + } + + /** Qualify schema parts at SET time: well-known -> SYSTEM; else current catalog + namespace. */ + private def qualifySchemaParts( + parts: Seq[String], + systemCatalog: String, + currentCatalog: String): Seq[Seq[String]] = { + val wellKnown = Set( + CatalogManager.BUILTIN_NAMESPACE.toLowerCase(java.util.Locale.ROOT), + CatalogManager.SESSION_NAMESPACE.toLowerCase(java.util.Locale.ROOT)) + if (parts.isEmpty) return Seq.empty + if (parts.size == 1) { + val ns = parts.head + if (wellKnown.contains(ns.toLowerCase(java.util.Locale.ROOT))) { + Seq(Seq(systemCatalog, ns)) + } else { + Seq(Seq(currentCatalog, ns)) + } + } else { + Seq(Seq(parts.head, parts(1))) + } + } + +} diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala index 08dd212060762..f543f0bba90f9 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/BaseSessionStateBuilder.scala @@ -185,7 +185,7 @@ abstract class BaseSessionStateBuilder( * * Note: this depends on the `conf` and `catalog` fields. */ - protected def analyzer: Analyzer = new Analyzer(catalogManager, sharedRelationCache) { + protected def analyzer: Analyzer = new Analyzer(catalogManager, sharedRelationCache, Some(conf)) { override val hintResolutionRules: Seq[Rule[LogicalPlan]] = customHintResolutionRules diff --git a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md index f2c72fa18ed6d..14f36cbae055b 100644 --- a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md +++ b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md @@ -107,6 +107,7 @@ | org.apache.spark.sql.catalyst.expressions.CurrentDatabase | current_database | SELECT current_database() | struct | | org.apache.spark.sql.catalyst.expressions.CurrentDatabase | current_schema | SELECT current_schema() | struct | | org.apache.spark.sql.catalyst.expressions.CurrentDate | current_date | SELECT current_date() | struct | +| org.apache.spark.sql.catalyst.expressions.CurrentPath | current_path | SELECT current_path() | struct | | org.apache.spark.sql.catalyst.expressions.CurrentTime | current_time | SELECT current_time() | struct | | org.apache.spark.sql.catalyst.expressions.CurrentTimeZone | current_timezone | SELECT current_timezone() | struct | | org.apache.spark.sql.catalyst.expressions.CurrentTimestamp | current_timestamp | SELECT current_timestamp() | struct | diff --git a/sql/core/src/test/resources/sql-tests/results/keywords-enforced.sql.out b/sql/core/src/test/resources/sql-tests/results/keywords-enforced.sql.out index 95391bcc1f64c..c264e3f7cf75f 100644 --- a/sql/core/src/test/resources/sql-tests/results/keywords-enforced.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/keywords-enforced.sql.out @@ -72,7 +72,10 @@ CREATE true CROSS true CUBE false CURRENT false +CURRENT_DATABASE false CURRENT_DATE true +CURRENT_PATH false +CURRENT_SCHEMA false CURRENT_TIME true CURRENT_TIMESTAMP true CURRENT_USER true @@ -93,6 +96,7 @@ DEC false DECIMAL false DECLARE false DEFAULT false +DEFAULT_PATH false DEFINED false DEFINER false DELAY false @@ -262,6 +266,7 @@ OVERWRITE false PARTITION false PARTITIONED false PARTITIONS false +PATH false PERCENT false PIVOT false PLACING false @@ -343,6 +348,7 @@ STRUCT false SUBSTR false SUBSTRING false SYNC false +SYSTEM_PATH false SYSTEM_TIME false SYSTEM_VERSION false TABLE true diff --git a/sql/core/src/test/resources/sql-tests/results/keywords.sql.out b/sql/core/src/test/resources/sql-tests/results/keywords.sql.out index 5d5a572bd8fb7..8df0fa256d05e 100644 --- a/sql/core/src/test/resources/sql-tests/results/keywords.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/keywords.sql.out @@ -72,7 +72,10 @@ CREATE false CROSS false CUBE false CURRENT false +CURRENT_DATABASE false CURRENT_DATE false +CURRENT_PATH false +CURRENT_SCHEMA false CURRENT_TIME false CURRENT_TIMESTAMP false CURRENT_USER false @@ -93,6 +96,7 @@ DEC false DECIMAL false DECLARE false DEFAULT false +DEFAULT_PATH false DEFINED false DEFINER false DELAY false @@ -262,6 +266,7 @@ OVERWRITE false PARTITION false PARTITIONED false PARTITIONS false +PATH false PERCENT false PIVOT false PLACING false @@ -343,6 +348,7 @@ STRUCT false SUBSTR false SUBSTRING false SYNC false +SYSTEM_PATH false SYSTEM_TIME false SYSTEM_VERSION false TABLE false diff --git a/sql/core/src/test/resources/sql-tests/results/nonansi/keywords.sql.out b/sql/core/src/test/resources/sql-tests/results/nonansi/keywords.sql.out index 5d5a572bd8fb7..8df0fa256d05e 100644 --- a/sql/core/src/test/resources/sql-tests/results/nonansi/keywords.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/nonansi/keywords.sql.out @@ -72,7 +72,10 @@ CREATE false CROSS false CUBE false CURRENT false +CURRENT_DATABASE false CURRENT_DATE false +CURRENT_PATH false +CURRENT_SCHEMA false CURRENT_TIME false CURRENT_TIMESTAMP false CURRENT_USER false @@ -93,6 +96,7 @@ DEC false DECIMAL false DECLARE false DEFAULT false +DEFAULT_PATH false DEFINED false DEFINER false DELAY false @@ -262,6 +266,7 @@ OVERWRITE false PARTITION false PARTITIONED false PARTITIONS false +PATH false PERCENT false PIVOT false PLACING false @@ -343,6 +348,7 @@ STRUCT false SUBSTR false SUBSTRING false SYNC false +SYSTEM_PATH false SYSTEM_TIME false SYSTEM_VERSION false TABLE false diff --git a/sql/core/src/test/scala/org/apache/spark/sql/FunctionQualificationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/FunctionQualificationSuite.scala index ccd67e8bb850c..f2c12404d18dd 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/FunctionQualificationSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/FunctionQualificationSuite.scala @@ -813,6 +813,12 @@ class FunctionQualificationSuite extends QueryTest with SharedSparkSession { sql("DROP TEMPORARY FUNCTION current_user") } + test("SECTION 11e: current_path() is a builtin and returns path string") { + val pathStr = sql("SELECT current_path()").collect().head.getString(0) + assert(pathStr.nonEmpty && pathStr.contains("."), + s"current_path() should return a non-empty path with qualified schemas, got: $pathStr") + } + test("SECTION 12d: Extension - SHOW FUNCTIONS includes extension functions") { val functions = sql("SHOW FUNCTIONS").collect().map(_.getString(0)) assert(functions.contains("test_ext_func"), diff --git a/sql/core/src/test/scala/org/apache/spark/sql/PathResolutionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/PathResolutionSuite.scala new file mode 100644 index 0000000000000..b6dc5555ea6ad --- /dev/null +++ b/sql/core/src/test/scala/org/apache/spark/sql/PathResolutionSuite.scala @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.test.SharedSparkSession + +/** + * Tests for SQL Standard PATH: SET PATH, CURRENT_PATH(), and path-based routine resolution. + * Covers feature flag, DEFAULT_PATH, CURRENT_PATH() and USE SCHEMA, duplicate/error cases. + */ +class PathResolutionSuite extends QueryTest with SharedSparkSession { + + test("PATH disabled: CURRENT_PATH() returns default path") { + // With path disabled, CURRENT_PATH() still returns the effective path (default) + val pathStr = sql("SELECT current_path()").collect().head.getString(0) + assert(pathStr.contains("spark_catalog") && pathStr.contains("default"), + s"Expected default path to contain spark_catalog.default, got: $pathStr") + } + + test("PATH disabled: SET PATH has no effect") { + sql("SET PATH = spark_catalog.other") + val pathStr = sql("SELECT current_path()").collect().head.getString(0) + // Should still be default (SET PATH was no-op) + assert(pathStr.contains("spark_catalog") && pathStr.contains("default"), + s"SET PATH should have no effect when disabled, got: $pathStr") + } + + test("PATH enabled: SET PATH = DEFAULT_PATH restores default") { + withSQLConf(SQLConf.PATH_ENABLED.key -> "true") { + sql("SET PATH = spark_catalog.default") + sql("SET PATH = DEFAULT_PATH") + val pathStr = sql("SELECT current_path()").collect().head.getString(0) + assert(pathStr.contains("spark_catalog") && pathStr.contains("default"), + s"After SET PATH = DEFAULT_PATH expected default path, got: $pathStr") + } + } + + test("PATH enabled: SET PATH and CURRENT_PATH()") { + withSQLConf(SQLConf.PATH_ENABLED.key -> "true") { + sql("SET PATH = spark_catalog.default") + val pathStr = sql("SELECT current_path()").collect().head.getString(0) + assert(pathStr.contains("spark_catalog.default"), + s"Expected path to contain spark_catalog.default, got: $pathStr") + } + } + + test("PATH enabled: CURRENT_PATH() with DEFAULT_PATH contains current schema") { + withSQLConf(SQLConf.PATH_ENABLED.key -> "true") { + sql("SET PATH = DEFAULT_PATH") + val pathStr = sql("SELECT current_path()").collect().head.getString(0) + assert(pathStr.contains("default"), + s"With DEFAULT_PATH in default schema, path should contain default, got: $pathStr") + } + } + + test("PATH: direct SET of session path is rejected") { + val err = intercept[AnalysisException] { + sql("SET spark.sql.session.path = 'spark_catalog.default'") + } + assert(err.getMessage.contains("SET PATH"), + s"Expected SET_PATH_VIA_SET error, got: ${err.getMessage}") + } + + test("PATH enabled: duplicate path entry raises error") { + withSQLConf(SQLConf.PATH_ENABLED.key -> "true") { + val e = intercept[AnalysisException] { + sql("SET PATH = spark_catalog.default, spark_catalog.default") + } + assert(e.getMessage.contains("Duplicate path entry"), + s"Expected DUPLICATE_PATH_ENTRY, got: ${e.getMessage}") + } + } + + test("PATH enabled: non-existing schema in path is skipped at resolution") { + withSQLConf(SQLConf.PATH_ENABLED.key -> "true") { + sql("SET PATH = spark_catalog.nonexistent_schema_xyz, spark_catalog.default") + // abs is builtin; resolution should skip nonexistent and find from default/builtin + checkAnswer(sql("SELECT abs(-1)"), Row(1)) + } + } + + test("PATH enabled: SET PATH = PATH, schema appends to path") { + withSQLConf(SQLConf.PATH_ENABLED.key -> "true") { + sql("CREATE SCHEMA IF NOT EXISTS path_append_test") + try { + sql("SET PATH = spark_catalog.default") + sql("SET PATH = PATH, spark_catalog.path_append_test") + val pathStr = sql("SELECT current_path()").collect().head.getString(0) + assert(pathStr.contains("path_append_test"), + s"PATH, schema should append; got: $pathStr") + } finally { + sql("DROP SCHEMA IF EXISTS path_append_test") + } + } + } + + test("PATH enabled: SET PATH = current_schema / current_database (keywords, no parens)") { + withSQLConf(SQLConf.PATH_ENABLED.key -> "true") { + sql("USE spark_catalog.default") + sql("SET PATH = current_schema") + val pathStr = sql("SELECT current_path()").collect().head.getString(0) + assert(pathStr.contains("spark_catalog") && pathStr.contains("default"), + s"SET PATH = current_schema should expand to current schema, got: $pathStr") + sql("SET PATH = current_database") + val pathStr2 = sql("SELECT current_path()").collect().head.getString(0) + assert(pathStr2.contains("spark_catalog") && pathStr2.contains("default"), + s"SET PATH = current_database should expand to current schema, got: $pathStr2") + } + } +} diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala index 09743d92c30a2..ef65d02757244 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerWithSparkContextSuite.scala @@ -214,7 +214,7 @@ trait ThriftServerWithSparkContextSuite extends SharedThriftServer { val sessionHandle = client.openSession(user, "") val infoValue = client.getInfo(sessionHandle, GetInfoType.CLI_ODBC_KEYWORDS) // scalastyle:off line.size.limit - assert(infoValue.getStringValue == "ADD,AFTER,AGGREGATE,ALL,ALTER,ALWAYS,ANALYZE,AND,ANTI,ANY,ANY_VALUE,ARCHIVE,ARRAY,AS,ASC,ASENSITIVE,AT,ATOMIC,AUTHORIZATION,BEGIN,BETWEEN,BIGINT,BINARY,BINDING,BOOLEAN,BOTH,BUCKET,BUCKETS,BY,BYTE,CACHE,CALL,CALLED,CASCADE,CASE,CAST,CATALOG,CATALOGS,CHANGE,CHAR,CHARACTER,CHECK,CLEAR,CLOSE,CLUSTER,CLUSTERED,CODEGEN,COLLATE,COLLATION,COLLECTION,COLUMN,COLUMNS,COMMENT,COMMIT,COMPACT,COMPACTIONS,COMPENSATION,COMPUTE,CONCATENATE,CONDITION,CONSTRAINT,CONTAINS,CONTINUE,COST,CREATE,CROSS,CUBE,CURRENT,CURRENT_DATE,CURRENT_TIME,CURRENT_TIMESTAMP,CURRENT_USER,CURSOR,DATA,DATABASE,DATABASES,DATE,DATEADD,DATEDIFF,DATE_ADD,DATE_DIFF,DAY,DAYOFYEAR,DAYS,DBPROPERTIES,DEC,DECIMAL,DECLARE,DEFAULT,DEFINED,DEFINER,DELAY,DELETE,DELIMITED,DESC,DESCRIBE,DETERMINISTIC,DFS,DIRECTORIES,DIRECTORY,DISTINCT,DISTRIBUTE,DIV,DO,DOUBLE,DROP,ELSE,ELSEIF,END,ENFORCED,ESCAPE,ESCAPED,EVOLUTION,EXCEPT,EXCHANGE,EXCLUDE,EXECUTE,EXISTS,EXIT,EXPLAIN,EXPORT,EXTEND,EXTENDED,EXTERNAL,EXTRACT,FALSE,FETCH,FIELDS,FILEFORMAT,FILTER,FIRST,FLOAT,FLOW,FOLLOWING,FOR,FOREIGN,FORMAT,FORMATTED,FOUND,FROM,FULL,FUNCTION,FUNCTIONS,GENERATED,GEOGRAPHY,GEOMETRY,GLOBAL,GRANT,GROUP,GROUPING,HANDLER,HAVING,HOUR,HOURS,IDENTIFIED,IDENTIFIER,IDENTITY,IF,IGNORE,ILIKE,IMMEDIATE,IMPORT,IN,INCLUDE,INCREMENT,INDEX,INDEXES,INNER,INPATH,INPUT,INPUTFORMAT,INSENSITIVE,INSERT,INT,INTEGER,INTERSECT,INTERVAL,INTO,INVOKER,IS,ITEMS,ITERATE,JOIN,JSON,KEY,KEYS,LANGUAGE,LAST,LATERAL,LAZY,LEADING,LEAVE,LEFT,LEVEL,LIKE,LIMIT,LINES,LIST,LOAD,LOCAL,LOCATION,LOCK,LOCKS,LOGICAL,LONG,LOOP,MACRO,MAP,MATCHED,MATERIALIZED,MAX,MEASURE,MERGE,METRICS,MICROSECOND,MICROSECONDS,MILLISECOND,MILLISECONDS,MINUS,MINUTE,MINUTES,MODIFIES,MONTH,MONTHS,MSCK,NAME,NAMESPACE,NAMESPACES,NANOSECOND,NANOSECONDS,NATURAL,NEXT,NO,NONE,NORELY,NOT,NULL,NULLS,NUMERIC,OF,OFFSET,ON,ONLY,OPEN,OPTION,OPTIONS,OR,ORDER,OUT,OUTER,OUTPUTFORMAT,OVER,OVERLAPS,OVERLAY,OVERWRITE,PARTITION,PARTITIONED,PARTITIONS,PERCENT,PIVOT,PLACING,POSITION,PRECEDING,PRIMARY,PRINCIPALS,PROCEDURE,PROCEDURES,PROPERTIES,PURGE,QUARTER,QUERY,RANGE,READ,READS,REAL,RECORDREADER,RECORDWRITER,RECOVER,RECURSION,RECURSIVE,REDUCE,REFERENCES,REFRESH,RELY,RENAME,REPAIR,REPEAT,REPEATABLE,REPLACE,RESET,RESPECT,RESTRICT,RETURN,RETURNS,REVOKE,RIGHT,ROLE,ROLES,ROLLBACK,ROLLUP,ROW,ROWS,SCHEMA,SCHEMAS,SECOND,SECONDS,SECURITY,SELECT,SEMI,SEPARATED,SERDE,SERDEPROPERTIES,SESSION_USER,SET,SETS,SHORT,SHOW,SINGLE,SKEWED,SMALLINT,SOME,SORT,SORTED,SOURCE,SPECIFIC,SQL,SQLEXCEPTION,SQLSTATE,START,STATISTICS,STORED,STRATIFY,STREAM,STREAMING,STRING,STRUCT,SUBSTR,SUBSTRING,SYNC,SYSTEM_TIME,SYSTEM_VERSION,TABLE,TABLES,TABLESAMPLE,TARGET,TBLPROPERTIES,TERMINATED,THEN,TIME,TIMEDIFF,TIMESTAMP,TIMESTAMPADD,TIMESTAMPDIFF,TIMESTAMP_LTZ,TIMESTAMP_NTZ,TINYINT,TO,TOUCH,TRAILING,TRANSACTION,TRANSACTIONS,TRANSFORM,TRIM,TRUE,TRUNCATE,TRY_CAST,TYPE,UNARCHIVE,UNBOUNDED,UNCACHE,UNION,UNIQUE,UNKNOWN,UNLOCK,UNPIVOT,UNSET,UNTIL,UPDATE,USE,USER,USING,VALUE,VALUES,VAR,VARCHAR,VARIABLE,VARIANT,VERSION,VIEW,VIEWS,VOID,WATERMARK,WEEK,WEEKS,WHEN,WHERE,WHILE,WINDOW,WITH,WITHIN,WITHOUT,X,YEAR,YEARS,ZONE") + assert(infoValue.getStringValue == "ADD,AFTER,AGGREGATE,ALL,ALTER,ALWAYS,ANALYZE,AND,ANTI,ANY,ANY_VALUE,ARCHIVE,ARRAY,AS,ASC,ASENSITIVE,AT,ATOMIC,AUTHORIZATION,BEGIN,BETWEEN,BIGINT,BINARY,BINDING,BOOLEAN,BOTH,BUCKET,BUCKETS,BY,BYTE,CACHE,CALL,CALLED,CASCADE,CASE,CAST,CATALOG,CATALOGS,CHANGE,CHAR,CHARACTER,CHECK,CLEAR,CLOSE,CLUSTER,CLUSTERED,CODEGEN,COLLATE,COLLATION,COLLECTION,COLUMN,COLUMNS,COMMENT,COMMIT,COMPACT,COMPACTIONS,COMPENSATION,COMPUTE,CONCATENATE,CONDITION,CONSTRAINT,CONTAINS,CONTINUE,COST,CREATE,CROSS,CUBE,CURRENT,CURRENT_DATABASE,CURRENT_DATE,CURRENT_PATH,CURRENT_SCHEMA,CURRENT_TIME,CURRENT_TIMESTAMP,CURRENT_USER,CURSOR,DATA,DATABASE,DATABASES,DATE,DATEADD,DATEDIFF,DATE_ADD,DATE_DIFF,DAY,DAYOFYEAR,DAYS,DBPROPERTIES,DEC,DECIMAL,DECLARE,DEFAULT,DEFAULT_PATH,DEFINED,DEFINER,DELAY,DELETE,DELIMITED,DESC,DESCRIBE,DETERMINISTIC,DFS,DIRECTORIES,DIRECTORY,DISTINCT,DISTRIBUTE,DIV,DO,DOUBLE,DROP,ELSE,ELSEIF,END,ENFORCED,ESCAPE,ESCAPED,EVOLUTION,EXCEPT,EXCHANGE,EXCLUDE,EXECUTE,EXISTS,EXIT,EXPLAIN,EXPORT,EXTEND,EXTENDED,EXTERNAL,EXTRACT,FALSE,FETCH,FIELDS,FILEFORMAT,FILTER,FIRST,FLOAT,FLOW,FOLLOWING,FOR,FOREIGN,FORMAT,FORMATTED,FOUND,FROM,FULL,FUNCTION,FUNCTIONS,GENERATED,GEOGRAPHY,GEOMETRY,GLOBAL,GRANT,GROUP,GROUPING,HANDLER,HAVING,HOUR,HOURS,IDENTIFIED,IDENTIFIER,IDENTITY,IF,IGNORE,ILIKE,IMMEDIATE,IMPORT,IN,INCLUDE,INCREMENT,INDEX,INDEXES,INNER,INPATH,INPUT,INPUTFORMAT,INSENSITIVE,INSERT,INT,INTEGER,INTERSECT,INTERVAL,INTO,INVOKER,IS,ITEMS,ITERATE,JOIN,JSON,KEY,KEYS,LANGUAGE,LAST,LATERAL,LAZY,LEADING,LEAVE,LEFT,LEVEL,LIKE,LIMIT,LINES,LIST,LOAD,LOCAL,LOCATION,LOCK,LOCKS,LOGICAL,LONG,LOOP,MACRO,MAP,MATCHED,MATERIALIZED,MAX,MEASURE,MERGE,METRICS,MICROSECOND,MICROSECONDS,MILLISECOND,MILLISECONDS,MINUS,MINUTE,MINUTES,MODIFIES,MONTH,MONTHS,MSCK,NAME,NAMESPACE,NAMESPACES,NANOSECOND,NANOSECONDS,NATURAL,NEXT,NO,NONE,NORELY,NOT,NULL,NULLS,NUMERIC,OF,OFFSET,ON,ONLY,OPEN,OPTION,OPTIONS,OR,ORDER,OUT,OUTER,OUTPUTFORMAT,OVER,OVERLAPS,OVERLAY,OVERWRITE,PARTITION,PARTITIONED,PARTITIONS,PATH,PERCENT,PIVOT,PLACING,POSITION,PRECEDING,PRIMARY,PRINCIPALS,PROCEDURE,PROCEDURES,PROPERTIES,PURGE,QUARTER,QUERY,RANGE,READ,READS,REAL,RECORDREADER,RECORDWRITER,RECOVER,RECURSION,RECURSIVE,REDUCE,REFERENCES,REFRESH,RELY,RENAME,REPAIR,REPEAT,REPEATABLE,REPLACE,RESET,RESPECT,RESTRICT,RETURN,RETURNS,REVOKE,RIGHT,ROLE,ROLES,ROLLBACK,ROLLUP,ROW,ROWS,SCHEMA,SCHEMAS,SECOND,SECONDS,SECURITY,SELECT,SEMI,SEPARATED,SERDE,SERDEPROPERTIES,SESSION_USER,SET,SETS,SHORT,SHOW,SINGLE,SKEWED,SMALLINT,SOME,SORT,SORTED,SOURCE,SPECIFIC,SQL,SQLEXCEPTION,SQLSTATE,START,STATISTICS,STORED,STRATIFY,STREAM,STREAMING,STRING,STRUCT,SUBSTR,SUBSTRING,SYNC,SYSTEM_PATH,SYSTEM_TIME,SYSTEM_VERSION,TABLE,TABLES,TABLESAMPLE,TARGET,TBLPROPERTIES,TERMINATED,THEN,TIME,TIMEDIFF,TIMESTAMP,TIMESTAMPADD,TIMESTAMPDIFF,TIMESTAMP_LTZ,TIMESTAMP_NTZ,TINYINT,TO,TOUCH,TRAILING,TRANSACTION,TRANSACTIONS,TRANSFORM,TRIM,TRUE,TRUNCATE,TRY_CAST,TYPE,UNARCHIVE,UNBOUNDED,UNCACHE,UNION,UNIQUE,UNKNOWN,UNLOCK,UNPIVOT,UNSET,UNTIL,UPDATE,USE,USER,USING,VALUE,VALUES,VAR,VARCHAR,VARIABLE,VARIANT,VERSION,VIEW,VIEWS,VOID,WATERMARK,WEEK,WEEKS,WHEN,WHERE,WHILE,WINDOW,WITH,WITHIN,WITHOUT,X,YEAR,YEARS,ZONE") // scalastyle:on line.size.limit } }