/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.spark.sql.catalyst.analysis

import java.util
import java.util.{LinkedHashMap, Locale}

import scala.collection.mutable
import scala.collection.mutable.ArrayBuffer
import scala.jdk.CollectionConverters._
import scala.util.{Failure, Random, Success, Try}

import org.apache.spark.{SparkException, SparkUnsupportedOperationException}
import org.apache.spark.internal.config.ConfigBindingPolicy
import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.catalyst._
import org.apache.spark.sql.catalyst.analysis.TableOutputResolver.DefaultValueFillMode._
import org.apache.spark.sql.catalyst.analysis.resolver.{
  AnalyzerBridgeState,
  HybridAnalyzer,
  ResolverExtension
}
import org.apache.spark.sql.catalyst.catalog._
import org.apache.spark.sql.catalyst.encoders.OuterScopes
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.SubExprUtils._
import org.apache.spark.sql.catalyst.expressions.aggregate._
import org.apache.spark.sql.catalyst.expressions.objects._
import org.apache.spark.sql.catalyst.optimizer.OptimizeUpdateFields
import org.apache.spark.sql.catalyst.plans._
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.rules._
import org.apache.spark.sql.catalyst.trees.AlwaysProcess
import org.apache.spark.sql.catalyst.trees.CurrentOrigin.withOrigin
import org.apache.spark.sql.catalyst.trees.TreePattern._
import org.apache.spark.sql.catalyst.types.DataTypeUtils
import org.apache.spark.sql.catalyst.util.{toPrettySQL, trimTempResolvedColumn, CharVarcharUtils}
import org.apache.spark.sql.catalyst.util.ResolveDefaultColumns._
// `View` is aliased to `V2View` to avoid clashing with the logical-plan `View` imported via
// `org.apache.spark.sql.catalyst.plans.logical._`.
import org.apache.spark.sql.connector.catalog.{View => V2View, _}
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
import org.apache.spark.sql.connector.catalog.TableChange.{After, ColumnPosition}
import org.apache.spark.sql.connector.catalog.functions.UnboundFunction
import org.apache.spark.sql.connector.catalog.procedures.{BoundProcedure, ProcedureParameter, UnboundProcedure}
import org.apache.spark.sql.connector.expressions.{FieldReference, IdentityTransform}
import org.apache.spark.sql.errors.QueryCompilationErrors
import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.internal.SQLConf.{PartitionOverwriteMode, StoreAssignmentPolicy}
import org.apache.spark.sql.internal.connector.V1Function
import org.apache.spark.sql.types._
import org.apache.spark.sql.util.CaseInsensitiveStringMap
import org.apache.spark.util.ArrayImplicits._

/**
 * A trivial [[Analyzer]] with a dummy [[SessionCatalog]] and
 * [[EmptyTableFunctionRegistry]]. Used for testing when all relations are already filled
 * in and the analyzer needs only to resolve attribute references.
 *
 * Built-in function registry is set for Spark Connect project to test unresolved
 * functions.
 */
object SimpleAnalyzer extends Analyzer(
  new DefaultCatalogManager(
    FakeV2SessionCatalog,
    new SessionCatalog(
      new InMemoryCatalog,
      FunctionRegistry.builtin,
      TableFunctionRegistry.builtin) {
      override def createDatabase(dbDefinition: CatalogDatabase, ignoreIfExists: Boolean): Unit = {}
    }),
  RelationCache.empty) {
  override def resolver: Resolver = caseSensitiveResolution
}

object FakeV2SessionCatalog extends TableCatalog with FunctionCatalog with SupportsNamespaces {
  private def fail() = throw SparkUnsupportedOperationException()
  override def listTables(namespace: Array[String]): Array[Identifier] = fail()
  override def loadTable(ident: Identifier): Table = {
    throw new NoSuchTableException(ident.asMultipartIdentifier)
  }
  override def alterTable(ident: Identifier, changes: TableChange*): Table = fail()
  override def dropTable(ident: Identifier): Boolean = fail()
  override def renameTable(oldIdent: Identifier, newIdent: Identifier): Unit = fail()
  override def initialize(name: String, options: CaseInsensitiveStringMap): Unit = {}
  override def name(): String = CatalogManager.SESSION_CATALOG_NAME
  override def listFunctions(namespace: Array[String]): Array[Identifier] = fail()
  override def loadFunction(ident: Identifier): UnboundFunction = fail()
  override def listNamespaces(): Array[Array[String]] = fail()
  override def listNamespaces(namespace: Array[String]): Array[Array[String]] = fail()
  override def loadNamespaceMetadata(namespace: Array[String]): util.Map[String, String] = {
    if (namespace.length == 1) {
      mutable.HashMap[String, String]().asJava
    } else {
      throw new NoSuchNamespaceException(namespace)
    }
  }
  override def createNamespace(
    namespace: Array[String], metadata: util.Map[String, String]): Unit = fail()
  override def alterNamespace(namespace: Array[String], changes: NamespaceChange*): Unit = fail()
  override def dropNamespace(namespace: Array[String], cascade: Boolean): Boolean = fail()
}

/**
 * Provides a way to keep state during the analysis, mostly for resolving views and subqueries.
 * This enables us to decouple the concerns of analysis environment from the catalog and resolve
 * star expressions in subqueries that reference the outer query plans.
 * The state that is kept here is per-query.
 *
 * Note this is thread local.
 *
 * @param catalogAndNamespace The catalog and namespace used in the view resolution. This overrides
 *                            the current catalog and namespace when resolving relations inside
 *                            views.
 * @param nestedViewDepth The nested depth in the view resolution, this enables us to limit the
 *                        depth of nested views.
 * @param maxNestedViewDepth The maximum allowed depth of nested view resolution.
 * @param relationCache A mapping from qualified table names and time travel spec to resolved
 *                      relations. This can ensure that the table is resolved only once if a table
 *                      is used multiple times in a query.
 * @param referredTempViewNames All the temp view names referred by the current view we are
 *                              resolving. It's used to make sure the relation resolution is
 *                              consistent between view creation and view resolution. For example,
 *                              if `t` was a permanent table when the current view was created, it
 *                              should still be a permanent table when resolving the current view,
 *                              even if a temp view `t` has been created.
 * @param outerPlan The query plan from the outer query that can be used to resolve star
 *                  expressions in a subquery.
 * @param resolutionPathEntries When resolving a view or SQL function body, the ordered frozen
 *                              path for unqualified relation/function names (if persisted in
 *                              metadata). Outside views/functions, compute from session
 *                              [[CatalogManager.sqlResolutionPathEntries]].
 */
case class AnalysisContext(
    isDefault: Boolean = false,
    catalogAndNamespace: Seq[String] = Nil,
    resolutionPathEntries: Option[Seq[Seq[String]]] = None,
    nestedViewDepth: Int = 0,
    maxNestedViewDepth: Int = -1,
    relationCache: mutable.Map[(Seq[String], Option[TimeTravelSpec]), LogicalPlan] =
      mutable.Map.empty,
    referredTempViewNames: Seq[Seq[String]] = Seq.empty,
    // 1. If we are resolving a view, this field will be restored from the view metadata,
    //    by calling `AnalysisContext.withAnalysisContext(viewDesc)`.
    // 2. If we are not resolving a view, this field will be updated everytime the analyzer
    //    lookup a temporary function. And export to the view metadata.
    referredTempFunctionNames: mutable.Set[String] = mutable.Set.empty,
    referredTempVariableNames: Seq[Seq[String]] = Seq.empty,
    outerPlan: Option[LogicalPlan] = None,
    collation: Option[String] = None,

    /**
     * This is a bridge state between this fixed-point [[Analyzer]] and a single-pass [[Resolver]].
     * It's managed ([[setSinglePassResolverBridgeState]] method) by the [[HybridAnalyzer]] - the
     * goal is to preserve it correctly between the fixed-point and single-pass runs.
     * [[AnalysisContext.reset]] simply propagates it to prevent it from being reset in
     * [[Analyzer.execute]]. Normally it's always [[None]], unless
     * [[ANALYZER_DUAL_RUN_LEGACY_AND_SINGLE_PASS_RESOLVER]] is set to [[true]].
     *
     * See [[AnalyzerBridgeState]] and [[HybridAnalyzer]] for more info.
     */
    private var singlePassResolverBridgeState: Option[AnalyzerBridgeState] = None) {

    def setSinglePassResolverBridgeState(bridgeState: Option[AnalyzerBridgeState]): Unit =
      singlePassResolverBridgeState = bridgeState

    def getSinglePassResolverBridgeState: Option[AnalyzerBridgeState] =
      singlePassResolverBridgeState

    /**
     * Per-pass memo of the SQL resolution search path (SPARK-57758). Function resolution computes
     * the ordered path once per analysis pass and reuses it for every [[UnresolvedFunction]],
     * instead of rebuilding and re-iterating it per node -- the cost that, under Spark Connect's
     * repeated re-analysis of the growing plan, scaled with plan size x analyze calls.
     *
     * The memo lives on the context so it shares the context's per-pass lifetime. `SET PATH` /
     * `USE` / conf changes all produce a fresh context ([[reset]], [[withNewAnalysisContext]], or
     * the `copy` / construction for a view or SQL-function body), and a body-level field is not
     * carried over by `copy`, so a new pass automatically starts with an empty memo and the memo is
     * collected with the context. It is therefore safe without an identity key or weak reference,
     * but only for values derived from this context's immutable fields (the path derives from
     * `resolutionPathEntries` / `catalogAndNamespace`); never memoize anything derived from the
     * mutable fields above (`relationCache`, `referredTempFunctionNames`, ...).
     *
     * INVARIANT: keep this a body `var`, never a constructor parameter. `.copy()` (used by
     * `withAnalysisContext(function)` and `withOuterPlan`) deliberately does not carry a body
     * field, which is what gives a SQL-function-body / outer-plan context a fresh memo. Promoting
     * it to a parameter would copy a stale path across that boundary and silently mis-resolve
     * (SECTION 17f of `FunctionQualificationSuite` is the regression guard).
     */
    private var resolutionPathMemo: Seq[Seq[String]] = _

    def memoizedResolutionPath(compute: => Seq[Seq[String]]): Seq[Seq[String]] = {
      if (resolutionPathMemo == null) {
        resolutionPathMemo = compute
      }
      resolutionPathMemo
    }
}

object AnalysisContext {
  private val value = new ThreadLocal[AnalysisContext]() {
    override def initialValue: AnalysisContext = AnalysisContext(isDefault = true)
  }

  def get: AnalysisContext = value.get()

  def reset(): Unit = {
    // We need to preserve the single-pass resolver bridge state here, since it's managed by the
    // [[HybridAnalyzer]] (set or reset to `None`) to avoid it being reset in [[execute]].
    // It acts as a bridge between the single-pass and fixed-point analyzers in the absence of any
    // other explicit state.
    val prevSinglePassResolverBridgeState = value.get.getSinglePassResolverBridgeState
    value.remove()
    value.get.setSinglePassResolverBridgeState(prevSinglePassResolverBridgeState)
  }

  private def set(context: AnalysisContext): Unit = value.set(context)

  def withAnalysisContext[A](viewDesc: CatalogTable)(f: => A): A = {
    val originContext = value.get()
    val maxNestedViewDepth = if (originContext.maxNestedViewDepth == -1) {
      SQLConf.get.maxNestedViewDepth
    } else {
      originContext.maxNestedViewDepth
    }
    val context = AnalysisContext(
      isDefault = false,
      catalogAndNamespace = viewDesc.viewCatalogAndNamespace,
      resolutionPathEntries = viewDesc.viewStoredResolutionPath
        .map(CatalogManager.deserializePathEntriesOrFail(
          _, "view", viewDesc.identifier.unquotedString)),
      nestedViewDepth = originContext.nestedViewDepth + 1,
      maxNestedViewDepth = maxNestedViewDepth,
      relationCache = originContext.relationCache,
      referredTempViewNames = viewDesc.viewReferredTempViewNames,
      referredTempFunctionNames = mutable.Set(viewDesc.viewReferredTempFunctionNames: _*),
      referredTempVariableNames = viewDesc.viewReferredTempVariableNames,
      collation = viewDesc.collation)
    set(context)
    try f finally { set(originContext) }
  }

  def withAnalysisContext[A](function: SQLFunction)(f: => A): A = {
    val originContext = value.get()
    // Function body analysis should not inherit any caller-pinned path; use only function metadata.
    val context = originContext.copy(
      resolutionPathEntries = function.functionStoredResolutionPath
        .map(CatalogManager.deserializePathEntriesOrFail(
          _, "SQL function", function.name.unquotedString)),
      collation = function.collation)
    set(context)
    try f finally { set(originContext) }
  }

  def withNewAnalysisContext[A](f: => A): A = {
    val originContext = value.get()
    reset()
    set(get.copy(isDefault = false))
    try f finally { set(originContext) }
  }

  def withOuterPlan[A](outerPlan: LogicalPlan)(f: => A): A = {
    val originContext = value.get()
    val context = originContext.copy(isDefault = false, outerPlan = Some(outerPlan))
    set(context)
    try f finally { set(originContext) }
  }
}

object Analyzer {
  // Configs with bindingPolicy SESSION or NOT_APPLICABLE are retained when resolving views and
  // SQL UDFs, so that their values propagate from the active session rather than falling back to
  // Spark defaults. Note: configs defined in lazily-loaded modules (e.g., sql/hive) will only
  // be included if their holding Scala object has been initialized before this set is computed.
  def retainResolutionConfigsForAnalysis(
      newConf: SQLConf,
      existingConf: SQLConf,
      createSparkVersion: String = ""): Unit = {
    val retainedConfigKeys = SQLConf.getConfigEntries().asScala
      .filter(entry =>
        entry.bindingPolicy.contains(ConfigBindingPolicy.SESSION) ||
        entry.bindingPolicy.contains(ConfigBindingPolicy.NOT_APPLICABLE))
      .map(_.key)
      .toSet

    val retainedConfigs = existingConf.getAllConfs.filter { case (key, _) =>
      retainedConfigKeys.contains(key) || key.startsWith("spark.sql.catalog.")
    }

    retainedConfigs.foreach { case (k, v) =>
      newConf.settings.put(k, v)
    }

    trySetAnsiValue(newConf, createSparkVersion)
  }

  /**
   * In case ANSI value wasn't persisted for a view or a UDF, we set it to `true` in case Spark
   * version used to create the view is 4.0.0 or higher. We set it to `false` in case Spark version
   * is lower than 4.0.0 or if the Spark version wasn't stored (in that case we assume that the
   * value is `false`)
   */
  def trySetAnsiValue(sqlConf: SQLConf, createSparkVersion: String = ""): Unit = {
    if (conf.getConf(SQLConf.ASSUME_ANSI_FALSE_IF_NOT_PERSISTED) &&
      !sqlConf.settings.containsKey(SQLConf.ANSI_ENABLED.key)) {
      if (createSparkVersion.startsWith("4.")) {
        sqlConf.settings.put(SQLConf.ANSI_ENABLED.key, "true")
      } else {
        sqlConf.settings.put(SQLConf.ANSI_ENABLED.key, "false")
      }
    }
  }
}

/**
 * Provides a logical query plan analyzer, which translates [[UnresolvedAttribute]]s and
 * [[UnresolvedRelation]]s into fully typed objects using information in a [[SessionCatalog]].
 */
class Analyzer(
    override val catalogManager: CatalogManager,
    private[sql] val sharedRelationCache: RelationCache = RelationCache.empty,
    private[sql] val sessionConf: Option[SQLConf] = None)
  extends RuleExecutor[LogicalPlan]
  with CheckAnalysis with AliasHelper with SQLConfHelper with ColumnResolutionHelper {

  private val v1SessionCatalog: SessionCatalog = catalogManager.v1SessionCatalog
  private val relationResolution =
    new RelationResolution(catalogManager, sharedRelationCache)
  private val functionResolution = new FunctionResolution(catalogManager, relationResolution)

  override protected def validatePlanChanges(
      previousPlan: LogicalPlan,
      currentPlan: LogicalPlan): Option[String] = {
    LogicalPlanIntegrity.validateExprIdUniqueness(currentPlan)
  }

  override def isView(nameParts: Seq[String]): Boolean = v1SessionCatalog.isView(nameParts)

  // Only for tests.
  def this(catalog: SessionCatalog) = {
    this(new DefaultCatalogManager(FakeV2SessionCatalog, catalog))
  }

  def getRelationResolution: RelationResolution = relationResolution

  def executeAndCheck(plan: LogicalPlan, tracker: QueryPlanningTracker): LogicalPlan = {
    if (plan.analyzed) {
      plan
    } else {
      def runAnalysis(): LogicalPlan = HybridAnalyzer.fromLegacyAnalyzer(
        legacyAnalyzer = this, tracker = tracker).apply(plan)
      if (AnalysisContext.get.isDefault) {
        AnalysisContext.reset()
        try {
          AnalysisHelper.markInAnalyzer {
            runWithSessionConf(runAnalysis())
          }
        } finally {
          AnalysisContext.reset()
        }
      } else {
        AnalysisContext.withNewAnalysisContext {
          AnalysisHelper.markInAnalyzer {
            runWithSessionConf(runAnalysis())
          }
        }
      }
    }
  }

  /**
   * Runs `thunk` under the analyzer's [[sessionConf]] for analyzer isolation, but yields to any
   * outer [[SQLConf.withExistingConf]] scope (e.g. a SQL UDF / view body that pinned the
   * creation-time configs). Falls through unchanged when [[sessionConf]] is unset, or when the
   * outer scope already installed a different conf -- otherwise the outer scope's conf would be
   * silently clobbered.
   */
  private def runWithSessionConf[T](thunk: => T): T = sessionConf match {
    case None => thunk
    case Some(c) =>
      SQLConf.getExistingConfIfSet match {
        case Some(outer) if outer ne c => thunk
        case _ => SQLConf.withExistingConf(c) { thunk }
      }
  }

  /**
   * Returns a copy of this analyzer that uses the given [[CatalogManager]] for all catalog
   * lookups. All other configuration (extended rules, checks, etc.) is preserved. Used by
   * [[QueryExecution]] to create a per-query analyzer for transactional operations for
   * transaction-aware catalog resolution.
   *
   * IMPORTANT: any new extension point added to Analyzer must also be copied here, otherwise
   * transaction-aware analyzer clones (created by QueryExecution) will silently miss those rules.
   */
  def withCatalogManager(newCatalogManager: CatalogManager): Analyzer = {
    val self = this
    new Analyzer(newCatalogManager, sharedRelationCache) {
      override val hintResolutionRules: Seq[Rule[LogicalPlan]] = self.hintResolutionRules
      override val extendedResolutionRules: Seq[Rule[LogicalPlan]] = self.extendedResolutionRules
      override val postHocResolutionRules: Seq[Rule[LogicalPlan]] = self.postHocResolutionRules
      override val extendedCheckRules: Seq[LogicalPlan => Unit] = self.extendedCheckRules
      override val singlePassResolverExtensions: Seq[ResolverExtension] =
        self.singlePassResolverExtensions
      override val singlePassMetadataResolverExtensions: Seq[ResolverExtension] =
        self.singlePassMetadataResolverExtensions
      override val singlePassPostHocResolutionRules: Seq[Rule[LogicalPlan]] =
        self.singlePassPostHocResolutionRules
      override val singlePassExtendedResolutionChecks: Seq[LogicalPlan => Unit] =
        self.singlePassExtendedResolutionChecks
    }
  }

  override def execute(plan: LogicalPlan): LogicalPlan = {
    AnalysisContext.withNewAnalysisContext {
      executeSameContext(plan)
    }
  }

  private def executeSameContext(plan: LogicalPlan): LogicalPlan =
    runWithSessionConf(super.execute(plan))

  def resolver: Resolver = conf.resolver

  /**
   * If the plan cannot be resolved within maxIterations, analyzer will throw exception to inform
   * user to increase the value of SQLConf.ANALYZER_MAX_ITERATIONS.
   */
  protected def fixedPoint =
    FixedPoint(
      conf.analyzerMaxIterations,
      errorOnExceed = true,
      maxIterationsSetting = SQLConf.ANALYZER_MAX_ITERATIONS.key)

  /**
   * Extensions for the single-pass analyzer.
   *
   * See [[ResolverExtension]] for more info.
   */
  val singlePassResolverExtensions: Seq[ResolverExtension] = Nil

  /**
   * Extensions used for early resolution of the single-pass analyzer.
   *
   * See [[ResolverExtension]] for more info.
   */
  val singlePassMetadataResolverExtensions: Seq[ResolverExtension] = Nil

  /**
   * Override to provide rules to do post-hoc resolution. This batch is to run right after the
   * single-pass resolution is complete and execute its rules in one pass.
   *
   * Using this mechanism is discouraged. Prefer to implement a given feature in the single-pass
   * Resolver directly.
   */
  val singlePassPostHocResolutionRules: Seq[Rule[LogicalPlan]] = Nil

  /**
   * Override to provide additional checks that will run after the single-pass resolution on the
   * resolved logical plan. These are simple callbacks, and not Catalyst rules.
   *
   * For historical reasons, some of these checks are heavy - issuing RPCs to external services or
   * mutating a global state.
   * [[ANALYZER_SINGLE_PASS_RESOLVER_RUN_HEAVY_EXTENDED_RESOLUTION_CHECKS]] enables checks that
   * are considered heavy.
   *
   * Using this mechanism is discouraged. Prefer to implement a given check in the single-pass
   * Resolver directly.
   */
  val singlePassExtendedResolutionChecks: Seq[LogicalPlan => Unit] = Nil

  /**
   * Override to provide additional rules for the "Resolution" batch.
   */
  val extendedResolutionRules: Seq[Rule[LogicalPlan]] = Nil

  /**
   * Override to provide additional rules for the "Hints" resolution batch.
   */
  val hintResolutionRules: Seq[Rule[LogicalPlan]] = Nil

  /**
   * Override to provide rules to do post-hoc resolution. Note that these rules will be executed
   * in an individual batch. This batch is to run right after the normal resolution batch and
   * execute its rules in one pass.
   */
  val postHocResolutionRules: Seq[Rule[LogicalPlan]] = Nil

  private def typeCoercionRules(): List[Rule[LogicalPlan]] = if (conf.ansiEnabled) {
    AnsiTypeCoercion.typeCoercionRules
  } else {
    TypeCoercion.typeCoercionRules
  }

  private def earlyBatches: Seq[Batch] = Seq(
    Batch("Substitution", fixedPoint,
      // This rule optimizes `UpdateFields` expression chains so looks more like optimization rule.
      // However, when manipulating deeply nested schema, `UpdateFields` expression tree could be
      // very complex and make analysis impossible. Thus we need to optimize `UpdateFields` early
      // at the beginning of analysis.
      OptimizeUpdateFields,
      CTESubstitution,
      WindowsSubstitution,
      EliminateUnions,
      EliminateLazyExpression),
    Batch("Apply Limit All", Once, ApplyLimitAll),
    Batch("Disable Hints", Once,
      new ResolveHints.DisableHints),
    Batch("Hints", fixedPoint,
      Seq(ResolveHints.ResolveJoinStrategyHints,
        ResolveHints.ResolveCoalesceHints) ++
        hintResolutionRules: _*),
    Batch("Simple Sanity Check", Once,
      LookupFunctions),
    Batch("Keep Legacy Outputs", Once,
      KeepLegacyOutputs),
    Batch("Unresolve Relations", Once,
      new UnresolveRelationsInTransaction(catalogManager))
  )

  override def batches: Seq[Batch] = earlyBatches ++ Seq(
    Batch("Resolution", fixedPoint,
      new ResolveCatalogs(catalogManager) ::
      ResolveInsertInto ::
      ResolveRelations ::
      ResolveChangelogTable ::
      ResolvePartitionSpec ::
      ResolveFieldNameAndPosition ::
      AddMetadataColumns ::
      DeduplicateRelations ::
      ResolveCollationName ::
      ResolveSchemaEvolution ::
      ValidateEventTimeWatermarkColumn ::
      new ResolveReferences(catalogManager) ::
      // Please do not insert any other rules in between. See the TODO comments in rule
      // ResolveLateralColumnAliasReference for more details.
      ResolveLateralColumnAliasReference ::
      ResolveExpressionsWithNamePlaceholders ::
      ResolveDeserializer ::
      ResolveNewInstance ::
      ResolveUpCast ::
      ResolveGroupingAnalytics ::
      ResolvePivot ::
      ResolveUnpivot ::
      ResolveOrdinalInOrderByAndGroupBy ::
      ExtractGenerator ::
      ResolveGenerate ::
      ResolveFunctions ::
      ResolveProcedures ::
      BindProcedures ::
      ResolveTableSpec ::
      ValidateAndStripPipeExpressions ::
      ResolveSQLFunctions ::
      ResolveSQLTableFunctions ::
      ResolveAliases ::
      ResolveSubquery ::
      ResolveSubqueryColumnAliases ::
      ApplyDefaultCollation ::
      ResolveWindowOrder ::
      ResolveWindowFrame ::
      ResolveNaturalAndUsingJoin ::
      ResolveOutputRelation ::
      new ResolveTableConstraints(catalogManager) ::
      new ResolveSetVariable(catalogManager) ::
      new ResolveFetchCursor(catalogManager) ::
      new ResolveCursors() ::
      ExtractWindowExpressions ::
      GlobalAggregates ::
      ResolveAggregateFunctions ::
      ResolveQualify ::
      TimeWindowing ::
      SessionWindowing ::
      ResolveWindowTime ::
      ResolveInlineTables ::
      ResolveLambdaVariables ::
      ResolveTimeZone ::
      ResolveRandomSeed ::
      ResolveBinaryArithmetic ::
      new ResolveIdentifierClause(earlyBatches) ::
      ResolveUnion ::
      FlattenSequentialStreamingUnion ::
      ValidateSequentialStreamingUnion ::
      ResolveRowLevelCommandAssignments ::
      RewriteDeleteFromTable ::
      RewriteUpdateTable ::
      RewriteMergeIntoTable ::
      MoveParameterizedQueriesDown ::
      BindParameters ::
      typeCoercionRules() ++
      Seq(
        ResolveWithCTE,
        ExtractDistributedSequenceID) ++
      Seq(ResolveUpdateEventTimeWatermarkColumn) ++
      extendedResolutionRules ++
      Seq(NameStreamingSources) : _*),
    Batch("Remove TempResolvedColumn", Once, RemoveTempResolvedColumn),
    Batch("Post-Hoc Resolution", Once,
      Seq(ResolveCommandsWithIfExists) ++
      postHocResolutionRules: _*),
    Batch("Remove Unresolved Hints", Once,
      new ResolveHints.RemoveAllHints),
    Batch("Nondeterministic", Once,
      PullOutNondeterministic),
    Batch("ScalaUDF Null Handling", fixedPoint,
      // `HandleNullInputsForUDF` may wrap the `ScalaUDF` with `If` expression to return null for
      // null inputs, so the result can be null even if `ScalaUDF#nullable` is false. We need to
      // run `UpdateAttributeNullability` to update nullability of the UDF output attribute in
      // downstream operators. After updating attribute nullability, `ScalaUDF`s in downstream
      // operators may need null handling as well, so we should run these two rules repeatedly.
      HandleNullInputsForUDF,
      UpdateAttributeNullability),
    Batch("UDF", Once,
      ResolveEncodersInUDF),
    Batch("Subquery", Once,
      UpdateOuterReferences),
    Batch("Cleanup", fixedPoint,
      CleanupAliases),
    Batch("HandleSpecialCommand", Once,
      HandleSpecialCommand),
    Batch("Remove watermark for batch query", Once,
      EliminateEventTimeWatermark),
    Batch("ResolveUnresolvedHaving", Once, ResolveUnresolvedHaving)
  )

  /**
   * For [[Add]]:
   * 1. if both side are interval, stays the same;
   * 2. else if one side is date and the other is interval,
   *    turns it to [[DateAddInterval]];
   * 3. else if one side is interval, turns it to [[TimestampAddInterval]];
   * 4. else if one side is date, turns it to [[DateAdd]] ;
   * 5. else stays the same.
   *
   * For [[Subtract]]:
   * 1. if both side are interval, stays the same;
   * 2. else if the left side is date and the right side is interval,
   *    turns it to [[DateAddInterval(l, -r)]];
   * 3. else if the right side is an interval, turns it to [[TimestampAddInterval(l, -r)]];
   * 4. else if one side is timestamp, turns it to [[SubtractTimestamps]];
   * 5. else if the right side is date, turns it to [[DateDiff]]/[[SubtractDates]];
   * 6. else if the left side is date, turns it to [[DateSub]];
   * 7. else turns it to stays the same.
   *
   * For [[Multiply]]:
   * 1. If one side is interval, turns it to [[MultiplyInterval]];
   * 2. otherwise, stays the same.
   *
   * For [[Divide]]:
   * 1. If the left side is interval, turns it to [[DivideInterval]];
   * 2. otherwise, stays the same.
   */
  object ResolveBinaryArithmetic extends Rule[LogicalPlan] {
    override def apply(plan: LogicalPlan): LogicalPlan =
      plan.resolveExpressionsUpWithPruning(_.containsPattern(BINARY_ARITHMETIC), ruleId) {
        case expr @ (_: Add | _: Subtract | _: Multiply | _: Divide) if expr.childrenResolved =>
          BinaryArithmeticWithDatetimeResolver.resolve(expr)
      }
  }

  /**
   * Substitute child plan with WindowSpecDefinitions.
   */
  object WindowsSubstitution extends Rule[LogicalPlan] {
    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
      _.containsAnyPattern(WITH_WINDOW_DEFINITION, UNRESOLVED_WINDOW_EXPRESSION), ruleId) {
      // Lookup WindowSpecDefinitions. This rule works with unresolved children.
      case WithWindowDefinition(windowDefinitions, child, forPipeSQL) =>
        val resolveWindowExpression: PartialFunction[Expression, Expression] = {
          case UnresolvedWindowExpression(c, WindowSpecReference(windowName)) =>
            val windowSpecDefinition = windowDefinitions.getOrElse(windowName,
              throw QueryCompilationErrors.windowSpecificationNotDefinedError(windowName))
            WindowExpression(c, windowSpecDefinition)
        }
        if (forPipeSQL) {
          child.transformExpressions(resolveWindowExpression)
        } else {
          child.resolveExpressions(resolveWindowExpression)
        }
    }
  }

  /**
   * Replaces [[UnresolvedAlias]]s with concrete aliases.
   */
  object ResolveAliases extends Rule[LogicalPlan] {
    def apply(plan: LogicalPlan): LogicalPlan = {
      val collatedPlan =
        if (conf.getConf(SQLConf.RUN_COLLATION_TYPE_CASTS_BEFORE_ALIAS_ASSIGNMENT)) {
          CollationRulesRunner(plan)
        } else {
          plan
        }
      doApply(collatedPlan)
    }

    private def doApply(plan: LogicalPlan): LogicalPlan = {
      plan.resolveOperatorsUpWithPruning(_.containsPattern(UNRESOLVED_ALIAS), ruleId) {
        case Aggregate(groups, aggs, child, _)
            if child.resolved && AliasResolution.hasUnresolvedAlias(aggs) =>
          Aggregate(groups, AliasResolution.assignAliases(aggs), child)

        case Pivot(groupByOpt, pivotColumn, pivotValues, aggregates, child)
            if child.resolved &&
            groupByOpt.isDefined &&
            AliasResolution.hasUnresolvedAlias(groupByOpt.get) =>
          Pivot(
            Some(AliasResolution.assignAliases(groupByOpt.get)),
            pivotColumn,
            pivotValues,
            aggregates,
            child
          )

        case up: Unpivot
            if up.child.resolved &&
            (up.ids.exists(AliasResolution.hasUnresolvedAlias) || up.values.exists(
              _.exists(AliasResolution.hasUnresolvedAlias)
            )) =>
          up.copy(
            ids = up.ids.map(AliasResolution.assignAliases),
            values = up.values.map(_.map(AliasResolution.assignAliases))
          )

        case Project(projectList, child)
            if child.resolved && AliasResolution.hasUnresolvedAlias(projectList) =>
          Project(AliasResolution.assignAliases(projectList), child)

        case c: CollectMetrics
            if c.child.resolved && AliasResolution.hasUnresolvedAlias(c.metrics) =>
          c.copy(metrics = AliasResolution.assignAliases(c.metrics))

        case u: UnresolvedEventTimeWatermark
          if u.child.resolved && AliasResolution.hasUnresolvedAlias(Seq(u.eventTimeColExpr)) =>
          u.copy(eventTimeColExpr = AliasResolution.assignAliases(Seq(u.eventTimeColExpr)).head)
      }
    }
  }

  object ResolveGroupingAnalytics extends Rule[LogicalPlan] {
    private[analysis] def hasGroupingFunction(e: Expression): Boolean = {
      e.exists (g => g.isInstanceOf[Grouping] || g.isInstanceOf[GroupingID])
    }

    /*
     * Construct [[Aggregate]] operator from Cube/Rollup/GroupingSets.
     */
    private def constructAggregate(
        operator: LogicalPlan,
        selectedGroupByExprs: Seq[Seq[Expression]],
        groupByExprs: Seq[Expression],
        aggregationExprs: Seq[NamedExpression],
        child: LogicalPlan): LogicalPlan = {
      val aggregationExprsNoAlias = aggregationExprs.map(trimNonTopLevelAliases)

      if (groupByExprs.size > GroupingID.dataType.defaultSize * 8) {
        throw QueryCompilationErrors.groupingSizeTooLargeError(GroupingID.dataType.defaultSize * 8)
      }

      if (groupByExprs.exists(_.exists(_.isInstanceOf[Generator]))) {
        throw QueryCompilationErrors.generatorOutsideSelectError(operator)
      }

      GroupingAnalyticsTransformer(
        newAlias = (child, name, qualifier) =>
          Alias(child, name.get)(qualifier = qualifier),
        childOutput = child.output,
        groupByExpressions = groupByExprs,
        selectedGroupByExpressions = selectedGroupByExprs,
        child = child,
        aggregationExpressions = aggregationExprsNoAlias
      )
    }

    private def findGroupingExprs(plan: LogicalPlan): Seq[Expression] = {
      plan.collectFirst {
        case a: Aggregate => GroupingAnalyticsTransformer.collectGroupingExpressions(a)
      }.getOrElse {
        throw QueryCompilationErrors.groupingMustWithGroupingSetsOrCubeOrRollupError()
      }
    }

    private def tryResolveHavingCondition(
        h: UnresolvedHaving,
        aggregate: Aggregate,
        selectedGroupByExprs: Seq[Seq[Expression]],
        groupByExprs: Seq[Expression]): LogicalPlan = {
      // For CUBE/ROLLUP expressions, to avoid resolving repeatedly, here we delete them from
      // groupingExpressions for condition resolving.
      val aggForResolving = aggregate.copy(groupingExpressions = groupByExprs)
      // HACK ALTER! Ideally we should only resolve GROUPING SETS + HAVING when the having condition
      // is fully resolved, similar to the rule `ResolveAggregateFunctions`. However, Aggregate
      // with GROUPING SETS is marked as unresolved and many analyzer rules can't apply to
      // UnresolvedHaving because its child is not resolved. Here we explicitly resolve columns
      // and subqueries of UnresolvedHaving so that the rewrite works in most cases.
      // TODO: mark Aggregate as resolved even if it has GROUPING SETS. We can expand it at the end
      //       of the analysis phase.
      val colResolved = h.mapExpressions { e =>
        resolveExpressionByPlanOutput(
          resolveColWithAgg(e, aggForResolving), aggForResolving, includeLastResort = true)
      }
      val cond = if (SubqueryExpression.hasSubquery(colResolved.havingCondition)) {
        val fake = Project(Alias(colResolved.havingCondition, "fake")() :: Nil, aggregate.child)
        ResolveSubquery(fake).asInstanceOf[Project].projectList.head.asInstanceOf[Alias].child
      } else {
        colResolved.havingCondition
      }
      // `cond` might contain unresolved aggregate functions so defer its resolution to
      // `ResolveAggregateFunctions` rule if needed.
      if (!cond.resolved) return colResolved

      // Try resolving the condition of the filter as though it is in the aggregate clause
      val (extraAggExprs, Seq(resolvedHavingCond)) =
        ResolveAggregateFunctions.resolveExprsWithAggregate(Seq(cond), aggForResolving)

      // Push the aggregate expressions into the aggregate (if any).
      val newChild = constructAggregate(h, selectedGroupByExprs, groupByExprs,
        aggregate.aggregateExpressions ++ extraAggExprs, aggregate.child)

      // Since the output exprId will be changed in the constructed aggregate, here we build an
      // attrMap to resolve the condition again.
      val attrMap = AttributeMap((aggForResolving.output ++ extraAggExprs.map(_.toAttribute))
        .zip(newChild.output))
      val newCond = resolvedHavingCond.transform {
        case a: AttributeReference => attrMap.getOrElse(a, a)
      }

      if (extraAggExprs.isEmpty) {
        Filter(newCond, newChild)
      } else {
        Project(newChild.output.dropRight(extraAggExprs.length),
          Filter(newCond, newChild))
      }
    }

    // This require transformDown to resolve having condition when generating aggregate node for
    // CUBE/ROLLUP/GROUPING SETS. This also replace grouping()/grouping_id() in resolved
    // Filter/Sort.
    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsDownWithPruning(
      _.containsPattern(GROUPING_ANALYTICS), ruleId) {
      case h @ UnresolvedHaving(_, agg @ Aggregate(
        GroupingAnalytics(selectedGroupByExprs, groupByExprs), aggExprs, _, _))
        if agg.childrenResolved && aggExprs.forall(_.resolved) =>
        tryResolveHavingCondition(h, agg, selectedGroupByExprs, groupByExprs)

      // Make sure all of the children are resolved.
      // We can't put this at the beginning, because `Aggregate` with GROUPING SETS is unresolved
      // but we need to resolve `UnresolvedHaving` above it.
      case a if !a.childrenResolved => a

      // Ensure group by expressions and aggregate expressions have been resolved.
      case a @ Aggregate(GroupingAnalytics(selectedGroupByExprs, groupByExprs), aggExprs, child, _)
        if aggExprs.forall(_.resolved) =>
        constructAggregate(a, selectedGroupByExprs, groupByExprs, aggExprs, child)

      // We should make sure all expressions in condition have been resolved.
      case f @ Filter(cond, child) if hasGroupingFunction(cond) && cond.resolved =>
        val groupingExprs = findGroupingExprs(child)
        // For the grand total this is a resolved Literal(0); otherwise the unresolved
        // spark_grouping_id attribute is resolved by ResolveReferences.
        val newCond = GroupingAnalyticsTransformer.replaceGroupingFunction(
          expression = cond,
          groupByExpressions = groupingExprs,
          gid = GroupingAnalyticsTransformer.groupingIdExpression(groupingExprs),
          newAlias = (child, name, qualifier) =>
            Alias(child, name.get)(qualifier = qualifier)
        )
        f.copy(condition = newCond)

      // We should make sure all [[SortOrder]]s have been resolved.
      case s @ Sort(order, _, child, _)
        if order.exists(hasGroupingFunction) && order.forall(_.resolved) =>
        val groupingExprs = findGroupingExprs(child)
        val gid = GroupingAnalyticsTransformer.groupingIdExpression(groupingExprs)
        // For the grand total this is a resolved Literal(0); otherwise the unresolved
        // spark_grouping_id attribute is resolved by ResolveReferences.
        val newOrder = order.map { expression =>
          GroupingAnalyticsTransformer.replaceGroupingFunction(
            expression = expression,
            groupByExpressions = groupingExprs,
            gid = gid,
            newAlias = (child, name, qualifier) =>
              Alias(child, name.get)(qualifier = qualifier)
          ).asInstanceOf[SortOrder]
        }
        s.copy(order = newOrder)
    }
  }

  object ResolvePivot extends Rule[LogicalPlan] {
    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsWithPruning(
      _.containsPattern(PIVOT), ruleId) {
      case p: Pivot if !p.childrenResolved || !p.aggregates.forall(_.resolved)
        || (p.groupByExprsOpt.isDefined && !p.groupByExprsOpt.get.forall(_.resolved))
        || !p.pivotColumn.resolved || !p.pivotValues.forall(_.resolved) => p
      case Pivot(groupByExprsOpt, pivotColumn, pivotValues, aggregates, child) =>
        if (!RowOrdering.isOrderable(pivotColumn.dataType)) {
          throw QueryCompilationErrors.unorderablePivotColError(pivotColumn)
        }
        // Check all aggregate expressions.
        aggregates.foreach(checkValidAggregateExpression)
        PivotTransformer(
          child = child,
          pivotValues = pivotValues,
          pivotColumn = pivotColumn,
          groupByExpressionsOpt = groupByExprsOpt,
          aggregates = aggregates,
          childOutput = child.output,
          newAlias = (child, name) =>
            Alias(child, name.get)()
        )
    }

    // Support any aggregate expression that can appear in an Aggregate plan except Pandas UDF.
    // TODO: Support Pandas UDF.
    private def checkValidAggregateExpression(expr: Expression): Unit = expr match {
      case a: AggregateExpression =>
        if (a.aggregateFunction.isInstanceOf[PythonUDAF]) {
          throw QueryCompilationErrors.pandasUDFAggregateNotSupportedInPivotError()
        } else {
          // OK and leave the argument check to CheckAnalysis.
        }
      case e: Attribute =>
        throw QueryCompilationErrors.aggregateExpressionRequiredForPivotError(e.sql)
      case e => e.children.foreach(checkValidAggregateExpression)
    }
  }

  object ResolveUnpivot extends Rule[LogicalPlan] {
    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsWithPruning(
      _.containsPattern(UNPIVOT), ruleId) {

      // once children are resolved, we can determine values from ids and vice versa
      // if only either is given, and only AttributeReference are given
      case up @ Unpivot(Some(ids), None, _, _, _, _) if up.childrenResolved &&
        ids.forall(_.resolved) &&
        ids.forall(_.isInstanceOf[AttributeReference]) =>
        val idAttrs = AttributeSet(up.ids.get)
        val values = up.child.output.filterNot(idAttrs.contains)
        up.copy(values = Some(values.map(Seq(_))))
      case up @ Unpivot(None, Some(values), _, _, _, _) if up.childrenResolved &&
        values.forall(_.forall(_.resolved)) &&
        values.forall(_.forall(_.isInstanceOf[AttributeReference])) =>
        val valueAttrs = AttributeSet(up.values.get.flatten)
        val ids = up.child.output.filterNot(valueAttrs.contains)
        up.copy(ids = Some(ids))

      case up: Unpivot if !up.childrenResolved || !up.ids.exists(_.forall(_.resolved)) ||
        !up.values.exists(_.nonEmpty) || !up.values.exists(_.forall(_.forall(_.resolved))) ||
        !up.values.get.forall(_.length == up.valueColumnNames.length) ||
        !up.valuesTypeCoercioned => up

      // TypeCoercionBase.UnpivotCoercion determines valueType
      // and casts values once values are set and resolved
      case Unpivot(Some(ids), Some(values), aliases, variableColumnName, valueColumnNames, child) =>
        UnpivotTransformer(ids, values, aliases, variableColumnName, valueColumnNames, child)
    }
  }

  /**
   * Adds metadata columns to output for child relations when nodes are missing resolved attributes.
   *
   * References to metadata columns are resolved using columns from [[LogicalPlan.metadataOutput]],
   * but the relation's output does not include the metadata columns until the relation is replaced.
   * Unless this rule adds metadata to the relation's output, the analyzer will detect that nothing
   * produces the columns.
   *
   * This rule only adds metadata columns when a node is resolved but is missing input from its
   * children. This ensures that metadata columns are not added to the plan unless they are used. By
   * checking only resolved nodes, this ensures that * expansion is already done so that metadata
   * columns are not accidentally selected by *. This rule resolves operators downwards to avoid
   * projecting away metadata columns prematurely.
   */
  object AddMetadataColumns extends Rule[LogicalPlan] {
    import org.apache.spark.sql.catalyst.util._

    def apply(plan: LogicalPlan): LogicalPlan = {
      val onlyUniqueAndNecessaryMetadataColumns =
        conf.getConf(SQLConf.ONLY_NECESSARY_AND_UNIQUE_METADATA_COLUMNS)
      plan.resolveOperatorsDownWithPruning(AlwaysProcess.fn, ruleId) {
        case hint: UnresolvedHint => hint
        // Add metadata output to all node types
        case node if node.children.nonEmpty && node.resolved && hasMetadataCol(node) =>
          val inputAttrs = AttributeSet(node.children.flatMap(_.output))
          val metaCols = getMetadataAttributes(node).filterNot(inputAttrs.contains)
          if (metaCols.isEmpty) {
            node
          } else {
            val newNode = node.mapChildren(
              addMetadataCol(_, metaCols.map(_.exprId).toSet, onlyUniqueAndNecessaryMetadataColumns)
            )
            // We should not change the output schema of the plan. We should project away the extra
            // metadata columns if necessary.
            if (newNode.sameOutput(node)) {
              newNode
            } else {
              Project(node.output, newNode)
            }
          }
      }
    }

    private def getMetadataAttributes(plan: LogicalPlan): Seq[Attribute] = {
      plan.expressions.flatMap(_.collect {
        case a: Attribute if a.isMetadataCol => a
        case a: Attribute
          if plan.children.exists(c => c.metadataOutput.exists(_.exprId == a.exprId)) =>
          plan.children.collectFirst {
            case c if c.metadataOutput.exists(_.exprId == a.exprId) =>
              c.metadataOutput.find(_.exprId == a.exprId).get
          }.get
      })
    }

    private def hasMetadataCol(plan: LogicalPlan): Boolean = {
      plan.expressions.exists(_.exists {
        case a: Attribute =>
          // If an attribute is resolved before being labeled as metadata
          // (i.e. from the originating Dataset), we check with expression ID
          a.isMetadataCol ||
            plan.children.exists(c => c.metadataOutput.exists(_.exprId == a.exprId))
        case _ => false
      })
    }

    private def addMetadataCol(
        plan: LogicalPlan,
        requiredAttrIds: Set[ExprId],
        onlyUniqueAndNecessaryMetadataColumns: Boolean = true): LogicalPlan = plan match {
      case s: ExposesMetadataColumns if s.metadataOutput.exists( a =>
        requiredAttrIds.contains(a.exprId)) =>
        s.withMetadataColumns()
      case p: Project if p.metadataOutput.exists(a => requiredAttrIds.contains(a.exprId)) =>
        val uniqueMetadataColumns = if (onlyUniqueAndNecessaryMetadataColumns) {
          val actualRequiredExprIds = new util.HashSet[ExprId](requiredAttrIds.asJava)
          p.projectList.foreach(ne => actualRequiredExprIds.remove(ne.exprId))
          p.metadataOutput.filter(attr => actualRequiredExprIds.contains(attr.exprId))
        } else {
          p.metadataOutput
        }

        val newProj = p.copy(
          // Do not leak the qualified-access-only restriction to normal plan outputs.
          projectList = p.projectList ++ uniqueMetadataColumns.map(_.markAsAllowAnyAccess()),
          child = addMetadataCol(p.child, requiredAttrIds, onlyUniqueAndNecessaryMetadataColumns)
        )
        newProj.copyTagsFrom(p)
        newProj
      case _ =>
        plan.withNewChildren(
          plan.children
            .map(addMetadataCol(_, requiredAttrIds, onlyUniqueAndNecessaryMetadataColumns))
        )
    }
  }

  /**
   * Replaces unresolved relations (tables and views) with concrete relations from the catalog.
   */
  object ResolveRelations extends Rule[LogicalPlan] {
    // The current catalog and namespace may be different from when the view was created, we must
    // resolve the view logical plan here, with the catalog and namespace stored in view metadata.
    // This is done by keeping the catalog and namespace in `AnalysisContext`, and analyzer will
    // look at `AnalysisContext.catalogAndNamespace` when resolving relations with single-part name.
    // If `AnalysisContext.catalogAndNamespace` is non-empty, analyzer will expand single-part names
    // with it, instead of current catalog and namespace. For views/functions with persisted frozen
    // PATH, `AnalysisContext.resolutionPathEntries` drives unqualified relation lookup order.
    private def resolveViews(
        plan: LogicalPlan,
        options: CaseInsensitiveStringMap): LogicalPlan = plan match {
      case view: View if !view.child.resolved =>
        ViewResolution
          .resolve(view, options, resolveChild = executeSameContext, checkAnalysis = checkAnalysis)
      // V2TableReference is a placeholder for DSv2 tables that needs to be resolved to
      // DataSourceV2Relation on each view access. Only dataframe temp view may contain it
      // as it stores resolved plans directly.
      case view: View if view.isTempViewStoringAnalyzedPlan =>
        view.copy(child = resolveTableReferencesInTempView(view.child))
      case p @ SubqueryAlias(_, view: View) =>
        p.copy(child = resolveViews(view, options))
      case _ => plan
    }

    // Unwrap temp views storing analyzed plans and resolve V2TableReference nodes in the child.
    private def unwrapRelationPlan(plan: LogicalPlan): LogicalPlan = {
      EliminateSubqueryAliases(plan) match {
        case v: View if v.isTempViewStoringAnalyzedPlan => resolveTableReferencesInTempView(v.child)
        case other => other
      }
    }

    // Resolve V2TableReference nodes inside temp view plans. These are created by
    // V2TableReference.createForTempView. We only need to resolve it when returning
    // the plan of temp views (in resolveViews and unwrapRelationPlan).
    private def resolveTableReferencesInTempView(plan: LogicalPlan): LogicalPlan = {
      plan.resolveOperatorsUp {
        case r: V2TableReference if r.context.isInstanceOf[V2TableReference.TemporaryViewContext] =>
          relationResolution.resolveReference(r)
      }
    }

    def apply(plan: LogicalPlan)
        : LogicalPlan = plan.resolveOperatorsUpWithPruning(AlwaysProcess.fn, ruleId) {
      case i @ InsertIntoStatement(table, _, _, _, _, _, _, _, _) =>
        val relation = table match {
          case u: UnresolvedRelation if !u.isStreaming =>
            resolveRelation(u).getOrElse(u)
          case other => other
        }

        // Inserting into a file-based temporary view is allowed.
        // (e.g., spark.read.parquet("path").createOrReplaceTempView("t").
        // Thus, we need to look at the raw plan if `relation` is a temporary view.
        // unwrapRelationPlan also resolves V2TableReference nodes in temp view plans.
        unwrapRelationPlan(relation) match {
          case v: View =>
            throw QueryCompilationErrors.insertIntoViewNotAllowedError(v.desc.identifier, table)
          case other => i.copy(table = other)
        }

      case write: V2StreamingWriteCommand =>
        write.table match {
          case ref: V2TableReference =>
            relationResolution.resolveReference(ref) match {
              case r: NamedRelation => write.withNewTable(r)
              case other => throw SparkException.internalError(
                s"Expected V2TableReference write target to resolve to a NamedRelation, " +
                  s"but got ${other.getClass.getName}")
            }
          case _ => write
        }

      case write: V2WriteCommand =>
        write.table match {
          case u: UnresolvedRelation if !u.isStreaming =>
            resolveRelation(u).map(unwrapRelationPlan).map {
              case v: View => throw QueryCompilationErrors.writeIntoViewNotAllowedError(
                v.desc.identifier, write)
              case u: UnresolvedCatalogRelation =>
                throw QueryCompilationErrors.writeIntoV1TableNotAllowedError(
                  u.tableMeta.identifier, write)
              case r: DataSourceV2Relation => write.withNewTable(r)
              case _ =>
                throw QueryCompilationErrors.writeIntoTempViewNotAllowedError(
                  u.multipartIdentifier.quoted)
            }.getOrElse(write)
          case _ => write
        }

      case u: UnresolvedRelation =>
        resolveRelation(u).map(resolveViews(_, u.options)).getOrElse(u)

      case r: V2TableReference =>
        relationResolution.resolveReference(r)

      case r @ RelationTimeTravel(u: UnresolvedRelation, timestamp, version)
          if timestamp.forall(ts => ts.resolved && !SubqueryExpression.hasSubquery(ts)) =>
        val timeTravelSpec = TimeTravelSpec.create(timestamp, version, conf.sessionLocalTimeZone)
        resolveRelation(u, timeTravelSpec).getOrElse(r)

      case r @ RelationChanges(u: UnresolvedRelation, ctx) =>
        relationResolution.resolveChangelog(u, ctx).getOrElse(r)

      case u @ UnresolvedTable(identifier, cmd, suggestAlternative) =>
        lookupTableOrView(identifier).map {
          case v: ResolvedPersistentView =>
            val nameParts = v.catalog.name() +: v.identifier.asMultipartIdentifier
            throw QueryCompilationErrors.expectTableNotViewError(
              nameParts, cmd, suggestAlternative, u)
          case _: ResolvedTempView =>
            throw QueryCompilationErrors.expectTableNotViewError(
              identifier, cmd, suggestAlternative, u)
          case table => table
        }.getOrElse(u)

      case u @ UnresolvedView(identifier, cmd, allowTemp, suggestAlternative) =>
        lookupTableOrView(identifier, viewOnly = true).map {
          case _: ResolvedTempView if !allowTemp =>
            throw QueryCompilationErrors.expectPermanentViewNotTempViewError(
              identifier, cmd, u)
          case t: ResolvedTable =>
            val nameParts = t.catalog.name() +: t.identifier.asMultipartIdentifier
            throw QueryCompilationErrors.expectViewNotTableError(
              nameParts, cmd, suggestAlternative, u)
          case other => other
        }.getOrElse(u)

      case u @ UnresolvedTableOrView(identifier, cmd, allowTempView, _) =>
        lookupTableOrView(identifier).map {
          case _: ResolvedTempView if !allowTempView =>
            throw QueryCompilationErrors.expectPermanentViewNotTempViewError(
              identifier, cmd, u)
          case other => other
        }.getOrElse(u)
    }

    /**
     * Resolves relations to `ResolvedTable` or `Resolved[Temp/Persistent]View`. This is
     * for resolving DDL and misc commands. UnresolvedView callers reject non-view results
     * downstream via `expectViewNotTableError`.
     *
     * When `viewOnly=true`, non-session catalogs that do not implement [[ViewCatalog]] are
     * rejected up front with MISSING_CATALOG_ABILITY.VIEWS -- they cannot host views at all,
     * so surfacing a downstream "view not found" would hide the real reason.
     *
     * Lookup order against a non-session catalog:
     *   1. If the catalog is a [[RelationCatalog]], [[RelationCatalog.loadRelation]] is called
     *      once. A returned [[org.apache.spark.sql.connector.catalog.View]] is interpreted as a
     *      view; a [[Table]] is a table.
     *   2. Otherwise, [[TableCatalog.loadTable]] is tried (when implemented), then
     *      [[ViewCatalog.loadView]] as the fallback view-resolution path (when implemented).
     */
    private def lookupTableOrView(
        identifier: Seq[String],
        viewOnly: Boolean = false): Option[LogicalPlan] = {
      relationResolution.lookupTempView(identifier).map { tempView =>
        ResolvedTempView(identifier.asIdentifier, tempView.tableMeta)
      }.orElse {
        relationResolution.expandIdentifier(identifier) match {
          case CatalogAndIdentifier(catalog, ident) =>
            if (viewOnly && !CatalogV2Util.isSessionCatalog(catalog) &&
                !catalog.isInstanceOf[ViewCatalog]) {
              throw QueryCompilationErrors.missingCatalogViewsAbilityError(catalog)
            }
            catalog match {
              case mc: RelationCatalog =>
                // Single-RPC perf path: loadRelation returns a Table for a table or a View
                // for a view. NoSuchTable means neither exists.
                try {
                  Some(mc.loadRelation(ident) match {
                    case v: V2View =>
                      ResolvedPersistentView(catalog, ident, v)
                    case table: Table =>
                      ResolvedTable.create(catalog.asTableCatalog, ident, table)
                    case other => throw SparkException.internalError(
                      s"Catalog ${catalog.name} returned an unexpected relation type for " +
                        s"$ident: ${other.getClass.getName}. Expected a Table or a View.")
                  })
                } catch {
                  case _: NoSuchTableException => None
                }
              case _ =>
                // Skip the table-side lookup entirely for view-only catalogs (no
                // `TableCatalog` mixin): `CatalogV2Util.loadTable` would call `asTableCatalog`
                // and throw MISSING_CATALOG_ABILITY.TABLES, masking the legitimate view-
                // resolution path.
                val tableResolved: Option[LogicalPlan] = if (
                  CatalogV2Util.isSessionCatalog(catalog) || catalog.isInstanceOf[TableCatalog]
                ) {
                  CatalogV2Util.loadTable(catalog, ident).map {
                    case v1Table: V1Table if CatalogV2Util.isSessionCatalog(catalog) &&
                      v1Table.v1Table.isViewLike =>
                      val v1Ident = v1Table.catalogTable.identifier
                      val v2Ident = Identifier.of(v1Ident.database.toArray, v1Ident.identifier)
                      ResolvedPersistentView(
                        catalog, v2Ident, new V1View(v1Table.catalogTable))
                    case table =>
                      ResolvedTable.create(catalog.asTableCatalog, ident, table)
                  }
                } else {
                  None
                }
                tableResolved.orElse {
                  catalog match {
                    case vc: ViewCatalog =>
                      try {
                        Some(ResolvedPersistentView(catalog, ident, vc.loadView(ident)))
                      } catch {
                        case _: NoSuchViewException => None
                      }
                    case _ => None
                  }
                }
            }
          case _ => None
        }
      }
    }

    def resolveRelation(
        unresolvedRelation: UnresolvedRelation,
        timeTravelSpec: Option[TimeTravelSpec] = None): Option[LogicalPlan] = {
      relationResolution
        .resolveRelation(
          unresolvedRelation,
          timeTravelSpec
        )
        .map { relation =>
          // We put the synchronously resolved relation into the [[AnalyzerBridgeState]] for
          // it to be later reused by the single-pass [[Resolver]] to avoid resolving the relation
          // metadata twice.
          AnalysisContext.get.getSinglePassResolverBridgeState.foreach { bridgeState =>
            bridgeState.addUnresolvedRelation(unresolvedRelation, relation)
          }
          relation
        }
    }
  }

  /** Handle INSERT INTO for DSv2 */
  object ResolveInsertInto extends ResolveInsertionBase {
    override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsWithPruning(
      AlwaysProcess.fn, ruleId) {
      case i: InsertIntoStatement
          if i.table.isInstanceOf[DataSourceV2Relation] &&
            i.query.resolved &&
            i.replaceCriteriaOpt.isDefined =>
        throw QueryCompilationErrors.unsupportedInsertReplaceOnOrUsing(
          i.table.asInstanceOf[DataSourceV2Relation].table.name())

      case i: InsertIntoStatement
          if i.table.isInstanceOf[DataSourceV2Relation] &&
            i.query.resolved &&
            i.replaceCriteriaOpt.isEmpty =>
        val r = i.table.asInstanceOf[DataSourceV2Relation]
        // ifPartitionNotExists is append with validation, but validation is not supported
        if (i.ifPartitionNotExists) {
          throw QueryCompilationErrors.unsupportedIfNotExistsError(r.table.name)
        }

        // Create a project if this is an INSERT INTO BY NAME query.
        val projectByName = if (i.userSpecifiedCols.nonEmpty) {
          Some(createProjectForByNameQuery(r.table.name, i))
        } else {
          None
        }
        val isByName = projectByName.nonEmpty || i.byName

        val partCols = partitionColumnNames(r.table)
        validatePartitionSpec(partCols, i.partitionSpec)

        val staticPartitions = i.partitionSpec.filter(_._2.isDefined).transform((_, v) => v.get)
        val query = addStaticPartitionColumns(r, projectByName.getOrElse(i.query), staticPartitions,
          isByName)

        if (!i.overwrite) {
          if (isByName) {
            AppendData.byName(
              r,
              query,
              withSchemaEvolution = i.withSchemaEvolution)
          } else {
            AppendData.byPosition(
              r,
              query,
              withSchemaEvolution = i.withSchemaEvolution)
          }
        } else if (conf.partitionOverwriteMode == PartitionOverwriteMode.DYNAMIC) {
          if (isByName) {
            OverwritePartitionsDynamic.byName(
              r,
              query,
              withSchemaEvolution = i.withSchemaEvolution)
          } else {
            OverwritePartitionsDynamic.byPosition(
              r,
              query,
              withSchemaEvolution = i.withSchemaEvolution)
          }
        } else {
          if (isByName) {
            OverwriteByExpression.byName(
              table = r,
              df = query,
              deleteExpr = staticDeleteExpression(r, staticPartitions),
              withSchemaEvolution = i.withSchemaEvolution)
          } else {
            OverwriteByExpression.byPosition(
              table = r,
              query = query,
              deleteExpr = staticDeleteExpression(r, staticPartitions),
              withSchemaEvolution = i.withSchemaEvolution)
          }
        }
    }

    private def partitionColumnNames(table: Table): Seq[String] = {
      // get partition column names. in v2, partition columns are columns that are stored using an
      // identity partition transform because the partition values and the column values are
      // identical. otherwise, partition values are produced by transforming one or more source
      // columns and cannot be set directly in a query's PARTITION clause.
      table.partitioning.flatMap {
        case IdentityTransform(FieldReference(Seq(name))) => Some(name)
        case _ => None
      }.toImmutableArraySeq
    }

    private def validatePartitionSpec(
        partitionColumnNames: Seq[String],
        partitionSpec: Map[String, Option[String]]): Unit = {
      // check that each partition name is a partition column. otherwise, it is not valid
      partitionSpec.keySet.foreach { partitionName =>
        partitionColumnNames.find(name => conf.resolver(name, partitionName)) match {
          case Some(_) =>
          case None =>
            throw QueryCompilationErrors.nonPartitionColError(partitionName)
        }
      }
    }

    private def addStaticPartitionColumns(
        relation: DataSourceV2Relation,
        query: LogicalPlan,
        staticPartitions: Map[String, String],
        isByName: Boolean): LogicalPlan = {

      if (staticPartitions.isEmpty) {
        query

      } else {
        // add any static value as a literal column
        val withStaticPartitionValues = {
          // for each static name, find the column name it will replace and check for unknowns.
          val outputNameToStaticName = staticPartitions.keySet.map { staticName =>
            if (isByName) {
              // If this is INSERT INTO BY NAME, the query output's names will be the user specified
              // column names. We need to make sure the static partition column name doesn't appear
              // there to catch the following ambiguous query:
              // INSERT OVERWRITE t PARTITION (c='1') (c) VALUES ('2')
              if (query.output.exists(col => conf.resolver(col.name, staticName))) {
                throw QueryCompilationErrors.staticPartitionInUserSpecifiedColumnsError(staticName)
              }
            }
            relation.output.find(col => conf.resolver(col.name, staticName)) match {
              case Some(attr) =>
                attr.name -> staticName
              case _ =>
                throw QueryCompilationErrors.missingStaticPartitionColumn(staticName)
            }
          }.toMap

          val queryColumns = query.output.iterator

          // for each output column, add the static value as a literal, or use the next input
          // column. this does not fail if input columns are exhausted and adds remaining columns
          // at the end. both cases will be caught by ResolveOutputRelation and will fail the
          // query with a helpful error message.
          relation.output.flatMap { col =>
            outputNameToStaticName.get(col.name).flatMap(staticPartitions.get) match {
              case Some(staticValue) =>
                // SPARK-30844: try our best to follow StoreAssignmentPolicy for static partition
                // values but not completely follow because we can't do static type checking due to
                // the reason that the parser has erased the type info of static partition values
                // and converted them to string.
                val cast = Cast(Literal(staticValue), col.dataType, ansiEnabled = true)
                cast.setTagValue(Cast.BY_TABLE_INSERTION, ())
                Some(Alias(cast, col.name)())
              case _ if queryColumns.hasNext =>
                Some(queryColumns.next())
              case _ =>
                None
            }
          } ++ queryColumns
        }

        Project(withStaticPartitionValues, query)
      }
    }

    private def staticDeleteExpression(
        relation: DataSourceV2Relation,
        staticPartitions: Map[String, String]): Expression = {
      if (staticPartitions.isEmpty) {
        Literal(true)
      } else {
        staticPartitions.map { case (name, value) =>
          relation.output.find(col => conf.resolver(col.name, name)) match {
            case Some(attr) =>
              // the delete expression must reference the table's column names, but these attributes
              // are not available when CheckAnalysis runs because the relation is not a child of
              // the logical operation. instead, expressions are resolved after
              // ResolveOutputRelation runs, using the query's column names that will match the
              // table names at that point. because resolution happens after a future rule, create
              // an UnresolvedAttribute.
              EqualNullSafe(
                UnresolvedAttribute.quoted(attr.name),
                Cast(Literal(value), attr.dataType))
            case None =>
              throw QueryCompilationErrors.missingStaticPartitionColumn(name)
          }
        }.reduce(And)
      }
    }
  }

  /**
   * Resolves column references in the query plan. Basically it transform the query plan tree bottom
   * up, and only try to resolve references for a plan node if all its children nodes are resolved,
   * and there is no conflicting attributes between the children nodes (see `hasConflictingAttrs`
   * for details).
   *
   * The general workflow to resolve references:
   * 1. Expands the star in Project/Aggregate/Generate.
   * 2. Resolves the columns to [[AttributeReference]] with the output of the children plans. This
   *    includes metadata columns as well.
   * 3. Resolves the columns to literal function which is allowed to be invoked without braces,
   *    e.g. `SELECT col, current_date FROM t`.
   * 4. Resolves the columns to outer references with the outer plan if we are resolving subquery
   *    expressions.
   * 5. Resolves the columns to SQL variables.
   *
   * Some plan nodes have special column reference resolution logic, please read these sub-rules for
   * details:
   *  - [[ResolveReferencesInAggregate]]
   *  - [[ResolveReferencesInUpdate]]
   *  - [[ResolveReferencesInSort]]
   *
   * Note: even if we use a single rule to resolve columns, it's still non-trivial to have a
   *       reliable column resolution order, as the rule will be executed multiple times, with other
   *       rules in the same batch. We should resolve columns with the next option only if all the
   *       previous options are permanently not applicable. If the current option can be applicable
   *       in the next iteration (other rules update the plan), we should not try the next option.
   */
  class ResolveReferences(val catalogManager: CatalogManager)
    extends Rule[LogicalPlan] with ColumnResolutionHelper {

    private val resolveColumnDefaultInCommandInputQuery =
      new ResolveColumnDefaultInCommandInputQuery(catalogManager)
    private val resolveReferencesInAggregate =
      new ResolveReferencesInAggregate(catalogManager)
    private val resolveReferencesInUpdate =
      new ResolveReferencesInUpdate(catalogManager)
    private val resolveReferencesInSort =
      new ResolveReferencesInSort(catalogManager)
    private val resolveDataFrameDropColumns =
      new ResolveDataFrameDropColumns(catalogManager)

    /**
     * Return true if there're conflicting attributes among children's outputs of a plan
     *
     * The children logical plans may output columns with conflicting attribute IDs. This may happen
     * in cases such as self-join. We should wait for the rule [[DeduplicateRelations]] to eliminate
     * conflicting attribute IDs, otherwise we can't resolve columns correctly due to ambiguity.
     */
    def hasConflictingAttrs(p: LogicalPlan): Boolean = {
      p.children.length > 1 && {
        // Note that duplicated attributes are allowed within a single node,
        // e.g., df.select($"a", $"a"), so we should only check conflicting
        // attributes between nodes.
        val uniqueAttrs = mutable.HashSet[ExprId]()
        p.children.head.outputSet.foreach(a => uniqueAttrs.add(a.exprId))
        p.children.tail.exists { child =>
          val uniqueSize = uniqueAttrs.size
          val childSize = child.outputSet.size
          child.outputSet.foreach(a => uniqueAttrs.add(a.exprId))
          uniqueSize + childSize > uniqueAttrs.size
        }
      }
    }

    def apply(plan: LogicalPlan): LogicalPlan = {
      val collatedPlan =
        if (conf.getConf(SQLConf.RUN_COLLATION_TYPE_CASTS_BEFORE_ALIAS_ASSIGNMENT)) {
          CollationRulesRunner(plan)
        } else {
          plan
        }
      doApply(collatedPlan)
    }

    def doApply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
      // `InsertIntoStatement.table` and `V2WriteCommand.table` are non-child `LogicalPlan`
      // slots (`child = query`), so the default `resolveOperatorsUp` + `mapExpressions`
      // traversal never resolves expressions placed inside them. For a
      // `PlanWithUnresolvedIdentifier`, `identifierExpr` (e.g. an `UnresolvedAttribute`
      // referring to a SQL variable in `INSERT INTO IDENTIFIER(target_table) ...`) must
      // be resolved here before `ResolveIdentifierClause` can materialize the relation.
      // Mirror the structural recursion into the non-child `.table` slot that
      // `BindParameters` and `ResolveIdentifierClause` already do for the same shape
      // (SPARK-46625); unlike those rules, this one performs attribute resolution rather
      // than parameter binding or placeholder materialization. Resolve against `p` (whose
      // `children` are `Nil` on the INSERT / `OverwriteByExpression` path built by
      // `buildWriteTableSlot`) so the IDENTIFIER expression cannot see query output
      // columns -- only the last-resort variable resolution path fires. The
      // `!identifierExpr.resolved` guard makes the case idempotent under bottom-up
      // traversal.
      case i: InsertIntoStatement
          if i.table.isInstanceOf[PlanWithUnresolvedIdentifier] &&
             !i.table.asInstanceOf[PlanWithUnresolvedIdentifier].identifierExpr.resolved =>
        val p = i.table.asInstanceOf[PlanWithUnresolvedIdentifier]
        val resolvedExpr = resolveExpressionByPlanChildren(
          p.identifierExpr, p, includeLastResort = true)
        i.copy(table = p.copy(identifierExpr = resolvedExpr))

      case w: V2WriteCommand
          if w.table.isInstanceOf[PlanWithUnresolvedIdentifier] &&
             !w.table.asInstanceOf[PlanWithUnresolvedIdentifier].identifierExpr.resolved =>
        val p = w.table.asInstanceOf[PlanWithUnresolvedIdentifier]
        val resolvedExpr = resolveExpressionByPlanChildren(
          p.identifierExpr, p, includeLastResort = true)
        w.withNewTable(p.copy(identifierExpr = resolvedExpr))

      // Don't wait other rules to resolve the child plans of `InsertIntoStatement` as we need
      // to resolve column "DEFAULT" in the child plans so that they must be unresolved.
      case i: InsertIntoStatement => resolveColumnDefaultInCommandInputQuery(i)

      // Don't wait other rules to resolve the child plans of `SetVariable` as we need
      // to resolve column "DEFAULT" in the child plans so that they must be unresolved.
      case s: SetVariable => resolveColumnDefaultInCommandInputQuery(s)

      // Skip FetchCursor - let ResolveFetchCursor handle variable resolution
      // This prevents ResolveReferences from trying to resolve target variables as columns
      case s: SingleStatement if s.parsedPlan.isInstanceOf[FetchCursor] => s
      case f: FetchCursor => f

      // Wait for other rules to resolve child plans first
      case p: LogicalPlan if !p.childrenResolved => p

      // Wait for the rule `DeduplicateRelations` to resolve conflicting attrs first.
      case p: LogicalPlan if hasConflictingAttrs(p) => p

      // If the projection list contains Stars, expand it.
      case p: Project if containsStar(p.projectList) =>
        val expanded = p.copy(projectList = buildExpandedProjectList(p.projectList, p.child))
        if (expanded.projectList.size < p.projectList.size) {
          checkTrailingCommaInSelect(expanded, starRemoved = true)
        }
        expanded
      // If the filter list contains Stars, expand it.
      case p: Filter if containsStar(Seq(p.condition)) =>
        p.copy(expandStarExpression(p.condition, p.child))
      // If the aggregate function argument contains Stars, expand it.
      case a: Aggregate if containsStar(a.aggregateExpressions) =>
        if (a.groupingExpressions.exists(_.isInstanceOf[UnresolvedOrdinal])) {
          throw QueryCompilationErrors.starNotAllowedWhenGroupByOrdinalPositionUsedError()
        } else {
          val expanded = a.copy(aggregateExpressions =
            buildExpandedProjectList(a.aggregateExpressions, a.child))
          if (expanded.aggregateExpressions.size < a.aggregateExpressions.size) {
            checkTrailingCommaInSelect(expanded, starRemoved = true)
          }
          expanded
        }
      case c: CollectMetrics if containsStar(c.metrics) =>
        c.copy(metrics = buildExpandedProjectList(c.metrics, c.child))
      case g: Generate if containsStar(g.generator.children) =>
        throw QueryCompilationErrors.invalidStarUsageError("explode/json_tuple/UDTF",
          extractStar(g.generator.children))
      // If the Unpivot ids or values contain Stars, expand them.
      case up: Unpivot if up.ids.exists(containsStar) ||
        // Only expand Stars in one-dimensional values
        up.values.exists(values => values.exists(_.length == 1) && values.exists(containsStar)) =>
        up.copy(
          ids = up.ids.map(buildExpandedProjectList(_, up.child)),
          // The inner exprs in Option[[exprs] is one-dimensional, e.g. Optional[[["*"]]].
          // The single NamedExpression turns into multiple, which we here have to turn into
          // Optional[[["col1"], ["col2"]]]
          values = up.values.map(_.flatMap(buildExpandedProjectList(_, up.child)).map(Seq(_)))
        )

      case u @ Union(children, _, _)
          // if there are duplicate output columns, give them unique expr ids
          if (u.allChildrenCompatible &&
          conf.getConf(SQLConf.ENFORCE_TYPE_COERCION_BEFORE_UNION_DEDUPLICATION)) &&
          children.exists(c => c.output.map(_.exprId).distinct.length < c.output.length) =>
        DeduplicateUnionChildOutput.deduplicateOutputPerChild(u)

      // A special case for Generate, because the output of Generate should not be resolved by
      // ResolveReferences. Attributes in the output will be resolved by ResolveGenerate.
      case g @ Generate(generator, _, _, _, _, _) if generator.resolved => g

      case g @ Generate(generator, join, outer, qualifier, output, child) =>
        val newG = resolveExpressionByPlanOutput(
          generator, child, throws = true, includeLastResort = true)
        if (newG.fastEquals(generator)) {
          g
        } else {
          Generate(newG.asInstanceOf[Generator], join, outer, qualifier, output, child)
        }

      case mg: MapGroups if mg.dataOrder.exists(!_.resolved) =>
        // Resolve against `AppendColumns`'s children, instead of `AppendColumns`,
        // because `AppendColumns`'s serializer might produce conflict attribute
        // names leading to ambiguous references exception.
        val planForResolve = mg.child match {
          case appendColumns: AppendColumns => appendColumns.child
          case plan => plan
        }
        val resolvedOrder = mg.dataOrder
          .map(resolveExpressionByPlanOutput(_, planForResolve).asInstanceOf[SortOrder])
        mg.copy(dataOrder = resolvedOrder)

      // Left and right sort expression have to be resolved against the respective child plan only
      case cg: CoGroup if cg.leftOrder.exists(!_.resolved) || cg.rightOrder.exists(!_.resolved) =>
        // Resolve against `AppendColumns`'s children, instead of `AppendColumns`,
        // because `AppendColumns`'s serializer might produce conflict attribute
        // names leading to ambiguous references exception.
        val (leftPlanForResolve, rightPlanForResolve) = Seq(cg.left, cg.right).map {
          case appendColumns: AppendColumns => appendColumns.child
          case plan => plan
        } match {
          case Seq(left, right) => (left, right)
        }

        val resolvedLeftOrder = cg.leftOrder
          .map(resolveExpressionByPlanOutput(_, leftPlanForResolve).asInstanceOf[SortOrder])
        val resolvedRightOrder = cg.rightOrder
          .map(resolveExpressionByPlanOutput(_, rightPlanForResolve).asInstanceOf[SortOrder])

        cg.copy(leftOrder = resolvedLeftOrder, rightOrder = resolvedRightOrder)

      // Skips plan which contains deserializer expressions, as they should be resolved by another
      // rule: ResolveDeserializer.
      case plan if containsDeserializer(plan.expressions) => plan

      case a: Aggregate => resolveReferencesInAggregate(a)

      // Special case for Project as it supports lateral column alias.
      case p: Project =>
        val resolvedBasic = p.projectList.map(resolveExpressionByPlanChildren(_, p))
        // Lateral column alias has higher priority than outer reference.
        val resolvedWithLCA = resolveLateralColumnAlias(resolvedBasic)
        val resolvedFinal = resolvedWithLCA.map(resolveColsLastResort)
        p.copy(projectList = resolvedFinal.map(_.asInstanceOf[NamedExpression]))

      case o: OverwriteByExpression if o.table.resolved =>
        // The delete condition of `OverwriteByExpression` will be passed to the table
        // implementation and should be resolved based on the table schema.
        o.copy(deleteExpr = resolveExpressionByPlanOutput(o.deleteExpr, o.table))

      case u: UpdateTable => resolveReferencesInUpdate(u)

      case m @ MergeIntoTable(targetTable, sourceTable, _, _, _, _, _)
        if !m.resolved && targetTable.resolved && sourceTable.resolved =>

        // Do not throw exception for schema evolution case.
        // This allows unresolved assignment keys a chance to be resolved by a second pass
        // by newly column/nested fields added by schema evolution.
        val throws = !m.schemaEvolutionEnabled

        EliminateSubqueryAliases(targetTable) match {
          case r: NamedRelation if r.skipSchemaResolution =>
            // Do not resolve the expression if the target table accepts any schema.
            // This allows data sources to customize their own resolution logic using
            // custom resolution rules.
            m

          case _ =>
            def findAttrInTarget(name: String): Option[Attribute] = {
              targetTable.output.find(targetAttr => conf.resolver(name, targetAttr.name))
            }
            val newMatchedActions = m.matchedActions.map {
              case DeleteAction(deleteCondition) =>
                val resolvedDeleteCondition = deleteCondition.map(
                  resolveExpressionByPlanChildren(_, m))
                DeleteAction(resolvedDeleteCondition)
              case UpdateAction(updateCondition, assignments, fromStar) =>
                val resolvedUpdateCondition = updateCondition.map(
                  resolveExpressionByPlanChildren(_, m))
                UpdateAction(
                  resolvedUpdateCondition,
                  // The update value can access columns from both target and source tables.
                  resolveAssignments(assignments, m, MergeResolvePolicy.BOTH, throws),
                  fromStar)
              case UpdateStarAction(updateCondition) =>
                // Expand star to top level source columns.  If source has less columns than target,
                // assignments will be added by ResolveRowLevelCommandAssignments later.
                val assignments = if (m.schemaEvolutionEnabled) {
                  // For schema evolution case, generate assignments for missing target columns.
                  // These columns will be added by ResolveSchemaEvolution later.
                  sourceTable.output.map { sourceAttr =>
                    val key = findAttrInTarget(sourceAttr.name).getOrElse(
                      UnresolvedAttribute.quoted(sourceAttr.name))
                    Assignment(key, sourceAttr)
                  }
                } else {
                  targetTable.output.map { attr =>
                    Assignment(attr, UnresolvedAttribute(Seq(attr.name)))
                  }
                }
                UpdateAction(
                  updateCondition.map(resolveExpressionByPlanChildren(_, m)),
                  // For UPDATE *, the value must be from source table.
                  resolveAssignments(assignments, m, MergeResolvePolicy.SOURCE, throws),
                  fromStar = true)
              case o => o
            }
            val newNotMatchedActions = m.notMatchedActions.map {
              case InsertAction(insertCondition, assignments) =>
                // The insert action is used when not matched, so its condition and value can only
                // access columns from the source table.
                val resolvedInsertCondition = insertCondition.map(
                  resolveExpressionByPlanOutput(_, m.sourceTable))
                InsertAction(
                  resolvedInsertCondition,
                  resolveAssignments(assignments, m, MergeResolvePolicy.SOURCE, throws))
              case InsertStarAction(insertCondition) =>
                // The insert action is used when not matched, so its condition and value can only
                // access columns from the source table.
                val resolvedInsertCondition = insertCondition.map(
                  resolveExpressionByPlanOutput(_, m.sourceTable))
                // Expand star to top level source columns.  If source has less columns than target,
                // assignments will be added by ResolveRowLevelCommandAssignments later.
                val assignments = if (m.schemaEvolutionEnabled) {
                  // For schema evolution case, generate assignments for missing target columns.
                  // These columns will be added by ResolveSchemaEvolution later.
                  sourceTable.output.map { sourceAttr =>
                    val key = findAttrInTarget(sourceAttr.name).getOrElse(
                      UnresolvedAttribute.quoted(sourceAttr.name))
                    Assignment(key, sourceAttr)
                  }
                } else {
                  targetTable.output.map { attr =>
                    Assignment(attr, UnresolvedAttribute(Seq(attr.name)))
                  }
                }
                InsertAction(
                  resolvedInsertCondition,
                  resolveAssignments(assignments, m, MergeResolvePolicy.SOURCE, throws))
              case o => o
            }
            val newNotMatchedBySourceActions = m.notMatchedBySourceActions.map {
              case DeleteAction(deleteCondition) =>
                val resolvedDeleteCondition = deleteCondition.map(
                  resolveExpressionByPlanOutput(_, targetTable))
                DeleteAction(resolvedDeleteCondition)
              case UpdateAction(updateCondition, assignments, fromStar) =>
                val resolvedUpdateCondition = updateCondition.map(
                  resolveExpressionByPlanOutput(_, targetTable))
                UpdateAction(
                  resolvedUpdateCondition,
                  // The update value can access columns from the target table only.
                  resolveAssignments(assignments, m, MergeResolvePolicy.TARGET, throws),
                  fromStar)
              case o => o
            }

            val resolvedMergeCondition = resolveExpressionByPlanChildren(m.mergeCondition, m)
            m.copy(mergeCondition = resolvedMergeCondition,
              matchedActions = newMatchedActions,
              notMatchedActions = newNotMatchedActions,
              notMatchedBySourceActions = newNotMatchedBySourceActions)
        }

      // UnresolvedHaving can host grouping expressions and aggregate functions. We should resolve
      // columns with `agg.output` and the rule `ResolveAggregateFunctions` will push them down to
      // Aggregate later.
      case u @ UnresolvedHaving(cond, agg: Aggregate) if !cond.resolved =>
        u.mapExpressions { e =>
          // Columns in HAVING should be resolved with `agg.child.output` first, to follow the SQL
          // standard. See more details in SPARK-31519.
          val resolvedWithAgg = resolveColWithAgg(e, agg)
          resolveExpressionByPlanChildren(resolvedWithAgg, u, includeLastResort = true)
        }

      // RepartitionByExpression can host missing attributes that are from a descendant node.
      // For example, `spark.table("t").select($"a").repartition($"b")`. We can resolve `b` with
      // table `t` even if there is a Project node between the table scan node and Sort node.
      // We also need to propagate the missing attributes from the descendant node to the current
      // node, and project them way at the end via an extra Project.
      case r @ RepartitionByExpression(partitionExprs, child, _, _)
        if !r.resolved || r.missingInput.nonEmpty =>
        val resolvedBasic = partitionExprs.map(resolveExpressionByPlanChildren(_, r))
        val (newPartitionExprs, newChild) = resolveExprsAndAddMissingAttrs(resolvedBasic, child)
        // Missing columns should be resolved right after basic column resolution.
        // See the doc of `ResolveReferences`.
        val resolvedFinal = newPartitionExprs.map(resolveColsLastResort)
        if (child.output == newChild.output) {
          r.copy(resolvedFinal, newChild)
        } else {
          Project(child.output, r.copy(resolvedFinal, newChild))
        }

      // Skip the qualify clause here. It will be handled in ResolveQualify.
      case q: UnresolvedQualify => q

      // Filter can host both grouping expressions/aggregate functions and missing attributes.
      // The grouping expressions/aggregate functions resolution takes precedence over missing
      // attributes. See the classdoc of `ResolveReferences` for details.
      case f @ Filter(cond, child) if !cond.resolved || f.missingInput.nonEmpty =>
        val resolvedBasic = resolveExpressionByPlanChildren(cond, f)
        val resolvedWithAgg = resolveColWithAgg(resolvedBasic, child)
        val (newCond, newChild) = resolveExprsAndAddMissingAttrs(Seq(resolvedWithAgg), child)
        // Missing columns should be resolved right after basic column resolution.
        // See the doc of `ResolveReferences`.
        val resolvedFinal = resolveColsLastResort(newCond.head)
        if (child.output == newChild.output) {
          f.copy(condition = resolvedFinal)
        } else {
          // Add missing attributes and then project them away.
          val newFilter = Filter(resolvedFinal, newChild)
          Project(child.output, newFilter)
        }

      case s: Sort if !s.resolved || s.missingInput.nonEmpty =>
        resolveReferencesInSort(s)


      case d: DataFrameDropColumns if !d.resolved =>
        resolveDataFrameDropColumns(d)

      case q: LogicalPlan =>
        logTrace(s"Attempting to resolve ${q.simpleString(conf.maxToStringFields)}")
        q.mapExpressions(resolveExpressionByPlanChildren(_, q, includeLastResort = true))
    }

    private object MergeResolvePolicy extends Enumeration {
      val BOTH, SOURCE, TARGET = Value
    }

    def resolveAssignments(
        assignments: Seq[Assignment],
        mergeInto: MergeIntoTable,
        resolvePolicy: MergeResolvePolicy.Value,
        throws: Boolean): Seq[Assignment] = {
      assignments.map { assign =>
        val resolvedKey = assign.key match {
          case c if !c.resolved =>
            resolveMergeExpr(c, Project(Nil, mergeInto.targetTable), throws)
          case o => o
        }
        val resolvedValue = assign.value match {
          case c if !c.resolved =>
            val resolvePlan = resolvePolicy match {
              case MergeResolvePolicy.BOTH => mergeInto
              case MergeResolvePolicy.SOURCE => Project(Nil, mergeInto.sourceTable)
              case MergeResolvePolicy.TARGET => Project(Nil, mergeInto.targetTable)
            }
            val resolvedExpr = resolveExprInAssignment(c, resolvePlan)
            val withDefaultResolved = if (conf.enableDefaultColumns) {
              resolveColumnDefaultInAssignmentValue(
                resolvedKey,
                resolvedExpr,
                QueryCompilationErrors
                  .defaultReferencesNotAllowedInComplexExpressionsInMergeInsertsOrUpdates())
            } else {
              resolvedExpr
            }
            if (throws) {
              checkResolvedMergeExpr(withDefaultResolved, resolvePlan)
            }
            withDefaultResolved
          case o => o
        }
        Assignment(resolvedKey, resolvedValue)
      }
    }

    private def resolveMergeExpr(e: Expression, p: LogicalPlan, throws: Boolean): Expression = {
      val resolved = resolveExprInAssignment(e, p, throws)
      if (throws) {
        checkResolvedMergeExpr(resolved, p)
      }
      resolved
    }

    private def checkResolvedMergeExpr(e: Expression, p: LogicalPlan): Unit = {
      e.references.filter(!_.resolved).foreach { a =>
        // Note: This will throw error only on unresolved attribute issues,
        // not other resolution errors like mismatched data types.
        val cols = p.inputSet.toSeq.map(attr => toSQLId(attr.name)).mkString(", ")
        a.failAnalysis(
          errorClass = "UNRESOLVED_COLUMN.WITH_SUGGESTION",
          messageParameters = Map(
            "objectName" -> toSQLId(a.name),
            "proposal" -> cols))
      }
    }

    // Expand the star expression using the input plan first. If failed, try resolve
    // the star expression using the outer query plan and wrap the resolved attributes
    // in outer references. Otherwise throw the original exception.
    private def expand(s: Star, plan: LogicalPlan): Seq[NamedExpression] = {
      withPosition(s) {
        try {
          s.expand(plan, resolver)
        } catch {
          case e: AnalysisException =>
            AnalysisContext.get.outerPlan.map {
              // Only Project, Aggregate, CollectMetrics can host star expressions.
              case u @ (_: Project | _: Aggregate | _: CollectMetrics) =>
                Try(s.expand(u.children.head, resolver)) match {
                  case Success(expanded) => expanded.map(wrapOuterReference)
                  case Failure(_) => throw e
                }
              // Do not use the outer plan to resolve the star expression
              // since the star usage is invalid.
              case _ => throw e
            }.getOrElse { throw e }
        }
      }
    }

    /**
     * Build a project list for Project/Aggregate and expand the star if possible
     */
    private def buildExpandedProjectList(
      exprs: Seq[NamedExpression],
      child: LogicalPlan): Seq[NamedExpression] = {
      exprs.flatMap {
        // Using Dataframe/Dataset API: testData2.groupBy($"a", $"b").agg($"*")
        case s: Star => expand(s, child).map(aliasIfOuterReference)
        // Using SQL API without running ResolveAlias: SELECT * FROM testData2 group by a, b
        case UnresolvedAlias(s: Star, _) => expand(s, child).map(aliasIfOuterReference)
        case o if containsStar(o :: Nil) => expandStarExpression(o, child) :: Nil
        case o => o :: Nil
      }.map(_.asInstanceOf[NamedExpression])
    }

    /**
     * Wrap an outer-scope star expansion result in [[Alias]] so that the [[OuterReference]]
     * attribute gets a fresh ExprId in the subquery's scope. This prevents the outer ExprId from
     * leaking through [[Project.output]] when the expansion goes through a derived table.
     * Struct star expansion already produces [[Alias]] nodes, so those are left unchanged.
     */
    private def aliasIfOuterReference(e: NamedExpression): NamedExpression = e match {
      case _: Alias => e
      case outerReference: OuterReference =>
        Alias(outerReference, toPrettySQL(outerReference.e))()
      case _ => e
    }

    /**
     * Returns true if `exprs` contains a [[Star]].
     */
    def containsStar(exprs: Seq[Expression]): Boolean =
      exprs.exists(_.collectFirst { case _: Star => true }.nonEmpty)

    private def extractStar(exprs: Seq[Expression]): Seq[Star] =
      exprs.flatMap(_.collect { case s: Star => s })

    private def isCountStarExpansionAllowed(arguments: Seq[Expression]): Boolean = arguments match {
      case Seq(UnresolvedStar(None)) => true
      case Seq(_: ResolvedStar) => true
      case _ => false
    }

    /**
     * Checks if the given function name parts match the expected builtin function name.
     * This is used for special syntax transformations (e.g., COUNT(*) -> COUNT(1)) that
     * should only apply to builtin functions, not to user-defined functions.
     *
     * When the effective SQL PATH puts `system.session` before `system.builtin`, temp
     * functions shadow builtins, so an unqualified name that matches a temp function
     * should NOT be treated as builtin.
     */
    private def matchesFunctionName(nameParts: Seq[String], expectedName: String): Boolean = {
      if (!FunctionResolution.isUnqualifiedOrBuiltinFunctionName(nameParts, expectedName)) {
        return false
      }
      if (nameParts.size == 1 && functionResolution.isSessionBeforeBuiltinInPath) {
        val v1Catalog = catalogManager.v1SessionCatalog
        !v1Catalog.isTemporaryFunction(FunctionIdentifier(nameParts.head))
      } else {
        true
      }
    }

    /**
     * Expands the matching attribute.*'s in `child`'s output.
     */
    def expandStarExpression(expr: Expression, child: LogicalPlan): Expression = {
      expr.transformUp {
        case f0: UnresolvedFunction if !f0.isDistinct &&
          matchesFunctionName(f0.nameParts, "count") &&
          isCountStarExpansionAllowed(f0.arguments) =>
          // Transform COUNT(*) into COUNT(1).
          // We do not normalize the name to "count"; we keep the original name parts
          // (e.g. builtin.count, system.builtin.count) so that resolution still sees
          // the same qualification.
          f0.copy(arguments = Seq(Literal(1)))
        case f1: UnresolvedFunction if containsStar(f1.arguments) =>
          // SPECIAL CASE: We want to block count(tblName.*) because in spark, count(tblName.*) will
          // be expanded while count(*) will be converted to count(1). They will produce different
          // results and confuse users if there are any null values. For count(t1.*, t2.*), it is
          // still allowed, since it's well-defined in spark.
          if (!conf.allowStarWithSingleTableIdentifierInCount &&
              matchesFunctionName(f1.nameParts, "count") &&
              f1.arguments.length == 1) {
            f1.arguments.foreach {
              case u: UnresolvedStar if u.isQualifiedByTable(child.output, resolver) =>
                throw QueryCompilationErrors
                  .singleTableStarInCountNotAllowedError(u.target.get.mkString("."))
              case _ => // do nothing
            }
          }
          f1.copy(arguments = f1.arguments.flatMap {
            case s: Star => expand(s, child)
            case o => o :: Nil
          })
        case c: CreateNamedStruct if containsStar(c.valExprs) =>
          val newChildren = c.children.grouped(2).flatMap {
            case Seq(k, s : Star) => CreateStruct(expand(s, child)).children
            case kv => kv
          }
          c.copy(children = newChildren.toList )
        case c: CreateArray if containsStar(c.children) =>
          c.copy(children = c.children.flatMap {
            case s: Star => expand(s, child)
            case o => o :: Nil
          })
        case p: Murmur3Hash if containsStar(p.children) =>
          p.copy(children = p.children.flatMap {
            case s: Star => expand(s, child)
            case o => o :: Nil
          })
        case p: XxHash64 if containsStar(p.children) =>
          p.copy(children = p.children.flatMap {
            case s: Star => expand(s, child)
            case o => o :: Nil
          })
        case p: In if containsStar(p.children) =>
          p.copy(list = p.list.flatMap {
            case s: Star => expand(s, child)
            case o => o :: Nil
          })
        // count(*) has been replaced by count(1)
        case o if containsStar(o.children) =>
          throw QueryCompilationErrors.invalidStarUsageError(s"expression `${o.prettyName}`",
            extractStar(o.children))
      }
    }
  }

  private def containsDeserializer(exprs: Seq[Expression]): Boolean = {
    exprs.exists(_.exists(_.isInstanceOf[UnresolvedDeserializer]))
  }

  /**
   * In many dialects of SQL it is valid to use ordinal positions in order/sort by and group by
   * clauses. This rule is to convert ordinal positions to the corresponding expressions in the
   * select list. This support is introduced in Spark 2.0.
   *
   * - When the sort references or group by expressions are not integer but foldable expressions,
   * just ignore them.
   * - When spark.sql.orderByOrdinal/spark.sql.groupByOrdinal is set to false, ignore the position
   * numbers too.
   *
   * Before the release of Spark 2.0, the literals in order/sort by and group by clauses
   * have no effect on the results.
   */
  object ResolveOrdinalInOrderByAndGroupBy extends Rule[LogicalPlan] {
    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
      _.containsPattern(UNRESOLVED_ORDINAL), ruleId) {
      case p if !p.childrenResolved => p
      // Replace the index with the related attribute for ORDER BY,
      // which is a 1-base position of the projection list.
      case Sort(orders, global, child, hint)
        if orders.exists(_.child.isInstanceOf[UnresolvedOrdinal]) =>
        val newOrders = orders map {
          case s @ SortOrder(UnresolvedOrdinal(index), direction, nullOrdering, _) =>
            if (index > 0 && index <= child.output.size) {
              SortOrder(child.output(index - 1), direction, nullOrdering, Seq.empty)
            } else {
              throw QueryCompilationErrors.orderByPositionRangeError(index, child.output.size, s)
            }
          case o => o
        }
        Sort(newOrders, global, child, hint)

      // Replace the index with the corresponding expression in aggregateExpressions. The index is
      // a 1-base position of aggregateExpressions, which is output columns (select expression)
      case Aggregate(groups, aggs, child, hint)
        if aggs
          .filter(!containUnresolvedPipeAggregateOrdinal(_))
          .forall(_.resolved) &&
        groups.exists(containUnresolvedOrdinal) =>
        val newAggs = aggs.map(resolvePipeAggregateExpressionOrdinal(_, child.output))
        val newGroups = groups.map(resolveGroupByExpressionOrdinal(_, newAggs))
        Aggregate(newGroups, newAggs, child, hint)
    }

    private def containUnresolvedOrdinal(e: Expression): Boolean = e match {
      case _: UnresolvedOrdinal => true
      case gs: BaseGroupingSets => gs.children.exists(containUnresolvedOrdinal)
      case _ => false
    }

    private def containUnresolvedPipeAggregateOrdinal(e: Expression): Boolean = e match {
      case _: UnresolvedPipeAggregateOrdinal => true
      case _ => false
    }

    private def resolveGroupByExpressionOrdinal(
        expr: Expression,
        aggs: Seq[Expression]): Expression = expr match {
      case ordinal @ UnresolvedOrdinal(index) =>
        withPosition(ordinal) {
          if (index > 0 && index <= aggs.size) {
            val ordinalExpr = aggs(index - 1)

            if (ordinalExpr.exists(_.isInstanceOf[AggregateExpression])) {
              throw QueryCompilationErrors.groupByPositionRefersToAggregateFunctionError(
                index, ordinalExpr)
            }

            ordinalExpr
          } else {
            throw QueryCompilationErrors.groupByPositionRangeError(index, aggs.size)
          }
        }
      case gs: BaseGroupingSets =>
        gs.withNewChildren(gs.children.map(resolveGroupByExpressionOrdinal(_, aggs)))
      case others => others
    }
  }

  private def resolvePipeAggregateExpressionOrdinal(
      expr: NamedExpression,
      inputs: Seq[Attribute]): NamedExpression = expr match {
    case UnresolvedPipeAggregateOrdinal(index) =>
      // In this case, the user applied the SQL pipe aggregate operator ("|> AGGREGATE") and used
      // ordinals in its GROUP BY clause. This expression then refers to the i-th attribute of the
      // child operator (one-based). Here we resolve the ordinal to the corresponding attribute.
      inputs(index - 1)
    case other =>
      other
  }

  /**
   * Checks whether a function identifier referenced by an [[UnresolvedFunction]] is defined in the
   * function registry. Note that this rule doesn't try to resolve the [[UnresolvedFunction]]. It
   * only performs simple existence check according to the function identifier to quickly identify
   * undefined functions without triggering relation resolution, which may incur potentially
   * expensive partition/schema discovery process in some cases.
   *
   * To avoid duplicate external catalog lookups, this rule maintains a per-plan cache of
   * persistent function names (externalFunctionNameSet). Builtin and temporary functions are
   * validated on every occurrence since they're fast in-memory lookups, but persistent functions
   * are cached after the first validation to avoid repeated external catalog calls for the same
   * function within a single plan.
   *
   * @see [[ResolveFunctions]]
   * @see https://issues.apache.org/jira/browse/SPARK-19737
   */
  object LookupFunctions extends Rule[LogicalPlan] {
    override def apply(plan: LogicalPlan): LogicalPlan = {
      val externalFunctionNameSet = new mutable.HashSet[Seq[String]]()

      plan.resolveExpressionsWithPruning(_.containsAnyPattern(UNRESOLVED_FUNCTION)) {
        case f @ UnresolvedFunction(nameParts, _, _, _, _, _, _) =>
          // For builtin/temp functions, we can do a quick check without catalog lookup
          val quickCheck = if (nameParts.size == 1 ||
              FunctionResolution.sessionNamespaceKind(nameParts).isDefined) {
            functionResolution.lookupBuiltinOrTempFunction(nameParts, Some(f))
          } else {
            None
          }

          if (quickCheck.isDefined) {
            // It's a builtin or temp function - no need for catalog lookup or caching
            f
          } else {
            // Might be a persistent function - compute full name and check cache first
            val CatalogAndIdentifier(catalog, ident) =
              relationResolution.expandIdentifier(nameParts)

            val fullName = normalizeFuncName(
              (catalog.name +: ident.namespace :+ ident.name).toImmutableArraySeq)

            if (externalFunctionNameSet.contains(fullName)) {
              // Already validated this function exists - skip lookup
              f
            } else {
              // Not in cache - do full lookup to determine type
              val functionType =
                functionResolution.lookupFunctionType(nameParts, Some(f))

              functionType match {
                case FunctionType.Local =>
                  throw SparkException.internalError(
                    s"Logic inconsistency: Function ${nameParts.mkString(".")} was " +
                    s"classified as $functionType by full lookup but not found by quick check. " +
                    s"Check sessionNamespaceKind logic.")

                case FunctionType.Persistent =>
                  externalFunctionNameSet.add(fullName)
                  f

                case FunctionType.TableOnly =>
                  throw QueryCompilationErrors.notAScalarFunctionError(nameParts.mkString("."), f)

                case FunctionType.NotFound =>
                  val searchPath =
                    functionResolution.sqlResolutionPathEntriesForAnalysis.map(_.quoted)
                  throw QueryCompilationErrors.unresolvedRoutineError(
                    nameParts,
                    searchPath,
                    f.origin)
              }
            }
          }
      }
    }

    def normalizeFuncName(name: Seq[String]): Seq[String] = {
      if (conf.caseSensitiveAnalysis) {
        name
      } else {
        name.map(_.toLowerCase(Locale.ROOT))
      }
    }
  }

  /**
   * Replaces [[UnresolvedFunctionName]]s with concrete [[LogicalPlan]]s.
   * Replaces [[UnresolvedFunction]]s with concrete [[Expression]]s.
   * Replaces [[UnresolvedGenerator]]s with concrete [[Expression]]s.
   * Replaces [[UnresolvedTableValuedFunction]]s with concrete [[LogicalPlan]]s.
   */
  object ResolveFunctions extends Rule[LogicalPlan] {
    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
      _.containsAnyPattern(UNRESOLVED_FUNC, UNRESOLVED_FUNCTION, GENERATOR,
        UNRESOLVED_TABLE_VALUED_FUNCTION, UNRESOLVED_TVF_ALIASES), ruleId) {
      // Resolve scalar/table functions and get the function metadata for DESCRIBE FUNCTION.
      case u @ UnresolvedFunctionName(nameParts, _, _) =>
        functionResolution.lookupBuiltinOrTempFunction(nameParts, None)
          .orElse(functionResolution.lookupBuiltinOrTempTableFunction(nameParts)).map { info =>
          ResolvedNonPersistentFunc(nameParts.head, V1Function.metadataOnly(info))
        }.getOrElse {
          val CatalogAndIdentifier(catalog, ident) =
            relationResolution.expandIdentifier(nameParts)
          val fullName = catalog.name +: ident.namespace :+ ident.name
          CatalogV2Util.loadFunction(catalog, ident).map { func =>
            ResolvedPersistentFunc(catalog.asFunctionCatalog, ident, func)
          }.getOrElse(u.copy(possibleQualifiedName = Some(fullName.toImmutableArraySeq)))
        }

      // Resolve table-valued function references.
      case u: UnresolvedTableValuedFunction if u.functionArgs.forall(_.resolved) =>
        withPosition(u) {
          val resolvedFunc = functionResolution.resolveTableFunction(u.name, u.functionArgs)
            .getOrElse {
              u.failAnalysis(
                errorClass = "UNRESOLVABLE_TABLE_VALUED_FUNCTION",
                messageParameters = Map("name" -> toSQLId(u.name)))
            }

          resolvedFunc.transformAllExpressionsWithPruning(
            _.containsPattern(FUNCTION_TABLE_RELATION_ARGUMENT_EXPRESSION))  {
            case t: FunctionTableSubqueryArgumentExpression =>
              resolvedFunc match {
                case Generate(_: PythonUDTF, _, _, _, _, _) =>
                case Generate(_: UnresolvedPolymorphicPythonUDTF, _, _, _, _, _) =>
                case _ =>
                  assert(!t.hasRepartitioning,
                    "Cannot evaluate the table-valued function call because it included the " +
                      "PARTITION BY clause, but only Python table functions support this " +
                      "clause")
              }
              t
          }
        }

      // Resolve table-valued functions' output column aliases.
      case u: UnresolvedTVFAliases if u.child.resolved =>
        // Add `Project` with the aliases.
        val outputAttrs = u.child.output
        // Checks if the number of the aliases is equal to expected one
        if (u.outputNames.size != outputAttrs.size) {
          u.failAnalysis(
            errorClass = "NUM_TABLE_VALUE_ALIASES_MISMATCH",
            messageParameters = Map(
              "funcName" -> toSQLId(u.name),
              "aliasesNum" -> u.outputNames.size.toString,
              "outColsNum" -> outputAttrs.size.toString))
        }
        val aliases = outputAttrs.zip(u.outputNames).map {
          case (attr, name) => Alias(attr, name)()
        }
        Project(aliases, u.child)

      case p: LogicalPlan
          if p.resolved && p.containsPattern(FUNCTION_TABLE_RELATION_ARGUMENT_EXPRESSION) =>
        withPosition(p) {
          val tableArgs =
            mutable.ArrayBuffer.empty[(FunctionTableSubqueryArgumentExpression, LogicalPlan)]

          val tvf = p.transformExpressionsWithPruning(
            _.containsPattern(FUNCTION_TABLE_RELATION_ARGUMENT_EXPRESSION)) {
            case t: FunctionTableSubqueryArgumentExpression =>
              val alias = SubqueryAlias.generateSubqueryName(s"_${tableArgs.size}")
              tableArgs.append((t, SubqueryAlias(alias, t.evaluable)))
              UnresolvedAttribute(Seq(alias, "c"))
          }

          assert(tableArgs.nonEmpty)
          if (!conf.tvfAllowMultipleTableArguments && tableArgs.size > 1) {
            throw QueryCompilationErrors.tableValuedFunctionTooManyTableArgumentsError(
              tableArgs.size)
          }
          val alias = SubqueryAlias.generateSubqueryName(s"_${tableArgs.size}")

          // Propagate the column indexes for TABLE arguments to the PythonUDTF instance.
          val f: FunctionTableSubqueryArgumentExpression = tableArgs.head._1
          val tvfWithTableColumnIndexes = tvf match {
            case g @ Generate(pyudtf: PythonUDTF, _, _, _, _, _)
                if f.extraProjectedPartitioningExpressions.nonEmpty =>
              val partitionColumnIndexes = if (f.selectedInputExpressions.isEmpty) {
                PythonUDTFPartitionColumnIndexes(f.partitioningExpressionIndexes)
              } else {
                // If the UDTF specified 'select' expression(s), we added a projection to compute
                // them plus the 'partitionBy' expression(s) afterwards.
                PythonUDTFPartitionColumnIndexes(
                  (0 until f.extraProjectedPartitioningExpressions.length)
                    .map(_ + f.selectedInputExpressions.length))
              }
              g.copy(generator = pyudtf.copy(
                pythonUDTFPartitionColumnIndexes = Some(partitionColumnIndexes)))
            case _ => tvf
          }

          val lateralJoin = LateralJoin(
            tableArgs.map(_._2).reduceLeft(Join(_, _, Inner, None, JoinHint.NONE)),
            LateralSubquery(SubqueryAlias(alias, tvfWithTableColumnIndexes)), Inner, None)

          // Set the tag so that it can be used to differentiate lateral join added by
          // TABLE argument vs added by user.
          lateralJoin.setTagValue(LateralJoin.BY_TABLE_ARGUMENT, ())

          Project(Seq(UnresolvedStar(Some(Seq(alias)))), lateralJoin)
        }

      case q: LogicalPlan =>
        q.transformExpressionsUpWithPruning(
          _.containsAnyPattern(UNRESOLVED_FUNCTION, GENERATOR),
          ruleId) {
          case u @ UnresolvedFunction(nameParts, arguments, _, _, _, _, _)
              if functionResolution.hasLambdaAndResolvedArguments(arguments) => withPosition(u) {
            functionResolution.resolveFunction(u) match {
              case func: HigherOrderFunction => func
              case other => other.failAnalysis(
                errorClass = "INVALID_LAMBDA_FUNCTION_CALL.NON_HIGHER_ORDER_FUNCTION",
                messageParameters = Map(
                  "class" -> other.getClass.getCanonicalName))
            }
          }

          case u if !u.childrenResolved => u // Skip until children are resolved.

          case u @ UnresolvedGenerator(name, arguments) => withPosition(u) {
            // For generator function, the parser only accepts v1 function name and creates
            // `FunctionIdentifier`.
            v1SessionCatalog.lookupFunction(name, arguments) match {
              case generator: Generator => generator
              case other => throw QueryCompilationErrors.generatorNotExpectedError(
                name, other.getClass.getCanonicalName)
            }
          }

          case u: UnresolvedFunction => functionResolution.resolveFunction(u)

          case u: UnresolvedPolymorphicPythonUDTF => withPosition(u) {
            // Check if this is a call to a Python user-defined table function whose polymorphic
            // 'analyze' method returned metadata indicated requested partitioning and/or
            // ordering properties of the input relation. In that event, make sure that the UDTF
            // call did not include any explicit PARTITION BY and/or ORDER BY clauses for the
            // corresponding TABLE argument, and then update the TABLE argument representation
            // to apply the requested partitioning and/or ordering.
            val analyzeResult = u.resolveElementMetadata(u.func, u.children)
            val newChildren = u.children.map {
              case NamedArgumentExpression(key, t: FunctionTableSubqueryArgumentExpression) =>
                NamedArgumentExpression(key, analyzeResult.applyToTableArgument(u.name, t))
              case t: FunctionTableSubqueryArgumentExpression =>
                analyzeResult.applyToTableArgument(u.name, t)
              case c => c
            }
            PythonUDTF(
              u.name, u.func, analyzeResult.schema, Some(analyzeResult.pickledAnalyzeResult),
              newChildren, u.evalType, u.udfDeterministic, u.resultId, None, u.tableArguments)
          }
        }
    }
  }

  /**
   * A rule that resolves procedures.
   */
  object ResolveProcedures extends Rule[LogicalPlan] {
    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsWithPruning(
      _.containsPattern(UNRESOLVED_PROCEDURE), ruleId) {
      case u: UnresolvedProcedure =>
        functionResolution.resolveProcedure(u)
    }
  }

  /**
   * A rule that binds procedures to the input types and rearranges arguments as needed.
   */
  object BindProcedures extends Rule[LogicalPlan] {
    def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
      case Call(ResolvedProcedure(catalog, ident, unbound: UnboundProcedure), args, execute)
          if args.forall(_.resolved) =>
        val inputType = extractInputType(args)
        val bound = unbound.bind(inputType)
        validateParameterModes(bound)
        val rearrangedArgs =
          NamedParametersSupport.defaultRearrange(bound, args, SQLConf.get.resolver)
        Call(ResolvedProcedure(catalog, ident, bound), rearrangedArgs, execute)
    }

    private def extractInputType(args: Seq[Expression]): StructType = {
      val fields = args.zipWithIndex.map {
        case (NamedArgumentExpression(name, value), _) =>
          StructField(name, value.dataType, value.nullable, byNameMetadata)
        case (arg, index) =>
          StructField(s"param$index", arg.dataType, arg.nullable)
      }
      StructType(fields)
    }

    private def byNameMetadata: Metadata = {
      new MetadataBuilder()
        .putBoolean(ProcedureParameter.BY_NAME_METADATA_KEY, value = true)
        .build()
    }

   private def validateParameterModes(procedure: BoundProcedure): Unit = {
     procedure.parameters.find(_.mode != ProcedureParameter.Mode.IN).foreach { param =>
       throw SparkException.internalError(s"Unsupported parameter mode: ${param.mode}")
     }
   }
  }

  /**
   * This rule resolves and rewrites subqueries inside expressions.
   *
   * Note: CTEs are handled in CTESubstitution.
   */
  object ResolveSubquery extends Rule[LogicalPlan] {
    /**
     * Resolves the subquery plan that is referenced in a subquery expression, by invoking the
     * entire analyzer recursively. We set outer plan in `AnalysisContext`, so that the analyzer
     * can resolve outer references.
     *
     * Outer references of the subquery are updated as children of Subquery expression.
     */
    private def resolveSubQuery(
        e: SubqueryExpression,
        outer: LogicalPlan)(
        f: (LogicalPlan, Seq[Expression]) => SubqueryExpression): SubqueryExpression = {
      val newSubqueryPlan = SQLFunctionContext.withNewContext {
        AnalysisContext.withOuterPlan(outer) {
          executeSameContext(e.plan)
        }
      }

      // If the subquery plan is fully resolved, pull the outer references and record
      // them as children of SubqueryExpression.
      if (newSubqueryPlan.resolved) {
        // Record the outer references as children of subquery expression.
        f(newSubqueryPlan, SubExprUtils.getOuterReferences(newSubqueryPlan))
      } else {
        e.withNewPlan(newSubqueryPlan)
      }
    }

    /**
     * Resolves the subquery. Apart of resolving the subquery and outer references (if any)
     * in the subquery plan, the children of subquery expression are updated to record the
     * outer references. This is needed to make sure
     * (1) The column(s) referred from the outer query are not pruned from the plan during
     *     optimization.
     * (2) Any aggregate expression(s) that reference outer attributes are pushed down to
     *     outer plan to get evaluated.
     */
    private def resolveSubQueries(plan: LogicalPlan, outer: LogicalPlan): LogicalPlan = {
      plan.transformAllExpressionsWithPruning(_.containsPattern(PLAN_EXPRESSION), ruleId) {
        case s @ ScalarSubquery(sub, _, exprId, _, _, _, _) if !sub.resolved =>
          resolveSubQuery(s, outer)(ScalarSubquery(_, _, exprId))
        case e @ Exists(sub, _, exprId, _, _) if !sub.resolved =>
          resolveSubQuery(e, outer)(Exists(_, _, exprId))
        case InSubquery(values, l @ ListQuery(_, _, exprId, _, _, _))
            if values.forall(_.resolved) && !l.resolved =>
          val expr = resolveSubQuery(l, outer)((plan, exprs) => {
            ListQuery(plan, exprs, exprId, plan.output.length)
          })
          InSubquery(values, expr.asInstanceOf[ListQuery])
        case s @ LateralSubquery(sub, _, exprId, _, _) if !sub.resolved =>
          resolveSubQuery(s, outer)(LateralSubquery(_, _, exprId))
        case a: FunctionTableSubqueryArgumentExpression if !a.plan.resolved =>
          resolveSubQuery(a, outer)(
            (plan, outerAttrs) => a.copy(plan = plan, outerAttrs = outerAttrs))
        // The subquery's plan is already resolved. Replace any V2TableReferences without
        // re-running any analyzer rules.
        case se: SubqueryExpression
            if se.plan.resolved &&
               se.plan.collectFirstWithSubqueries { case _: V2TableReference => () }.isDefined =>
          val newPlan = se.plan.transformWithSubqueries {
            case r: V2TableReference => relationResolution.resolveReference(r)
          }
          se.withNewPlan(newPlan)
      }
    }

    /**
     * Resolve and rewrite all subqueries in an operator tree..
     */
    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
      _.containsPattern(PLAN_EXPRESSION), ruleId) {
      case j: LateralJoin if j.left.resolved =>
        // We can't pass `LateralJoin` as the outer plan, as its right child is not resolved yet
        // and we can't call `LateralJoin.resolveChildren` to resolve outer references. Here we
        // create a fake Project node as the outer plan.
        resolveSubQueries(j, Project(Nil, j.left))
      // Skip the qualify clause here. It will be handled in ResolveQualify.
      case q: UnresolvedQualify => q
      // Only a few unary nodes (Project/Filter/Aggregate) can contain subqueries.
      case q: UnaryNode if q.childrenResolved =>
        resolveSubQueries(q, q)
      case r: RelationTimeTravel =>
        resolveSubQueries(r, r)
      case j: Join if j.childrenResolved && j.duplicateResolved =>
        resolveSubQueries(j, j)
      case tvf: UnresolvedTableValuedFunction =>
        resolveSubQueries(tvf, tvf)
      case s: SupportsSubquery if s.childrenResolved =>
        resolveSubQueries(s, s)
    }
  }

  /**
   * Replaces unresolved column aliases for a subquery with projections.
   */
  object ResolveSubqueryColumnAliases extends Rule[LogicalPlan] {

     def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
       _.containsPattern(UNRESOLVED_SUBQUERY_COLUMN_ALIAS), ruleId) {
      case u @ UnresolvedSubqueryColumnAliases(columnNames, child) if child.resolved =>
        // Resolves output attributes if a query has alias names in its subquery:
        // e.g., SELECT * FROM (SELECT 1 AS a, 1 AS b) t(col1, col2)
        val outputAttrs = child.output
        // Checks if the number of the aliases equals to the number of output columns
        // in the subquery.
        if (columnNames.size != outputAttrs.size) {
          throw QueryCompilationErrors.aliasNumberNotMatchColumnNumberError(
            columnNames.size, outputAttrs.size, u)
        }
        val aliases = outputAttrs.zip(columnNames).map { case (attr, aliasName) =>
          Alias(attr, aliasName)()
        }
        Project(aliases, child)
    }
  }

  /**
   * This rule resolves SQL function expressions. It pulls out function inputs and place them
   * in a separate [[Project]] node below the operator and replace the SQL function with its
   * actual function body. SQL function expressions in [[Aggregate]] are handled in a special
   * way. Non-aggregated SQL functions in the aggregate expressions of an Aggregate need to be
   * pulled out into a Project above the Aggregate before replacing the SQL function expressions
   * with actual function bodies. For example:
   *
   * Before:
   *   Aggregate [c1] [foo(c1), foo(max(c2)), sum(foo(c2)) AS sum]
   *   +- Relation [c1, c2]
   *
   * After:
   *   Project [foo(c1), foo(max_c2), sum]
   *   +- Aggregate [c1] [c1, max(c2) AS max_c2, sum(foo(c2)) AS sum]
   *      +- Relation [c1, c2]
   */
  object ResolveSQLFunctions extends Rule[LogicalPlan] {

    private def hasSQLFunctionExpression(exprs: Seq[Expression]): Boolean = {
      exprs.exists(_.find(_.isInstanceOf[SQLFunctionExpression]).nonEmpty)
    }

    /**
     * Check if the function input contains aggregate expressions.
     */
    private def checkFunctionInput(f: SQLFunctionExpression): Unit = {
      if (f.inputs.exists(AggregateExpression.containsAggregate)) {
        // The input of a SQL function should not contain aggregate functions after
        // `extractAndRewrite`. If there are aggregate functions, it means they are
        // nested in another aggregate function, which is not allowed.
        // For example: SELECT sum(foo(sum(c1))) FROM t
        // We have to throw the error here because otherwise the query plan after
        // resolving the SQL function will not be valid.
        throw new AnalysisException(
          errorClass = "NESTED_AGGREGATE_FUNCTION",
          messageParameters = Map.empty)
      }
    }

    /**
     * Resolve a SQL function expression as a logical plan check if it can be analyzed.
     */
    private def resolve(f: SQLFunctionExpression): LogicalPlan = {
      // Validate the SQL function input.
      checkFunctionInput(f)
      val plan = v1SessionCatalog.makeSQLFunctionPlan(f.name, f.function, f.inputs)
      val resolved = SQLFunctionContext.withSQLFunction {
        // Resolve the SQL function plan using its context.
        val newConf = new SQLConf()
        f.function.getSQLConfigs.foreach { case (k, v) => newConf.settings.put(k, v) }
        if (conf.getConf(SQLConf.APPLY_SESSION_CONF_OVERRIDES_TO_FUNCTION_RESOLUTION)) {
          Analyzer.retainResolutionConfigsForAnalysis(newConf = newConf, existingConf = conf)
        }
        SQLConf.withExistingConf(newConf) {
          AnalysisContext.withAnalysisContext(f.function) {
            executeSameContext(plan)
          }
        }
      }
      // Fail the analysis eagerly if a SQL function cannot be resolved using its input.
      SimpleAnalyzer.checkAnalysis(resolved)
      resolved
    }

    /**
     * Rewrite SQL function expressions into actual resolved function bodies and extract
     * function inputs into the given project list.
     */
    private def rewriteSQLFunctions[E <: Expression](
        expression: E,
        projectList: ArrayBuffer[NamedExpression]): E = {
      val newExpr = expression match {
        case f: SQLFunctionExpression if !hasSQLFunctionExpression(f.inputs) &&
          // Make sure LateralColumnAliasReference in parameters is resolved and eliminated first.
          // Otherwise, the projectList can contain the LateralColumnAliasReference, which will be
          // pushed down to a Project without the 'referenced' alias by LCA present, leaving it
          // unresolved.
          !f.inputs.exists(_.containsPattern(LATERAL_COLUMN_ALIAS_REFERENCE)) =>
          withPosition(f) {
            // Set CurrentOrigin to the SQL function call site so that input-binding
            // Casts constructed inside makeSQLFunctionPlan capture the call-site
            // position in their queryContext snapshot (see Cast.initQueryContext).
            withOrigin(f.origin) {
              val plan = resolve(f)
              // Extract the function input project list from the SQL function plan and
              // inline the SQL function expression.
              plan match {
                case Project(body :: Nil, Project(aliases, _: OneRowRelation)) =>
                  val inputs = aliases.map(stripOuterReference)
                  projectList ++= inputs
                  SQLScalarFunction(f.function, inputs.map(_.toAttribute), body)
                case o =>
                  throw new AnalysisException(
                    errorClass = "INVALID_SQL_FUNCTION_PLAN_STRUCTURE",
                    messageParameters = Map("plan" -> o.toString))
              }
            }
          }
        case o => o.mapChildren(rewriteSQLFunctions(_, projectList))
      }
      newExpr.asInstanceOf[E]
    }

    /**
     * Check if the given expression contains expressions that should be extracted,
     * i.e. non-aggregated SQL functions with non-foldable inputs.
     */
    private def shouldExtract(e: Expression): Boolean = e match {
      // Return false if the expression is already an aggregate expression.
      case _: AggregateExpression => false
      case _: SQLFunctionExpression => true
      case _: LeafExpression => false
      case o => o.children.exists(shouldExtract)
    }

    /**
     * Extract aggregate expressions from the given expression and replace
     * them with attribute references.
     * Example:
     *   Before: foo(c1) + foo(max(c2)) + max(foo(c2))
     *   After: foo(c1) + foo(max_c2) + max_foo_c2
     *   Extracted expressions: [c1, max(c2) AS max_c2, max(foo(c2)) AS max_foo_c2]
     */
    private def extractAndRewrite[T <: Expression](
        expression: T,
        extractedExprs: ArrayBuffer[NamedExpression]): T = {
      val newExpr = expression match {
        case e if !shouldExtract(e) =>
          val exprToAdd: NamedExpression = e match {
            case o: OuterReference => Alias(o, toPrettySQL(o.e))()
            case ne: NamedExpression => ne
            case o => Alias(o, toPrettySQL(o))()
          }
          extractedExprs += exprToAdd
          exprToAdd.toAttribute
        case f: SQLFunctionExpression =>
          val newInputs = f.inputs.map(extractAndRewrite(_, extractedExprs))
          f.copy(inputs = newInputs)
        case o => o.mapChildren(extractAndRewrite(_, extractedExprs))
      }
      newExpr.asInstanceOf[T]
    }

    /**
     * Replace all [[SQLFunctionExpression]]s in an expression with attribute references
     * from the aliasMap.
     */
    private def replaceSQLFunctionWithAttr[T <: Expression](
        expr: T,
        aliasMap: mutable.HashMap[Expression, Alias]): T = {
      expr.transform {
        case f: SQLFunctionExpression if aliasMap.contains(f.canonicalized) =>
          aliasMap(f.canonicalized).toAttribute
      }.asInstanceOf[T]
    }

    private def rewrite(plan: LogicalPlan): LogicalPlan = plan match {
      // Return if a sub-tree does not contain SQLFunctionExpression.
      case p: LogicalPlan if !p.containsPattern(SQL_FUNCTION_EXPRESSION) => p

      case f @ Filter(cond, a: Aggregate)
        if !f.resolved || AggregateExpression.containsAggregate(cond) ||
          ResolveGroupingAnalytics.hasGroupingFunction(cond) ||
          cond.containsPattern(TEMP_RESOLVED_COLUMN) =>
        // If the filter's condition contains aggregate expressions or grouping expressions or temp
        // resolved column, we cannot rewrite both the filter and the aggregate until they are
        // resolved by ResolveAggregateFunctions or ResolveGroupingAnalytics, because rewriting SQL
        // functions in aggregate can add an additional project on top of the aggregate
        // which breaks the pattern matching in those rules.
        f.copy(child = a.copy(child = rewrite(a.child)))

      case h @ UnresolvedHaving(_, a: Aggregate) =>
        // Similarly UnresolvedHaving should be resolved by ResolveAggregateFunctions first
        // before rewriting aggregate.
        h.copy(child = a.copy(child = rewrite(a.child)))

      case a: Aggregate if a.resolved && hasSQLFunctionExpression(a.expressions) =>
        val child = rewrite(a.child)
        // Extract SQL functions in the grouping expressions and place them in a project list
        // below the current aggregate. Also update their appearances in the aggregate expressions.
        val bottomProjectList = ArrayBuffer.empty[NamedExpression]
        val aliasMap = mutable.HashMap.empty[Expression, Alias]
        val newGrouping = a.groupingExpressions.map { expr =>
          expr.transformDown {
            case f: SQLFunctionExpression =>
              val alias = aliasMap.getOrElseUpdate(f.canonicalized, Alias(f, f.name)())
              bottomProjectList += alias
              alias.toAttribute
          }
        }
        val aggregateExpressions = a.aggregateExpressions.map(
          replaceSQLFunctionWithAttr(_, aliasMap))

        // Rewrite SQL functions in the aggregate expressions that are not wrapped in
        // aggregate functions. They need to be extracted into a project list above the
        // current aggregate.
        val aggExprs = ArrayBuffer.empty[NamedExpression]
        val topProjectList = aggregateExpressions.map(extractAndRewrite(_, aggExprs))

        // Rewrite SQL functions in the new aggregate expressions that are wrapped inside
        // aggregate functions.
        val newAggExprs = aggExprs.map(rewriteSQLFunctions(_, bottomProjectList))

        val bottomProject = if (bottomProjectList.nonEmpty) {
          Project(child.output ++ bottomProjectList, child)
        } else {
          child
        }
        val newAgg = if (newGrouping.nonEmpty || newAggExprs.nonEmpty) {
          a.copy(
            groupingExpressions = newGrouping,
            aggregateExpressions = newAggExprs.toSeq,
            child = bottomProject)
        } else {
          bottomProject
        }
        if (topProjectList.nonEmpty) Project(topProjectList, newAgg) else newAgg

      case p: Project if p.resolved && hasSQLFunctionExpression(p.expressions) =>
        val newChild = rewrite(p.child)
        val projectList = ArrayBuffer.empty[NamedExpression]
        val newPList = p.projectList.map(rewriteSQLFunctions(_, projectList))
        if (newPList != newChild.output) {
          p.copy(newPList, Project(newChild.output ++ projectList, newChild))
        } else {
          assert(projectList.isEmpty)
          p.copy(child = newChild)
        }

      case f: Filter if f.resolved && hasSQLFunctionExpression(f.expressions) =>
        val newChild = rewrite(f.child)
        val projectList = ArrayBuffer.empty[NamedExpression]
        val newCond = rewriteSQLFunctions(f.condition, projectList)
        if (newCond != f.condition) {
          Project(f.output, Filter(newCond, Project(newChild.output ++ projectList, newChild)))
        } else {
          assert(projectList.isEmpty)
          f.copy(child = newChild)
        }

      case j: Join if j.resolved && hasSQLFunctionExpression(j.expressions) =>
        val newLeft = rewrite(j.left)
        val newRight = rewrite(j.right)
        val projectList = ArrayBuffer.empty[NamedExpression]
        val joinCond = j.condition.map(rewriteSQLFunctions(_, projectList))
        if (joinCond != j.condition) {
          // Join condition cannot have non-deterministic expressions. We can safely
          // replace the aliases with the original SQL function input expressions.
          val aliasMap = projectList.collect { case a: Alias => a.toAttribute -> a.child }.toMap
          val newJoinCond = joinCond.map(_.transform {
            case a: Attribute => aliasMap.getOrElse(a, a)
          })
          j.copy(left = newLeft, right = newRight, condition = newJoinCond)
        } else {
          assert(projectList.isEmpty)
          j.copy(left = newLeft, right = newRight)
        }

      case o: LogicalPlan if o.resolved && hasSQLFunctionExpression(o.expressions) =>
        o.transformExpressionsWithPruning(_.containsPattern(SQL_FUNCTION_EXPRESSION)) {
          case f: SQLFunctionExpression =>
            f.failAnalysis(
              errorClass = "UNSUPPORTED_SQL_UDF_USAGE",
              messageParameters = Map(
                "functionName" -> toSQLId(f.function.name.nameParts),
                "nodeName" -> o.nodeName.toString))
        }

      case p: LogicalPlan => p.mapChildren(rewrite)
    }

    def apply(plan: LogicalPlan): LogicalPlan = {
      // Only rewrite SQL functions when they are not in nested function calls.
      if (SQLFunctionContext.get.nestedSQLFunctionDepth > 0) {
        plan
      } else {
        rewrite(plan)
      }
    }
  }

  /*
   * This rule resolves SQL table functions.
   */
  object ResolveSQLTableFunctions extends Rule[LogicalPlan] with AliasHelper {

    /**
     * Check if a subquery plan is subject to the COUNT bug that can cause wrong results.
     * A lateral correlation can only be removed if the lateral subquery is not subject to
     * the COUNT bug. Currently only lateral correlation can handle it correctly.
     */
    private def hasCountBug(sub: LogicalPlan): Boolean = sub.find {
      // The COUNT bug occurs when there is an Aggregate that satisfies all the following
      // conditions:
      // 1) is on the correlation path
      // 2) has non-empty group by expressions
      // 3) has one or more output columns that evaluate to non-null values with empty input.
      //    E.g: COUNT(empty row) = 0.
      // For simplicity, we use a stricter criteria (1 and 2 only) to determine if a query
      // is subject to the COUNT bug.
      case a: Aggregate if a.groupingExpressions.nonEmpty => hasOuterReferences(a.child)
      case _ => false
    }.nonEmpty

    /**
     * Rewrite a resolved SQL table function plan by removing unnecessary lateral joins:
     * Before:
     *   LateralJoin lateral-subquery [a], Inner
     *   :  +- Project [c1, c2]
     *   :     +- Filter [outer(a) == c1]
     *   :        +- Relation [c1, c2]
     *   +- Project [1 AS a]
     *      +- OneRowRelation
     * After:
     *   Project [c1, c2]
     *   +- Filter [1 == c1]  <---- Replaced outer(a)
     *      +- Relation [c1, c2]
     */
    private def rewrite(plan: LogicalPlan): LogicalPlan = {
      (plan transformUp {
        case j @ LateralJoin(Project(aliases, _: OneRowRelation), sub: LateralSubquery, Inner, None)
            if j.resolved && aliases.forall(_.deterministic) =>
          val attrMap = AttributeMap(aliases.collect { case a: Alias => a.toAttribute -> a.child })
          val newPlan = sub.plan.transformAllExpressionsWithPruning(
            _.containsPattern(OUTER_REFERENCE)) {
            // Avoid replacing outer references that do not belong to the current outer plan.
            // This can happen if the child of an alias also contains outer references (nested
            // table function references). E.g:
            // LateralJoin
            // :  +- Filter [outer(a) == x]
            // :     +- Relation [x, y]
            // +- Project [outer(c) AS a]
            //    +- OneRowRelation
            case OuterReference(a: Attribute) if attrMap.contains(a) => attrMap(a) match {
              case ne: NamedExpression => ne
              case o => Alias(o, a.name)(exprId = a.exprId, qualifier = a.qualifier)
            }
          }
          // Keep the original lateral join if the new plan is subject to the count bug.
          if (hasCountBug(newPlan)) j else newPlan
      }).transformWithPruning(_.containsPattern(ALIAS)) {
        // As a result of the above rewriting, we may end-up introducing nested Aliases (i.e.,
        // Aliases defined inside Aliases). This is problematic for the plan canonicalization as
        // it doesn't expect nested Aliases. Therefore, here we remove non-top level Aliases.
        case node => node.mapExpressions(trimNonTopLevelAliases)
      }
    }

    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
      _.containsPattern(SQL_TABLE_FUNCTION)) {
      case SQLTableFunction(name, function, inputs, output) =>
        // Resolve the SQL table function plan using its function context.
        val newConf = new SQLConf()
        function.getSQLConfigs.foreach { case (k, v) => newConf.settings.put(k, v) }
        if (conf.getConf(SQLConf.APPLY_SESSION_CONF_OVERRIDES_TO_FUNCTION_RESOLUTION)) {
          Analyzer.retainResolutionConfigsForAnalysis(newConf = newConf, existingConf = conf)
        }
        val resolved = SQLConf.withExistingConf(newConf) {
          val plan = v1SessionCatalog.makeSQLTableFunctionPlan(name, function, inputs, output)
          SQLFunctionContext.withSQLFunction {
            AnalysisContext.withAnalysisContext(function) {
              executeSameContext(plan)
            }
          }
        }
        // Remove unnecessary lateral joins that are used to resolve the SQL function.
        val newPlan = rewrite(resolved)
        // Fail the analysis eagerly if a SQL table function cannot be resolved using its input.
        SimpleAnalyzer.checkAnalysis(newPlan)
        newPlan
    }
  }

  /**
   * Turns projections that contain aggregate expressions into aggregations.
   */
  object GlobalAggregates extends Rule[LogicalPlan] {
    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
      t => t.containsAnyPattern(AGGREGATE_EXPRESSION, PYTHON_UDF) && t.containsPattern(PROJECT),
      ruleId) {
      case Project(projectList, child) if containsAggregates(projectList) =>
        Aggregate(Nil, projectList, child)
    }

    def containsAggregates(exprs: Seq[Expression]): Boolean = {
      // Collect all Windowed Aggregate Expressions.
      val windowedAggExprs: Set[Expression] = exprs.flatMap { expr =>
        expr.collect {
          case WindowExpression(ae: AggregateExpression, _) => ae
          case UnresolvedWindowExpression(ae: AggregateExpression, _) => ae
        }
      }.toSet

      // Find the first Aggregate Expression that is not Windowed.
      exprs.exists(_.exists {
        case ae: AggregateExpression => !windowedAggExprs.contains(ae)
        case _ => false
      })
    }
  }

  /**
   * This rule finds aggregate expressions that are not in an aggregate operator.  For example,
   * those in a HAVING clause or ORDER BY clause.  These expressions are pushed down to the
   * underlying aggregate operator and then projected away after the original operator.
   *
   * We need to make sure the expressions all fully resolved before looking for aggregate functions
   * and group by expressions from them.
   */
  object ResolveAggregateFunctions extends Rule[LogicalPlan] {
    def apply(plan: LogicalPlan): LogicalPlan = {
      val collatedPlan =
        if (conf.getConf(SQLConf.RUN_COLLATION_TYPE_CASTS_BEFORE_ALIAS_ASSIGNMENT)) {
          CollationRulesRunner(plan)
        } else {
          plan
        }
      doApply(collatedPlan)
    }

    def doApply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
      _.containsPattern(AGGREGATE), ruleId) {
      case UnresolvedHaving(cond, agg: Aggregate) if agg.resolved && cond.resolved =>
        resolveOperatorWithAggregate(Seq(cond), agg, (newExprs, newChild) => {
          val newCond = newExprs.head
          if (newCond.resolved) {
            Filter(newCond, newChild)
          } else {
            // The condition can be unresolved after the resolution, as we may mark
            // `TempResolvedColumn` as unresolved if it's not aggregate function inputs or grouping
            // expressions. We should remain `UnresolvedHaving` as the rule `ResolveReferences` can
            // re-resolve `TempResolvedColumn` and `UnresolvedHaving` has a special column
            // resolution order.
            UnresolvedHaving(newCond, newChild)
          }
        })

      case Filter(cond, agg: Aggregate) if agg.resolved && cond.resolved =>
        resolveOperatorWithAggregate(Seq(cond), agg, (newExprs, newChild) => {
          Filter(newExprs.head, newChild)
        })

      case s @ Sort(_, _, agg: Aggregate, _) if agg.resolved && s.order.forall(_.resolved) =>
        resolveOperatorWithAggregate(s.order.map(_.child), agg, (newExprs, newChild) => {
          val newSortOrder = s.order.zip(newExprs).map {
            case (sortOrder, expr) => sortOrder.copy(child = expr)
          }
          s.copy(order = newSortOrder, child = newChild)
        })

      case s @ Sort(_, _, f @ Filter(cond, agg: Aggregate), _)
          if agg.resolved && cond.resolved && s.order.forall(_.resolved) =>
        resolveOperatorWithAggregate(s.order.map(_.child), agg, (newExprs, newChild) => {
          val newSortOrder = s.order.zip(newExprs).map {
            case (sortOrder, expr) => sortOrder.copy(child = expr)
          }
          s.copy(order = newSortOrder, child = f.copy(child = newChild))
        })
    }

    /**
     * Resolves the given expressions as if they are in the given Aggregate operator, which means
     * the column can be resolved using `agg.child` and aggregate functions/grouping columns are
     * allowed. It returns a list of named expressions that need to be appended to
     * `agg.aggregateExpressions`, and the list of resolved expressions.
     */
    def resolveExprsWithAggregate(
        exprs: Seq[Expression],
        agg: Aggregate): (Seq[NamedExpression], Seq[Expression]) = {
      val extraAggExprs = new LinkedHashMap[Expression, NamedExpression]
      val transformed = exprs.map { e =>
        if (!e.resolved) {
          e
        } else {
          buildAggExprList(e, agg, extraAggExprs)
        }
      }
      (extraAggExprs.values().asScala.toSeq, transformed)
    }

    private def buildAggExprList(
        expr: Expression,
        agg: Aggregate,
        aggExprMap: LinkedHashMap[Expression, NamedExpression]): Expression = {
      // Avoid adding an extra aggregate expression if it's already present in
      // `agg.aggregateExpressions`. Trim inner aliases from aggregate expressions because of
      // expressions like `spark_grouping_id` that can have inner aliases.
      val replacement: Option[NamedExpression] =
        agg.aggregateExpressions.foldLeft(Option.empty[NamedExpression]) {
          case (None, alias: Alias) if expr.semanticEquals(trimAliases(alias.child)) => Some(alias)
          case (None | Some(_: Alias), aggExpr) if expr.semanticEquals(aggExpr) => Some(aggExpr)
          case (current, _) => current
        }
      if (replacement.isDefined) {
        replacement.get.toAttribute
      } else {
        expr match {
          case ae: AggregateExpression =>
            val cleaned = trimTempResolvedColumn(ae)
            val resultAlias = aggExprMap.computeIfAbsent(
              cleaned.canonicalized,
              _ => Alias(cleaned, toPrettySQL(e = cleaned, shouldTrimTempResolvedColumn = true))()
            )
            resultAlias.toAttribute
          case grouping: Expression if agg.groupingExpressions.exists(grouping.semanticEquals) =>
            trimTempResolvedColumn(grouping) match {
              case ne: NamedExpression =>
                val resultAttribute = aggExprMap.computeIfAbsent(ne.canonicalized, _ => ne)
                resultAttribute.toAttribute
              case other =>
                val resultAlias = aggExprMap.computeIfAbsent(
                  other.canonicalized,
                  _ => Alias(other, toPrettySQL(e = other, shouldTrimTempResolvedColumn = true))()
                )
                resultAlias.toAttribute
            }
          case t: TempResolvedColumn =>
            if (t.child.isInstanceOf[Attribute]) {
              // This column is neither inside aggregate functions nor a grouping column. It
              // shouldn't be resolved with `agg.child.output`. Mark it as "hasTried", so that it
              // can be re-resolved later or go back to `UnresolvedAttribute` at the end.
              withOrigin(t.origin)(t.copy(hasTried = true))
            } else {
              // This is a nested column, we still have a chance to match grouping expressions with
              // the top-level column. Here we wrap the underlying `Attribute` with
              // `TempResolvedColumn` and try again.
              val childWithTempCol = t.child.transformUp {
                case a: Attribute => TempResolvedColumn(a, Seq(a.name))
              }
              val newChild = buildAggExprList(childWithTempCol, agg, aggExprMap)
              if (newChild.containsPattern(TEMP_RESOLVED_COLUMN)) {
                withOrigin(t.origin)(t.copy(hasTried = true))
              } else {
                newChild
              }
            }
          case other =>
            other.withNewChildren(other.children.map(buildAggExprList(_, agg, aggExprMap)))
        }
      }
    }

    def resolveOperatorWithAggregate(
        exprs: Seq[Expression],
        agg: Aggregate,
        buildOperator: (Seq[Expression], Aggregate) => LogicalPlan): LogicalPlan = {
      val (extraAggExprs, resolvedExprs) = resolveExprsWithAggregate(exprs, agg)
      if (extraAggExprs.isEmpty) {
        buildOperator(resolvedExprs, agg)
      } else {
        Project(agg.output, buildOperator(resolvedExprs, agg.copy(
          aggregateExpressions = agg.aggregateExpressions ++ extraAggExprs)))
      }
    }
  }

  /**
   * Extracts [[Generator]] from the projectList of a [[Project]] operator and creates [[Generate]]
   * operator under [[Project]].
   *
   * This rule will throw [[AnalysisException]] for following cases:
   * 1. [[Generator]] is nested in expressions, e.g. `SELECT explode(list) + 1 FROM tbl`
   * 2. more than one [[Generator]] is found in projectList,
   *    e.g. `SELECT explode(list), explode(list) FROM tbl`
   * 3. [[Generator]] is found in other operators that are not [[Project]] or [[Generate]],
   *    e.g. `SELECT * FROM tbl SORT BY explode(list)`
   */
  object ExtractGenerator extends Rule[LogicalPlan] {
    def hasGenerator(expr: Expression): Boolean = {
      expr.exists(_.isInstanceOf[Generator])
    }

    private def hasNestedGenerator(expr: NamedExpression): Boolean = {
      @scala.annotation.tailrec
      def hasInnerGenerator(g: Generator): Boolean = g match {
        // Since `GeneratorOuter` is just a wrapper of generators, we skip it here
        case go: GeneratorOuter =>
          hasInnerGenerator(go.child)
        case _ =>
          g.children.exists { _.exists {
            case _: Generator => true
            case _ => false
          } }
      }
      trimNonTopLevelAliases(expr) match {
        case UnresolvedAlias(g: Generator, _) => hasInnerGenerator(g)
        case Alias(g: Generator, _) => hasInnerGenerator(g)
        case MultiAlias(g: Generator, _) => hasInnerGenerator(g)
        case other => hasGenerator(other)
      }
    }

    private def hasAggFunctionInGenerator(ne: Seq[NamedExpression]): Boolean = {
      ne.exists(_.exists {
        case g: Generator =>
          g.children.exists(_.exists(_.isInstanceOf[AggregateFunction]))
        case _ =>
          false
      })
    }

    private def trimAlias(expr: NamedExpression): Expression = expr match {
      case UnresolvedAlias(child, _) => child
      case Alias(child, _) => child
      case MultiAlias(child, _) => child
      case _ => expr
    }

    private object AliasedGenerator {
      /**
       * Extracts a [[Generator]] expression, any names assigned by aliases to the outputs
       * and the outer flag. The outer flag is used when joining the generator output.
       * @param e the [[Expression]]
       * @return (the [[Generator]], seq of output names, outer flag)
       */
      def unapply(e: Expression): Option[(Generator, Seq[String], Boolean)] = e match {
        case Alias(GeneratorOuter(g: Generator), name) if g.resolved => Some((g, name :: Nil, true))
        case MultiAlias(GeneratorOuter(g: Generator), names) if g.resolved => Some((g, names, true))
        case Alias(g: Generator, name) if g.resolved => Some((g, name :: Nil, false))
        case MultiAlias(g: Generator, names) if g.resolved => Some((g, names, false))
        case _ => None
      }
    }

    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
      _.containsPattern(GENERATOR), ruleId) {
      case p @ Project(Seq(UnresolvedStarWithColumns(_, _, _)), _) =>
        // UnresolvedStarWithColumns should be resolved before extracting.
        p

      case Project(projectList, _) if projectList.exists(hasNestedGenerator) =>
        val nestedGenerator = projectList.find(hasNestedGenerator).get
        throw QueryCompilationErrors.nestedGeneratorError(trimAlias(nestedGenerator))

      case Aggregate(_, aggList, _, _) if aggList.exists(hasNestedGenerator) =>
        val nestedGenerator = aggList.find(hasNestedGenerator).get
        throw QueryCompilationErrors.nestedGeneratorError(trimAlias(nestedGenerator))

      case Aggregate(_, aggList, _, _) if aggList.count(hasGenerator) > 1 =>
        val generators = aggList.filter(hasGenerator).map(trimAlias)
        throw QueryCompilationErrors.moreThanOneGeneratorError(generators)

      case Aggregate(groupList, aggList, child, _) if
        aggList.forall {
          case AliasedGenerator(_, _, _) => true
          case other => other.resolved
        } && aggList.exists(hasGenerator) =>
        // If generator in the aggregate list was visited, set the boolean flag true.
        var generatorVisited = false

        val projectExprs = Array.ofDim[NamedExpression](aggList.length)
        val newAggList = aggList
          .toIndexedSeq
          .map(trimNonTopLevelAliases)
          .zipWithIndex
          .flatMap {
            case (AliasedGenerator(generator, names, outer), idx) =>
              // It's a sanity check, this should not happen as the previous case will throw
              // exception earlier.
              assert(!generatorVisited, "More than one generator found in aggregate.")
              generatorVisited = true

              val newGenChildren: Seq[Expression] = generator.children.zipWithIndex.map {
                case (e, idx) => if (e.foldable) e else Alias(e, s"_gen_input_${idx}")()
              }
              val newGenerator = {
                val g = generator.withNewChildren(newGenChildren.map { e =>
                  if (e.foldable) e else e.asInstanceOf[Alias].toAttribute
                }).asInstanceOf[Generator]
                if (outer) GeneratorOuter(g) else g
              }
              val newAliasedGenerator = if (names.length == 1) {
                Alias(newGenerator, names(0))()
              } else {
                MultiAlias(newGenerator, names)
              }
              projectExprs(idx) = newAliasedGenerator
              newGenChildren.filter(!_.foldable).asInstanceOf[Seq[NamedExpression]]
            case (other, idx) =>
              projectExprs(idx) = other.toAttribute
              other :: Nil
          }

        val newAgg = Aggregate(groupList, newAggList, child)
        Project(projectExprs.toList, newAgg)

      case p @ Project(projectList, _) if hasAggFunctionInGenerator(projectList) =>
        // If a generator has any aggregate function, we need to apply the `GlobalAggregates` rule
        // first for replacing `Project` with `Aggregate`.
        p

      // The star will be expanded differently if we insert `Generate` under `Project` too early.
      case p @ Project(projectList, child) if !projectList.exists(_.exists(_.isInstanceOf[Star])) =>
        val (resolvedGenerator, newProjectList) = projectList
          .map(trimNonTopLevelAliases)
          .foldLeft((None: Option[Generate], Nil: Seq[NamedExpression])) { (res, e) =>
            e match {
              // If there are more than one generator, we only rewrite the first one and wait for
              // the next analyzer iteration to rewrite the next one.
              case AliasedGenerator(generator, names, outer) if res._1.isEmpty &&
                  generator.childrenResolved =>
                val g = Generate(
                  generator,
                  unrequiredChildIndex = Nil,
                  outer = outer,
                  qualifier = None,
                  generatorOutput = GeneratorResolution.makeGeneratorOutput(generator, names),
                  child)
                (Some(g), res._2 ++ g.nullableOutput)
              case other =>
                (res._1, res._2 :+ other)
            }
          }

        if (resolvedGenerator.isDefined) {
          Project(newProjectList, resolvedGenerator.get)
        } else {
          p
        }

      case g @ Generate(GeneratorOuter(generator), _, _, _, _, _) =>
        g.copy(generator = generator, outer = true)

      case g: Generate => g

      case u: UnresolvedTableValuedFunction => u

      case p: Project => p

      case a: Aggregate => a

      case p if p.expressions.exists(hasGenerator) =>
        throw QueryCompilationErrors.generatorOutsideSelectError(p)
    }
  }

  /**
   * Rewrites table generating expressions that either need one or more of the following in order
   * to be resolved:
   *  - concrete attribute references for their output.
   *  - to be relocated from a SELECT clause (i.e. from  a [[Project]]) into a [[Generate]]).
   *
   * Names for the output [[Attribute]]s are extracted from [[Alias]] or [[MultiAlias]] expressions
   * that wrap the [[Generator]].
   */
  object ResolveGenerate extends Rule[LogicalPlan] {
    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
      _.containsPattern(GENERATE), ruleId) {
      case g: Generate if !g.child.resolved || !g.generator.resolved => g
      case g: Generate if !g.resolved => withPosition(g) {
        // Check nested generators.
        if (g.generator.children.exists(ExtractGenerator.hasGenerator)) {
          throw QueryCompilationErrors.nestedGeneratorError(g.generator)
        }
        g.copy(generatorOutput =
          GeneratorResolution.makeGeneratorOutput(
            g.generator, g.generatorOutput.map {
              case ua: UnresolvedAttribute =>
                // LATERAL VIEW parser always emits single-part names via
                // UnresolvedAttribute.quoted; assert to fail loudly if that ever changes.
                assert(ua.nameParts.length == 1, s"unexpected multi-part name: ${ua.nameParts}")
                ua.nameParts.head
              case a => a.name
            }))
      }
    }
  }

  /**
   * Extracts [[WindowExpression]]s from the projectList of a [[Project]] operator and
   * aggregateExpressions of an [[Aggregate]] operator and creates individual [[Window]]
   * operators for every distinct [[WindowSpecDefinition]].
   *
   * This rule handles three cases:
   *  - A [[Project]] having [[WindowExpression]]s in its projectList;
   *  - An [[Aggregate]] having [[WindowExpression]]s in its aggregateExpressions.
   *  - A [[Filter]]->[[Aggregate]] pattern representing GROUP BY with a HAVING
   *    clause and the [[Aggregate]] has [[WindowExpression]]s in its aggregateExpressions.
   * Note: If there is a GROUP BY clause in the query, aggregations and corresponding
   * filters (expressions in the HAVING clause) should be evaluated before any
   * [[WindowExpression]]. If a query has SELECT DISTINCT, the DISTINCT part should be
   * evaluated after all [[WindowExpression]]s.
   *
   * Note: [[ResolveLateralColumnAliasReference]] rule is applied before this rule. To guarantee
   * this order, we make sure this rule applies only when the [[Project]] or [[Aggregate]] doesn't
   * contain any [[LATERAL_COLUMN_ALIAS_REFERENCE]].
   *
   * For every case, the transformation works as follows:
   * 1. For a list of [[Expression]]s (a projectList or an aggregateExpressions), partitions
   *    it two lists of [[Expression]]s, one for all [[WindowExpression]]s and another for
   *    all regular expressions.
   * 2. For all [[WindowExpression]]s, groups them based on their [[WindowSpecDefinition]]s
   *    and [[WindowFunctionType]]s.
   * 3. For every distinct [[WindowSpecDefinition]] and [[WindowFunctionType]], creates a
   *    [[Window]] operator and inserts it into the plan tree.
   */
  object ExtractWindowExpressions extends Rule[LogicalPlan] {
    type Spec = (Seq[Expression], Seq[SortOrder], WindowFunctionType)

    private def hasWindowFunction(exprs: Seq[Expression]): Boolean =
      exprs.exists(hasWindowFunction)

    private def hasWindowFunction(expr: Expression): Boolean = {
      expr.exists {
        case window: WindowExpression => true
        case _ => false
      }
    }

    /**
     * From a Seq of [[NamedExpression]]s, extract expressions containing window expressions and
     * other regular expressions that do not contain any window expression. For example, for
     * `col1, Sum(col2 + col3) OVER (PARTITION BY col4 ORDER BY col5)`, we will extract
     * `col1`, `col2 + col3`, `col4`, and `col5` out and replace their appearances in
     * the window expression as attribute references. So, the first returned value will be
     * `[Sum(_w0) OVER (PARTITION BY _w1 ORDER BY _w2)]` and the second returned value will be
     * [col1, col2 + col3 as _w0, col4 as _w1, col5 as _w2].
     *
     * @return (seq of expressions containing at least one window expression,
     *          seq of non-window expressions)
     */
    private def extract(
        expressions: Seq[NamedExpression]): (Seq[NamedExpression], Seq[NamedExpression]) = {
      // First, we partition the input expressions to two part. For the first part,
      // every expression in it contain at least one WindowExpression.
      // Expressions in the second part do not have any WindowExpression.
      val (expressionsWithWindowFunctions, regularExpressions) =
        expressions.partition(hasWindowFunction)

      // Then, we need to extract those regular expressions used in the WindowExpression.
      // For example, when we have col1 - Sum(col2 + col3) OVER (PARTITION BY col4 ORDER BY col5),
      // we need to make sure that col1 to col5 are all projected from the child of the Window
      // operator.
      val extractedExprMap = mutable.LinkedHashMap.empty[Expression, NamedExpression]
      def getOrExtract(key: Expression, value: Expression): Expression = {
        extractedExprMap.getOrElseUpdate(key.canonicalized,
          Alias(value, s"_w${extractedExprMap.size}")()).toAttribute
      }
      def extractExpr(expr: Expression): Expression = expr match {
        case ne: NamedExpression =>
          // If a named expression is not in regularExpressions, add it to
          // extractedExprMap and replace it with an AttributeReference.
          val missingExpr =
            AttributeSet(Seq(expr)) -- (regularExpressions ++ extractedExprMap.values)
          if (missingExpr.nonEmpty) {
            extractedExprMap += ne.canonicalized -> ne
          }
          // alias will be cleaned in the rule CleanupAliases
          ne
        case e: Expression if e.foldable =>
          e // No need to create an attribute reference if it will be evaluated as a Literal.
        case e: SortOrder =>
          // For SortOder just recursively extract the from child expression.
          e.copy(child = extractExpr(e.child))
        case e: NamedArgumentExpression =>
          // For NamedArgumentExpression, we extract the value and replace it with
          // an AttributeReference (with an internal column name, e.g. "_w0").
          NamedArgumentExpression(e.key, getOrExtract(e, e.value))
        case e: Expression =>
          // For other expressions, we extract it and replace it with an AttributeReference (with
          // an internal column name, e.g. "_w0").
          getOrExtract(e, e)
      }

      // Now, we extract regular expressions from expressionsWithWindowFunctions
      // by using extractExpr.
      val seenWindowAggregates = new ArrayBuffer[AggregateExpression]
      val newExpressionsWithWindowFunctions = expressionsWithWindowFunctions.map {
        _.transform {
          // Extracts children expressions of a WindowFunction (input parameters of
          // a WindowFunction).
          case wf: WindowFunction =>
            val newChildren = wf.children.map(extractExpr)
            wf.withNewChildren(newChildren)

          // Extracts expressions from the partition spec and order spec.
          case wsc @ WindowSpecDefinition(partitionSpec, orderSpec, _) =>
            val newPartitionSpec = partitionSpec.map(extractExpr)
            val newOrderSpec = orderSpec.map { so =>
              val newChild = extractExpr(so.child)
              so.copy(child = newChild)
            }
            wsc.copy(partitionSpec = newPartitionSpec, orderSpec = newOrderSpec)

          // Extract Windowed AggregateExpression
          case we @ WindowExpression(
              ae @ AggregateExpression(function, _, _, filter, _),
              spec: WindowSpecDefinition) =>
            val newChildren = function.children.map(extractExpr)
            val newFunction = function.withNewChildren(newChildren).asInstanceOf[AggregateFunction]
            val newFilter = filter.map(extractExpr)
            val newAgg = ae.copy(aggregateFunction = newFunction, filter = newFilter)
            seenWindowAggregates += newAgg
            WindowExpression(newAgg, spec)

          case AggregateExpression(aggFunc, _, _, _, _) if hasWindowFunction(aggFunc.children) =>
            throw QueryCompilationErrors.windowFunctionInsideAggregateFunctionNotAllowedError()

          // Extracts AggregateExpression. For example, for SUM(x) - Sum(y) OVER (...),
          // we need to extract SUM(x).
          case agg: AggregateExpression if !seenWindowAggregates.contains(agg) =>
            extractedExprMap.getOrElseUpdate(agg.canonicalized,
              Alias(agg, s"_w${extractedExprMap.size}")()).toAttribute

          // Extracts other attributes
          case attr: Attribute => extractExpr(attr)

        }.asInstanceOf[NamedExpression]
      }

      (newExpressionsWithWindowFunctions, regularExpressions ++ extractedExprMap.values)
    } // end of extract

    /**
     * Adds operators for Window Expressions. Every Window operator handles a single Window Spec.
     */
    private def addWindow(
        expressionsWithWindowFunctions: Seq[NamedExpression],
        child: LogicalPlan): LogicalPlan = {
      // First, we need to extract all WindowExpressions from expressionsWithWindowFunctions
      // and put those extracted WindowExpressions to extractedWindowExprBuffer.
      // This step is needed because it is possible that an expression contains multiple
      // WindowExpressions with different Window Specs.
      // After extracting WindowExpressions, we need to construct a project list to generate
      // expressionsWithWindowFunctions based on extractedWindowExprBuffer.
      // For example, for "sum(a) over (...) / sum(b) over (...)", we will first extract
      // "sum(a) over (...)" and "sum(b) over (...)" out, and assign "_we0" as the alias to
      // "sum(a) over (...)" and "_we1" as the alias to "sum(b) over (...)".
      // Then, the projectList will be [_we0/_we1].
      val extractedWindowExprBuffer = new ArrayBuffer[NamedExpression]()
      val newExpressionsWithWindowFunctions = expressionsWithWindowFunctions.map {
        // We need to use transformDown because we want to trigger
        // "case alias @ Alias(window: WindowExpression, _)" first.
        _.transformDown {
          case alias @ Alias(window: WindowExpression, _) =>
            // If a WindowExpression has an assigned alias, just use it.
            extractedWindowExprBuffer += alias
            alias.toAttribute
          case window: WindowExpression =>
            // If there is no alias assigned to the WindowExpressions. We create an
            // internal column.
            val withName = Alias(window, s"_we${extractedWindowExprBuffer.length}")()
            extractedWindowExprBuffer += withName
            withName.toAttribute
        }.asInstanceOf[NamedExpression]
      }

      // SPARK-32616: Use a linked hash map to maintains the insertion order of the Window
      // operators, so the query with multiple Window operators can have the determined plan.
      val groupedWindowExpressions = mutable.LinkedHashMap.empty[Spec, ArrayBuffer[NamedExpression]]
      // Second, we group extractedWindowExprBuffer based on their Partition and Order Specs.
      extractedWindowExprBuffer.foreach { expr =>
        val distinctWindowSpec = expr.collect {
          case window: WindowExpression => window.windowSpec
        }.distinct

        // We do a final check and see if we only have a single Window Spec defined in an
        // expressions.
        if (distinctWindowSpec.isEmpty) {
          throw QueryCompilationErrors.expressionWithoutWindowExpressionError(expr)
        } else if (distinctWindowSpec.length > 1) {
          // newExpressionsWithWindowFunctions only have expressions with a single
          // WindowExpression. If we reach here, we have a bug.
          throw QueryCompilationErrors.expressionWithMultiWindowExpressionsError(
            expr, distinctWindowSpec)
        } else {
          val spec = distinctWindowSpec.head
          val specKey = (spec.partitionSpec, spec.orderSpec, WindowFunctionType.functionType(expr))
          val windowExprs = groupedWindowExpressions
            .getOrElseUpdate(specKey, new ArrayBuffer[NamedExpression])
          windowExprs += expr
        }
      }

      // Third, we aggregate them by adding each Window operator for each Window Spec and then
      // setting this to the child of the next Window operator.
      val windowOps =
        groupedWindowExpressions.foldLeft(child) {
          case (last, ((partitionSpec, orderSpec, _), windowExpressions)) =>
            Window(windowExpressions.toSeq, partitionSpec, orderSpec, last)
        }

      // Finally, we create a Project to output windowOps's output
      // newExpressionsWithWindowFunctions.
      Project(windowOps.output ++ newExpressionsWithWindowFunctions, windowOps)
    } // end of addWindow

    // We have to use transformDown at here to make sure the rule of
    // "Aggregate with Having clause" will be triggered.
    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsDownWithPruning(
      _.containsPattern(WINDOW_EXPRESSION), ruleId) {

      case Filter(condition, _) if hasWindowFunction(condition) =>
        throw QueryCompilationErrors.windowFunctionNotAllowedError("WHERE")

      case UnresolvedHaving(condition, _) if hasWindowFunction(condition) =>
        throw QueryCompilationErrors.windowFunctionNotAllowedError("HAVING")

      // Aggregate with Having clause. This rule works with an unresolved Aggregate because
      // a resolved Aggregate will not have Window Functions.
      case f @ UnresolvedHaving(condition, a @ Aggregate(groupingExprs, aggregateExprs, child, _))
        if child.resolved &&
          hasWindowFunction(aggregateExprs) &&
          a.expressions.forall(_.resolved) =>
        aggregateExprs.foreach(_.transformDownWithPruning(
          _.containsPattern(LATERAL_COLUMN_ALIAS_REFERENCE)) {
          case lcaRef: LateralColumnAliasReference =>
            throw QueryCompilationErrors.lateralColumnAliasInAggWithWindowAndHavingUnsupportedError(
              lcaRef.nameParts)
        })
        val (windowExpressions, aggregateExpressions) = extract(aggregateExprs)
        // Create an Aggregate operator to evaluate aggregation functions.
        val withAggregate = Aggregate(groupingExprs, aggregateExpressions, child)
        // Add a Filter operator for conditions in the Having clause.
        val withFilter = Filter(condition, withAggregate)
        val withWindow = addWindow(windowExpressions, withFilter)

        // Finally, generate output columns according to the original projectList.
        val finalProjectList = aggregateExprs.map(_.toAttribute)
        Project(finalProjectList, withWindow)

      case p: LogicalPlan if !p.childrenResolved => p

      // Aggregate without Having clause.
      // Make sure the lateral column aliases are properly handled first.
      case a @ Aggregate(groupingExprs, aggregateExprs, child, _)
        if hasWindowFunction(aggregateExprs) &&
          a.expressions.forall(_.resolved) &&
          !aggregateExprs.exists(_.containsPattern(LATERAL_COLUMN_ALIAS_REFERENCE)) =>
        val (windowExpressions, aggregateExpressions) = extract(aggregateExprs)
        // Create an Aggregate operator to evaluate aggregation functions.
        val withAggregate = Aggregate(groupingExprs, aggregateExpressions, child)
        // Add Window operators.
        val withWindow = addWindow(windowExpressions, withAggregate)

        // Finally, generate output columns according to the original projectList.
        val finalProjectList = aggregateExprs.map(_.toAttribute)
        Project(finalProjectList, withWindow)

      // We only extract Window Expressions after all expressions of the Project
      // have been resolved, and lateral column aliases are properly handled first.
      case p @ Project(projectList, child)
        if hasWindowFunction(projectList) &&
          p.expressions.forall(_.resolved) &&
          !projectList.exists(_.containsPattern(LATERAL_COLUMN_ALIAS_REFERENCE)) =>
        val (windowExpressions, regularExpressions) = extract(projectList.toIndexedSeq)
        // We add a project to get all needed expressions for window expressions from the child
        // of the original Project operator.
        val withProject = Project(regularExpressions, child)
        // Add Window operators.
        val withWindow = addWindow(windowExpressions, withProject)

        // Finally, generate output columns according to the original projectList.
        val finalProjectList = projectList.map(_.toAttribute)
        Project(finalProjectList, withWindow)
    }
  }

  /**
   * Set the seed for random number generation.
   */
  object ResolveRandomSeed extends Rule[LogicalPlan] {
    private lazy val random = new Random()

    override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
      _.containsPattern(EXPRESSION_WITH_RANDOM_SEED), ruleId) {
      case p if p.resolved => p
      case p => p.transformExpressionsUpWithPruning(
        _.containsPattern(EXPRESSION_WITH_RANDOM_SEED), ruleId) {
        case e: ExpressionWithRandomSeed if e.seedExpression == UnresolvedSeed =>
          e.withNewSeed(random.nextLong())
      }
    }
  }

  /**
   * Correctly handle null primitive inputs for UDF by adding extra [[If]] expression to do the
   * null check.  When user defines a UDF with primitive parameters, there is no way to tell if the
   * primitive parameter is null or not, so here we assume the primitive input is null-propagatable
   * and we should return null if the input is null.
   */
  object HandleNullInputsForUDF extends Rule[LogicalPlan] {
    override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
      _.containsPattern(SCALA_UDF)) {
      case p if !p.resolved => p // Skip unresolved nodes.

      case p => p.transformExpressionsUpWithPruning(_.containsPattern(SCALA_UDF)) {

        case udf: ScalaUDF if udf.inputPrimitives.contains(true) =>
          // Otherwise, add special handling of null for fields that can't accept null.
          // The result of operations like this, when passed null, is generally to return null.
          assert(udf.inputPrimitives.length == udf.children.length)

          val inputPrimitivesPair = udf.inputPrimitives.zip(udf.children)
          val inputNullCheck = inputPrimitivesPair.collect {
            case (isPrimitive, input) if isPrimitive && input.nullable =>
              IsNull(input)
          }.reduceLeftOption[Expression](Or)

          if (inputNullCheck.isDefined) {
            // Once we add an `If` check above the udf, it is safe to mark those checked inputs
            // as null-safe (i.e., wrap with `KnownNotNull`), because the null-returning
            // branch of `If` will be called if any of these checked inputs is null. Thus we can
            // prevent this rule from being applied repeatedly.
            val newInputs = inputPrimitivesPair.map {
              case (isPrimitive, input) =>
                if (isPrimitive && input.nullable) {
                  KnownNotNull(input)
                } else {
                  input
                }
            }
            val newUDF = udf.copy(children = newInputs)
            If(inputNullCheck.get, Literal.create(null, udf.dataType), newUDF)
          } else {
            udf
          }
      }
    }
  }

  /**
   * Resolve the encoders for the UDF by explicitly given the attributes. We give the
   * attributes explicitly in order to handle the case where the data type of the input
   * value is not the same with the internal schema of the encoder, which could cause
   * data loss. For example, the encoder should not cast the input value to Decimal(38, 18)
   * if the actual data type is Decimal(30, 0).
   *
   * The resolved encoders then will be used to deserialize the internal row to Scala value.
   */
  object ResolveEncodersInUDF extends Rule[LogicalPlan] {
    override def apply(plan: LogicalPlan): LogicalPlan =
      plan.resolveOperatorsUpWithSubqueriesAndPruning(_.containsPattern(SCALA_UDF), ruleId) {
      case p if !p.resolved => p // Skip unresolved nodes.

        case p => p.transformExpressionsUpWithPruning(_.containsPattern(SCALA_UDF), ruleId) {

          case udf: ScalaUDF if udf.inputEncoders.nonEmpty =>
            val boundEncoders = udf.inputEncoders.zipWithIndex.map { case (encOpt, i) =>
              val dataType = udf.children(i).dataType
              encOpt.map { enc =>
                val attrs = if (enc.isSerializedAsStructForTopLevel) {
                  // Value class that has been replaced with its underlying type
                  if (enc.schema.fields.length == 1 &&
                    enc.schema.fields.head.dataType == dataType) {
                    DataTypeUtils.toAttributes(enc.schema)
                  } else {
                    DataTypeUtils.toAttributes(dataType.asInstanceOf[StructType])
                  }
                } else {
                  // the field name doesn't matter here, so we use
                  // a simple literal to avoid any overhead
                  DataTypeUtils.toAttribute(StructField("input", dataType)) :: Nil
                }
                enc.resolveAndBind(attrs)
              }
            }
          udf.copy(inputEncoders = boundEncoders)
      }
    }
  }

  /**
   * Check and add proper window frames for all window functions.
   */
  object ResolveWindowFrame extends Rule[LogicalPlan] {
    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveExpressionsWithPruning(
      _.containsPattern(WINDOW_EXPRESSION), ruleId) {
      case we: WindowExpression => WindowResolution.resolveFrame(we)
    }
  }

  /**
   * Check and add order to [[AggregateWindowFunction]]s.
   */
  object ResolveWindowOrder extends Rule[LogicalPlan] {
    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveExpressionsWithPruning(
      _.containsPattern(WINDOW_EXPRESSION), ruleId) {
      case we: WindowExpression => WindowResolution.resolveOrder(we)
    }
  }

  /**
   * Removes natural or using joins by calculating output columns based on output from two sides,
   * Then apply a Project on a normal Join to eliminate natural or using join.
   */
  object ResolveNaturalAndUsingJoin extends Rule[LogicalPlan] {
    override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
      _.containsPattern(NATURAL_LIKE_JOIN), ruleId) {
      case j @ Join(left, right, UsingJoin(joinType, usingCols), _, hint)
          if left.resolved && right.resolved && j.duplicateResolved =>
        val project = commonNaturalJoinProcessing(
          left, right, joinType, usingCols, None, hint)
        j.getTagValue(LogicalPlan.PLAN_ID_TAG)
          .foreach(project.setTagValue(LogicalPlan.PLAN_ID_TAG, _))
        project
      case j @ Join(left, right, NaturalJoin(joinType), condition, hint)
          if j.resolvedExceptNatural =>
        // find common column names from both sides
        val joinNames = NaturalAndUsingJoinResolution.canonicalizedIntersect(
            left.output.map(_.name), right.output.map(_.name))
        val project = commonNaturalJoinProcessing(
          left, right, joinType, joinNames, condition, hint)
        j.getTagValue(LogicalPlan.PLAN_ID_TAG)
          .foreach(project.setTagValue(LogicalPlan.PLAN_ID_TAG, _))
        project
    }
  }

  /**
   * Resolves columns of an output table from the data in a logical plan. This rule will:
   *
   * - Reorder columns when the write is by name
   * - Insert casts when data types do not match
   * - Insert aliases when column names do not match
   * - Detect plans that are not compatible with the output table and throw AnalysisException
   */
  object ResolveOutputRelation extends Rule[LogicalPlan] {
    override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsWithPruning(
      _.containsPattern(COMMAND), ruleId) {
      case v2Write: V2WriteCommand
          if v2Write.table.resolved && v2Write.query.resolved && !v2Write.outputResolved &&
            v2Write.pendingSchemaChanges.isEmpty =>
        validateStoreAssignmentPolicy()
        TableOutputResolver.suitableForByNameCheck(v2Write.isByName,
          expected = v2Write.table.output, queryOutput = v2Write.query.output)
        // With schema evolution + coercion flag, missing top-level columns AND missing nested
        // struct fields are filled with defaults/null (RECURSE mode). Otherwise, only missing
        // top-level columns are filled via FILL mode; missing nested struct fields still cause
        // schema enforcement errors.
        val defaultValueFillMode =
          if (conf.coerceInsertNestedTypes && v2Write.schemaEvolutionEnabled) RECURSE
          else FILL
        val projection = TableOutputResolver.resolveOutputColumns(
          v2Write.table.name, v2Write.table.output, v2Write.query, v2Write.isByName, conf,
          defaultValueFillMode)
        if (projection != v2Write.query) {
          val cleanedTable = v2Write.table match {
            case r: DataSourceV2Relation =>
              r.copy(output = r.output.map(CharVarcharUtils.cleanAttrMetadata))
            case other => other
          }
          v2Write.withNewQuery(projection).withNewTable(cleanedTable)
        } else {
          v2Write
        }
    }
  }

  private def validateStoreAssignmentPolicy(): Unit = {
    // SPARK-28730: LEGACY store assignment policy is disallowed in data source v2.
    if (conf.storeAssignmentPolicy == StoreAssignmentPolicy.LEGACY) {
      throw QueryCompilationErrors.legacyStoreAssignmentPolicyError()
    }
  }

  private def commonNaturalJoinProcessing(
      left: LogicalPlan,
      right: LogicalPlan,
      joinType: JoinType,
      joinNames: Seq[String],
      condition: Option[Expression],
      hint: JoinHint): LogicalPlan = {
    import org.apache.spark.sql.catalyst.util._

    val (projectList, hiddenList, newCondition) =
      NaturalAndUsingJoinResolution.computeJoinOutputsAndNewCondition(
        left,
        left.output,
        right,
        right.output,
        joinType,
        joinNames,
        condition,
        (attributeName, keyName) => resolver(attributeName, keyName)
      )

    // use Project to hide duplicated common keys
    // propagate hidden columns from nested USING/NATURAL JOINs
    val project = Project(projectList, Join(left, right, joinType, newCondition, hint))
    project.setTagValue(
      Project.hiddenOutputTag,
      hiddenList.map(_.markAsQualifiedAccessOnly()) ++
        project.child.metadataOutput.filter(_.qualifiedAccessOnly))
    project
  }

  /**
   * Replaces [[UnresolvedDeserializer]] with the deserialization expression that has been resolved
   * to the given input attributes.
   */
  object ResolveDeserializer extends Rule[LogicalPlan] {
    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
      _.containsPattern(UNRESOLVED_DESERIALIZER), ruleId) {
      case p if !p.childrenResolved => p
      case p if p.resolved => p

      case p => p.transformExpressionsWithPruning(
        _.containsPattern(UNRESOLVED_DESERIALIZER), ruleId) {
        case UnresolvedDeserializer(deserializer, inputAttributes) =>
          val inputs = if (inputAttributes.isEmpty) {
            p.children.flatMap(_.output)
          } else {
            inputAttributes
          }

          validateTopLevelTupleFields(deserializer, inputs)
          val resolved = resolveExpressionByPlanOutput(
            deserializer, LocalRelation(inputs), throws = true)
          val result = resolved transformDown {
            case UnresolvedMapObjects(func, inputData, cls) if inputData.resolved =>
              inputData.dataType match {
                case ArrayType(et, cn) =>
                  MapObjects(func, inputData, et, cn, cls) transformUp {
                    case UnresolvedExtractValue(child, fieldName) if child.resolved =>
                      ExtractValue(child, fieldName, resolver)
                  }
                case other =>
                  throw QueryCompilationErrors.dataTypeMismatchForDeserializerError(other,
                    "array")
              }
            case u: UnresolvedCatalystToExternalMap if u.child.resolved =>
              u.child.dataType match {
                case _: MapType =>
                  CatalystToExternalMap(u) transformUp {
                    case UnresolvedExtractValue(child, fieldName) if child.resolved =>
                      ExtractValue(child, fieldName, resolver)
                  }
                case other =>
                  throw QueryCompilationErrors.dataTypeMismatchForDeserializerError(other, "map")
              }
          }
          validateNestedTupleFields(result)
          result
      }
    }

    private def fail(schema: StructType, maxOrdinal: Int): Unit = {
      throw QueryCompilationErrors.fieldNumberMismatchForDeserializerError(schema, maxOrdinal)
    }

    /**
     * For each top-level Tuple field, we use [[GetColumnByOrdinal]] to get its corresponding column
     * by position.  However, the actual number of columns may be different from the number of Tuple
     * fields.  This method is used to check the number of columns and fields, and throw an
     * exception if they do not match.
     */
    private def validateTopLevelTupleFields(
        deserializer: Expression, inputs: Seq[Attribute]): Unit = {
      val ordinals = deserializer.collect {
        case GetColumnByOrdinal(ordinal, _) => ordinal
      }.distinct.sorted

      if (ordinals.nonEmpty && ordinals != inputs.indices) {
        fail(inputs.toStructType, ordinals.last)
      }
    }

    /**
     * For each nested Tuple field, we use [[GetStructField]] to get its corresponding struct field
     * by position.  However, the actual number of struct fields may be different from the number
     * of nested Tuple fields.  This method is used to check the number of struct fields and nested
     * Tuple fields, and throw an exception if they do not match.
     */
    private def validateNestedTupleFields(deserializer: Expression): Unit = {
      val structChildToOrdinals = deserializer
        // There are 2 kinds of `GetStructField`:
        //   1. resolved from `UnresolvedExtractValue`, and it will have a `name` property.
        //   2. created when we build deserializer expression for nested tuple, no `name` property.
        // Here we want to validate the ordinals of nested tuple, so we should only catch
        // `GetStructField` without the name property.
        .collect { case g: GetStructField if g.name.isEmpty => g }
        .groupBy(_.child)
        .transform((_, v) => v.map(_.ordinal).distinct.sorted)

      structChildToOrdinals.foreach { case (expr, ordinals) =>
        val schema = expr.dataType.asInstanceOf[StructType]
        if (ordinals != schema.indices) {
          fail(schema, ordinals.last)
        }
      }
    }
  }

  /**
   * Resolves [[NewInstance]] by finding and adding the outer scope to it if the object being
   * constructed is an inner class.
   */
  object ResolveNewInstance extends Rule[LogicalPlan] {
    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
      _.containsPattern(NEW_INSTANCE), ruleId) {
      case p if !p.childrenResolved => p
      case p if p.resolved => p

      case p => p.transformExpressionsUpWithPruning(_.containsPattern(NEW_INSTANCE), ruleId) {
        case n: NewInstance if n.childrenResolved && !n.resolved =>
          val outer = OuterScopes.getOuterScope(n.cls)
          if (outer == null) {
            throw QueryCompilationErrors.outerScopeFailureForNewInstanceError(n.cls.getName)
          }
          n.copy(outerPointer = Some(outer))
      }
    }
  }

  /**
   * Replace the [[UpCast]] expression by [[Cast]], and throw exceptions if the cast may truncate.
   */
  object ResolveUpCast extends Rule[LogicalPlan] {
    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
      _.containsPattern(UP_CAST), ruleId) {
      case p if !p.childrenResolved => p
      case p if p.resolved => p

      case p => p.transformExpressionsWithPruning(_.containsPattern(UP_CAST), ruleId) {
        case unresolvedUpCast @ UpCast(child, _, _) if !child.resolved =>
          unresolvedUpCast
        case unresolvedUpCast: UpCast =>
          UpCastResolution.resolve(unresolvedUpCast)
      }
    }
  }

  /**
   * Rule to resolve, normalize and rewrite field names based on case sensitivity for commands.
   */
  object ResolveFieldNameAndPosition extends Rule[LogicalPlan] {
    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
      case cmd: CreateIndex if cmd.table.resolved &&
          cmd.columns.exists(_._1.isInstanceOf[UnresolvedFieldName]) =>
        val table = cmd.table.asInstanceOf[ResolvedTable]
        cmd.copy(columns = cmd.columns.map {
          case (u: UnresolvedFieldName, prop) => resolveFieldNames(table, u.name, u) -> prop
          case other => other
        })

      case a: DropColumns if a.table.resolved && hasUnresolvedFieldName(a) && a.ifExists =>
        // for DropColumn with IF EXISTS clause, we should resolve and ignore missing column errors
        val table = a.table.asInstanceOf[ResolvedTable]
        val columnsToDrop = a.columnsToDrop
        a.copy(columnsToDrop = columnsToDrop.flatMap(c => resolveFieldNamesOpt(table, c.name, c)))

      case a: AlterTableCommand if a.table.resolved && hasUnresolvedFieldName(a) =>
        val table = a.table.asInstanceOf[ResolvedTable]
        a.transformExpressions {
          case u: UnresolvedFieldName => resolveFieldNames(table, u.name, u)
        }

      case a @ AddColumns(r: ResolvedTable, cols) if !a.resolved =>
        // 'colsToAdd' keeps track of new columns being added. It stores a mapping from a
        // normalized parent name of fields to field names that belong to the parent.
        // For example, if we add columns "a.b.c", "a.b.d", and "a.c", 'colsToAdd' will become
        // Map(Seq("a", "b") -> Seq("c", "d"), Seq("a") -> Seq("c")).
        val colsToAdd = mutable.Map.empty[Seq[String], Seq[String]]
        def resolvePosition(
            col: QualifiedColType,
            parentSchema: StructType,
            resolvedParentName: Seq[String]): Option[FieldPosition] = {
          val fieldsAdded = colsToAdd.getOrElse(resolvedParentName, Nil)
          val resolvedPosition = col.position.map {
            case u: UnresolvedFieldPosition => u.position match {
              case after: After =>
                val allFields = parentSchema.fieldNames ++ fieldsAdded
                allFields.find(n => conf.resolver(n, after.column())) match {
                  case Some(colName) =>
                    ResolvedFieldPosition(ColumnPosition.after(colName))
                  case None =>
                    throw QueryCompilationErrors.referenceColNotFoundForAlterTableChangesError(
                      col.colName, allFields)
                }
              case _ => ResolvedFieldPosition(u.position)
            }
            case resolved => resolved
          }
          colsToAdd(resolvedParentName) = fieldsAdded :+ col.colName
          resolvedPosition
        }
        val schema = r.table.columns.asSchema
        val resolvedCols = cols.map { col =>
          col.path match {
            case Some(parent: UnresolvedFieldName) =>
              // Adding a nested field, need to resolve the parent column and position.
              val resolvedParent = resolveFieldNames(r, parent.name, parent)
              val parentSchema = resolvedParent.field.dataType match {
                case s: StructType => s
                case _ => throw QueryCompilationErrors.invalidFieldName(
                  col.name, parent.name, parent.origin)
              }
              val resolvedPosition = resolvePosition(col, parentSchema, resolvedParent.name)
              col.copy(path = Some(resolvedParent), position = resolvedPosition)
            case _ =>
              // Adding to the root. Just need to resolve position.
              val resolvedPosition = resolvePosition(col, schema, Nil)
              col.copy(position = resolvedPosition)
          }
        }
        val resolved = a.copy(columnsToAdd = resolvedCols)
        resolved.copyTagsFrom(a)
        resolved

      case a @ AlterColumns(table: ResolvedTable, specs) =>
        val resolvedSpecs = specs.map {
          case s @ AlterColumnSpec(
              ResolvedFieldName(path, field), dataType, _, _, position, _, _) =>
            val newDataType = dataType.flatMap { dt =>
              // Hive style syntax provides the column type, even if it may not have changed.
              val existing = CharVarcharUtils.getRawType(field.metadata).getOrElse(field.dataType)
              if (existing == dt) None else Some(dt)
            }
            val newPosition = position map {
              case u @ UnresolvedFieldPosition(after: After) =>
                // TODO: since the field name is already resolved, it's more efficient if
                //       `ResolvedFieldName` carries the parent struct and we resolve column
                //       position based on the parent struct, instead of re-resolving the entire
                //       column path.
                val resolved = resolveFieldNames(table, path :+ after.column(), u)
                ResolvedFieldPosition(ColumnPosition.after(resolved.field.name))
              case u: UnresolvedFieldPosition => ResolvedFieldPosition(u.position)
              case other => other
            }
            s.copy(newDataType = newDataType, newPosition = newPosition)
          case spec => spec
        }
        val resolved = a.copy(specs = resolvedSpecs)
        resolved.copyTagsFrom(a)
        resolved
    }

    /**
     * Returns the resolved field name if the field can be resolved, returns None if the column is
     * not found. An error will be thrown in CheckAnalysis for columns that can't be resolved.
     */
    private def resolveFieldNames(
        table: ResolvedTable,
        fieldName: Seq[String],
        context: Expression): ResolvedFieldName = {
      resolveFieldNamesOpt(table, fieldName, context)
        .getOrElse {
          throw QueryCompilationErrors.unresolvedColumnError(fieldName, table.schema.fieldNames,
            context.origin)
        }
    }

    private def resolveFieldNamesOpt(
        table: ResolvedTable,
        fieldName: Seq[String],
        context: Expression): Option[ResolvedFieldName] = {
      table.schema.findNestedField(
        fieldName, includeCollections = true, conf.resolver, context.origin
      ).map {
        case (path, field) => ResolvedFieldName(path, field)
      }
    }

    private def hasUnresolvedFieldName(a: AlterTableCommand): Boolean = {
      a.expressions.exists(_.exists(_.isInstanceOf[UnresolvedFieldName]))
    }
  }

  /**
   * A rule to handle special commands that need to be notified when analysis is done. This rule
   * should run after all other analysis rules are run.
   */
  object HandleSpecialCommand extends Rule[LogicalPlan] {
    override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsWithPruning(
      _.containsPattern(COMMAND)) {
      case c: AnalysisOnlyCommand if c.resolved =>
        checkAnalysis(c)
        c.markAsAnalyzed(AnalysisContext.get)
      case c: KeepAnalyzedQuery if c.resolved =>
        c.storeAnalyzedQuery()
      case p: LogicalPlan if p.resolved =>
        p.transformAllExpressionsWithPruning(_.containsPattern(ANALYSIS_AWARE_EXPRESSION)) {
          case e: AnalysisAwareExpression[_] => e.markAsAnalyzed()
        }
    }
  }

  /**
   * This rule resolves the QUALIFY clause. It resolves the qualify condition against the child
   * plan's output, extracts window expressions into aliases, validates that at least one window
   * function is present, and builds a Filter placed after window materialization.
   *
   * The resolution follows these steps:
   * 1. Resolve the qualify condition using the child plan's output (Project or Aggregate).
   * 2. Extract canonically-unique WindowExpressions from the condition into aliases.
   * 3. Reject standalone aggregate functions in the condition.
   * 4. Validate that at least one window function exists in SELECT or in the condition.
   * 5. Build: Filter(newCond, Project(child.output ++ windowAliases, newChild)) and trim output.
   *
   * The resulting Project node contains window expressions that will be handled by
   * ExtractWindowExpressions in a subsequent analyzer iteration.
   */
  object ResolveQualify extends Rule[LogicalPlan] {
    /**
     * Resolve subqueries in the condition expression using the fake Project pattern.
     * This reuses the same approach as HAVING resolution (see line ~722).
     */
    private def resolveConditionSubqueries(
        cond: Expression, outer: LogicalPlan): Expression = {
      if (SubqueryExpression.hasSubquery(cond)) {
        val fake = Project(Alias(cond, "fake")() :: Nil, outer)
        ResolveSubquery(fake).asInstanceOf[Project].projectList.head.asInstanceOf[Alias].child
      } else {
        cond
      }
    }

    /**
     * Resolve the qualify condition by recursing through the child plan tree
     * (Project/Aggregate/Window/Filter) and adding missing attribute references
     * to each intermediate node so they are preserved through window extraction.
     */
    private def resolveQualifyCondition(
        cond: Expression,
        candidateAttrs: AttributeSet,
        plan: LogicalPlan): (Expression, LogicalPlan) = plan match {
      case agg: Aggregate =>
        val resolved = resolveExpressionByPlanChildren(cond, agg)
        val subqueryResolved = resolveConditionSubqueries(resolved, agg.child)
        val extraAggExprs = mutable.ArrayBuffer.empty[NamedExpression]
        val newCond = subqueryResolved.transform {
          case grouping if grouping.resolved &&
            agg.groupingExpressions.exists(grouping.semanticEquals) =>
            val allAggExprs = agg.aggregateExpressions ++ extraAggExprs.toSeq
            val index = allAggExprs.indexWhere {
              case Alias(child, _) => child.semanticEquals(grouping)
              case o => o.semanticEquals(grouping)
            }
            if (index >= 0) {
              allAggExprs(index).toAttribute
            } else {
              grouping match {
                case ne: NamedExpression =>
                  extraAggExprs += ne
                  ne.toAttribute
                case o: Expression =>
                  val alias = Alias(o, o.sql)()
                  extraAggExprs += alias
                  alias.toAttribute
              }
            }
          case a: Attribute if a.resolved && !candidateAttrs.contains(a) =>
            a.failAnalysis(
              errorClass = "UNRESOLVED_COLUMN.WITH_SUGGESTION",
              messageParameters = Map(
                "objectName" -> a.sql,
                "proposal" -> candidateAttrs.map(_.sql).mkString(", ")))
        }
        val newAgg = agg.copy(aggregateExpressions = agg.aggregateExpressions ++ extraAggExprs)
        (newCond, newAgg)

      case p: Project =>
        // First resolve against this Project's children output.
        val resolved = resolveExpressionByPlanChildren(cond, p)
        val subqueryResolved = resolveConditionSubqueries(resolved, p.child)
        // Recurse into child to resolve any remaining unresolved references and
        // propagate missing attributes through intermediate nodes.
        val (newCond, newChild) = resolveQualifyCondition(
          subqueryResolved, candidateAttrs, p.child)
        val missingAttrs = (newCond.references -- p.outputSet).intersect(newChild.outputSet)
        val newProject = p.copy(projectList = p.projectList ++ missingAttrs, child = newChild)
        (newCond, newProject)

      case w: Window =>
        val (newCond, newChild) = resolveQualifyCondition(cond, candidateAttrs, w.child)
        (newCond, w.copy(child = newChild))

      case f: Filter =>
        val (newCond, newChild) = resolveQualifyCondition(cond, candidateAttrs, f.child)
        (newCond, f.copy(child = newChild))

      case other =>
        // Base case (SubqueryAlias, View, etc.): try resolving the condition
        // using a fake UnresolvedQualify wrapper so resolveExpressionByPlanChildren
        // can resolve against this node's output.
        val resolved = cond.transformUp {
          case u: UnresolvedAttribute =>
            other.output.resolve(u.nameParts, conf.resolver).getOrElse(u)
        }
        (resolved, other)
    }

    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsWithPruning(
      _.containsPattern(UNRESOLVED_QUALIFY)) {
      case q @ UnresolvedQualify(cond, child) if child.resolved =>
        // Save the original output before missing attrs are added.
        val originalOutput = child.output
        val (resolvedByChildren, newChild) =
          resolveQualifyCondition(cond, child.outputSet, child)
        // Also try resolving by the current child's output (for SELECT aliases).
        val maybeResolved = if (resolvedByChildren.resolved) {
          resolvedByChildren
        } else {
          resolveExpressionByPlanChildren(resolvedByChildren, q)
        }
        val newPlan = if (maybeResolved.resolved) {
          // Extract canonically unique window functions in the qualify condition.
          val windowExpressionToAliasMap = new java.util.LinkedHashMap[Expression, Alias]()
          val newCond = maybeResolved.transform {
            case windowExpression: WindowExpression =>
              val canonicalized = windowExpression.canonicalized
              if (windowExpressionToAliasMap.containsKey(canonicalized)) {
                windowExpressionToAliasMap.get(canonicalized).toAttribute
              } else {
                val alias = Alias(windowExpression, windowExpression.sql)()
                windowExpressionToAliasMap.put(canonicalized, alias)
                alias.toAttribute
              }
            case a: AggregateExpression =>
              throw QueryCompilationErrors.aggregateInQualifyNotAllowedError(a)
          }
          // Ensure at least one window function in SELECT or QUALIFY condition.
          if (windowExpressionToAliasMap.size() == 0 && !hasWindowInPlan(child)) {
            throw QueryCompilationErrors.qualifyRequiresWindowFunctionError()
          }
          if (windowExpressionToAliasMap.size() > 0) {
            val projectList =
              windowExpressionToAliasMap.values().asScala.toSeq
            Filter(newCond, Project(newChild.output ++ projectList, newChild))
          } else {
            Filter(newCond, newChild)
          }
        } else {
          UnresolvedQualify(maybeResolved, newChild)
        }
        // Trim additional attributes to preserve original SELECT output.
        if (newPlan.output.map(_.exprId) != originalOutput.map(_.exprId)) {
          Project(originalOutput, newPlan)
        } else {
          newPlan
        }
    }

    /** Check if the resolved child plan already contains a Window node (from SELECT list). */
    private def hasWindowInPlan(plan: LogicalPlan): Boolean = plan match {
      case _: Window => true
      case p: Project => hasWindowInPlan(p.child)
      case f: Filter => hasWindowInPlan(f.child)
      case _ => false
    }
  }
}

/**
 * Removes [[SubqueryAlias]] operators from the plan. Subqueries are only required to provide
 * scoping information for attributes and can be removed once analysis is complete.
 */
object EliminateSubqueryAliases extends Rule[LogicalPlan] {
  // This is also called in the beginning of the optimization phase, and as a result
  // is using transformUp rather than resolveOperators.
  def apply(plan: LogicalPlan): LogicalPlan = AnalysisHelper.allowInvokingTransformsInAnalyzer {
    plan.transformUpWithPruning(_.containsPattern(SUBQUERY_ALIAS), ruleId) {
      case SubqueryAlias(_, child) => child
    }
  }
}

/**
 * Removes [[Union]] operators from the plan if it just has one child.
 */
object EliminateUnions extends Rule[LogicalPlan] {
  def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsWithPruning(
    _.containsPattern(UNION), ruleId) {
    case u: Union if u.children.size == 1 => u.children.head
  }
}

/**
 * Cleans up unnecessary Aliases inside the plan. Basically we only need Alias as a top level
 * expression in Project(project list) or Aggregate(aggregate expressions) or
 * Window(window expressions). Notice that if an expression has other expression parameters which
 * are not in its `children`, e.g. `RuntimeReplaceable`, the transformation for Aliases in this
 * rule can't work for those parameters.
 */
object CleanupAliases extends Rule[LogicalPlan] with AliasHelper {
  override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUpWithPruning(
    // trimNonTopLevelAliases can transform Alias and MultiAlias.
    _.containsAnyPattern(ALIAS, MULTI_ALIAS)) {
    case Project(projectList, child) =>
      val cleanedProjectList = projectList.map(trimNonTopLevelAliases)
      Project(cleanedProjectList, child)

    case Aggregate(grouping, aggs, child, hint) =>
      val cleanedAggs = aggs.map(trimNonTopLevelAliases)
      Aggregate(grouping.map(trimAliases), cleanedAggs, child, hint)

    case Window(windowExprs, partitionSpec, orderSpec, child, hint) =>
      val cleanedWindowExprs = windowExprs.map(trimNonTopLevelAliases)
      Window(cleanedWindowExprs, partitionSpec.map(trimAliases),
        orderSpec.map(trimAliases(_).asInstanceOf[SortOrder]), child, hint)

    case CollectMetrics(name, metrics, child, dataframeId) =>
      val cleanedMetrics = metrics.map(trimNonTopLevelAliases)
      CollectMetrics(name, cleanedMetrics, child, dataframeId)

    case Unpivot(ids, values, aliases, variableColumnName, valueColumnNames, child) =>
      val cleanedIds = ids.map(_.map(trimNonTopLevelAliases))
      val cleanedValues = values.map(_.map(_.map(trimNonTopLevelAliases)))
      Unpivot(
        cleanedIds,
        cleanedValues,
        aliases,
        variableColumnName,
        valueColumnNames,
        child)

    // Operators that operate on objects should only have expressions from encoders, which should
    // never have extra aliases.
    case o: ObjectConsumer => o
    case o: ObjectProducer => o
    case a: AppendColumns => a

    case other =>
      other.transformExpressionsDownWithPruning(_.containsAnyPattern(ALIAS, MULTI_ALIAS)) {
        case Alias(child, _) => child
      }
  }
}

/**
 * Validates that the event time column in EventTimeWatermark is a top-level column reference
 * (e.g. a single name), not a nested field (e.g. "struct_col.field").
 *
 * Multi-part names are allowed when they resolve to a top-level attribute via a table alias
 * (e.g. "alias.column"), but rejected when they resolve to a nested struct field extraction.
 */
object ValidateEventTimeWatermarkColumn extends Rule[LogicalPlan] {
  override def apply(plan: LogicalPlan): LogicalPlan = {
    if (conf.getConf(SQLConf.STREAMING_VALIDATE_EVENT_TIME_WATERMARK_COLUMN)) {
      plan.resolveOperatorsWithPruning(_.containsPattern(EVENT_TIME_WATERMARK)) {
        case etw: EventTimeWatermark =>
          etw.eventTime match {
            case u: UnresolvedAttribute if u.nameParts.length > 1 =>
              // Try to resolve the multi-part name against the child output.
              // An alias-qualified column (e.g. "a.eventTime") resolves to an Attribute,
              // while a nested struct field (e.g. "struct_col.field") resolves to an
              // Alias(ExtractValue(...)) which is not an Attribute.
              etw.child.resolve(u.nameParts, conf.resolver) match {
                case Some(_: Attribute) => etw
                case _ =>
                  etw.failAnalysis(
                    errorClass = "EVENT_TIME_MUST_BE_TOP_LEVEL_COLUMN",
                    messageParameters = Map("eventExpr" -> u.sql)
                  )
              }
            case _ => etw
          }
      }
    } else {
      plan
    }
  }
}

/**
 * Ignore event time watermark in batch query, which is only supported in Structured Streaming.
 */
object EliminateEventTimeWatermark extends Rule[LogicalPlan] {
  override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsWithPruning(
    _.containsPattern(EVENT_TIME_WATERMARK)) {
    case EventTimeWatermark(_, _, _, child) if child.resolved && !child.isStreaming => child
    case UpdateEventTimeWatermarkColumn(_, _, child) if child.resolved && !child.isStreaming =>
      child
  }
}

/**
 * Resolve expressions if they contains [[NamePlaceholder]]s.
 */
object ResolveExpressionsWithNamePlaceholders extends Rule[LogicalPlan] {
  override def apply(plan: LogicalPlan): LogicalPlan = plan.resolveExpressionsWithPruning(
    _.containsAnyPattern(ARRAYS_ZIP, CREATE_NAMED_STRUCT), ruleId) {
    case e: ArraysZip if !e.resolved =>
      val names = e.children.zip(e.names).map {
        case (e: NamedExpression, NamePlaceholder) if e.resolved =>
          Literal(e.name)
        case (_, other) => other
      }
      ArraysZip(e.children, names)

    case e: CreateNamedStruct if !e.resolved =>
      val children = e.children.grouped(2).flatMap {
        case Seq(NamePlaceholder, e: NamedExpression) if e.resolved =>
          Seq(Literal(e.name), e)
        case kv =>
          kv
      }
      CreateNamedStruct(children.toList)
  }
}

/**
 * The aggregate expressions from subquery referencing outer query block are pushed
 * down to the outer query block for evaluation. This rule below updates such outer references
 * as AttributeReference referring attributes from the parent/outer query block.
 *
 * For example (SQL):
 * {{{
 *   SELECT l.a FROM l GROUP BY 1 HAVING EXISTS (SELECT 1 FROM r WHERE r.d < min(l.b))
 * }}}
 * Plan before the rule.
 *    Project [a#226]
 *    +- Filter exists#245 [min(b#227)#249]
 *       :  +- Project [1 AS 1#247]
 *       :     +- Filter (d#238 < min(outer(b#227)))       <-----
 *       :        +- SubqueryAlias r
 *       :           +- Project [_1#234 AS c#237, _2#235 AS d#238]
 *       :              +- LocalRelation [_1#234, _2#235]
 *       +- Aggregate [a#226], [a#226, min(b#227) AS min(b#227)#249]
 *          +- SubqueryAlias l
 *             +- Project [_1#223 AS a#226, _2#224 AS b#227]
 *                +- LocalRelation [_1#223, _2#224]
 * Plan after the rule.
 *    Project [a#226]
 *    +- Filter exists#245 [min(b#227)#249]
 *       :  +- Project [1 AS 1#247]
 *       :     +- Filter (d#238 < outer(min(b#227)#249))   <-----
 *       :        +- SubqueryAlias r
 *       :           +- Project [_1#234 AS c#237, _2#235 AS d#238]
 *       :              +- LocalRelation [_1#234, _2#235]
 *       +- Aggregate [a#226], [a#226, min(b#227) AS min(b#227)#249]
 *          +- SubqueryAlias l
 *             +- Project [_1#223 AS a#226, _2#224 AS b#227]
 *                +- LocalRelation [_1#223, _2#224]
 */
object UpdateOuterReferences extends Rule[LogicalPlan] {
  private def stripAlias(expr: Expression): Expression = expr match { case a: Alias => a.child }

  private def updateOuterReferenceInSubquery(
      plan: LogicalPlan,
      refExprs: Seq[Expression]): LogicalPlan = {
    plan resolveExpressions {
      case e if e.containsPattern(OUTER_REFERENCE) =>
        val outerAlias =
          refExprs.find(stripAlias(_).semanticEquals(stripOuterReference(e)))
        outerAlias match {
          case Some(a: Alias) => OuterReference(a.toAttribute)
          case _ => e
        }
    }
  }

  def apply(plan: LogicalPlan): LogicalPlan = {
    plan.resolveOperatorsWithPruning(
      _.containsAllPatterns(PLAN_EXPRESSION, FILTER, AGGREGATE), ruleId) {
      case f @ Filter(_, a: Aggregate) if f.resolved =>
        f.transformExpressionsWithPruning(_.containsPattern(PLAN_EXPRESSION), ruleId) {
          case s: SubqueryExpression if s.children.nonEmpty =>
            // Collect the aliases from output of aggregate.
            val outerAliases = a.aggregateExpressions collect { case a: Alias => a }
            // Update the subquery plan to record the OuterReference to point to outer query plan.
            s.withNewPlan(updateOuterReferenceInSubquery(s.plan, outerAliases))
      }
    }
  }
}

/**
 * The rule `ResolveReferences` in the main resolution batch creates [[TempResolvedColumn]] in
 * UnresolvedHaving/Filter/Sort to hold the temporarily resolved column with `agg.child`.
 *
 * If the expression hosting [[TempResolvedColumn]] is fully resolved, the rule
 * `ResolveAggregationFunctions` will
 * - Replace [[TempResolvedColumn]] with [[AttributeReference]] if it's inside aggregate functions
 *   or grouping expressions.
 * - Mark [[TempResolvedColumn]] as `hasTried` if not inside aggregate functions or grouping
 *   expressions, hoping other rules can re-resolve it.
 * `ResolveReferences` will re-resolve [[TempResolvedColumn]] if `hasTried` is true, and keep it
 * unchanged if the resolution fails. We should turn it back to [[UnresolvedAttribute]] so that the
 * analyzer can report missing column error later.
 *
 * If the expression hosting [[TempResolvedColumn]] is not resolved, [[TempResolvedColumn]] will
 * remain with `hasTried` as false. We should strip [[TempResolvedColumn]], so that users can see
 * the reason why the expression is not resolved, e.g. type mismatch.
 */
object RemoveTempResolvedColumn extends Rule[LogicalPlan] {
  override def apply(plan: LogicalPlan): LogicalPlan = {
    plan.resolveExpressionsWithPruning(_.containsPattern(TEMP_RESOLVED_COLUMN)) {
      case t: TempResolvedColumn =>
        if (t.hasTried) {
          UnresolvedAttribute(t.nameParts)
        } else {
          t.child
        }
    }
  }
}

/**
 * Rule that's used to handle `UnresolvedHaving` nodes with resolved `condition` and `child`.
 * It's placed outside the main batch to avoid conflicts with other rules that resolve
 * `UnresolvedHaving` in the main batch.
 */
object ResolveUnresolvedHaving extends Rule[LogicalPlan] {
  override def apply(plan: LogicalPlan): LogicalPlan = {
    plan.resolveOperatorsWithPruning(_.containsPattern(UNRESOLVED_HAVING), ruleId) {
      case u @ UnresolvedHaving(havingCondition, child)
        if havingCondition.resolved && child.resolved =>
        Filter(condition = havingCondition, child = child)
    }
  }
}
