/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.spark.sql.execution.datasources.parquet

import java.util.Locale

import org.apache.hadoop.conf.Configuration
import org.apache.parquet.io.{ColumnIO, ColumnIOFactory, GroupColumnIO, PrimitiveColumnIO}
import org.apache.parquet.schema._
import org.apache.parquet.schema.LogicalTypeAnnotation._
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName._
import org.apache.parquet.schema.Type.Repetition._

import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.errors.QueryCompilationErrors
import org.apache.spark.sql.execution.datasources.VariantMetadata
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._

/**
 * This converter class is used to convert Parquet [[MessageType]] to Spark SQL [[StructType]]
 * (via the `convert` method) as well as [[ParquetColumn]] (via the `convertParquetColumn`
 * method). The latter contains richer information about the Parquet type, including its
 * associated repetition & definition level, column path, column descriptor etc.
 *
 * Parquet format backwards-compatibility rules are respected when converting Parquet
 * [[MessageType]] schemas.
 *
 * @see https://github.com/apache/parquet-format/blob/master/LogicalTypes.md
 *
 * @param assumeBinaryIsString Whether unannotated BINARY fields should be assumed to be Spark SQL
 *        [[StringType]] fields.
 * @param assumeInt96IsTimestamp Whether unannotated INT96 fields should be assumed to be Spark SQL
 *        [[TimestampType]] fields.
 * @param caseSensitive Whether use case sensitive analysis when comparing Spark catalyst read
 *                      schema with Parquet schema.
 * @param inferTimestampNTZ Whether TimestampNTZType type is enabled.
 * @param nanosAsLong Whether timestamps with nanos are converted to long.
 */
class ParquetToSparkSchemaConverter(
    assumeBinaryIsString: Boolean = SQLConf.PARQUET_BINARY_AS_STRING.defaultValue.get,
    assumeInt96IsTimestamp: Boolean = SQLConf.PARQUET_INT96_AS_TIMESTAMP.defaultValue.get,
    caseSensitive: Boolean = SQLConf.CASE_SENSITIVE.defaultValue.get,
    inferTimestampNTZ: Boolean = SQLConf.PARQUET_INFER_TIMESTAMP_NTZ_ENABLED.defaultValue.get,
    nanosAsLong: Boolean = SQLConf.LEGACY_PARQUET_NANOS_AS_LONG.defaultValue.get,
    useFieldId: Boolean = SQLConf.PARQUET_FIELD_ID_READ_ENABLED.defaultValue.get,
    val ignoreVariantAnnotation: Boolean =
      SQLConf.PARQUET_IGNORE_VARIANT_ANNOTATION.defaultValue.get,
    val respectUnknownTypeAnnotation: Boolean =
      SQLConf.PARQUET_READER_RESPECT_UNKNOWN_TYPE_ANNOTATION.defaultValue.get) {

  def this(conf: SQLConf) = this(
    assumeBinaryIsString = conf.isParquetBinaryAsString,
    assumeInt96IsTimestamp = conf.isParquetINT96AsTimestamp,
    caseSensitive = conf.caseSensitiveAnalysis,
    inferTimestampNTZ = conf.parquetInferTimestampNTZEnabled,
    nanosAsLong = conf.legacyParquetNanosAsLong,
    useFieldId = conf.parquetFieldIdReadEnabled,
    ignoreVariantAnnotation = conf.parquetIgnoreVariantAnnotation,
    respectUnknownTypeAnnotation =
      conf.parquetReaderRespectUnknownTypeAnnotation)

  def this(conf: Configuration) = this(
    assumeBinaryIsString = conf.get(SQLConf.PARQUET_BINARY_AS_STRING.key).toBoolean,
    assumeInt96IsTimestamp = conf.get(SQLConf.PARQUET_INT96_AS_TIMESTAMP.key).toBoolean,
    caseSensitive = conf.get(SQLConf.CASE_SENSITIVE.key).toBoolean,
    inferTimestampNTZ = conf.get(SQLConf.PARQUET_INFER_TIMESTAMP_NTZ_ENABLED.key).toBoolean,
    nanosAsLong = conf.get(SQLConf.LEGACY_PARQUET_NANOS_AS_LONG.key).toBoolean,
    useFieldId = conf.getBoolean(SQLConf.PARQUET_FIELD_ID_READ_ENABLED.key,
      SQLConf.PARQUET_FIELD_ID_READ_ENABLED.defaultValue.get),
    ignoreVariantAnnotation = conf.getBoolean(SQLConf.PARQUET_IGNORE_VARIANT_ANNOTATION.key,
      SQLConf.PARQUET_IGNORE_VARIANT_ANNOTATION.defaultValue.get),
    respectUnknownTypeAnnotation = conf.getBoolean(
      SQLConf.PARQUET_READER_RESPECT_UNKNOWN_TYPE_ANNOTATION.key,
      SQLConf.PARQUET_READER_RESPECT_UNKNOWN_TYPE_ANNOTATION.defaultValue.get))

  /**
   * Converts Parquet [[MessageType]] `parquetSchema` to a Spark SQL [[StructType]].
   */
  def convert(parquetSchema: MessageType): StructType = {
    val column = new ColumnIOFactory().getColumnIO(parquetSchema)
    val converted = convertInternal(column)
    converted.sparkType.asInstanceOf[StructType]
  }

  /**
   * Convert `parquetSchema` into a [[ParquetColumn]] which contains its corresponding Spark
   * SQL [[StructType]] along with other information such as the maximum repetition and definition
   * level of each node, column descriptor for the leave nodes, etc.
   *
   * If `sparkReadSchema` is not empty, when deriving Spark SQL type from a Parquet field this will
   * check if the same field also exists in the schema. If so, it will use the Spark SQL type.
   * This is necessary since conversion from Parquet to Spark could cause precision loss. For
   * instance, Spark read schema is smallint/tinyint but Parquet only support int.
   */
  def convertParquetColumn(
      parquetSchema: MessageType,
      sparkReadSchema: Option[StructType] = None): ParquetColumn = {
    val column = new ColumnIOFactory().getColumnIO(parquetSchema)
    convertInternal(column, sparkReadSchema)
  }

  private def convertInternal(
      groupColumn: GroupColumnIO,
      sparkReadSchema: Option[StructType] = None): ParquetColumn = {
    // First convert the read schema into a map from field name to the field itself, to avoid O(n)
    // lookup cost below.
    val schemaMapOpt = sparkReadSchema.map { schema =>
      schema.map(f => normalizeFieldName(f.name) -> f).toMap
    }
    // Use ID mapping only when the name mapping doesn't find a match.
    lazy val schemaIdMapOpt = sparkReadSchema match {
      case Some(schema) if useFieldId =>
        Some(schema.fields.flatMap { f =>
          if (ParquetUtils.hasFieldId(f)) {
            Some((ParquetUtils.getFieldId(f), f))
          } else {
            None
          }
        }.toMap)
      case _ => None
    }

    val converted = (0 until groupColumn.getChildrenCount).map { i =>
      val field = groupColumn.getChild(i)
      val fieldFromReadSchema = schemaMapOpt.flatMap { schemaMap =>
        schemaMap.get(normalizeFieldName(field.getName))
      }.orElse {
        val parquetFieldId = Option(field.getType.getId).map(_.intValue())
        (parquetFieldId, schemaIdMapOpt) match {
          case (Some(id), Some(map)) => map.get(id)
          case _ => None
        }
      }
      var fieldReadType = fieldFromReadSchema.map(_.dataType)

      // If a field is repeated here then it is neither contained by a `LIST` nor `MAP`
      // annotated group (these should've been handled in `convertGroupField`), e.g.:
      //
      //  message schema {
      //    repeated int32 int_array;
      //  }
      // or
      //  message schema {
      //    repeated group struct_array {
      //      optional int32 field;
      //    }
      //  }
      //
      // the corresponding Spark read type should be an array and we should pass the element type
      // to the group or primitive type conversion method.
      if (field.getType.getRepetition == REPEATED) {
        fieldReadType = fieldReadType.flatMap {
          case at: ArrayType => Some(at.elementType)
          case _ =>
            throw QueryCompilationErrors.illegalParquetTypeError(groupColumn.toString)
        }
      }

      val convertedField = convertField(field, fieldReadType)
      val fieldName = fieldFromReadSchema.map(_.name).getOrElse(field.getType.getName)

      field.getType.getRepetition match {
        case OPTIONAL | REQUIRED =>
          val nullable = field.getType.getRepetition == OPTIONAL
          (StructField(fieldName, convertedField.sparkType, nullable = nullable),
              convertedField)

        case REPEATED =>
          // A repeated field that is neither contained by a `LIST`- or `MAP`-annotated group nor
          // annotated by `LIST` or `MAP` should be interpreted as a required list of required
          // elements where the element type is the type of the field.
          val arrayType = ArrayType(convertedField.sparkType, containsNull = false)
          (StructField(fieldName, arrayType, nullable = false),
              ParquetColumn(arrayType, None, convertedField.repetitionLevel - 1,
                convertedField.definitionLevel - 1, required = true, convertedField.path,
                Seq(convertedField.copy(required = true))))
      }
    }

    ParquetColumn(StructType(converted.map(_._1)), groupColumn, converted.map(_._2))
  }

  private def normalizeFieldName(name: String): String =
    if (caseSensitive) name else name.toLowerCase(Locale.ROOT)

  /**
   * Converts a Parquet [[Type]] to a [[ParquetColumn]] which wraps a Spark SQL [[DataType]] with
   * additional information such as the Parquet column's repetition & definition level, column
   * path, column descriptor etc.
   */
  def convertField(
      field: ColumnIO,
      sparkReadType: Option[DataType] = None): ParquetColumn = {
    val targetType = sparkReadType.map {
      _.transformRecursively { case t: UserDefinedType[_] => t.sqlType }
    }
    field match {
      case primitiveColumn: PrimitiveColumnIO => convertPrimitiveField(primitiveColumn, targetType)
      case groupColumn: GroupColumnIO if targetType.contains(VariantType) =>
        if (SQLConf.get.getConf(SQLConf.VARIANT_ALLOW_READING_SHREDDED)) {
          // We need the underlying file type regardless of the config.
          val col = convertGroupField(groupColumn, ignoreVariantAnnotation = true)
          col.copy(sparkType = VariantType, variantFileType = Some(col))
        } else {
          convertVariantField(groupColumn)
        }
      case groupColumn: GroupColumnIO if targetType.exists(VariantMetadata.isVariantStruct) =>
        val col = convertGroupField(groupColumn, ignoreVariantAnnotation = true)
        col.copy(sparkType = targetType.get, variantFileType = Some(col))
      case groupColumn: GroupColumnIO =>
        convertGroupField(groupColumn, ignoreVariantAnnotation, targetType)
    }
  }

  private def convertPrimitiveField(
      primitiveColumn: PrimitiveColumnIO,
      sparkReadType: Option[DataType] = None): ParquetColumn = {
    val parquetType = primitiveColumn.getType.asPrimitiveType()
    val typeAnnotation = primitiveColumn.getType.getLogicalTypeAnnotation match {
      case unknown: UnknownLogicalTypeAnnotation =>
        if (respectUnknownTypeAnnotation) unknown else null
      case other => other
    }
    val typeName = primitiveColumn.getPrimitive

    def typeString =
      if (typeAnnotation == null) s"$typeName" else s"$typeName ($typeAnnotation)"

    def typeNotImplemented() =
      throw QueryCompilationErrors.parquetTypeUnsupportedYetError(typeString)

    def illegalType() =
      throw QueryCompilationErrors.illegalParquetTypeError(typeString)

    // When maxPrecision = -1, we skip precision range check, and always respect the precision
    // specified in field.getDecimalMetadata.  This is useful when interpreting decimal types stored
    // as binaries with variable lengths.
    def makeDecimalType(maxPrecision: Int = -1): DecimalType = {
      val decimalLogicalTypeAnnotation = typeAnnotation
        .asInstanceOf[DecimalLogicalTypeAnnotation]
      val precision = decimalLogicalTypeAnnotation.getPrecision
      val scale = decimalLogicalTypeAnnotation.getScale

      ParquetSchemaConverter.checkConversionRequirement(
        maxPrecision == -1 || 1 <= precision && precision <= maxPrecision,
        s"Invalid decimal precision: $typeName cannot store $precision digits (max $maxPrecision)")

      DecimalType(precision, scale)
    }

    val isUnknownType = typeAnnotation.isInstanceOf[UnknownLogicalTypeAnnotation]
    val nullTypeOpt = Option.when(isUnknownType)(NullType)
    val sparkType = sparkReadType.orElse(nullTypeOpt).getOrElse(typeName match {
      case BOOLEAN => BooleanType

      case FLOAT => FloatType

      case DOUBLE => DoubleType

      case INT32 =>
        typeAnnotation match {
          case intTypeAnnotation: IntLogicalTypeAnnotation if intTypeAnnotation.isSigned =>
            intTypeAnnotation.getBitWidth match {
              case 8 => ByteType
              case 16 => ShortType
              case 32 => IntegerType
              case _ => illegalType()
            }
          case null => IntegerType
          case _: DateLogicalTypeAnnotation => DateType
          case _: DecimalLogicalTypeAnnotation => makeDecimalType(Decimal.MAX_INT_DIGITS)
          case intTypeAnnotation: IntLogicalTypeAnnotation if !intTypeAnnotation.isSigned =>
            intTypeAnnotation.getBitWidth match {
              case 8 => ShortType
              case 16 => IntegerType
              case 32 => LongType
              case _ => illegalType()
            }
          case t: TimestampLogicalTypeAnnotation if t.getUnit == TimeUnit.MILLIS =>
            typeNotImplemented()
          case _ => illegalType()
        }

      case INT64 =>
        typeAnnotation match {
          case intTypeAnnotation: IntLogicalTypeAnnotation if intTypeAnnotation.isSigned =>
            intTypeAnnotation.getBitWidth match {
              case 64 => LongType
              case _ => illegalType()
            }
          case null => LongType
          case _: DecimalLogicalTypeAnnotation => makeDecimalType(Decimal.MAX_LONG_DIGITS)
          case intTypeAnnotation: IntLogicalTypeAnnotation if !intTypeAnnotation.isSigned =>
            intTypeAnnotation.getBitWidth match {
              // The precision to hold the largest unsigned long is:
              // `java.lang.Long.toUnsignedString(-1).length` = 20
              case 64 => DecimalType(20, 0)
              case _ => illegalType()
            }
          case timestamp: TimestampLogicalTypeAnnotation
            if timestamp.getUnit == TimeUnit.MICROS || timestamp.getUnit == TimeUnit.MILLIS =>
            if (timestamp.isAdjustedToUTC || !inferTimestampNTZ) {
              TimestampType
            } else {
              TimestampNTZType
            }
          // SPARK-40819: NANOS are not supported as a Timestamp, convert to LongType without
          // timezone awareness to address behaviour regression introduced by SPARK-34661
          case timestamp: TimestampLogicalTypeAnnotation
            if timestamp.getUnit == TimeUnit.NANOS && nanosAsLong =>
            LongType
          case time: TimeLogicalTypeAnnotation
            if time.getUnit == TimeUnit.MICROS && !time.isAdjustedToUTC =>
            TimeType(TimeType.MICROS_PRECISION)
          case _ =>
            illegalType()
        }

      case INT96 =>
        ParquetSchemaConverter.checkConversionRequirement(
          assumeInt96IsTimestamp,
          "INT96 is not supported unless it's interpreted as timestamp. " +
            s"Please try to set ${SQLConf.PARQUET_INT96_AS_TIMESTAMP.key} to true.")
        TimestampType

      case BINARY =>
        typeAnnotation match {
          case _: StringLogicalTypeAnnotation | _: EnumLogicalTypeAnnotation |
               _: JsonLogicalTypeAnnotation => StringType
          case null if assumeBinaryIsString => StringType
          case null => BinaryType
          case _: BsonLogicalTypeAnnotation => BinaryType
          case _: DecimalLogicalTypeAnnotation => makeDecimalType()
          case _ => illegalType()
        }

      case FIXED_LEN_BYTE_ARRAY =>
        typeAnnotation match {
          case _: DecimalLogicalTypeAnnotation =>
            makeDecimalType(Decimal.maxPrecisionForBytes(parquetType.getTypeLength))
          case _: IntervalLogicalTypeAnnotation => typeNotImplemented()
          case null => BinaryType
          case _ => illegalType()
        }

      case _ => illegalType()
    })

    ParquetColumn(sparkType, primitiveColumn)
  }

  private def convertGroupField(
      groupColumn: GroupColumnIO,
      ignoreVariantAnnotation: Boolean,
      sparkReadType: Option[DataType] = None): ParquetColumn = {
    val field = groupColumn.getType.asGroupType()

    /*
     * We need to use the Spark SQL type if available, since conversion from Parquet to Spark could
     * cause precision loss. For instance, Spark read schema is smallint/tinyint but Parquet only
     * supports int. This is only applicable to primitive types, as the Spark type and the Parquet
     * converted type for complex types should be largely the same (the only difference can be at
     * the leaves or if we are adding non-requested struct fields for detecting struct nullability).
     * @param sparkReadType Matching Spark schema field type for the Parquet field
     * @param converted ParquetColumn that is recursively converted from the Parquet field
     * @return `sparkReadType` if it is defined and not a complex type, otherwise the type of
     *         `converted`
     */
    def getSparkTypeIfApplicable(
        sparkReadType: Option[DataType],
        converted: ParquetColumn): DataType = {
      sparkReadType
        .filterNot(ParquetSchemaConverter.isComplexType)
        .getOrElse(converted.sparkType)
    }

    Option(field.getLogicalTypeAnnotation).fold(
      convertInternal(groupColumn, sparkReadType.map(_.asInstanceOf[StructType]))) {
      case v: VariantLogicalTypeAnnotation if v.getSpecVersion == 1 =>
        if (ignoreVariantAnnotation) {
          convertInternal(groupColumn)
        } else {
          ParquetSchemaConverter.checkConversionRequirement(
            sparkReadType.forall(_.isInstanceOf[VariantType]),
            s"Invalid Spark read type: expected $field to be variant type but found " +
              s"${if (sparkReadType.isEmpty) { "None" } else {sparkReadType.get.sql} }")
          if (SQLConf.get.getConf(SQLConf.VARIANT_ALLOW_READING_SHREDDED)) {
            val col = convertInternal(groupColumn)
            col.copy(sparkType = VariantType, variantFileType = Some(col))
          } else {
            convertVariantField(groupColumn)
          }
        }

      // A Parquet list is represented as a 3-level structure:
      //
      //   <list-repetition> group <name> (LIST) {
      //     repeated group list {
      //       <element-repetition> <element-type> element;
      //     }
      //   }
      //
      // However, according to the most recent Parquet format spec (not released yet up until
      // writing), some 2-level structures are also recognized for backwards-compatibility.  Thus,
      // we need to check whether the 2nd level or the 3rd level refers to list element type.
      //
      // See: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#lists
      case _: ListLogicalTypeAnnotation =>
        ParquetSchemaConverter.checkConversionRequirement(
          field.getFieldCount == 1, s"Invalid list type $field")
        ParquetSchemaConverter.checkConversionRequirement(
          sparkReadType.forall(_.isInstanceOf[ArrayType]),
          s"Invalid Spark read type: expected $field to be list type but found $sparkReadType")

        val repeated = groupColumn.getChild(0)
        val repeatedType = repeated.getType
        ParquetSchemaConverter.checkConversionRequirement(
          repeatedType.isRepetition(REPEATED), s"Invalid list type $field")
        val sparkReadElementType = sparkReadType.map(_.asInstanceOf[ArrayType].elementType)

        if (isElementType(repeatedType, field.getName)) {
          var converted = convertField(repeated, sparkReadElementType)
          val convertedType = getSparkTypeIfApplicable(sparkReadElementType, converted)

          // legacy format such as:
          //   optional group my_list (LIST) {
          //     repeated int32 element;
          //   }
          // we should mark the primitive field as required
          if (repeatedType.isPrimitive) converted = converted.copy(required = true)

          ParquetColumn(ArrayType(convertedType, containsNull = false),
            groupColumn, Seq(converted))
        } else {
          val element = repeated.asInstanceOf[GroupColumnIO].getChild(0)
          val converted = convertField(element, sparkReadElementType)
          val convertedType = getSparkTypeIfApplicable(sparkReadElementType, converted)
          val optional = element.getType.isRepetition(OPTIONAL)
          ParquetColumn(ArrayType(convertedType, containsNull = optional),
            groupColumn, Seq(converted))
        }

      // scalastyle:off
      // `MAP_KEY_VALUE` is for backwards-compatibility
      // See: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#backward-compatibility-rules-1
      // scalastyle:on
      case _: MapLogicalTypeAnnotation | _: MapKeyValueTypeAnnotation =>
        ParquetSchemaConverter.checkConversionRequirement(
          field.getFieldCount == 1 && !field.getType(0).isPrimitive,
          s"Invalid map type: $field")
        ParquetSchemaConverter.checkConversionRequirement(
          sparkReadType.forall(_.isInstanceOf[MapType]),
          s"Invalid Spark read type: expected $field to be map type but found $sparkReadType")

        val keyValue = groupColumn.getChild(0).asInstanceOf[GroupColumnIO]
        val keyValueType = keyValue.getType.asGroupType()
        ParquetSchemaConverter.checkConversionRequirement(
          keyValueType.isRepetition(REPEATED) && keyValueType.getFieldCount == 2,
          s"Invalid map type: $field")

        val key = keyValue.getChild(0)
        val value = keyValue.getChild(1)
        val sparkReadKeyType = sparkReadType.map(_.asInstanceOf[MapType].keyType)
        val sparkReadValueType = sparkReadType.map(_.asInstanceOf[MapType].valueType)
        val convertedKey = convertField(key, sparkReadKeyType)
        val convertedValue = convertField(value, sparkReadValueType)
        val convertedKeyType = getSparkTypeIfApplicable(sparkReadKeyType, convertedKey)
        val convertedValueType = getSparkTypeIfApplicable(sparkReadValueType, convertedValue)
        val valueOptional = value.getType.isRepetition(OPTIONAL)
        ParquetColumn(
          MapType(convertedKeyType, convertedValueType,
            valueContainsNull = valueOptional),
          groupColumn, Seq(convertedKey, convertedValue))
      case _ =>
        throw QueryCompilationErrors.unrecognizedParquetTypeError(field.toString)
    }
  }

  private def convertVariantField(groupColumn: GroupColumnIO): ParquetColumn = {
    if (groupColumn.getChildrenCount != 2) {
      // We may allow more than two children in the future, so consider this unsupported.
      throw QueryCompilationErrors.invalidVariantWrongNumFieldsError()
    }
    // Find the binary columns, and validate that they have the correct type.
    val valueAndMetadata = Seq("value", "metadata").map { colName =>
      val idx = (0 until groupColumn.getChildrenCount)
          .find(groupColumn.getChild(_).getName == colName)
      if (idx.isEmpty) {
        throw QueryCompilationErrors.invalidVariantMissingFieldError(colName)
      }
      val child = groupColumn.getChild(idx.get)
      // The value and metadata cannot be individually null, only the full struct can.
      if (child.getType.getRepetition != REQUIRED ||
          !child.isInstanceOf[PrimitiveColumnIO] ||
          child.asInstanceOf[PrimitiveColumnIO].getPrimitive != BINARY) {
        throw QueryCompilationErrors.invalidVariantNullableOrNotBinaryFieldError(colName)
      }
      child
    }
    ParquetColumn(VariantType, groupColumn, Seq(
      convertField(valueAndMetadata(0), Some(BinaryType)),
      convertField(valueAndMetadata(1), Some(BinaryType))
    ))
  }

  // scalastyle:off
  // Here we implement Parquet LIST backwards-compatibility rules.
  // See: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#backward-compatibility-rules
  // scalastyle:on
  private[parquet] def isElementType(repeatedType: Type, parentName: String): Boolean = {
    {
      // For legacy 2-level list types with primitive element type, e.g.:
      //
      //    // ARRAY<INT> (nullable list, non-null elements)
      //    optional group my_list (LIST) {
      //      repeated int32 element;
      //    }
      //
      repeatedType.isPrimitive
    } || {
      // For legacy 2-level list types whose element type is a group type with 2 or more fields,
      // e.g.:
      //
      //    // ARRAY<STRUCT<str: STRING, num: INT>> (nullable list, non-null elements)
      //    optional group my_list (LIST) {
      //      repeated group element {
      //        required binary str (UTF8);
      //        required int32 num;
      //      };
      //    }
      //
      repeatedType.asGroupType().getFieldCount > 1
    } || {
      // For legacy 2-level list types generated by parquet-avro (Parquet version < 1.6.0), e.g.:
      //
      //    // ARRAY<STRUCT<str: STRING>> (nullable list, non-null elements)
      //    optional group my_list (LIST) {
      //      repeated group array {
      //        required binary str (UTF8);
      //      };
      //    }
      //
      repeatedType.getName == "array"
    } || {
      // For Parquet data generated by parquet-thrift, e.g.:
      //
      //    // ARRAY<STRUCT<str: STRING>> (nullable list, non-null elements)
      //    optional group my_list (LIST) {
      //      repeated group my_list_tuple {
      //        required binary str (UTF8);
      //      };
      //    }
      //
      repeatedType.getName == s"${parentName}_tuple"
    }
  }
}

/**
 * This converter class is used to convert Spark SQL [[StructType]] to Parquet [[MessageType]].
 *
 * @param writeLegacyParquetFormat Whether to use legacy Parquet format compatible with Spark 1.4
 *        and prior versions when converting a Catalyst [[StructType]] to a Parquet [[MessageType]].
 *        When set to false, use standard format defined in parquet-format spec.  This argument only
 *        affects Parquet write path.
 * @param outputTimestampType which parquet timestamp type to use when writing.
 * @param useFieldId whether we should include write field id to Parquet schema. Set this to false
 *        via `spark.sql.parquet.fieldId.write.enabled = false` to disable writing field ids.
 */
class SparkToParquetSchemaConverter(
    writeLegacyParquetFormat: Boolean = SQLConf.PARQUET_WRITE_LEGACY_FORMAT.defaultValue.get,
    outputTimestampType: SQLConf.ParquetOutputTimestampType.Value =
      SQLConf.ParquetOutputTimestampType.INT96,
    useFieldId: Boolean = SQLConf.PARQUET_FIELD_ID_WRITE_ENABLED.defaultValue.get,
    annotateVariantLogicalType: Boolean =
      SQLConf.PARQUET_ANNOTATE_VARIANT_LOGICAL_TYPE.defaultValue.get) {

  def this(conf: SQLConf) = this(
    writeLegacyParquetFormat = conf.writeLegacyParquetFormat,
    outputTimestampType = conf.parquetOutputTimestampType,
    useFieldId = conf.parquetFieldIdWriteEnabled)

  def this(conf: Configuration) = this(
    writeLegacyParquetFormat = conf.get(SQLConf.PARQUET_WRITE_LEGACY_FORMAT.key).toBoolean,
    outputTimestampType = SQLConf.ParquetOutputTimestampType.withName(
      conf.get(SQLConf.PARQUET_OUTPUT_TIMESTAMP_TYPE.key)),
    useFieldId = conf.get(SQLConf.PARQUET_FIELD_ID_WRITE_ENABLED.key).toBoolean,
    annotateVariantLogicalType =
      conf.get(SQLConf.PARQUET_ANNOTATE_VARIANT_LOGICAL_TYPE.key).toBoolean)

  /**
   * Converts a Spark SQL [[StructType]] to a Parquet [[MessageType]].
   */
  def convert(catalystSchema: StructType): MessageType = {
    Types
      .buildMessage()
      .addFields(catalystSchema.map(f => convertField(f, inShredded = false)): _*)
      .named(ParquetSchemaConverter.SPARK_PARQUET_SCHEMA_NAME)
  }

  /**
   * Converts a Spark SQL [[StructField]] to a Parquet [[Type]]. `inShredded` indicates whether
   * the field is within a shredded Variant schema.
   */
  def convertField(field: StructField, inShredded: Boolean): Type = {
    val converted = convertField(field, if (field.nullable) OPTIONAL else REQUIRED, inShredded)
    if (useFieldId && ParquetUtils.hasFieldId(field)) {
      converted.withId(ParquetUtils.getFieldId(field))
    } else {
      converted
    }
  }

  private def convertField(
      field: StructField,
      repetition: Type.Repetition,
      inShredded: Boolean): Type = {

    field.dataType match {
      // ===================
      // Simple atomic types
      // ===================

      case BooleanType =>
        Types.primitive(BOOLEAN, repetition).named(field.name)

      case ByteType =>
        Types.primitive(INT32, repetition)
          .as(LogicalTypeAnnotation.intType(8, true)).named(field.name)

      case ShortType =>
        Types.primitive(INT32, repetition)
          .as(LogicalTypeAnnotation.intType(16, true)).named(field.name)

      case IntegerType | _: YearMonthIntervalType =>
        Types.primitive(INT32, repetition).named(field.name)

      case LongType | _: DayTimeIntervalType =>
        Types.primitive(INT64, repetition).named(field.name)

      case FloatType =>
        Types.primitive(FLOAT, repetition).named(field.name)

      case DoubleType =>
        Types.primitive(DOUBLE, repetition).named(field.name)

      case _: StringType =>
        Types.primitive(BINARY, repetition)
          .as(LogicalTypeAnnotation.stringType()).named(field.name)

      case DateType =>
        Types.primitive(INT32, repetition)
          .as(LogicalTypeAnnotation.dateType()).named(field.name)

      case _: TimeType =>
        Types.primitive(INT64, repetition)
          .as(LogicalTypeAnnotation.timeType(false, TimeUnit.MICROS)).named(field.name)

      // NOTE: Spark SQL can write timestamp values to Parquet using INT96, TIMESTAMP_MICROS or
      // TIMESTAMP_MILLIS. TIMESTAMP_MICROS is recommended but INT96 is the default to keep the
      // behavior same as before.
      //
      // As stated in PARQUET-323, Parquet `INT96` was originally introduced to represent nanosecond
      // timestamp in Impala for some historical reasons.  It's not recommended to be used for any
      // other types and will probably be deprecated in some future version of parquet-format spec.
      // That's the reason why parquet-format spec only defines `TIMESTAMP_MILLIS` and
      // `TIMESTAMP_MICROS` which are both logical types annotating `INT64`.
      //
      // Originally, Spark SQL uses the same nanosecond timestamp type as Impala and Hive.  Starting
      // from Spark 1.5.0, we resort to a timestamp type with microsecond precision so that we can
      // store a timestamp into a `Long`.  This design decision is subject to change though, for
      // example, we may resort to nanosecond precision in the future.
      //
      // The Parquet Variant Shredding spec states that timestamps coming from shredded Variants
      // must be stored as INT64, and the Spark Timestamp type always has microsecond precision,
      // therefore, the TIMESTAMP_MICROS configuration is used when writing shredded variant
      // timestamp subfields.
      case TimestampType =>
        if (inShredded) {
          Types.primitive(INT64, repetition)
            .as(LogicalTypeAnnotation.timestampType(true, TimeUnit.MICROS)).named(field.name)
        } else {
          outputTimestampType match {
            case SQLConf.ParquetOutputTimestampType.INT96 =>
              Types.primitive(INT96, repetition).named(field.name)
            case SQLConf.ParquetOutputTimestampType.TIMESTAMP_MICROS =>
              Types.primitive(INT64, repetition)
                .as(LogicalTypeAnnotation.timestampType(true, TimeUnit.MICROS)).named(field.name)
            case SQLConf.ParquetOutputTimestampType.TIMESTAMP_MILLIS =>
              Types.primitive(INT64, repetition)
                .as(LogicalTypeAnnotation.timestampType(true, TimeUnit.MILLIS)).named(field.name)
          }
        }

      case TimestampNTZType =>
        Types.primitive(INT64, repetition)
          .as(LogicalTypeAnnotation.timestampType(false, TimeUnit.MICROS)).named(field.name)
      case BinaryType =>
        Types.primitive(BINARY, repetition).named(field.name)

      // ======================
      // Decimals (legacy mode)
      // ======================

      // Spark 1.4.x and prior versions only support decimals with a maximum precision of 18 and
      // always store decimals in fixed-length byte arrays.  To keep compatibility with these older
      // versions, here we convert decimals with all precisions to `FIXED_LEN_BYTE_ARRAY` annotated
      // by `DECIMAL`.
      case DecimalType.Fixed(precision, scale) if writeLegacyParquetFormat =>
        Types
          .primitive(FIXED_LEN_BYTE_ARRAY, repetition)
          .as(LogicalTypeAnnotation.decimalType(scale, precision))
          .length(Decimal.minBytesForPrecision(precision))
          .named(field.name)

      // ========================
      // Decimals (standard mode)
      // ========================

      // Uses INT32 for 1 <= precision <= 9
      case DecimalType.Fixed(precision, scale)
          if precision <= Decimal.MAX_INT_DIGITS && !writeLegacyParquetFormat =>
        Types
          .primitive(INT32, repetition)
          .as(LogicalTypeAnnotation.decimalType(scale, precision))
          .named(field.name)

      // Uses INT64 for 1 <= precision <= 18
      case DecimalType.Fixed(precision, scale)
          if precision <= Decimal.MAX_LONG_DIGITS && !writeLegacyParquetFormat =>
        Types
          .primitive(INT64, repetition)
          .as(LogicalTypeAnnotation.decimalType(scale, precision))
          .named(field.name)

      // Uses FIXED_LEN_BYTE_ARRAY for all other precisions
      case DecimalType.Fixed(precision, scale) if !writeLegacyParquetFormat =>
        Types
          .primitive(FIXED_LEN_BYTE_ARRAY, repetition)
          .as(LogicalTypeAnnotation.decimalType(scale, precision))
          .length(Decimal.minBytesForPrecision(precision))
          .named(field.name)

      // ===================================
      // ArrayType and MapType (legacy mode)
      // ===================================

      // Spark 1.4.x and prior versions convert `ArrayType` with nullable elements into a 3-level
      // `LIST` structure.  This behavior is somewhat a hybrid of parquet-hive and parquet-avro
      // (1.6.0rc3): the 3-level structure is similar to parquet-hive while the 3rd level element
      // field name "array" is borrowed from parquet-avro.
      case ArrayType(elementType, nullable @ true) if writeLegacyParquetFormat =>
        // <list-repetition> group <name> (LIST) {
        //   optional group bag {
        //     repeated <element-type> array;
        //   }
        // }

        // This should not use `listOfElements` here because this new method checks if the
        // element name is `element` in the `GroupType` and throws an exception if not.
        // As mentioned above, Spark prior to 1.4.x writes `ArrayType` as `LIST` but with
        // `array` as its element name as below. Therefore, we build manually
        // the correct group type here via the builder. (See SPARK-16777)
        Types
          .buildGroup(repetition).as(LogicalTypeAnnotation.listType())
          .addField(Types
            .buildGroup(REPEATED)
            .addField(convertField(StructField("array", elementType, nullable), inShredded))
            .named("bag"))
          .named(field.name)

      // Spark 1.4.x and prior versions convert ArrayType with non-nullable elements into a 2-level
      // LIST structure.  This behavior mimics parquet-avro (1.6.0rc3).  Note that this case is
      // covered by the backwards-compatibility rules implemented in `isElementType()`.
      case ArrayType(elementType, nullable @ false) if writeLegacyParquetFormat =>
        // <list-repetition> group <name> (LIST) {
        //   repeated <element-type> element;
        // }

        // Here too, we should not use `listOfElements`. (See SPARK-16777)
        Types
          .buildGroup(repetition).as(LogicalTypeAnnotation.listType())
          // "array" is the name chosen by parquet-avro (1.7.0 and prior version)
          .addField(convertField(StructField("array", elementType, nullable), REPEATED, inShredded))
          .named(field.name)

      // Spark 1.4.x and prior versions convert MapType into a 3-level group annotated by
      // MAP_KEY_VALUE.  This is covered by `convertGroupField(field: GroupType): DataType`.
      case MapType(keyType, valueType, valueContainsNull) if writeLegacyParquetFormat =>
        // <map-repetition> group <name> (MAP) {
        //   repeated group map (MAP_KEY_VALUE) {
        //     required <key-type> key;
        //     <value-repetition> <value-type> value;
        //   }
        // }
        ConversionPatterns.mapType(
          repetition,
          field.name,
          convertField(StructField("key", keyType, nullable = false), inShredded),
          convertField(StructField("value", valueType, valueContainsNull), inShredded))

      // =====================================
      // ArrayType and MapType (standard mode)
      // =====================================

      case ArrayType(elementType, containsNull) if !writeLegacyParquetFormat =>
        // <list-repetition> group <name> (LIST) {
        //   repeated group list {
        //     <element-repetition> <element-type> element;
        //   }
        // }
        Types
          .buildGroup(repetition).as(LogicalTypeAnnotation.listType())
          .addField(
            Types.repeatedGroup()
              .addField(convertField(StructField("element", elementType, containsNull), inShredded))
              .named("list"))
          .named(field.name)

      case MapType(keyType, valueType, valueContainsNull) =>
        // <map-repetition> group <name> (MAP) {
        //   repeated group key_value {
        //     required <key-type> key;
        //     <value-repetition> <value-type> value;
        //   }
        // }
        Types
          .buildGroup(repetition).as(LogicalTypeAnnotation.mapType())
          .addField(
            Types
              .repeatedGroup()
              .addField(convertField(StructField("key", keyType, nullable = false), inShredded))
              .addField(
                convertField(StructField("value", valueType, valueContainsNull), inShredded)
              ).named("key_value"))
          .named(field.name)

      // ===========
      // Other types
      // ===========

      case VariantType =>
        (if (annotateVariantLogicalType) {
          Types.buildGroup(repetition).as(LogicalTypeAnnotation.variantType(1))
        } else {
          Types.buildGroup(repetition)
        })
          .addField(convertField(StructField("value", BinaryType, nullable = false), inShredded))
          .addField(convertField(StructField("metadata", BinaryType, nullable = false), inShredded))
          .named(field.name)

      case s: StructType if SparkShreddingUtils.isVariantShreddingStruct(s) =>
        // Variant struct takes a Variant and writes to Parquet as a shredded schema.
        val group = if (annotateVariantLogicalType) {
          Types.buildGroup(repetition).as(LogicalTypeAnnotation.variantType(1))
        } else {
          Types.buildGroup(repetition)
        }
        s.fields.foreach { f =>
          group.addField(convertField(f, inShredded = true))
        }
        group.named(field.name)

      case StructType(fields) =>
        fields.foldLeft(Types.buildGroup(repetition)) { (builder, field) =>
          builder.addField(convertField(field, inShredded))
        }.named(field.name)

      case udt: UserDefinedType[_] =>
        convertField(field.copy(dataType = udt.sqlType), inShredded)

      case NullType => // Selected primitive type here doesn't have significance.
        Types.primitive(BOOLEAN, repetition).named(field.name)
          .withLogicalTypeAnnotation(LogicalTypeAnnotation.unknownType())

      case _ =>
        throw QueryCompilationErrors.cannotConvertDataTypeToParquetTypeError(field)
    }
  }
}

private[sql] object ParquetSchemaConverter {
  val SPARK_PARQUET_SCHEMA_NAME = "spark_schema"

  val EMPTY_MESSAGE: MessageType =
    Types.buildMessage().named(ParquetSchemaConverter.SPARK_PARQUET_SCHEMA_NAME)

  def checkConversionRequirement(f: => Boolean, message: String): Unit = {
    if (!f) {
      throw new AnalysisException(
        errorClass = "_LEGACY_ERROR_TEMP_3071",
        messageParameters = Map("msg" -> message))
    }
  }

  /**
   * Whether a [[DataType]] is complex. Complex [[DataType]] is not equivalent to
   * non-[[AtomicType]]. For example, [[CalendarIntervalType]] is not complex, but it's not an
   * [[AtomicType]] either.
   */
  def isComplexType(dataType: DataType): Boolean = {
    dataType match {
      case _: ArrayType | _: MapType | _: StructType => true
      case _ => false
    }
  }
}
