/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.spark.sql.hive

import java.io.File
import java.time.{Duration, Period}
import java.time.temporal.ChronoUnit

import org.apache.hadoop.fs.Path
import org.apache.parquet.hadoop.ParquetFileReader

import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.execution.datasources.parquet.{ParquetCompressionCodec, ParquetTest}
import org.apache.spark.sql.hive.test.TestHiveSingleton
import org.apache.spark.sql.internal.SQLConf

case class Cases(lower: String, UPPER: String)

class HiveParquetSuite extends QueryTest
  with ParquetTest
  with TestHiveSingleton {

  test("Case insensitive attribute names") {
    withParquetTable((1 to 4).map(i => Cases(i.toString, i.toString)), "cases") {
      val expected = (1 to 4).map(i => Row(i.toString))
      checkAnswer(sql("SELECT upper FROM cases"), expected)
      checkAnswer(sql("SELECT LOWER FROM cases"), expected)
    }
  }

  test("SELECT on Parquet table") {
    val data = (1 to 4).map(i => (i, s"val_$i"))
    withParquetTable(data, "t") {
      checkAnswer(sql("SELECT * FROM t"), data.map(Row.fromTuple))
    }
  }

  test("Simple column projection + filter on Parquet table") {
    withParquetTable((1 to 4).map(i => (i % 2 == 0, i, s"val_$i")), "t") {
      checkAnswer(
        sql("SELECT `_1`, `_3` FROM t WHERE `_1` = true"),
        Seq(Row(true, "val_2"), Row(true, "val_4")))
    }
  }

  test("Converting Hive to Parquet Table via saveAsParquetFile") {
    withTempPath { dir =>
      sql("SELECT * FROM src").write.parquet(dir.getCanonicalPath)
      spark.read.parquet(dir.getCanonicalPath).createOrReplaceTempView("p")
      withTempView("p") {
        checkAnswer(
          sql("SELECT * FROM src ORDER BY key"),
          sql("SELECT * from p ORDER BY key").collect().toSeq)
      }
    }
  }

  test("INSERT OVERWRITE TABLE Parquet table") {
    // Don't run with vectorized: currently relies on UnsafeRow.
    withParquetTable((1 to 10).map(i => (i, s"val_$i")), "t", false) {
      withTempPath { file =>
        sql("SELECT * FROM t LIMIT 1").write.parquet(file.getCanonicalPath)
        spark.read.parquet(file.getCanonicalPath).createOrReplaceTempView("p")
        withTempView("p") {
          // let's do three overwrites for good measure
          sql("INSERT OVERWRITE TABLE p SELECT * FROM t")
          sql("INSERT OVERWRITE TABLE p SELECT * FROM t")
          sql("INSERT OVERWRITE TABLE p SELECT * FROM t")
          checkAnswer(sql("SELECT * FROM p"), sql("SELECT * FROM t").collect().toSeq)
        }
      }
    }
  }

  test("SPARK-25206: wrong records are returned by filter pushdown " +
    "when Hive metastore schema and parquet schema are in different letter cases") {
    withSQLConf(SQLConf.PARQUET_FILTER_PUSHDOWN_ENABLED.key -> true.toString) {
      withTempPath { path =>
        val data = spark.range(1, 10).toDF("id")
        data.write.parquet(path.getCanonicalPath)
        withTable("SPARK_25206") {
          sql("CREATE TABLE SPARK_25206 (ID LONG) USING parquet LOCATION " +
            s"'${path.getCanonicalPath}'")
          checkAnswer(sql("select id from SPARK_25206 where id > 0"), data)
        }
      }
    }
  }

  test("SPARK-25271: write empty map into hive parquet table") {
    import testImplicits._

    Seq(Map(1 -> "a"), Map.empty[Int, String]).toDF("m").createOrReplaceTempView("p")
    withTempView("p") {
      val targetTable = "targetTable"
      withTable(targetTable) {
        sql(s"CREATE TABLE $targetTable STORED AS PARQUET AS SELECT m FROM p")
        checkAnswer(sql(s"SELECT m FROM $targetTable"),
          Row(Map(1 -> "a")) :: Row(Map.empty[Int, String]) :: Nil)
      }
    }
  }

  test("SPARK-33323: Add query resolved check before convert hive relation") {
    withTable("t") {
      val query =
        s"""
           |CREATE TABLE t STORED AS PARQUET AS
           |SELECT * FROM (
           | SELECT c3 FROM (
           |  SELECT c1, c2 from values(1,2) t(c1, c2)
           |  )
           |)
           |""".stripMargin
      val ex = intercept[AnalysisException] {
        sql(query)
      }
      checkError(
        exception = ex,
        condition = "UNRESOLVED_COLUMN.WITH_SUGGESTION",
        parameters = Map("objectName" -> "`c3`", "proposal" -> "`c1`, `c2`"),
        context = ExpectedContext(
          fragment = "c3",
          start = 61,
          stop = 62)
       )
    }
  }

  test("SPARK-36948: Create a table with ANSI intervals using Hive external catalog") {
    val tbl = "tbl_with_ansi_intervals"
    withTable(tbl) {
      sql(s"CREATE TABLE $tbl (ym INTERVAL YEAR TO MONTH, dt INTERVAL DAY TO SECOND) USING PARQUET")
      sql(
        s"""INSERT INTO $tbl VALUES (
           |  INTERVAL '1-1' YEAR TO MONTH,
           |  INTERVAL '1 02:03:04.123456' DAY TO SECOND)""".stripMargin)
      checkAnswer(
        sql(s"SELECT * FROM $tbl"),
        Row(
          Period.ofYears(1).plusMonths(1),
          Duration.ofDays(1).plusHours(2).plusMinutes(3).plusSeconds(4)
            .plus(123456, ChronoUnit.MICROS)))
    }
  }

  test("SPARK-37098: Alter table properties should invalidate cache") {
    // specify the compression in case we change it in future
    withSQLConf(
      SQLConf.PARQUET_COMPRESSION.key -> ParquetCompressionCodec.SNAPPY.lowerCaseName()) {
      withTempPath { dir =>
        withTable("t") {
          sql(s"CREATE TABLE t (c int) STORED AS PARQUET LOCATION '${dir.getCanonicalPath}'")
          // cache table metadata
          sql("SELECT * FROM t")
          sql("ALTER TABLE t SET TBLPROPERTIES('parquet.compression'='zstd')")
          sql("INSERT INTO TABLE t values(1)")
          val files1 = dir.listFiles().filter(_.getName.endsWith("zstd.parquet"))
          assert(files1.length == 1)

          // cache table metadata again
          sql("SELECT * FROM t")
          sql("ALTER TABLE t UNSET TBLPROPERTIES('parquet.compression')")
          sql("INSERT INTO TABLE t values(1)")
          val files2 = dir.listFiles().filter(_.getName.endsWith("snappy.parquet"))
          assert(files2.length == 1)
        }
      }
    }
  }

  test("SPARK-52574: Ensure compression codec is correctly applied in Hive tables and dirs") {
    withSQLConf(
      HiveUtils.CONVERT_METASTORE_PARQUET.key -> "false",
      HiveUtils.CONVERT_METASTORE_INSERT_DIR.key -> "false",
      SQLConf.PARQUET_COMPRESSION.key -> ParquetCompressionCodec.SNAPPY.lowerCaseName()) {
      withTable("tbl") {
        sql("CREATE TABLE tbl(id int) STORED AS PARQUET")
        sql("INSERT INTO tbl SELECT id AS part FROM range(10)")
        val tblMata = spark.sessionState.catalog.getTableMetadata(TableIdentifier("tbl"))
        checkCompressionCodec(new File(tblMata.storage.locationUri.get))
      }

      withTempPath { dir =>
        sql(
          s"""
             |INSERT OVERWRITE LOCAL DIRECTORY '${dir.getCanonicalPath}'
             |STORED AS parquet
             |SELECT id FROM range(10)
             |""".stripMargin)
        checkCompressionCodec(dir)
      }
    }

    def checkCompressionCodec(dir: File): Unit = {
      val parquetFiles = dir.listFiles().filter(_.getName.startsWith("part-"))
      assert(parquetFiles.nonEmpty, "No Parquet files found")

      val conf = spark.sessionState.newHadoopConf()
      val file = parquetFiles.head
      val footer = ParquetFileReader.readFooter(conf, new Path(file.getAbsolutePath))

      val codec = footer.getBlocks.get(0).getColumns.get(0).getCodec.name()
      assert(codec.equalsIgnoreCase(ParquetCompressionCodec.SNAPPY.lowerCaseName()),
        s"Expected ${ParquetCompressionCodec.SNAPPY.lowerCaseName()} compression but found $codec")
    }
  }
}
