/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hudi.common.table;

import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.HoodieCommitMetadata;
import org.apache.hudi.common.model.HoodieLogFile;
import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.common.model.HoodieWriteStat;
import org.apache.hudi.common.schema.HoodieSchema;
import org.apache.hudi.common.schema.HoodieSchemaField;
import org.apache.hudi.common.schema.HoodieSchemaType;
import org.apache.hudi.common.table.log.HoodieLogFormat;
import org.apache.hudi.common.table.log.block.HoodieDataBlock;
import org.apache.hudi.common.table.log.block.HoodieLogBlock;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.table.timeline.versioning.v2.InstantComparatorV2;
import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.testutils.SchemaTestUtil;
import org.apache.hudi.common.util.FileFormatUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.internal.schema.HoodieSchemaException;
import org.apache.hudi.io.storage.HoodieIOFactory;
import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.hadoop.HadoopStorageConfiguration;

import org.apache.avro.Schema;
import org.apache.avro.generic.IndexedRecord;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.mockito.MockedStatic;

import java.io.IOException;
import java.net.URISyntaxException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Stream;

import static org.apache.hudi.common.table.log.block.HoodieLogBlock.HoodieLogBlockType.AVRO_DATA_BLOCK;
import static org.apache.hudi.common.testutils.HoodieCommonTestHarness.getDataBlock;
import static org.apache.hudi.common.testutils.SchemaTestUtil.getSimpleSchema;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.mockStatic;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.when;

/**
 * Tests {@link TableSchemaResolver}.
 */
class TestTableSchemaResolver {

  @TempDir
  java.nio.file.Path tempDir;

  @Test
  void testRecreateSchemaWhenDropPartitionColumns() {
    HoodieSchema originSchema = HoodieSchema.parse(HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA);

    // case2
    String[] pts1 = new String[0];
    HoodieSchema s2 = TableSchemaResolver.appendPartitionColumns(originSchema, Option.of(pts1));
    assertEquals(originSchema, s2);

    // case3: partition_path is in originSchema
    String[] pts2 = {"partition_path"};
    HoodieSchema s3 = TableSchemaResolver.appendPartitionColumns(originSchema, Option.of(pts2));
    assertEquals(originSchema, s3);

    // case4: user_partition is not in originSchema
    String[] pts3 = {"user_partition"};
    HoodieSchema s4 = TableSchemaResolver.appendPartitionColumns(originSchema, Option.of(pts3));
    assertNotEquals(originSchema, s4);
    assertTrue(s4.getFields().stream().anyMatch(f -> f.name().equals("user_partition")));
    HoodieSchemaField f = s4.getField("user_partition").get();
    assertEquals(f.schema(), HoodieSchema.createNullable(HoodieSchemaType.STRING));

    // case5: user_partition is in originSchema, but partition_path is in originSchema
    String[] pts4 = {"user_partition", "partition_path"};
    try {
      TableSchemaResolver.appendPartitionColumns(originSchema, Option.of(pts3));
    } catch (HoodieSchemaException e) {
      assertTrue(e.getMessage().contains("Partial partition fields are still in the schema"));
    }
  }

  @Test
  void testReadSchemaFromLogFile() throws IOException, URISyntaxException, InterruptedException {
    String testDir = initTestDir("read_schema_from_log_file");
    StoragePath partitionPath = new StoragePath(testDir, "partition1");
    HoodieSchema expectedSchema = getSimpleSchema();
    StoragePath logFilePath = writeLogFile(partitionPath, expectedSchema.toAvroSchema());
    assertEquals(expectedSchema, TableSchemaResolver.readSchemaFromLogFile(
        HoodieStorageUtils.getStorage(new StoragePath(logFilePath.toString()), HoodieTestUtils.getDefaultStorageConf()),
        logFilePath));
  }

  @Test
  void testHasOperationFieldFileInspectionOrdering() throws IOException {
    StorageConfiguration conf = new HadoopStorageConfiguration(false);
    HoodieTableMetaClient metaClient = mock(HoodieTableMetaClient.class, RETURNS_DEEP_STUBS);
    when(metaClient.getStorageConf()).thenReturn(conf);

    when(metaClient.getTableType()).thenReturn(HoodieTableType.MERGE_ON_READ);
    when(metaClient.getBasePath()).thenReturn(new StoragePath("/tmp/hudi_table"));
    TableSchemaResolver schemaResolver = new TableSchemaResolver(metaClient);
    HoodieCommitMetadata commitMetadata = new HoodieCommitMetadata();
    // create 3 base files and 2 log files
    HoodieWriteStat baseFileWriteStat = buildWriteStat("partition1/baseFile1.parquet", 10, 100);
    HoodieWriteStat logFileWriteStat = buildWriteStat("partition1/" + FSUtils.makeLogFileName("001", ".log", "100", 2, "1-0-1"), 0, 10);
    // we don't expect any interactions with this write stat since the code should exit as soon as a valid schema is found
    HoodieWriteStat baseFileWriteStat2 = mock(HoodieWriteStat.class);
    // files that only have deletes will be skipped
    HoodieWriteStat logFileWithOnlyDeletes = buildWriteStat("partition1/" + FSUtils.makeLogFileName("002", ".log", "100", 2, "1-0-1"), 0, 0);
    HoodieWriteStat baseFileWithOnlyDeletes = buildWriteStat("partition2/baseFile2.parquet", 0, 0);

    commitMetadata.addWriteStat("partition1", baseFileWriteStat);
    commitMetadata.addWriteStat("partition1", logFileWithOnlyDeletes);
    commitMetadata.addWriteStat("partition1", logFileWriteStat);
    commitMetadata.addWriteStat("partition1", baseFileWriteStat2);
    commitMetadata.addWriteStat("partition2", baseFileWithOnlyDeletes);
    HoodieInstant instant = new HoodieInstant(HoodieInstant.State.COMPLETED, HoodieTimeline.DELTA_COMMIT_ACTION, "001", InstantComparatorV2.COMPLETION_TIME_BASED_COMPARATOR);
    HoodieTimeline timeline = mock(HoodieTimeline.class);
    when(metaClient.getCommitsTimeline().filterCompletedInstants()).thenReturn(timeline);
    when(timeline.getReverseOrderedInstants()).thenReturn(Stream.of(instant));
    when(timeline.readCommitMetadata(instant)).thenReturn(commitMetadata);

    // mock calls to read schema
    try (MockedStatic<HoodieIOFactory> ioFactoryMockedStatic = mockStatic(HoodieIOFactory.class);
         MockedStatic<TableSchemaResolver> tableSchemaResolverMockedStatic = mockStatic(TableSchemaResolver.class)) {
      // return null for first parquet file to force iteration to inspect the next file
      HoodieSchema schema = HoodieSchema.createRecord("test_schema", null, "test_namespace", false,
          Arrays.asList(HoodieSchemaField.of("int_field", HoodieSchema.create(HoodieSchemaType.INT)),
              HoodieSchemaField.of("_hoodie_operation", HoodieSchema.create(HoodieSchemaType.STRING))));

      // mock parquet file schema reading to return null for the first base file to force iteration
      HoodieIOFactory ioFactory = mock(HoodieIOFactory.class);
      FileFormatUtils fileFormatUtils = mock(FileFormatUtils.class);
      StoragePath parquetPath = new StoragePath("/tmp/hudi_table/partition1/baseFile1.parquet");
      when(ioFactory.getFileFormatUtils(parquetPath)).thenReturn(fileFormatUtils);
      when(fileFormatUtils.readSchema(any(), eq(parquetPath))).thenReturn(null);
      ioFactoryMockedStatic.when(() -> HoodieIOFactory.getIOFactory(any())).thenReturn(ioFactory);
      // mock log file schema reading to return the expected schema
      tableSchemaResolverMockedStatic.when(() -> TableSchemaResolver.readSchemaFromLogFile(any(), eq(new StoragePath("/tmp/hudi_table/" + logFileWriteStat.getPath()))))
          .thenReturn(schema);

      assertTrue(schemaResolver.hasOperationField());
    }
    verifyNoInteractions(baseFileWriteStat2);
  }

  private static HoodieWriteStat buildWriteStat(String path, int numInserts, int numUpdateWrites) {
    HoodieWriteStat logFileWriteStat = new HoodieWriteStat();
    logFileWriteStat.setPath(path);
    logFileWriteStat.setNumInserts(numInserts);
    logFileWriteStat.setNumUpdateWrites(numUpdateWrites);
    logFileWriteStat.setNumDeletes(1);
    return logFileWriteStat;
  }

  private String initTestDir(String folderName) throws IOException {
    java.nio.file.Path basePath = tempDir.resolve(folderName);
    java.nio.file.Files.createDirectories(basePath);
    return basePath.toString();
  }

  private StoragePath writeLogFile(StoragePath partitionPath, Schema schema) throws IOException, URISyntaxException, InterruptedException {
    HoodieStorage storage = HoodieTestUtils.getStorage(partitionPath);
    HoodieLogFormat.Writer writer =
        HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath).withFileExtension(HoodieLogFile.DELTA_EXTENSION)
            .withFileId("test-fileid1").withInstantTime("100").withStorage(storage).build();
    List<IndexedRecord> records = SchemaTestUtil.generateTestRecords(0, 100);
    Map<HoodieLogBlock.HeaderMetadataType, String> header = new HashMap<>();
    header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "100");
    header.put(HoodieLogBlock.HeaderMetadataType.SCHEMA, schema.toString());
    HoodieDataBlock dataBlock = getDataBlock(AVRO_DATA_BLOCK, records, header);
    writer.appendBlock(dataBlock);
    writer.close();
    return writer.getLogFile().getPath();
  }

  @Test
  void testGetTableInternalSchemaFromCommitMetadataFindsLatestSchemaUpdateInstant() throws IOException {
    // Given: A timeline with multiple instants where some can update schema and some cannot
    HoodieTableMetaClient metaClient = mock(HoodieTableMetaClient.class, RETURNS_DEEP_STUBS);
    TableSchemaResolver schemaResolver = new TableSchemaResolver(metaClient);

    HoodieInstant clusterInstant = new HoodieInstant(HoodieInstant.State.COMPLETED, HoodieTimeline.REPLACE_COMMIT_ACTION, "001", InstantComparatorV2.COMPLETION_TIME_BASED_COMPARATOR);
    HoodieInstant insertInstant = new HoodieInstant(HoodieInstant.State.COMPLETED, HoodieTimeline.COMMIT_ACTION, "002", InstantComparatorV2.COMPLETION_TIME_BASED_COMPARATOR);
    HoodieInstant compactInstant = new HoodieInstant(HoodieInstant.State.COMPLETED, HoodieTimeline.COMMIT_ACTION, "003", InstantComparatorV2.COMPLETION_TIME_BASED_COMPARATOR);

    HoodieCommitMetadata clusterMetadata = new HoodieCommitMetadata();
    clusterMetadata.setOperationType(org.apache.hudi.common.model.WriteOperationType.CLUSTER);

    HoodieCommitMetadata insertMetadata = new HoodieCommitMetadata();
    insertMetadata.setOperationType(org.apache.hudi.common.model.WriteOperationType.INSERT);
    // Create a valid InternalSchema
    org.apache.hudi.internal.schema.InternalSchema internalSchema = new org.apache.hudi.internal.schema.InternalSchema(
        org.apache.hudi.internal.schema.Types.RecordType.get(
            org.apache.hudi.internal.schema.Types.Field.get(0, false, "id", org.apache.hudi.internal.schema.Types.IntType.get())));
    insertMetadata.addMetadata(org.apache.hudi.internal.schema.utils.SerDeHelper.LATEST_SCHEMA,
        org.apache.hudi.internal.schema.utils.SerDeHelper.toJson(internalSchema));

    HoodieCommitMetadata compactMetadata = new HoodieCommitMetadata();
    compactMetadata.setOperationType(org.apache.hudi.common.model.WriteOperationType.COMPACT);

    HoodieTimeline timeline = mock(HoodieTimeline.class);
    when(metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants()).thenReturn(timeline);
    // Timeline in reverse order: 003 (compact), 002 (insert), 001 (cluster)
    when(timeline.getReverseOrderedInstants()).thenReturn(Stream.of(compactInstant, insertInstant, clusterInstant));
    when(timeline.readCommitMetadata(compactInstant)).thenReturn(compactMetadata);
    when(timeline.readCommitMetadata(insertInstant)).thenReturn(insertMetadata);
    when(timeline.readCommitMetadata(clusterInstant)).thenReturn(clusterMetadata);

    // When: Get internal schema from commit metadata
    Option<org.apache.hudi.internal.schema.InternalSchema> result = schemaResolver.getTableInternalSchemaFromCommitMetadata();

    // Then: Should find the insert instant (002) which is the most recent schema-updating operation
    assertTrue(result.isPresent());
  }

  @Test
  void testGetTableInternalSchemaFromCommitMetadataSkipsNonSchemaUpdatingOperations() throws IOException {
    // Given: A timeline with only non-schema-updating operations (CLUSTER, COMPACT, INDEX, LOG_COMPACT)
    HoodieTableMetaClient metaClient = mock(HoodieTableMetaClient.class, RETURNS_DEEP_STUBS);
    TableSchemaResolver schemaResolver = new TableSchemaResolver(metaClient);

    HoodieInstant clusterInstant = new HoodieInstant(HoodieInstant.State.COMPLETED, HoodieTimeline.REPLACE_COMMIT_ACTION, "001", InstantComparatorV2.COMPLETION_TIME_BASED_COMPARATOR);
    HoodieInstant compactInstant = new HoodieInstant(HoodieInstant.State.COMPLETED, HoodieTimeline.COMMIT_ACTION, "002", InstantComparatorV2.COMPLETION_TIME_BASED_COMPARATOR);

    HoodieCommitMetadata clusterMetadata = new HoodieCommitMetadata();
    clusterMetadata.setOperationType(org.apache.hudi.common.model.WriteOperationType.CLUSTER);

    HoodieCommitMetadata compactMetadata = new HoodieCommitMetadata();
    compactMetadata.setOperationType(org.apache.hudi.common.model.WriteOperationType.COMPACT);

    HoodieTimeline timeline = mock(HoodieTimeline.class);
    when(metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants()).thenReturn(timeline);
    when(timeline.getReverseOrderedInstants()).thenReturn(Stream.of(compactInstant, clusterInstant));
    when(timeline.readCommitMetadata(compactInstant)).thenReturn(compactMetadata);
    when(timeline.readCommitMetadata(clusterInstant)).thenReturn(clusterMetadata);

    // When: Get internal schema from commit metadata
    Option<org.apache.hudi.internal.schema.InternalSchema> result = schemaResolver.getTableInternalSchemaFromCommitMetadata();

    // Then: Should return empty since no schema-updating operations exist
    assertTrue(result.isEmpty());
  }

  @Test
  void testGetTableInternalSchemaFromCommitMetadataHandlesEmptyTimeline() {
    // Given: An empty timeline
    HoodieTableMetaClient metaClient = mock(HoodieTableMetaClient.class, RETURNS_DEEP_STUBS);
    TableSchemaResolver schemaResolver = new TableSchemaResolver(metaClient);

    HoodieTimeline timeline = mock(HoodieTimeline.class);
    when(metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants()).thenReturn(timeline);
    when(timeline.getReverseOrderedInstants()).thenReturn(Stream.empty());

    // When: Get internal schema from commit metadata
    Option<org.apache.hudi.internal.schema.InternalSchema> result = schemaResolver.getTableInternalSchemaFromCommitMetadata();

    // Then: Should return empty for empty timeline
    assertTrue(result.isEmpty());
  }

  @Test
  void testGetTableInternalSchemaFromCommitMetadataStopsAtFirstMatch() throws IOException {
    // Given: A timeline with multiple schema-updating operations
    HoodieTableMetaClient metaClient = mock(HoodieTableMetaClient.class, RETURNS_DEEP_STUBS);
    TableSchemaResolver schemaResolver = new TableSchemaResolver(metaClient);

    HoodieInstant insertInstant1 = new HoodieInstant(HoodieInstant.State.COMPLETED, HoodieTimeline.COMMIT_ACTION, "003", InstantComparatorV2.COMPLETION_TIME_BASED_COMPARATOR);
    HoodieInstant insertInstant2 = new HoodieInstant(HoodieInstant.State.COMPLETED, HoodieTimeline.COMMIT_ACTION, "002", InstantComparatorV2.COMPLETION_TIME_BASED_COMPARATOR);
    // This instant should never be read due to short-circuit behavior
    HoodieInstant insertInstant3 = mock(HoodieInstant.class);

    HoodieCommitMetadata insertMetadata1 = new HoodieCommitMetadata();
    insertMetadata1.setOperationType(org.apache.hudi.common.model.WriteOperationType.INSERT);
    // Create a valid InternalSchema
    org.apache.hudi.internal.schema.InternalSchema internalSchema = new org.apache.hudi.internal.schema.InternalSchema(
        org.apache.hudi.internal.schema.Types.RecordType.get(
            org.apache.hudi.internal.schema.Types.Field.get(0, false, "id", org.apache.hudi.internal.schema.Types.IntType.get())));
    insertMetadata1.addMetadata(org.apache.hudi.internal.schema.utils.SerDeHelper.LATEST_SCHEMA,
        org.apache.hudi.internal.schema.utils.SerDeHelper.toJson(internalSchema));

    HoodieCommitMetadata insertMetadata2 = new HoodieCommitMetadata();
    insertMetadata2.setOperationType(org.apache.hudi.common.model.WriteOperationType.INSERT);

    HoodieTimeline timeline = mock(HoodieTimeline.class);
    when(metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants()).thenReturn(timeline);
    // Timeline in reverse order: 003, 002, 001
    when(timeline.getReverseOrderedInstants()).thenReturn(Stream.of(insertInstant1, insertInstant2, insertInstant3));
    when(timeline.readCommitMetadata(insertInstant1)).thenReturn(insertMetadata1);
    when(timeline.readCommitMetadata(insertInstant2)).thenReturn(insertMetadata2);
    // Should not call readCommitMetadata for insertInstant3 due to findFirst() short-circuit

    // When: Get internal schema from commit metadata
    Option<org.apache.hudi.internal.schema.InternalSchema> result = schemaResolver.getTableInternalSchemaFromCommitMetadata();

    // Then: Should find the first (most recent) schema-updating operation and stop
    assertTrue(result.isPresent());
    // Verify that insertInstant3 was never interacted with (proving short-circuit behavior)
    verifyNoInteractions(insertInstant3);
  }
}
