/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hudi.common.schema;

import org.apache.hudi.avro.AvroSchemaUtils;
import org.apache.hudi.avro.HoodieAvroUtils;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.internal.schema.HoodieSchemaException;

import org.apache.avro.JsonProperties;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericData;

import java.math.BigDecimal;
import java.math.BigInteger;
import java.math.MathContext;
import java.math.RoundingMode;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.Function;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

/**
 * Utility class for HoodieSchema operations including table schema manipulation,
 * compatibility checking, and schema evolution operations.
 *
 * <p>This class provides HoodieSchema equivalents of operations found in AvroSchemaUtils
 * and HoodieAvroUtils, focusing on table schema management rather than record-level operations.</p>
 *
 * <p>All methods in this class delegate to the corresponding Avro utilities internally
 * while providing a clean HoodieSchema-based API. This approach ensures consistency
 * with existing behavior while enabling migration to HoodieSchema types.</p>
 *
 * @since 1.2.0
 */
public final class HoodieSchemaUtils {

  // As per https://avro.apache.org/docs/current/spec.html#names
  private static final Pattern INVALID_AVRO_CHARS_IN_NAMES_PATTERN = Pattern.compile("[^A-Za-z0-9_]");
  private static final Pattern INVALID_AVRO_FIRST_CHAR_IN_NAMES_PATTERN = Pattern.compile("[^A-Za-z_]");
  private static final String MASK_FOR_INVALID_CHARS_IN_NAMES = "__";

  public static final HoodieSchema METADATA_FIELD_SCHEMA = HoodieSchema.createNullable(HoodieSchemaType.STRING);
  public static final HoodieSchema RECORD_KEY_SCHEMA = initRecordKeySchema();

  // Private constructor to prevent instantiation
  private HoodieSchemaUtils() {
    throw new UnsupportedOperationException("Utility class cannot be instantiated");
  }

  /**
   * Creates a write schema for Hudi operations, adding necessary metadata fields.
   *
   * @param schema             the base schema string (JSON format)
   * @param withOperationField whether to include operation metadata field
   * @return HoodieSchema configured for write operations
   * @throws IllegalArgumentException if schema is null or empty
   */
  public static HoodieSchema createHoodieWriteSchema(String schema, boolean withOperationField) {
    ValidationUtils.checkArgument(schema != null && !schema.trim().isEmpty(),
        "Schema string cannot be null or empty");

    return addMetadataFields(HoodieSchema.parse(schema), withOperationField);
  }

  /**
   * Adds Hudi metadata fields to the given schema with the withOperationField flag set as false.
   *
   * @param schema             the input schema
   * @return new HoodieSchema with metadata fields added
   * @throws IllegalArgumentException if schema is null
   */
  public static HoodieSchema addMetadataFields(HoodieSchema schema) {
    return addMetadataFields(schema, false);
  }

  /**
   * Adds Hudi metadata fields to the given schema.
   * This is equivalent to HoodieAvroUtils.addMetadataFields() but operates on HoodieSchema.
   *
   * @param schema             the input schema
   * @param withOperationField whether to include operation metadata field
   * @return new HoodieSchema with metadata fields added
   * @throws IllegalArgumentException if schema is null
   */
  public static HoodieSchema addMetadataFields(HoodieSchema schema, boolean withOperationField) {
    ValidationUtils.checkArgument(schema != null, "Schema cannot be null");

    if (schema.getType() == HoodieSchemaType.NULL) {
      return schema;
    }
    int newFieldsSize = HoodieRecord.HOODIE_META_COLUMNS.size() + (withOperationField ? 1 : 0);
    List<HoodieSchemaField> parentFields = new ArrayList<>(schema.getFields().size() + newFieldsSize);

    HoodieSchemaField commitTimeField =
        HoodieSchemaField.of(HoodieRecord.COMMIT_TIME_METADATA_FIELD, METADATA_FIELD_SCHEMA, "", HoodieSchema.NULL_VALUE);
    HoodieSchemaField commitSeqnoField =
        HoodieSchemaField.of(HoodieRecord.COMMIT_SEQNO_METADATA_FIELD, METADATA_FIELD_SCHEMA, "", HoodieSchema.NULL_VALUE);
    HoodieSchemaField recordKeyField =
        HoodieSchemaField.of(HoodieRecord.RECORD_KEY_METADATA_FIELD, METADATA_FIELD_SCHEMA, "", HoodieSchema.NULL_VALUE);
    HoodieSchemaField partitionPathField =
        HoodieSchemaField.of(HoodieRecord.PARTITION_PATH_METADATA_FIELD, METADATA_FIELD_SCHEMA, "", HoodieSchema.NULL_VALUE);
    HoodieSchemaField fileNameField =
        HoodieSchemaField.of(HoodieRecord.FILENAME_METADATA_FIELD, METADATA_FIELD_SCHEMA, "", HoodieSchema.NULL_VALUE);

    parentFields.add(commitTimeField);
    parentFields.add(commitSeqnoField);
    parentFields.add(recordKeyField);
    parentFields.add(partitionPathField);
    parentFields.add(fileNameField);

    if (withOperationField) {
      final HoodieSchemaField operationField =
          HoodieSchemaField.of(HoodieRecord.OPERATION_METADATA_FIELD, METADATA_FIELD_SCHEMA, "", HoodieSchema.NULL_VALUE);
      parentFields.add(operationField);
    }

    for (HoodieSchemaField field : schema.getFields()) {
      if (!HoodieSchemaUtils.isMetadataField(field.name())) {
        HoodieSchemaField newField = createNewSchemaField(field);
        for (Map.Entry<String, Object> prop : field.getObjectProps().entrySet()) {
          newField.addProp(prop.getKey(), prop.getValue());
        }
        parentFields.add(newField);
      }
    }
    return createNewSchemaFromFieldsWithReference(schema, parentFields);
  }

  /**
   * Removes Hudi metadata fields from the given schema.
   * This is equivalent to HoodieAvroUtils.removeMetadataFields() but operates on HoodieSchema.
   *
   * @param schema the input schema with metadata fields
   * @return new HoodieSchema with metadata fields removed
   * @throws IllegalArgumentException if schema is null
   */
  public static HoodieSchema removeMetadataFields(HoodieSchema schema) {
    ValidationUtils.checkArgument(schema != null, "Schema cannot be null");

    if (schema.getType() == HoodieSchemaType.NULL) {
      return schema;
    }
    return removeFields(schema, HoodieRecord.HOODIE_META_COLUMNS_WITH_OPERATION);
  }

  /**
   * Merges two schemas, combining fields from both with conflict resolution.
   *
   * @param sourceSchema source schema to merge from
   * @param targetSchema target schema to merge into
   * @return new HoodieSchema representing the merged result
   * @throws IllegalArgumentException if either schema is null
   */
  public static HoodieSchema mergeSchemas(HoodieSchema sourceSchema, HoodieSchema targetSchema) {
    ValidationUtils.checkArgument(sourceSchema != null, "Source schema cannot be null");
    ValidationUtils.checkArgument(targetSchema != null, "Target schema cannot be null");

    if (sourceSchema.getType() != HoodieSchemaType.RECORD) {
      return sourceSchema;
    }
    List<HoodieSchemaField> fields = new ArrayList<>();
    for (HoodieSchemaField f : sourceSchema.getFields()) {
      Option<HoodieSchemaField> foundField = targetSchema.getField(f.name());
      fields.add(createNewSchemaField(f.name(), foundField.map(field -> mergeSchemas(f.schema(), field.schema())).orElse(f.schema()),
          f.doc().orElse(null), f.defaultVal().orElse(null)));
    }
    for (HoodieSchemaField f : targetSchema.getFields()) {
      if (sourceSchema.getField(f.name()).isEmpty()) {
        fields.add(createNewSchemaField(f));
      }
    }
    return createNewSchemaFromFieldsWithReference(sourceSchema, fields);
  }

  /**
   * Creates a nullable version of the given schema (union with null).
   * This is equivalent to AvroSchemaUtils.createNullableSchema() but operates on HoodieSchema.
   *
   * @param schema the input schema
   * @return new HoodieSchema that allows null values
   * @throws IllegalArgumentException if schema is null
   */
  public static HoodieSchema createNullableSchema(HoodieSchema schema) {
    ValidationUtils.checkArgument(schema != null, "Schema cannot be null");

    // Delegate to AvroSchemaUtils
    Schema nullableAvro = AvroSchemaUtils.createNullableSchema(schema.toAvroSchema());
    return HoodieSchema.fromAvroSchema(nullableAvro);
  }

  /**
   * Create a new schema by force changing all the fields as nullable.
   * This is equivalent to AvroSchemaUtils.asNullable() but operates on HoodieSchema.
   *
   * @return a new schema with all the fields updated as nullable
   * @throws IllegalArgumentException if schema is null
   */
  public static HoodieSchema asNullable(HoodieSchema schema) {
    ValidationUtils.checkArgument(schema != null, "Schema cannot be null");

    // Delegate to AvroSchemaUtils
    Schema nullableAvro = AvroSchemaUtils.asNullable(schema.toAvroSchema());
    return HoodieSchema.fromAvroSchema(nullableAvro);
  }

  /**
   * Removes specified fields from a RECORD schema.
   * This is equivalent to HoodieAvroUtils.removeFields() but operates on HoodieSchema.
   *
   * @param schema original schema (must be RECORD type)
   * @param fieldNamesToRemove set of field names to remove
   * @return new HoodieSchema without the specified fields
   * @throws IllegalArgumentException if schema is null or not a RECORD type
   */
  public static HoodieSchema removeFields(HoodieSchema schema, Set<String> fieldNamesToRemove) {
    ValidationUtils.checkArgument(schema != null, "Schema cannot be null");
    ValidationUtils.checkArgument(schema.getType() == HoodieSchemaType.RECORD,
        () -> "Only RECORD schemas can have fields removed, got: " + schema.getType());

    if (fieldNamesToRemove == null || fieldNamesToRemove.isEmpty()) {
      return schema;
    }

    // Filter and copy fields (must create new instances, can't reuse Avro fields)
    List<HoodieSchemaField> filteredFields = schema.getFields().stream()
        .filter(field -> !fieldNamesToRemove.contains(field.name()))
        .map(HoodieSchemaUtils::createNewSchemaField)
        .collect(Collectors.toList());

    if (filteredFields.size() == schema.getFields().size()) {
      return schema; // No fields were removed
    }

    // Create record with isError flag preserved
    HoodieSchema newSchema = HoodieSchema.createRecord(
        schema.getName(),
        schema.getDoc().orElse(null),
        schema.getNamespace().orElse(null),
        schema.isError(),
        filteredFields
    );

    // Copy custom properties
    Map<String, Object> props = schema.getObjectProps();
    for (Map.Entry<String, Object> prop : props.entrySet()) {
      newSchema.addProp(prop.getKey(), prop.getValue());
    }

    return newSchema;
  }

  /**
   * Finds fields that are present in the table schema but missing in the writer schema.
   * This is equivalent to AvroSchemaUtils.findMissingFields() but operates on HoodieSchemas.
   *
   * @param tableSchema  the complete table schema
   * @param writerSchema the writer schema to check against
   * @return list of HoodieSchemaFields that are missing in writer schema
   * @throws IllegalArgumentException if either schema is null
   */
  public static List<HoodieSchemaField> findMissingFields(HoodieSchema tableSchema, HoodieSchema writerSchema) {
    return findMissingFields(tableSchema, writerSchema, Collections.emptySet());
  }

  /**
   * Finds fields that are present in the table schema but missing in the writer schema,
   * excluding partition columns from the check.
   * This is equivalent to AvroSchemaUtils.findMissingFields() but operates on HoodieSchemas.
   *
   * @param tableSchema    the complete table schema
   * @param writerSchema   the writer schema to check against
   * @param excludeColumns column names to exclude from missing field check
   * @return list of HoodieSchemaFields that are missing in writer schema
   * @throws IllegalArgumentException if either schema is null
   */
  public static List<HoodieSchemaField> findMissingFields(HoodieSchema tableSchema, HoodieSchema writerSchema,
                                                          Set<String> excludeColumns) {
    ValidationUtils.checkArgument(tableSchema != null, "Table schema cannot be null");
    ValidationUtils.checkArgument(writerSchema != null, "Writer schema cannot be null");

    if (tableSchema.getType() != HoodieSchemaType.RECORD || writerSchema.getType() != HoodieSchemaType.RECORD) {
      return Collections.emptyList();
    }

    Set<String> exclusions = excludeColumns != null ? excludeColumns : Collections.emptySet();
    Set<String> writerFieldNames = writerSchema.getFields().stream()
        .map(HoodieSchemaField::name)
        .collect(Collectors.toSet());

    // Find fields in table schema that are not present in writer schema and not excluded
    return tableSchema.getFields().stream()
        .filter(field -> !exclusions.contains(field.name()))
        .filter(field -> !writerFieldNames.contains(field.name()))
        .collect(Collectors.toList());
  }

  /**
   * Creates a new schema field with the specified properties.
   * This is equivalent to HoodieAvroUtils.createNewSchemaField() but returns HoodieSchemaField.
   *
   * @param name         field name
   * @param schema       field schema
   * @param doc          field documentation (can be null)
   * @param defaultValue default value (can be null)
   * @return new HoodieSchemaField instance
   * @throws IllegalArgumentException if name or schema is null/empty
   */
  public static HoodieSchemaField createNewSchemaField(String name, HoodieSchema schema,
                                                       String doc, Object defaultValue) {
    ValidationUtils.checkArgument(name != null && !name.isEmpty(), "Field name cannot be null or empty");
    ValidationUtils.checkArgument(schema != null, "Field schema cannot be null");

    return HoodieSchemaField.of(name, schema, doc, defaultValue);
  }

  /**
   * Creates a new schema field with the specified properties, including field order.
   * This is equivalent to HoodieAvroUtils.createNewSchemaField() but returns HoodieSchemaField.
   *
   * @param name         field name
   * @param schema       field schema
   * @param doc          field documentation (can be null)
   * @param defaultValue default value (can be null)
   * @param order        field order for sorting
   * @return new HoodieSchemaField instance
   * @throws IllegalArgumentException if name, schema, or order is null/empty
   * @since 1.2.0
   */
  public static HoodieSchemaField createNewSchemaField(String name, HoodieSchema schema,
                                                       String doc, Object defaultValue, HoodieFieldOrder order) {
    ValidationUtils.checkArgument(name != null && !name.isEmpty(), "Field name cannot be null or empty");
    ValidationUtils.checkArgument(schema != null, "Field schema cannot be null");
    ValidationUtils.checkArgument(order != null, "Field order cannot be null");

    // Delegate to HoodieAvroUtils
    Schema.Field avroField = HoodieAvroUtils.createNewSchemaField(
        name, schema.toAvroSchema(), doc, defaultValue, order.toAvroOrder());
    return HoodieSchemaField.fromAvroField(avroField);
  }

  /**
   * Creates a new HoodieSchemaField from an existing field.
   * This is equivalent to HoodieAvroUtils.createNewSchemaField() but returns HoodieSchemaField.
   *
   * @param field the original HoodieSchemaField to create a new field from
   * @return a new HoodieSchemaField with the same properties but properly formatted default value
   */
  public static HoodieSchemaField createNewSchemaField(HoodieSchemaField field) {
    return createNewSchemaField(field.name(), field.schema(), field.doc().orElse(null), field.defaultVal().orElse(null));
  }

  /**
   * Converts a byte array to a BigDecimal using the given decimal schema.
   *
   * @param value         the byte array to convert
   * @param decimalSchema the decimal schema containing precision and scale
   * @return the resulting BigDecimal
   * @throws IllegalArgumentException if the schema is not a DECIMAL type
   */
  public static BigDecimal convertBytesToBigDecimal(byte[] value, HoodieSchema decimalSchema) {
    ValidationUtils.checkArgument(decimalSchema != null, "Decimal schema cannot be null");
    ValidationUtils.checkArgument(decimalSchema.getType() == HoodieSchemaType.DECIMAL,
        () -> "Schema must be of DECIMAL type, but is " + decimalSchema.getType());

    HoodieSchema.Decimal decimal = (HoodieSchema.Decimal) decimalSchema;
    return convertBytesToBigDecimal(value, decimal.getPrecision(), decimal.getScale());
  }

  /**
   * Converts a byte array to a BigDecimal with the specified precision and scale.
   *
   * @param value     the byte array to convert
   * @param precision the precision of the decimal
   * @param scale     the scale of the decimal
   * @return the resulting BigDecimal
   */
  public static BigDecimal convertBytesToBigDecimal(byte[] value, int precision, int scale) {
    return new BigDecimal(new BigInteger(value),
        scale, new MathContext(precision, RoundingMode.HALF_UP));
  }

  /**
   * Gets a field (including nested fields) from the schema using dot notation.
   * This method delegates to {@link HoodieSchema#getNestedField(String)}.
   * <p>
   * Supports nested field access using dot notation. For example:
   * <ul>
   *   <li>"name" - retrieves top-level field</li>
   *   <li>"user.profile.displayName" - retrieves nested field</li>
   *   <li>"items.list.element" - retrieves array element schema </li>
   *   <li>"metadata.key_value.key" - retrieves map key schema</li>
   *   <li>"metadata.key_value.value" - retrieves map value schema</li>
   * </ul>
   *
   * @param schema    the schema to search in
   * @param fieldName the field name (may contain dots for nested fields)
   * @return Option containing Pair of canonical field name and the HoodieSchemaField, or Option.empty() if field not found
   * @throws IllegalArgumentException if schema or fieldName is null/empty
   * @since 1.2.0
   */
  public static Option<Pair<String, HoodieSchemaField>> getNestedField(HoodieSchema schema, String fieldName) {
    ValidationUtils.checkArgument(schema != null, "Schema cannot be null");
    ValidationUtils.checkArgument(fieldName != null && !fieldName.isEmpty(), "Field name cannot be null or empty");
    return schema.getNestedField(fieldName);
  }

  /**
   * Generates a projection schema from the original schema, including only the specified fields.
   *
   * @param originalSchema the source schema
   * @param fieldNames     the list of field names to include in the projection
   * @return new HoodieSchema containing only the specified fields
   * @throws IllegalArgumentException if schema is null or not a record type
   * @since 1.2.0
   */
  public static HoodieSchema generateProjectionSchema(HoodieSchema originalSchema, List<String> fieldNames) {
    ValidationUtils.checkArgument(originalSchema != null, "Original schema cannot be null");
    ValidationUtils.checkArgument(fieldNames != null, "Field names cannot be null");

    Map<String, HoodieSchemaField> schemaFieldsMap = originalSchema.getFields().stream()
        .map(r -> Pair.of(r.name().toLowerCase(), r)).collect(Collectors.toMap(Pair::getLeft, Pair::getRight));
    List<HoodieSchemaField> projectedFields = new ArrayList<>(fieldNames.size());
    for (String fn : fieldNames) {
      HoodieSchemaField field = schemaFieldsMap.get(fn.toLowerCase());
      if (field == null) {
        throw new HoodieException("Field " + fn + " not found in log schema. Query cannot proceed! "
            + "Derived Schema Fields: " + new ArrayList<>(schemaFieldsMap.keySet()));
      } else {
        projectedFields.add(createNewSchemaField(field));
      }
    }

    return HoodieSchema.createRecord(originalSchema.getName(), originalSchema.getNamespace().orElse(null), originalSchema.getDoc().orElse(null), projectedFields);
  }

  /**
   * Prunes the data schema to only include fields that are required by the required schema,
   * plus any mandatory fields specified.
   *
   * @param dataSchema      the full data schema
   * @param requiredSchema  the schema containing required fields
   * @param mandatoryFields set of field names that must be included regardless
   * @return new HoodieSchema with pruned fields
   * @throws IllegalArgumentException if either schema is null
   * @since 1.2.0
   */
  public static HoodieSchema pruneDataSchema(HoodieSchema dataSchema, HoodieSchema requiredSchema, Set<String> mandatoryFields) {
    ValidationUtils.checkArgument(dataSchema != null, "Data schema cannot be null");
    ValidationUtils.checkArgument(requiredSchema != null, "Required schema cannot be null");

    Set<String> mandatorySet = mandatoryFields != null ? mandatoryFields : Collections.emptySet();

    HoodieSchema prunedDataSchema = pruneDataSchemaInternal(dataSchema.getNonNullType(), requiredSchema.getNonNullType(), mandatorySet);
    if (dataSchema.isNullable() && !prunedDataSchema.isNullable()) {
      return HoodieSchema.createNullable(prunedDataSchema);
    }
    return prunedDataSchema;
  }

  private static HoodieSchema pruneDataSchemaInternal(HoodieSchema dataSchema, HoodieSchema requiredSchema, Set<String> mandatoryFields) {
    switch (requiredSchema.getType()) {
      case RECORD:
        // BLOB and VARIANT are represented as Avro RECORDs but carry a logical type
        // whose validate() contract requires the full canonical field layout
        // ({type,data,reference} and {metadata,value} respectively). Partially pruning
        // their inner fields would drop that contract. Spark's projection still prunes
        // these columns at eval time, so reading the full logical-type record here is
        // correct and cheap.
        if (dataSchema.getType() == HoodieSchemaType.BLOB
            || dataSchema.getType() == HoodieSchemaType.VARIANT) {
          return dataSchema;
        }
        if (dataSchema.getType() != HoodieSchemaType.RECORD) {
          throw new IllegalArgumentException("Data schema is not a record");
        }
        List<HoodieSchemaField> newFields = new ArrayList<>(requiredSchema.getFields().size());
        for (HoodieSchemaField requiredSchemaField : requiredSchema.getFields()) {
          if (mandatoryFields.contains(requiredSchemaField.name())) {
            newFields.add(createNewSchemaField(requiredSchemaField));
          } else {
            dataSchema.getField(requiredSchemaField.name()).ifPresent(dataSchemaField -> {
              HoodieSchema prunedFieldSchema = pruneDataSchema(dataSchemaField.schema(), requiredSchemaField.schema(), Collections.emptySet());
              HoodieSchemaField newField = createNewSchemaField(
                  dataSchemaField.name(),
                  prunedFieldSchema,
                  dataSchemaField.doc().orElse(null),
                  dataSchemaField.defaultVal().orElse(null)
              );
              newFields.add(newField);
            });
          }
        }
        HoodieSchema newRecord = HoodieSchema.createRecord(dataSchema.getName(), dataSchema.getNamespace().orElse(null), dataSchema.getDoc().orElse(null), newFields);
        copyProperties(dataSchema, newRecord);
        return newRecord;

      case ARRAY:
        if (dataSchema.getType() != HoodieSchemaType.ARRAY) {
          throw new IllegalArgumentException("Data schema is not an array");
        }
        return HoodieSchema.createArray(pruneDataSchema(dataSchema.getElementType(), requiredSchema.getElementType(), Collections.emptySet()));

      case MAP:
        if (dataSchema.getType() != HoodieSchemaType.MAP) {
          throw new IllegalArgumentException("Data schema is not a map");
        }
        return HoodieSchema.createMap(pruneDataSchema(dataSchema.getValueType(), requiredSchema.getValueType(), Collections.emptySet()));

      case UNION:
        throw new IllegalArgumentException("Data schema is a union");

      default:
        return dataSchema;
    }
  }

  /**
   * Helper to copy properties and logical types from source schema to target schema.
   */
  private static HoodieSchema copyProperties(HoodieSchema source, HoodieSchema target) {
    for (Map.Entry<String, Object> prop : source.getObjectProps().entrySet()) {
      target.addProp(prop.getKey(), prop.getValue());
    }
    return target;
  }

  /**
   * Adds newFields to the schema. Will add nested fields without duplicating the field
   * For example if your schema is "a.b.{c,e}" and newfields contains "a.{b.{d,e},x.y}",
   * It will stitch them together to be "a.{b.{c,d,e},x.y}
   *
   * @param schema    the original schema
   * @param newFields list of new fields to add
   * @return the updated schema with new fields added
   */
  public static HoodieSchema appendFieldsToSchemaDedupNested(HoodieSchema schema, List<HoodieSchemaField> newFields) {
    return appendFieldsToSchemaBase(schema, newFields, true);
  }

  /**
   * Appends provided new fields at the end of the given schema
   * <p>
   * NOTE: No deduplication is made, this method simply appends fields at the end of the list
   *       of the source schema as is
   * <p>
   */
  public static HoodieSchema appendFieldsToSchema(HoodieSchema schema, List<HoodieSchemaField> newFields) {
    return appendFieldsToSchemaBase(schema, newFields, false);
  }

  /**
   * Create a new schema but maintain all meta info from the old schema.
   *
   * @param schema schema to get the meta info from
   * @param fields list of fields in order that will be in the new schema
   *
   * @return schema with fields from fields, and metadata from schema
   */
  public static HoodieSchema createNewSchemaFromFieldsWithReference(HoodieSchema schema, List<HoodieSchemaField> fields) {
    if (schema == null) {
      throw new IllegalArgumentException("Schema must not be null");
    }
    Map<String, Object> schemaProps = schema.getObjectProps();
    HoodieSchema newSchema = HoodieSchema.createRecord(schema.getName(), schema.getNamespace().orElse(null), schema.getDoc().orElse(null), fields);
    for (Map.Entry<String, Object> prop : schemaProps.entrySet()) {
      newSchema.addProp(prop.getKey(), prop.getValue());
    }
    return newSchema;
  }

  /**
   * Get gets a field from a record, works on nested fields as well (if you provide the whole name, eg: toplevel.nextlevel.child)
   * @return the field, including its lineage.
   * For example, if you have a schema: record(a:int, b:record(x:int, y:long, z:record(z1: int, z2: float, z3: double), c:bool)
   * "fieldName" | output
   * ---------------------------------
   * "a"         | a:int
   * "b"         | b:record(x:int, y:long, z:record(z1: int, z2: int, z3: int)
   * "c"         | c:bool
   * "b.x"       | b:record(x:int)
   * "b.z.z2"    | b:record(z:record(z2:float))
   *
   * this is intended to be used with appendFieldsToSchemaDedupNested
   */
  public static Option<HoodieSchemaField> findNestedField(HoodieSchema schema, String fieldName) {
    return findNestedField(schema, fieldName.split("\\."), 0);
  }

  private static Option<HoodieSchemaField> findNestedField(HoodieSchema schema, String[] fieldParts, int index) {
    if (schema.getType() == HoodieSchemaType.UNION) {
      Option<HoodieSchemaField> notUnion = findNestedField(schema.getNonNullType(), fieldParts, index);
      if (!notUnion.isPresent()) {
        return Option.empty();
      }
      HoodieSchemaField nu = notUnion.get();
      return Option.of(createNewSchemaField(nu));
    }
    if (fieldParts.length <= index) {
      return Option.empty();
    }

    Option<HoodieSchemaField> foundFieldOpt = schema.getField(fieldParts[index]);
    if (foundFieldOpt.isEmpty()) {
      return Option.empty();
    }
    HoodieSchemaField foundField = foundFieldOpt.get();

    if (index == fieldParts.length - 1) {
      return Option.of(createNewSchemaField(foundField));
    }

    HoodieSchema foundSchema = foundField.schema();
    Option<HoodieSchemaField> nestedPart = findNestedField(foundSchema, fieldParts, index + 1);
    if (!nestedPart.isPresent()) {
      return Option.empty();
    }
    boolean isUnion = false;
    if (foundSchema.getType() == HoodieSchemaType.UNION) {
      isUnion = true;
      foundSchema = foundSchema.getNonNullType();
    }
    HoodieSchema newSchema = createNewSchemaFromFieldsWithReference(foundSchema, Collections.singletonList(nestedPart.get()));
    return Option.of(createNewSchemaField(foundField.name(), isUnion ? HoodieSchema.createNullable(newSchema) : newSchema, foundField.doc().orElse(null), foundField.defaultVal().orElse(null)));
  }

  private static HoodieSchema initRecordKeySchema() {
    HoodieSchemaField recordKeyField =
            createNewSchemaField(HoodieRecord.RECORD_KEY_METADATA_FIELD, METADATA_FIELD_SCHEMA, "", JsonProperties.NULL_VALUE);
    return HoodieSchema.createRecord(
            "HoodieRecordKey",
            "",
            "",
            false,
            Collections.singletonList(recordKeyField)
    );
  }

  public static HoodieSchema getRecordKeySchema() {
    return RECORD_KEY_SCHEMA;
  }

  /**
   * Converts field values for specific data types with logical type handling.
   * This is equivalent to HoodieAvroUtils.convertValueForSpecificDataTypes() but operates on HoodieSchema.
   * <p>
   * Handles special conversions for Avro logical types:
   * <ul>
   *   <li>Date type - converts epoch day integer to LocalDate</li>
   *   <li>Timestamp types - converts epoch milliseconds/microseconds to Timestamp</li>
   *   <li>Decimal type - converts bytes/fixed to BigDecimal</li>
   * </ul>
   *
   * @param fieldSchema the field schema
   * @param fieldValue the field value to convert
   * @param consistentLogicalTimestampEnabled whether to use consistent logical timestamp handling
   * @return converted value for logical types, or original value
   * @throws IllegalStateException if fieldValue is null but schema is not nullable
   * @since 1.2.0
   */
  public static Object convertValueForSpecificDataTypes(HoodieSchema fieldSchema,
                                                        Object fieldValue,
                                                        boolean consistentLogicalTimestampEnabled) {
    if (fieldSchema == null) {
      return fieldValue;
    } else if (fieldValue == null) {
      ValidationUtils.checkState(fieldSchema.isNullable(),
          "Field value is null but schema is not nullable");
      return null;
    }

    // Delegate to existing Avro utility
    return HoodieAvroUtils.convertValueForSpecificDataTypes(
        fieldSchema.toAvroSchema(),
        fieldValue,
        consistentLogicalTimestampEnabled
    );
  }

  /**
   * Fetch schema for record key and partition path.
   * This is equivalent to HoodieAvroUtils.getRecordKeyPartitionPathSchema() but returns HoodieSchema.
   *
   * @return HoodieSchema containing record key and partition path fields
   */
  public static HoodieSchema getRecordKeyPartitionPathSchema() {
    List<HoodieSchemaField> toBeAddedFields = new ArrayList<>(2);

    HoodieSchemaField recordKeyField =
        createNewSchemaField(HoodieRecord.RECORD_KEY_METADATA_FIELD, METADATA_FIELD_SCHEMA, "", JsonProperties.NULL_VALUE);
    HoodieSchemaField partitionPathField =
        createNewSchemaField(HoodieRecord.PARTITION_PATH_METADATA_FIELD, METADATA_FIELD_SCHEMA, "", JsonProperties.NULL_VALUE);

    toBeAddedFields.add(recordKeyField);
    toBeAddedFields.add(partitionPathField);
    return HoodieSchema.createRecord("HoodieRecordKey", "", "", false, toBeAddedFields);
  }

  /**
   * Fetches projected schema given list of fields to project. The field can be nested in format `a.b.c` where a is
   * the top level field, b is at second level and so on.
   * This is equivalent to {@link HoodieAvroUtils#projectSchema(Schema, List)} but operates on HoodieSchema.
   *
   * @param fileSchema the original schema
   * @param fields     list of fields to project
   * @return projected schema containing only specified fields
   */
  public static HoodieSchema projectSchema(HoodieSchema fileSchema, List<String> fields) {
    return HoodieSchema.fromAvroSchema(HoodieAvroUtils.projectSchema(fileSchema.toAvroSchema(), fields));
  }

  /**
   * Generates fully-qualified name for the Avro's schema based on the Table's name
   *
   * <p>The qualified name follows the pattern: hoodie.{tableName}.{tableName}_record
   * where tableName is sanitized for Avro compatibility.</p>
   *
   * NOTE: PLEASE READ CAREFULLY BEFORE CHANGING
   *       This method should not change for compatibility reasons as older versions
   *       of Avro might be comparing fully-qualified names rather than just the record
   *       names
   *
   * @param tableName the Hudi table name
   * @return the fully-qualified Avro record name (e.g., "hoodie.my_table.my_table_record")
   * @throws IllegalArgumentException if tableName is null or empty
   * @since 1.2.0
   */
  public static String getRecordQualifiedName(String tableName) {
    ValidationUtils.checkArgument(tableName != null && !tableName.trim().isEmpty(),
        "Table name cannot be null or empty");

    String sanitizedTableName = sanitizeName(tableName);
    return "hoodie." + sanitizedTableName + "." + sanitizedTableName + "_record";
  }

  public static boolean hasDecimalField(HoodieSchema schema) {
    return hasDecimalWithCondition(schema, unused -> true);
  }

  /**
   * Checks whether the provided schema contains a decimal with a precision less than or equal to 18,
   * which allows the decimal to be stored as int/long instead of a fixed size byte array in
   * <a href="https://github.com/apache/parquet-format/blob/master/LogicalTypes.md">parquet logical types</a>
   * @param schema the input schema to search
   * @return true if the schema contains a small precision decimal field and false otherwise
   */
  public static boolean hasSmallPrecisionDecimalField(HoodieSchema schema) {
    return hasDecimalWithCondition(schema, HoodieSchemaUtils::isSmallPrecisionDecimalField);
  }

  private static boolean hasDecimalWithCondition(HoodieSchema schema, Function<HoodieSchema.Decimal, Boolean> condition) {
    switch (schema.getType()) {
      case RECORD:
        for (HoodieSchemaField field : schema.getFields()) {
          if (hasDecimalWithCondition(field.schema(), condition)) {
            return true;
          }
        }
        return false;
      case ARRAY:
        return hasDecimalWithCondition(schema.getElementType(), condition);
      case MAP:
        return hasDecimalWithCondition(schema.getValueType(), condition);
      case UNION:
        return hasDecimalWithCondition(schema.getNonNullType(), condition);
      case DECIMAL:
        HoodieSchema.Decimal decimal = (HoodieSchema.Decimal) schema;
        return condition.apply(decimal);
      default:
        return false;
    }
  }

  private static boolean isSmallPrecisionDecimalField(HoodieSchema.Decimal decimal) {
    return decimal.getPrecision() <= 18;
  }

  /**
   * Resolves a union schema by finding the schema matching the given full name.
   * Handles both simple nullable unions (null + non-null) and complex unions with multiple types.
   *
   * <p>This method supports the following union types:
   * <ul>
   *   <li>Simple nullable unions: {@code ["null", "Type"]} - returns the non-null type</li>
   *   <li>Complex unions: {@code ["null", "TypeA", "TypeB"]} - returns the type matching fieldSchemaFullName</li>
   *   <li>Non-union schemas - returns the schema as-is</li>
   * </ul>
   *
   * @param schema the schema to resolve (may or may not be a union)
   * @param fieldSchemaFullName the full name of the schema to find within the union
   * @return the resolved schema
   * @throws HoodieSchemaException if the union cannot be resolved or no matching type is found
   */
  public static HoodieSchema resolveUnionSchema(HoodieSchema schema, String fieldSchemaFullName) {
    if (schema.getType() != HoodieSchemaType.UNION) {
      return schema;
    }

    List<HoodieSchema> innerTypes = schema.getTypes();
    if (innerTypes.size() == 2 && schema.isNullable()) {
      // this is a basic nullable field so handle it more efficiently
      return schema.getNonNullType();
    }

    HoodieSchema nonNullType = innerTypes.stream()
        .filter(it -> it.getType() != HoodieSchemaType.NULL && Objects.equals(it.getFullName(), fieldSchemaFullName))
        .findFirst()
        .orElse(null);

    if (nonNullType == null) {
      throw new HoodieSchemaException(
          String.format("Unsupported UNION type %s: Only UNION of a null type and a non-null type is supported", schema));
    }

    return nonNullType;
  }

  public static String addMetadataColumnTypes(String hiveColumnTypes) {
    return "string,string,string,string,string," + hiveColumnTypes;
  }

  public static boolean isMetadataField(String fieldName) {
    return HoodieRecord.HOODIE_META_COLUMNS_WITH_OPERATION.contains(fieldName);
  }

  /**
   * Converts a HoodieSchemaField's default value to its Java representation.
   *
   * <p>For primitive types (STRING, INT, LONG, FLOAT, DOUBLE, BOOLEAN, ENUM, BYTES, FIXED, DECIMAL)
   * and logical types (TIME, TIMESTAMP, DATE, UUID), the default value is returned as-is.
   * For complex types (ARRAY, MAP, RECORD), Avro's GenericData utility is used
   * to properly construct the default value.</p>
   *
   * @param field the HoodieSchemaField containing the default value
   * @return the Java representation of the default value, or null if no default value exists
   * @throws IllegalArgumentException if the field's type is not supported
   * @since 1.2.0
   */
  public static Object toJavaDefaultValue(HoodieSchemaField field) {
    ValidationUtils.checkArgument(field != null, "Field cannot be null");

    Option<Object> defaultValOpt = field.defaultVal();
    if (!defaultValOpt.isPresent() || defaultValOpt.get() == HoodieJsonProperties.NULL_VALUE) {
      return null;
    }

    Object defaultVal = defaultValOpt.get();
    HoodieSchemaType type = field.getNonNullSchema().getType();

    switch (type) {
      case STRING:
      case INT:
      case LONG:
      case FLOAT:
      case DOUBLE:
      case BOOLEAN:
      case ENUM:
      case BYTES:
      case FIXED:
      case DECIMAL:
      case TIME:
      case TIMESTAMP:
      case DATE:
      case UUID:
        return defaultVal;
      case ARRAY:
      case MAP:
      case RECORD:
        // Use Avro's standard GenericData utility for complex types
        // Delegate to the underlying Avro field
        return GenericData.get().getDefaultValue(field.getAvroField());
      default:
        throw new IllegalArgumentException("Unsupported HoodieSchema type: " + type);
    }
  }

  /**
   * Sanitizes Name according to Avro rule for names.
   * Removes characters other than the ones mentioned in <a href="https://avro.apache.org/docs/current/spec.html#names">avro spec</a> .
   *
   * @param name input name
   * @return sanitized name
   */
  public static String sanitizeName(String name) {
    return sanitizeName(name, MASK_FOR_INVALID_CHARS_IN_NAMES);
  }

  /**
   * Sanitizes Name according to Avro rule for names.
   * Removes characters other than the ones mentioned in <a href="https://avro.apache.org/docs/current/spec.html#names">avro spec</a>.
   *
   * @param name            input name
   * @param invalidCharMask replacement for invalid characters.
   * @return sanitized name
   */
  public static String sanitizeName(String name, String invalidCharMask) {
    if (INVALID_AVRO_FIRST_CHAR_IN_NAMES_PATTERN.matcher(name.substring(0, 1)).matches()) {
      name = INVALID_AVRO_FIRST_CHAR_IN_NAMES_PATTERN.matcher(name).replaceFirst(invalidCharMask);
    }
    return INVALID_AVRO_CHARS_IN_NAMES_PATTERN.matcher(name).replaceAll(invalidCharMask);
  }

  public static String createSchemaErrorString(String errorMessage, HoodieSchema writerSchema, HoodieSchema tableSchema) {
    return String.format("%s\nwriterSchema: %s\ntableSchema: %s", errorMessage, writerSchema, tableSchema);
  }

  private static HoodieSchema appendFieldsToSchemaBase(HoodieSchema schema, List<HoodieSchemaField> newFields, boolean dedupNested) {
    List<HoodieSchemaField> fields = schema.getFields().stream()
        .map(HoodieSchemaUtils::createNewSchemaField)
        .collect(Collectors.toList());
    if (dedupNested) {
      for (HoodieSchemaField f : newFields) {
        Option<HoodieSchemaField> field = schema.getField(f.name());
        if (field.isPresent()) {
          HoodieSchemaField foundField = field.get();
          fields.set(foundField.pos(), createNewSchemaField(foundField.name(), mergeSchemas(foundField.schema(), f.schema()), foundField.doc().orElse(null), foundField.defaultVal().orElse(null)));
        } else {
          fields.add(f);
        }
      }
    } else {
      fields.addAll(newFields);
    }

    return createNewSchemaFromFieldsWithReference(schema, fields);
  }
}
