/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.apache.hudi.common.util;

import org.apache.hudi.avro.HoodieAvroUtils;
import org.apache.hudi.common.config.HoodieReaderConfig;
import org.apache.hudi.common.model.HoodieAvroIndexedRecord;
import org.apache.hudi.common.model.HoodieFileFormat;
import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType;
import org.apache.hudi.common.schema.HoodieSchema;
import org.apache.hudi.common.schema.HoodieSchemaField;
import org.apache.hudi.common.util.collection.ClosableIterator;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.io.compress.CompressionCodec;
import org.apache.hudi.io.hfile.HFileContext;
import org.apache.hudi.io.hfile.HFileWriter;
import org.apache.hudi.io.hfile.HFileWriterImpl;
import org.apache.hudi.io.storage.HoodieAvroHFileReaderImplBase;
import org.apache.hudi.io.storage.HoodieFileReader;
import org.apache.hudi.io.storage.HoodieIOFactory;
import org.apache.hudi.keygen.BaseKeyGenerator;
import org.apache.hudi.metadata.HoodieIndexVersion;
import org.apache.hudi.stats.HoodieColumnRangeMetadata;
import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;

import lombok.extern.slf4j.Slf4j;
import org.apache.avro.generic.GenericRecord;

import java.io.ByteArrayOutputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;

import static org.apache.hudi.common.config.HoodieStorageConfig.HFILE_COMPRESSION_ALGORITHM_NAME;
import static org.apache.hudi.common.util.StringUtils.getUTF8Bytes;

/**
 * Utility functions for HFile files.
 */
@Slf4j
public class HFileUtils extends FileFormatUtils {

  private static final int DEFAULT_BLOCK_SIZE_FOR_LOG_FILE = 1024 * 1024;

  /**
   * @param paramsMap parameter map containing the compression codec config.
   * @return the {@link CompressionCodec} Enum.
   */
  public static CompressionCodec getHFileCompressionAlgorithm(Map<String, String> paramsMap) {
    String codecName = paramsMap.get(HFILE_COMPRESSION_ALGORITHM_NAME.key());
    if (StringUtils.isNullOrEmpty(codecName)) {
      // Use the default value from config instead of hardcoding GZIP
      return CompressionCodec.findCodecByName(HFILE_COMPRESSION_ALGORITHM_NAME.defaultValue());
    }
    return CompressionCodec.findCodecByName(codecName);
  }

  @Override
  public List<GenericRecord> readAvroRecords(HoodieStorage storage, StoragePath filePath) {
    throw new UnsupportedOperationException("HFileUtils does not support readAvroRecords");
  }

  @Override
  public List<GenericRecord> readAvroRecords(HoodieStorage storage, StoragePath filePath, HoodieSchema schema) {
    throw new UnsupportedOperationException("HFileUtils does not support readAvroRecords");
  }

  @Override
  public Map<String, String> readFooter(HoodieStorage storage, boolean required, StoragePath filePath, String... footerNames) {
    throw new UnsupportedOperationException("HFileUtils does not support readFooter");
  }

  @Override
  public long getRowCount(HoodieStorage storage, StoragePath filePath) {
    throw new UnsupportedOperationException("HFileUtils does not support getRowCount");
  }

  @Override
  public Set<Pair<String, Long>> filterRowKeys(HoodieStorage storage, StoragePath filePath, Set<String> filter) {
    throw new UnsupportedOperationException("HFileUtils does not support filterRowKeys");
  }

  @Override
  public ClosableIterator<Pair<HoodieKey, Long>> fetchRecordKeysWithPositions(HoodieStorage storage, StoragePath filePath) {
    throw new UnsupportedOperationException("HFileUtils does not support fetchRecordKeysWithPositions");
  }

  @Override
  public ClosableIterator<HoodieKey> getHoodieKeyIterator(HoodieStorage storage, StoragePath filePath, Option<BaseKeyGenerator> keyGeneratorOpt, Option<String> partitionPath) {
    try {
      HoodieFileReader reader = HoodieIOFactory
          .getIOFactory(storage)
          .getReaderFactory(HoodieRecord.HoodieRecordType.AVRO)
          .getFileReader(new HoodieReaderConfig(), filePath, HoodieFileFormat.HFILE);
      ClosableIterator<String> keyIterator = reader.getRecordKeyIterator();
      return new ClosableIterator<HoodieKey>() {
        @Override
        public void close() {
          keyIterator.close();
        }

        @Override
        public boolean hasNext() {
          return keyIterator.hasNext();
        }

        @Override
        public HoodieKey next() {
          String key = keyIterator.next();
          return new HoodieKey(key, partitionPath.orElse(null));
        }
      };
    } catch (IOException e) {
      throw new HoodieIOException("Unable to read the HFile: ", e);
    }
  }

  @Override
  public ClosableIterator<HoodieKey> getHoodieKeyIterator(HoodieStorage storage, StoragePath filePath) {
    throw new UnsupportedOperationException("HFileUtils does not support getHoodieKeyIterator");
  }

  @Override
  public ClosableIterator<Pair<HoodieKey, Long>> fetchRecordKeysWithPositions(HoodieStorage storage, StoragePath filePath, Option<BaseKeyGenerator> keyGeneratorOpt, Option<String> partitionPath) {
    throw new UnsupportedOperationException("HFileUtils does not support fetchRecordKeysWithPositions");
  }

  @Override
  public HoodieSchema readSchema(HoodieStorage storage, StoragePath filePath) {
    log.info("Reading schema from {}", filePath);

    try (HoodieFileReader fileReader =
             HoodieIOFactory.getIOFactory(storage)
                 .getReaderFactory(HoodieRecord.HoodieRecordType.AVRO)
                 .getFileReader(
                     ConfigUtils.DEFAULT_HUDI_CONFIG_FOR_READER,
                     filePath)) {
      return fileReader.getSchema();
    } catch (IOException e) {
      throw new HoodieIOException("Failed to read schema from HFile", e);
    }
  }

  @Override
  public List<HoodieColumnRangeMetadata<Comparable>> readColumnStatsFromMetadata(HoodieStorage storage, StoragePath filePath, List<String> columnList, HoodieIndexVersion indexVersion) {
    throw new UnsupportedOperationException(
        "Reading column statistics from metadata is not supported for HFile format yet");
  }

  @Override
  public HoodieFileFormat getFormat() {
    return HoodieFileFormat.HFILE;
  }

  @Override
  public void writeMetaFile(HoodieStorage storage, StoragePath filePath, Properties props) throws IOException {
    throw new UnsupportedOperationException("HFileUtils does not support writeMetaFile");
  }

  @Override
  public ByteArrayOutputStream serializeRecordsToLogBlock(HoodieStorage storage,
                                                          List<HoodieRecord> records,
                                                          HoodieSchema writerSchema,
                                                          HoodieSchema readerSchema,
                                                          String keyFieldName,
                                                          Map<String, String> paramsMap) throws IOException {
    CompressionCodec compressionCodec = getHFileCompressionAlgorithm(paramsMap);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try (OutputStream ostream = new DataOutputStream(baos)) {
      HFileContext context = HFileContext.builder()
          .blockSize(DEFAULT_BLOCK_SIZE_FOR_LOG_FILE)
          .compressionCodec(compressionCodec)
          .build();
      // Use simple incrementing counter as a key
      boolean useIntegerKey = !getRecordKey(records.get(0), readerSchema, keyFieldName).isPresent();
      // This is set here to avoid re-computing this in the loop
      int keyWidth = useIntegerKey ? (int) Math.ceil(Math.log(records.size())) + 1 : -1;
      int id = 0;
      Option<HoodieSchemaField> keyField = writerSchema.getField(keyFieldName);
      try (HFileWriter writer = new HFileWriterImpl(context, ostream)) {
        String previousRecordKey = null;
        // It is assumed that the input records are sorted based on the record key
        // for HFile block
        for (int i = 0; i < records.size(); i++) {
          HoodieRecord<?> record = records.get(i);
          String recordKey = useIntegerKey
              ? String.format("%" + keyWidth + "s", id++)
              : getRecordKey(record, readerSchema, keyFieldName).get();
          final byte[] recordBytes = serializeRecord(record, writerSchema, keyField);
          // Since the list is sorted, duplicates will be adjacent.
          if (i > 0 && recordKey.equals(previousRecordKey)) {
            log.error("Found duplicate record with recordKey: {}", recordKey);
            logRecordMetadata("Previous record",
                serializeRecord(records.get(i - 1), writerSchema, keyField), writerSchema);
            logRecordMetadata("Current record",
                serializeRecord(record, writerSchema, keyField), writerSchema);
            throw new HoodieException(String.format(
                "Writing multiple records with same key %s not supported for Hfile format with Metadata table", recordKey));
          }
          try {
            writer.append(recordKey, recordBytes);
          } catch (IOException e) {
            throw new HoodieIOException("IOException serializing records", e);
          }
          previousRecordKey = recordKey;
        }

        writer.appendFileInfo(
            HoodieAvroHFileReaderImplBase.SCHEMA_KEY,
            getUTF8Bytes(readerSchema.toString()));
      }
      ostream.flush();
    }
    return baos;
  }

  /**
   * Print the meta fields of the record of interest
   */
  private void logRecordMetadata(String msg, byte[] bs, HoodieSchema schema) throws IOException {
    GenericRecord record = HoodieAvroUtils.bytesToAvro(bs, schema.toAvroSchema());
    if (schema.getField(HoodieRecord.RECORD_KEY_METADATA_FIELD).isPresent()) {
      log.error("{}: Hudi meta field values -> Record key: {}, Partition Path: {}, FileName: {}, CommitTime: {}, CommitSeqNo: {}", msg,
          record.get(HoodieRecord.RECORD_KEY_METADATA_FIELD), record.get(HoodieRecord.PARTITION_PATH_METADATA_FIELD),
          record.get(HoodieRecord.FILENAME_METADATA_FIELD), record.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD),
          record.get(HoodieRecord.COMMIT_SEQNO_METADATA_FIELD));
    }
  }

  @Override
  public Pair<ByteArrayOutputStream, Object> serializeRecordsToLogBlock(
      HoodieStorage storage,
      Iterator<HoodieRecord> records,
      HoodieRecordType recordType,
      HoodieSchema writerSchema,
      HoodieSchema readerSchema,
      String keyFieldName,
      Map<String, String> paramsMap) throws IOException {
    throw new UnsupportedOperationException("HFileUtils does not support serializeRecordsToLogBlock returning HoodieColumnRangeMetadata.");
  }

  private static Option<String> getRecordKey(HoodieRecord record, HoodieSchema readerSchema, String keyFieldName) {
    return Option.ofNullable(record.getRecordKey(readerSchema, keyFieldName));
  }

  private static byte[] serializeRecord(HoodieRecord<?> record, HoodieSchema schema, Option<HoodieSchemaField> keyField) throws IOException {
    return record.toIndexedRecord(schema, CollectionUtils.emptyProps()).map(HoodieAvroIndexedRecord::getData).map(indexedRecord -> {
      keyField.ifPresent(field -> indexedRecord.put(field.pos(), StringUtils.EMPTY_STRING));
      return HoodieAvroUtils.avroToBytes(indexedRecord);
    }).orElseThrow(() -> new HoodieException("Unable to convert record to indexed record"));
  }
}
