/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hudi.configuration;

import org.apache.hudi.client.clustering.plan.strategy.FlinkSizeBasedClusteringPlanStrategy;
import org.apache.hudi.common.config.AdvancedConfig;
import org.apache.hudi.common.config.ConfigClassProperty;
import org.apache.hudi.common.config.ConfigGroups;
import org.apache.hudi.common.config.HoodieConfig;
import org.apache.hudi.common.config.HoodieMetadataConfig;
import org.apache.hudi.common.model.EventTimeAvroPayload;
import org.apache.hudi.common.model.HoodieAvroRecordMerger;
import org.apache.hudi.common.model.HoodieCleaningPolicy;
import org.apache.hudi.common.model.HoodieRecordMerger;
import org.apache.hudi.common.model.HoodieSyncTableStrategy;
import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.common.model.WriteOperationType;
import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.config.HoodieIndexConfig;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.hive.MultiPartKeysValueExtractor;
import org.apache.hudi.hive.ddl.HiveSyncMode;
import org.apache.hudi.index.HoodieIndex;
import org.apache.hudi.keygen.constant.KeyGeneratorOptions;
import org.apache.hudi.keygen.constant.KeyGeneratorType;
import org.apache.hudi.sink.overwrite.PartitionOverwriteMode;
import org.apache.hudi.table.action.cluster.ClusteringPlanPartitionFilterMode;
import org.apache.hudi.util.ClientIds;

import org.apache.flink.configuration.ConfigOption;
import org.apache.flink.configuration.ConfigOptions;
import org.apache.flink.configuration.Configuration;

import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import static org.apache.hudi.common.table.cdc.HoodieCDCSupplementalLoggingMode.DATA_BEFORE_AFTER;
import static org.apache.hudi.common.util.PartitionPathEncodeUtils.DEFAULT_PARTITION_PATH;

/**
 * Hoodie Flink config options.
 *
 * <p>It has the options for Hoodie table read and write. It also defines some utilities.
 */
@ConfigClassProperty(name = "Flink Options",
    groupName = ConfigGroups.Names.FLINK_SQL,
    description = "Flink jobs using the SQL can be configured through the options in WITH clause."
        + " The actual datasource level configs are listed below.")
public class FlinkOptions extends HoodieConfig {
  private FlinkOptions() {
  }

  // ------------------------------------------------------------------------
  //  Base Options
  // ------------------------------------------------------------------------

  public static final ConfigOption<String> PATH = ConfigOptions
      .key("path")
      .stringType()
      .noDefaultValue()
      .withDescription("Base path for the target hoodie table.\n"
          + "The path would be created if it does not exist,\n"
          + "otherwise a Hoodie table expects to be initialized successfully");

  // ------------------------------------------------------------------------
  //  Common Options
  // ------------------------------------------------------------------------

  public static final ConfigOption<String> DATABASE_NAME = ConfigOptions
      .key(HoodieTableConfig.DATABASE_NAME.key())
      .stringType()
      .noDefaultValue()
      .withDescription("Database name to register to Hive metastore");

  public static final ConfigOption<String> TABLE_NAME = ConfigOptions
      .key(HoodieWriteConfig.TBL_NAME.key())
      .stringType()
      .noDefaultValue()
      .withDescription("Table name to register to Hive metastore");

  public static final String TABLE_TYPE_COPY_ON_WRITE = HoodieTableType.COPY_ON_WRITE.name();
  public static final String TABLE_TYPE_MERGE_ON_READ = HoodieTableType.MERGE_ON_READ.name();
  public static final ConfigOption<String> TABLE_TYPE = ConfigOptions
      .key("table.type")
      .stringType()
      .defaultValue(TABLE_TYPE_COPY_ON_WRITE)
      .withFallbackKeys(HoodieTableConfig.TYPE.key())
      .withDescription("Type of table to write. COPY_ON_WRITE (or) MERGE_ON_READ");

  public static final String NO_PRE_COMBINE = "no_precombine";
  public static final ConfigOption<String> PRECOMBINE_FIELD = ConfigOptions
      .key("precombine.field")
      .stringType()
      .defaultValue("ts")
      .withFallbackKeys("write.precombine.field", HoodieWriteConfig.PRECOMBINE_FIELD_NAME.key())
      .withDescription("Field used in preCombining before actual write. When two records have the same\n"
          + "key value, we will pick the one with the largest value for the precombine field,\n"
          + "determined by Object.compareTo(..)");

  @AdvancedConfig
  public static final ConfigOption<String> PAYLOAD_CLASS_NAME = ConfigOptions
      .key("payload.class")
      .stringType()
      .defaultValue(EventTimeAvroPayload.class.getName())
      .withFallbackKeys("write.payload.class", HoodieWriteConfig.WRITE_PAYLOAD_CLASS_NAME.key())
      .withDescription("Payload class used. Override this, if you like to roll your own merge logic, when upserting/inserting.\n"
          + "This will render any value set for the option in-effective");

  @AdvancedConfig
  public static final ConfigOption<String> RECORD_MERGER_IMPLS = ConfigOptions
      .key("record.merger.impls")
      .stringType()
      .defaultValue(HoodieAvroRecordMerger.class.getName())
      .withFallbackKeys(HoodieWriteConfig.RECORD_MERGER_IMPLS.key())
      .withDescription("List of HoodieMerger implementations constituting Hudi's merging strategy -- based on the engine used. "
          + "These merger impls will filter by record.merger.strategy. "
          + "Hudi will pick most efficient implementation to perform merging/combining of the records (during update, reading MOR table, etc)");

  @AdvancedConfig
  public static final ConfigOption<String> RECORD_MERGER_STRATEGY = ConfigOptions
      .key("record.merger.strategy")
      .stringType()
      .defaultValue(HoodieRecordMerger.DEFAULT_MERGER_STRATEGY_UUID)
      .withFallbackKeys(HoodieWriteConfig.RECORD_MERGER_STRATEGY.key())
      .withDescription("Id of merger strategy. Hudi will pick HoodieRecordMerger implementations in record.merger.impls which has the same merger strategy id");

  @AdvancedConfig
  public static final ConfigOption<String> PARTITION_DEFAULT_NAME = ConfigOptions
      .key("partition.default_name")
      .stringType()
      .defaultValue(DEFAULT_PARTITION_PATH) // keep sync with hoodie style
      .withDescription("The default partition name in case the dynamic partition"
          + " column value is null/empty string");

  // ------------------------------------------------------------------------
  //  Changelog Capture Options
  // ------------------------------------------------------------------------

  public static final ConfigOption<Boolean> CHANGELOG_ENABLED = ConfigOptions
      .key("changelog.enabled")
      .booleanType()
      .defaultValue(false)
      .withDescription("Whether to keep all the intermediate changes, "
          + "we try to keep all the changes of a record when enabled:\n"
          + "1). The sink accept the UPDATE_BEFORE message;\n"
          + "2). The source try to emit every changes of a record.\n"
          + "The semantics is best effort because the compaction job would finally merge all changes of a record into one.\n"
          + " default false to have UPSERT semantics");

  public static final ConfigOption<Boolean> CDC_ENABLED = ConfigOptions
      .key("cdc.enabled")
      .booleanType()
      .defaultValue(false)
      .withFallbackKeys(HoodieTableConfig.CDC_ENABLED.key())
      .withDescription("When enable, persist the change data if necessary, and can be queried as a CDC query mode");

  public static final ConfigOption<String> SUPPLEMENTAL_LOGGING_MODE = ConfigOptions
      .key("cdc.supplemental.logging.mode")
      .stringType()
      .defaultValue(DATA_BEFORE_AFTER.name())
      .withFallbackKeys(HoodieTableConfig.CDC_SUPPLEMENTAL_LOGGING_MODE.key())
      .withDescription("Setting 'op_key_only' persists the 'op' and the record key only, "
          + "setting 'data_before' persists the additional 'before' image, "
          + "and setting 'data_before_after' persists the additional 'before' and 'after' images.");

  // ------------------------------------------------------------------------
  //  Metadata table Options
  // ------------------------------------------------------------------------

  public static final ConfigOption<Boolean> METADATA_ENABLED = ConfigOptions
      .key("metadata.enabled")
      .booleanType()
      .defaultValue(false)
      .withFallbackKeys(HoodieMetadataConfig.ENABLE.key())
      .withDescription("Enable the internal metadata table which serves table metadata like level file listings, default disabled");

  public static final ConfigOption<Integer> METADATA_COMPACTION_DELTA_COMMITS = ConfigOptions
      .key("metadata.compaction.delta_commits")
      .intType()
      .defaultValue(10)
      .withDescription("Max delta commits for metadata table to trigger compaction, default 10");

  // ------------------------------------------------------------------------
  //  Index Options
  // ------------------------------------------------------------------------

  public static final ConfigOption<String> INDEX_TYPE = ConfigOptions
      .key("index.type")
      .stringType()
      .defaultValue(HoodieIndex.IndexType.FLINK_STATE.name())
      .withFallbackKeys(HoodieIndexConfig.INDEX_TYPE.key())
      .withDescription("Index type of Flink write job, default is using state backed index.");

  @AdvancedConfig
  public static final ConfigOption<Boolean> INDEX_BOOTSTRAP_ENABLED = ConfigOptions
      .key("index.bootstrap.enabled")
      .booleanType()
      .defaultValue(false)
      .withDescription("Whether to bootstrap the index state from existing hoodie table, default false");

  @AdvancedConfig
  public static final ConfigOption<Double> INDEX_STATE_TTL = ConfigOptions
      .key("index.state.ttl")
      .doubleType()
      .defaultValue(0D)
      .withDescription("Index state ttl in days, default stores the index permanently");

  @AdvancedConfig
  public static final ConfigOption<Boolean> INDEX_GLOBAL_ENABLED = ConfigOptions
      .key("index.global.enabled")
      .booleanType()
      .defaultValue(true)
      .withDescription("Whether to update index for the old partition path\n"
          + "if same key record with different partition path came in, default true");

  @AdvancedConfig
  public static final ConfigOption<String> INDEX_PARTITION_REGEX = ConfigOptions
      .key("index.partition.regex")
      .stringType()
      .defaultValue(".*")
      .withDescription("Whether to load partitions in state if partition path matching， default `*`");

  // ------------------------------------------------------------------------
  //  Read Options
  // ------------------------------------------------------------------------

  @AdvancedConfig
  public static final ConfigOption<Integer> READ_TASKS = ConfigOptions
      .key("read.tasks")
      .intType()
      .noDefaultValue()
      .withDescription("Parallelism of tasks that do actual read, default is the parallelism of the execution environment");

  @AdvancedConfig
  public static final ConfigOption<String> SOURCE_AVRO_SCHEMA_PATH = ConfigOptions
      .key("source.avro-schema.path")
      .stringType()
      .noDefaultValue()
      .withDescription("Source avro schema file path, the parsed schema is used for deserialization");

  @AdvancedConfig
  public static final ConfigOption<String> SOURCE_AVRO_SCHEMA = ConfigOptions
      .key("source.avro-schema")
      .stringType()
      .noDefaultValue()
      .withDescription("Source avro schema string, the parsed schema is used for deserialization");

  public static final String QUERY_TYPE_SNAPSHOT = "snapshot";
  public static final String QUERY_TYPE_READ_OPTIMIZED = "read_optimized";
  public static final String QUERY_TYPE_INCREMENTAL = "incremental";
  public static final ConfigOption<String> QUERY_TYPE = ConfigOptions
      .key("hoodie.datasource.query.type")
      .stringType()
      .defaultValue(QUERY_TYPE_SNAPSHOT)
      .withDescription("Decides how data files need to be read, in\n"
          + "1) Snapshot mode (obtain latest view, based on row & columnar data);\n"
          + "2) incremental mode (new data since an instantTime);\n"
          + "3) Read Optimized mode (obtain latest view, based on columnar data)\n."
          + "Default: snapshot");

  public static final String REALTIME_SKIP_MERGE = "skip_merge";
  public static final String REALTIME_PAYLOAD_COMBINE = "payload_combine";
  @AdvancedConfig
  public static final ConfigOption<String> MERGE_TYPE = ConfigOptions
      .key("hoodie.datasource.merge.type")
      .stringType()
      .defaultValue(REALTIME_PAYLOAD_COMBINE)
      .withDescription("For Snapshot query on merge on read table. Use this key to define how the payloads are merged, in\n"
          + "1) skip_merge: read the base file records plus the log file records;\n"
          + "2) payload_combine: read the base file records first, for each record in base file, checks whether the key is in the\n"
          + "   log file records(combines the two records with same key for base and log file records), then read the left log file records");

  @AdvancedConfig
  public static final ConfigOption<Boolean> UTC_TIMEZONE = ConfigOptions
      .key("read.utc-timezone")
      .booleanType()
      .defaultValue(true)
      .withDescription("Use UTC timezone or local timezone to the conversion between epoch"
          + " time and LocalDateTime. Hive 0.x/1.x/2.x use local timezone. But Hive 3.x"
          + " use UTC timezone, by default true");

  public static final ConfigOption<Boolean> READ_AS_STREAMING = ConfigOptions
      .key("read.streaming.enabled")
      .booleanType()
      .defaultValue(false)// default read as batch
      .withDescription("Whether to read as streaming source, default false");

  @AdvancedConfig
  public static final ConfigOption<Integer> READ_STREAMING_CHECK_INTERVAL = ConfigOptions
      .key("read.streaming.check-interval")
      .intType()
      .defaultValue(60)// default 1 minute
      .withDescription("Check interval for streaming read of SECOND, default 1 minute");

  @AdvancedConfig
  // this option is experimental
  public static final ConfigOption<Boolean> READ_STREAMING_SKIP_COMPACT = ConfigOptions
      .key("read.streaming.skip_compaction")
      .booleanType()
      .defaultValue(false)// default read as batch
      .withDescription("Whether to skip compaction instants and avoid reading compacted base files for streaming read to improve read performance.\n"
          + "This option can be used to avoid reading duplicates when changelog mode is enabled, it is a solution to keep data integrity\n");

  @AdvancedConfig
  // this option is experimental
  public static final ConfigOption<Boolean> READ_STREAMING_SKIP_CLUSTERING = ConfigOptions
      .key("read.streaming.skip_clustering")
      .booleanType()
      .defaultValue(false)
      .withDescription("Whether to skip clustering instants to avoid reading base files of clustering operations for streaming read "
          + "to improve read performance.");

  public static final String START_COMMIT_EARLIEST = "earliest";
  public static final ConfigOption<String> READ_START_COMMIT = ConfigOptions
      .key("read.start-commit")
      .stringType()
      .noDefaultValue()
      .withDescription("Start commit instant for reading, the commit time format should be 'yyyyMMddHHmmss', "
          + "by default reading from the latest instant for streaming read");

  public static final ConfigOption<String> READ_END_COMMIT = ConfigOptions
      .key("read.end-commit")
      .stringType()
      .noDefaultValue()
      .withDescription("End commit instant for reading, the commit time format should be 'yyyyMMddHHmmss'");

  @AdvancedConfig
  public static final ConfigOption<Boolean> READ_DATA_SKIPPING_ENABLED = ConfigOptions
      .key("read.data.skipping.enabled")
      .booleanType()
      .defaultValue(false)
      .withDescription("Enables data-skipping allowing queries to leverage indexes to reduce the search space by"
          + "skipping over files");

  // ------------------------------------------------------------------------
  //  Write Options
  // ------------------------------------------------------------------------

  @AdvancedConfig
  public static final ConfigOption<Boolean> INSERT_CLUSTER = ConfigOptions
      .key("write.insert.cluster")
      .booleanType()
      .defaultValue(false)
      .withDescription("Whether to merge small files for insert mode, "
          + "if true, the write throughput will decrease because the read/write of existing small file, "
          + "only valid for COW table, default false");

  public static final ConfigOption<String> OPERATION = ConfigOptions
      .key("write.operation")
      .stringType()
      .defaultValue(WriteOperationType.UPSERT.value())
      .withDescription("The write operation, that this write should do");

  /**
   * Flag to indicate whether to drop duplicates before insert/upsert.
   * By default false to gain extra performance.
   */
  @AdvancedConfig
  public static final ConfigOption<Boolean> PRE_COMBINE = ConfigOptions
      .key("write.precombine")
      .booleanType()
      .defaultValue(false)
      .withDescription("Flag to indicate whether to drop duplicates before insert/upsert.\n"
          + "By default these cases will accept duplicates, to gain extra performance:\n"
          + "1) insert operation;\n"
          + "2) upsert for MOR table, the MOR table deduplicate on reading");

  @AdvancedConfig
  public static final ConfigOption<Integer> RETRY_TIMES = ConfigOptions
      .key("write.retry.times")
      .intType()
      .defaultValue(3)
      .withDescription("Flag to indicate how many times streaming job should retry for a failed checkpoint batch.\n"
          + "By default 3");

  @AdvancedConfig
  public static final ConfigOption<Long> RETRY_INTERVAL_MS = ConfigOptions
      .key("write.retry.interval.ms")
      .longType()
      .defaultValue(2000L)
      .withDescription("Flag to indicate how long (by millisecond) before a retry should issued for failed checkpoint batch.\n"
          + "By default 2000 and it will be doubled by every retry");

  @AdvancedConfig
  public static final ConfigOption<Boolean> IGNORE_FAILED = ConfigOptions
      .key("write.ignore.failed")
      .booleanType()
      .defaultValue(false)
      .withDescription("Flag to indicate whether to ignore any non exception error (e.g. writestatus error). within a checkpoint batch. \n"
          + "By default false. Turning this on, could hide the write status errors while the flink checkpoint moves ahead. \n"
          + "So, would recommend users to use this with caution.");

  public static final ConfigOption<String> RECORD_KEY_FIELD = ConfigOptions
      .key(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key())
      .stringType()
      .defaultValue("uuid")
      .withDescription("Record key field. Value to be used as the `recordKey` component of `HoodieKey`.\n"
          + "Actual value will be obtained by invoking .toString() on the field value. Nested fields can be specified using "
          + "the dot notation eg: `a.b.c`");

  @AdvancedConfig
  public static final ConfigOption<String> INDEX_KEY_FIELD = ConfigOptions
      .key(HoodieIndexConfig.BUCKET_INDEX_HASH_FIELD.key())
      .stringType()
      .defaultValue("")
      .withDescription("Index key field. Value to be used as hashing to find the bucket ID. Should be a subset of or equal to the recordKey fields.\n"
          + "Actual value will be obtained by invoking .toString() on the field value. Nested fields can be specified using "
          + "the dot notation eg: `a.b.c`");

  @AdvancedConfig
  public static final ConfigOption<String> BUCKET_INDEX_ENGINE_TYPE = ConfigOptions
      .key(HoodieIndexConfig.BUCKET_INDEX_ENGINE_TYPE.key())
      .stringType()
      .defaultValue("SIMPLE")
      .withDescription("Type of bucket index engine. Available options: [SIMPLE | CONSISTENT_HASHING]");

  @AdvancedConfig
  public static final ConfigOption<Integer> BUCKET_INDEX_NUM_BUCKETS = ConfigOptions
      .key(HoodieIndexConfig.BUCKET_INDEX_NUM_BUCKETS.key())
      .intType()
      .defaultValue(4) // default 4 buckets per partition
      .withDescription("Hudi bucket number per partition. Only affected if using Hudi bucket index.");

  public static final ConfigOption<String> PARTITION_PATH_FIELD = ConfigOptions
      .key(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key())
      .stringType()
      .defaultValue("")
      .withDescription("Partition path field. Value to be used at the `partitionPath` component of `HoodieKey`.\n"
          + "Actual value obtained by invoking .toString(), default ''");

  @AdvancedConfig
  public static final ConfigOption<Boolean> URL_ENCODE_PARTITIONING = ConfigOptions
      .key(KeyGeneratorOptions.URL_ENCODE_PARTITIONING.key())
      .booleanType()
      .defaultValue(false)
      .withDescription("Whether to encode the partition path url, default false");

  public static final ConfigOption<Boolean> HIVE_STYLE_PARTITIONING = ConfigOptions
      .key(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key())
      .booleanType()
      .defaultValue(false)
      .withDescription("Whether to use Hive style partitioning.\n"
          + "If set true, the names of partition folders follow <partition_column_name>=<partition_value> format.\n"
          + "By default false (the names of partition folders are only partition values)");

  @AdvancedConfig
  public static final ConfigOption<String> KEYGEN_CLASS_NAME = ConfigOptions
      .key(HoodieWriteConfig.KEYGENERATOR_CLASS_NAME.key())
      .stringType()
      .noDefaultValue()
      .withDescription("Key generator class, that implements will extract the key out of incoming record");

  @AdvancedConfig
  public static final ConfigOption<String> KEYGEN_TYPE = ConfigOptions
      .key(HoodieWriteConfig.KEYGENERATOR_TYPE.key())
      .stringType()
      .defaultValue(KeyGeneratorType.SIMPLE.name())
      .withDescription("Key generator type, that implements will extract the key out of incoming record. "
          + "**Note** This is being actively worked on. Please use "
          + "`hoodie.datasource.write.keygenerator.class` instead.");

  public static final String PARTITION_FORMAT_HOUR = "yyyyMMddHH";
  public static final String PARTITION_FORMAT_DAY = "yyyyMMdd";
  public static final String PARTITION_FORMAT_DASHED_DAY = "yyyy-MM-dd";
  @AdvancedConfig
  public static final ConfigOption<String> PARTITION_FORMAT = ConfigOptions
      .key("write.partition.format")
      .stringType()
      .noDefaultValue()
      .withDescription("Partition path format, only valid when 'write.datetime.partitioning' is true, default is:\n"
          + "1) 'yyyyMMddHH' for timestamp(3) WITHOUT TIME ZONE, LONG, FLOAT, DOUBLE, DECIMAL;\n"
          + "2) 'yyyyMMdd' for DATE and INT.");

  @AdvancedConfig
  public static final ConfigOption<Integer> INDEX_BOOTSTRAP_TASKS = ConfigOptions
      .key("write.index_bootstrap.tasks")
      .intType()
      .noDefaultValue()
      .withDescription("Parallelism of tasks that do index bootstrap, default same as the write task parallelism");

  @AdvancedConfig
  public static final ConfigOption<Integer> BUCKET_ASSIGN_TASKS = ConfigOptions
      .key("write.bucket_assign.tasks")
      .intType()
      .noDefaultValue()
      .withDescription("Parallelism of tasks that do bucket assign, default same as the write task parallelism");

  @AdvancedConfig
  public static final ConfigOption<Integer> WRITE_TASKS = ConfigOptions
      .key("write.tasks")
      .intType()
      .noDefaultValue()
      .withDescription("Parallelism of tasks that do actual write, default is the parallelism of the execution environment");

  @AdvancedConfig
  public static final ConfigOption<Double> WRITE_TASK_MAX_SIZE = ConfigOptions
      .key("write.task.max.size")
      .doubleType()
      .defaultValue(1024D) // 1GB
      .withDescription("Maximum memory in MB for a write task, when the threshold hits,\n"
          + "it flushes the max size data bucket to avoid OOM, default 1GB");

  @AdvancedConfig
  public static final ConfigOption<Long> WRITE_RATE_LIMIT = ConfigOptions
      .key("write.rate.limit")
      .longType()
      .defaultValue(0L) // default no limit
      .withDescription("Write record rate limit per second to prevent traffic jitter and improve stability, default 0 (no limit)");

  @AdvancedConfig
  public static final ConfigOption<Double> WRITE_BATCH_SIZE = ConfigOptions
      .key("write.batch.size")
      .doubleType()
      .defaultValue(256D) // 256MB
      .withDescription("Batch buffer size in MB to flush data into the underneath filesystem, default 256MB");

  @AdvancedConfig
  public static final ConfigOption<Integer> WRITE_LOG_BLOCK_SIZE = ConfigOptions
      .key("write.log_block.size")
      .intType()
      .defaultValue(128)
      .withDescription("Max log block size in MB for log file, default 128MB");

  @AdvancedConfig
  public static final ConfigOption<Long> WRITE_LOG_MAX_SIZE = ConfigOptions
      .key("write.log.max.size")
      .longType()
      .defaultValue(1024L)
      .withDescription("Maximum size allowed in MB for a log file before it is rolled over to the next version, default 1GB");

  @AdvancedConfig
  public static final ConfigOption<Integer> WRITE_PARQUET_BLOCK_SIZE = ConfigOptions
      .key("write.parquet.block.size")
      .intType()
      .defaultValue(120)
      .withDescription("Parquet RowGroup size. It's recommended to make this large enough that scan costs can be"
          + " amortized by packing enough column values into a single row group.");

  public static final ConfigOption<Integer> WRITE_PARQUET_MAX_FILE_SIZE = ConfigOptions
      .key("write.parquet.max.file.size")
      .intType()
      .defaultValue(120)
      .withDescription("Target size for parquet files produced by Hudi write phases. "
          + "For DFS, this needs to be aligned with the underlying filesystem block size for optimal performance.");

  @AdvancedConfig
  public static final ConfigOption<Integer> WRITE_PARQUET_PAGE_SIZE = ConfigOptions
      .key("write.parquet.page.size")
      .intType()
      .defaultValue(1)
      .withDescription("Parquet page size. Page is the unit of read within a parquet file. "
          + "Within a block, pages are compressed separately.");

  @AdvancedConfig
  public static final ConfigOption<Integer> WRITE_MERGE_MAX_MEMORY = ConfigOptions
      .key("write.merge.max_memory")
      .intType()
      .defaultValue(100) // default 100 MB
      .withDescription("Max memory in MB for merge, default 100MB");

  // this is only for internal use
  @AdvancedConfig
  public static final ConfigOption<Long> WRITE_COMMIT_ACK_TIMEOUT = ConfigOptions
      .key("write.commit.ack.timeout")
      .longType()
      .defaultValue(-1L) // default at least once
      .withDescription("Timeout limit for a writer task after it finishes a checkpoint and\n"
          + "waits for the instant commit success, only for internal use");

  @AdvancedConfig
  public static final ConfigOption<Boolean> WRITE_BULK_INSERT_SHUFFLE_INPUT = ConfigOptions
      .key("write.bulk_insert.shuffle_input")
      .booleanType()
      .defaultValue(true)
      .withDescription("Whether to shuffle the inputs by specific fields for bulk insert tasks, default true");

  @AdvancedConfig
  public static final ConfigOption<Boolean> WRITE_BULK_INSERT_SORT_INPUT = ConfigOptions
      .key("write.bulk_insert.sort_input")
      .booleanType()
      .defaultValue(true)
      .withDescription("Whether to sort the inputs by specific fields for bulk insert tasks, default true");

  @AdvancedConfig
  public static final ConfigOption<Boolean> WRITE_BULK_INSERT_SORT_INPUT_BY_RECORD_KEY = ConfigOptions
      .key("write.bulk_insert.sort_input.by_record_key")
      .booleanType()
      .defaultValue(false)
      .withDescription("Whether to sort the inputs by record keys for bulk insert tasks, default false");

  @AdvancedConfig
  public static final ConfigOption<Integer> WRITE_SORT_MEMORY = ConfigOptions
      .key("write.sort.memory")
      .intType()
      .defaultValue(128)
      .withDescription("Sort memory in MB, default 128MB");

  @AdvancedConfig
  public static final ConfigOption<String> WRITE_PARTITION_OVERWRITE_MODE = ConfigOptions
      .key("write.partition.overwrite.mode")
      .stringType()
      .defaultValue(PartitionOverwriteMode.STATIC.name())
      .withDescription("When INSERT OVERWRITE a partitioned data source table, we currently support 2 modes: static and dynamic. "
          + "Static mode deletes all the partitions that match the partition specification(e.g. PARTITION(a=1,b)) in the INSERT statement, before overwriting. "
          + "Dynamic mode doesn't delete partitions ahead, and only overwrite those partitions that have data written into it at runtime. "
          + "By default we use static mode to keep the same behavior of previous version.");

  // this is only for internal use
  @AdvancedConfig
  public static final ConfigOption<String> WRITE_CLIENT_ID = ConfigOptions
      .key("write.client.id")
      .stringType()
      .defaultValue(ClientIds.INIT_CLIENT_ID)
      .withDescription("Unique identifier used to distinguish different writer pipelines for concurrent mode");

  // ------------------------------------------------------------------------
  //  Compaction Options
  // ------------------------------------------------------------------------

  @AdvancedConfig
  public static final ConfigOption<Boolean> COMPACTION_SCHEDULE_ENABLED = ConfigOptions
      .key("compaction.schedule.enabled")
      .booleanType()
      .defaultValue(true) // default true for MOR write
      .withDescription("Schedule the compaction plan, enabled by default for MOR");

  public static final ConfigOption<Boolean> COMPACTION_ASYNC_ENABLED = ConfigOptions
      .key("compaction.async.enabled")
      .booleanType()
      .defaultValue(true) // default true for MOR write
      .withDescription("Async Compaction, enabled by default for MOR");

  @AdvancedConfig
  public static final ConfigOption<Integer> COMPACTION_TASKS = ConfigOptions
      .key("compaction.tasks")
      .intType()
      .noDefaultValue()
      .withDescription("Parallelism of tasks that do actual compaction, default same as the write task parallelism");

  public static final String NUM_COMMITS = "num_commits";
  public static final String TIME_ELAPSED = "time_elapsed";
  public static final String NUM_AND_TIME = "num_and_time";
  public static final String NUM_OR_TIME = "num_or_time";
  @AdvancedConfig
  public static final ConfigOption<String> COMPACTION_TRIGGER_STRATEGY = ConfigOptions
      .key("compaction.trigger.strategy")
      .stringType()
      .defaultValue(NUM_COMMITS) // default true for MOR write
      .withDescription("Strategy to trigger compaction, options are "
          + "'num_commits': trigger compaction when there are at least N delta commits after last completed compaction;\n"
          + "'num_commits_after_last_request': trigger compaction when there are at least N delta commits after last completed/requested compaction;\n"
          + "'time_elapsed': trigger compaction when time elapsed > N seconds since last compaction;\n"
          + "'num_and_time': trigger compaction when both NUM_COMMITS and TIME_ELAPSED are satisfied;\n"
          + "'num_or_time': trigger compaction when NUM_COMMITS or TIME_ELAPSED is satisfied.\n"
          + "Default is 'num_commits'");

  public static final ConfigOption<Integer> COMPACTION_DELTA_COMMITS = ConfigOptions
      .key("compaction.delta_commits")
      .intType()
      .defaultValue(5)
      .withDescription("Max delta commits needed to trigger compaction, default 5 commits");

  @AdvancedConfig
  public static final ConfigOption<Integer> COMPACTION_DELTA_SECONDS = ConfigOptions
      .key("compaction.delta_seconds")
      .intType()
      .defaultValue(3600) // default 1 hour
      .withDescription("Max delta seconds time needed to trigger compaction, default 1 hour");

  @AdvancedConfig
  public static final ConfigOption<Integer> COMPACTION_TIMEOUT_SECONDS = ConfigOptions
      .key("compaction.timeout.seconds")
      .intType()
      .defaultValue(1200) // default 20 minutes
      .withDescription("Max timeout time in seconds for online compaction to rollback, default 20 minutes");

  @AdvancedConfig
  public static final ConfigOption<Integer> COMPACTION_MAX_MEMORY = ConfigOptions
      .key("compaction.max_memory")
      .intType()
      .defaultValue(100) // default 100 MB
      .withDescription("Max memory in MB for compaction spillable map, default 100MB");

  @AdvancedConfig
  public static final ConfigOption<Long> COMPACTION_TARGET_IO = ConfigOptions
      .key("compaction.target_io")
      .longType()
      .defaultValue(500 * 1024L) // default 500 GB
      .withDescription("Target IO in MB for per compaction (both read and write), default 500 GB");

  public static final ConfigOption<Boolean> CLEAN_ASYNC_ENABLED = ConfigOptions
      .key("clean.async.enabled")
      .booleanType()
      .defaultValue(true)
      .withDescription("Whether to cleanup the old commits immediately on new commits, enabled by default");

  @AdvancedConfig
  public static final ConfigOption<String> CLEAN_POLICY = ConfigOptions
      .key("clean.policy")
      .stringType()
      .defaultValue(HoodieCleaningPolicy.KEEP_LATEST_COMMITS.name())
      .withDescription("Clean policy to manage the Hudi table. Available option: KEEP_LATEST_COMMITS, KEEP_LATEST_FILE_VERSIONS, KEEP_LATEST_BY_HOURS."
          + "Default is KEEP_LATEST_COMMITS.");

  public static final ConfigOption<Integer> CLEAN_RETAIN_COMMITS = ConfigOptions
      .key("clean.retain_commits")
      .intType()
      .defaultValue(30)// default 30 commits
      .withDescription("Number of commits to retain. So data will be retained for num_of_commits * time_between_commits (scheduled).\n"
          + "This also directly translates into how much you can incrementally pull on this table, default 30");

  @AdvancedConfig
  public static final ConfigOption<Integer> CLEAN_RETAIN_HOURS = ConfigOptions
      .key("clean.retain_hours")
      .intType()
      .defaultValue(24)// default 24 hours
      .withDescription("Number of hours for which commits need to be retained. This config provides a more flexible option as"
          + "compared to number of commits retained for cleaning service. Setting this property ensures all the files, but the latest in a file group,"
          + " corresponding to commits with commit times older than the configured number of hours to be retained are cleaned.");

  @AdvancedConfig
  public static final ConfigOption<Integer> CLEAN_RETAIN_FILE_VERSIONS = ConfigOptions
      .key("clean.retain_file_versions")
      .intType()
      .defaultValue(5)// default 5 version
      .withDescription("Number of file versions to retain. default 5");

  public static final ConfigOption<Integer> ARCHIVE_MAX_COMMITS = ConfigOptions
      .key("archive.max_commits")
      .intType()
      .defaultValue(50)// default max 50 commits
      .withDescription("Max number of commits to keep before archiving older commits into a sequential log, default 50");

  public static final ConfigOption<Integer> ARCHIVE_MIN_COMMITS = ConfigOptions
      .key("archive.min_commits")
      .intType()
      .defaultValue(40)// default min 40 commits
      .withDescription("Min number of commits to keep before archiving older commits into a sequential log, default 40");

  // ------------------------------------------------------------------------
  //  Clustering Options
  // ------------------------------------------------------------------------

  @AdvancedConfig
  public static final ConfigOption<Boolean> CLUSTERING_SCHEDULE_ENABLED = ConfigOptions
      .key("clustering.schedule.enabled")
      .booleanType()
      .defaultValue(false) // default false for pipeline
      .withDescription("Schedule the cluster plan, default false");

  public static final ConfigOption<Boolean> CLUSTERING_ASYNC_ENABLED = ConfigOptions
      .key("clustering.async.enabled")
      .booleanType()
      .defaultValue(false) // default false for pipeline
      .withDescription("Async Clustering, default false");

  @AdvancedConfig
  public static final ConfigOption<Integer> CLUSTERING_DELTA_COMMITS = ConfigOptions
      .key("clustering.delta_commits")
      .intType()
      .defaultValue(4)
      .withDescription("Max delta commits needed to trigger clustering, default 4 commits");

  @AdvancedConfig
  public static final ConfigOption<Integer> CLUSTERING_TASKS = ConfigOptions
      .key("clustering.tasks")
      .intType()
      .noDefaultValue()
      .withDescription("Parallelism of tasks that do actual clustering, default same as the write task parallelism");

  @AdvancedConfig
  public static final ConfigOption<Integer> CLUSTERING_TARGET_PARTITIONS = ConfigOptions
      .key("clustering.plan.strategy.daybased.lookback.partitions")
      .intType()
      .defaultValue(2)
      .withDescription("Number of partitions to list to create ClusteringPlan, default is 2");

  @AdvancedConfig
  public static final ConfigOption<Integer> CLUSTERING_PLAN_STRATEGY_SKIP_PARTITIONS_FROM_LATEST = ConfigOptions
      .key("clustering.plan.strategy.daybased.skipfromlatest.partitions")
      .intType()
      .defaultValue(0)
      .withDescription("Number of partitions to skip from latest when choosing partitions to create ClusteringPlan");

  @AdvancedConfig
  public static final ConfigOption<String> CLUSTERING_PLAN_STRATEGY_CLUSTER_BEGIN_PARTITION = ConfigOptions
      .key("clustering.plan.strategy.cluster.begin.partition")
      .stringType()
      .defaultValue("")
      .withDescription("Begin partition used to filter partition (inclusive)");

  @AdvancedConfig
  public static final ConfigOption<String> CLUSTERING_PLAN_STRATEGY_CLUSTER_END_PARTITION = ConfigOptions
      .key("clustering.plan.strategy.cluster.end.partition")
      .stringType()
      .defaultValue("")
      .withDescription("End partition used to filter partition (inclusive)");

  @AdvancedConfig
  public static final ConfigOption<String> CLUSTERING_PLAN_STRATEGY_PARTITION_REGEX_PATTERN = ConfigOptions
      .key("clustering.plan.strategy.partition.regex.pattern")
      .stringType()
      .defaultValue("")
      .withDescription("Filter clustering partitions that matched regex pattern");

  @AdvancedConfig
  public static final ConfigOption<String> CLUSTERING_PLAN_STRATEGY_PARTITION_SELECTED = ConfigOptions
      .key("clustering.plan.strategy.partition.selected")
      .stringType()
      .defaultValue("")
      .withDescription("Partitions to run clustering");

  @AdvancedConfig
  public static final ConfigOption<String> CLUSTERING_PLAN_STRATEGY_CLASS = ConfigOptions
      .key("clustering.plan.strategy.class")
      .stringType()
      .defaultValue(FlinkSizeBasedClusteringPlanStrategy.class.getName())
      .withDescription("Config to provide a strategy class (subclass of ClusteringPlanStrategy) to create clustering plan "
          + "i.e select what file groups are being clustered. Default strategy, looks at the last N (determined by "
          + CLUSTERING_TARGET_PARTITIONS.key() + ") day based partitions picks the small file slices within those partitions.");

  @AdvancedConfig
  public static final ConfigOption<String> CLUSTERING_PLAN_PARTITION_FILTER_MODE_NAME = ConfigOptions
      .key("clustering.plan.partition.filter.mode")
      .stringType()
      .defaultValue(ClusteringPlanPartitionFilterMode.NONE.name())
      .withDescription("Partition filter mode used in the creation of clustering plan. Available values are - "
          + "NONE: do not filter table partition and thus the clustering plan will include all partitions that have clustering candidate."
          + "RECENT_DAYS: keep a continuous range of partitions, worked together with configs '" + CLUSTERING_TARGET_PARTITIONS.key() + "' and '"
          + CLUSTERING_PLAN_STRATEGY_SKIP_PARTITIONS_FROM_LATEST.key() + "."
          + "SELECTED_PARTITIONS: keep partitions that are in the specified range ['" + CLUSTERING_PLAN_STRATEGY_CLUSTER_BEGIN_PARTITION.key() + "', '"
          + CLUSTERING_PLAN_STRATEGY_CLUSTER_END_PARTITION.key() + "']."
          + "DAY_ROLLING: clustering partitions on a rolling basis by the hour to avoid clustering all partitions each time, "
          + "which strategy sorts the partitions asc and chooses the partition of which index is divided by 24 and the remainder is equal to the current hour.");

  public static final ConfigOption<Long> CLUSTERING_PLAN_STRATEGY_TARGET_FILE_MAX_BYTES = ConfigOptions
      .key("clustering.plan.strategy.target.file.max.bytes")
      .longType()
      .defaultValue(1024 * 1024 * 1024L) // default 1 GB
      .withDescription("Each group can produce 'N' (CLUSTERING_MAX_GROUP_SIZE/CLUSTERING_TARGET_FILE_SIZE) output file groups, default 1 GB");

  public static final ConfigOption<Long> CLUSTERING_PLAN_STRATEGY_SMALL_FILE_LIMIT = ConfigOptions
      .key("clustering.plan.strategy.small.file.limit")
      .longType()
      .defaultValue(600L) // default 600 MB
      .withDescription("Files smaller than the size specified here are candidates for clustering, default 600 MB");

  @AdvancedConfig
  public static final ConfigOption<String> CLUSTERING_SORT_COLUMNS = ConfigOptions
      .key("clustering.plan.strategy.sort.columns")
      .stringType()
      .defaultValue("")
      .withDescription("Columns to sort the data by when clustering");

  @AdvancedConfig
  public static final ConfigOption<Integer> CLUSTERING_MAX_NUM_GROUPS = ConfigOptions
      .key("clustering.plan.strategy.max.num.groups")
      .intType()
      .defaultValue(30)
      .withDescription("Maximum number of groups to create as part of ClusteringPlan. Increasing groups will increase parallelism, default is 30");

  // ------------------------------------------------------------------------
  //  Hive Sync Options
  // ------------------------------------------------------------------------

  public static final ConfigOption<Boolean> HIVE_SYNC_ENABLED = ConfigOptions
      .key("hive_sync.enabled")
      .booleanType()
      .defaultValue(false)
      .withFallbackKeys("hive_sync.enable")
      .withDescription("Asynchronously sync Hive meta to HMS, default false");

  @AdvancedConfig
  public static final ConfigOption<String> HIVE_SYNC_DB = ConfigOptions
      .key("hive_sync.db")
      .stringType()
      .defaultValue("default")
      .withDescription("Database name for hive sync, default 'default'");

  @AdvancedConfig
  public static final ConfigOption<String> HIVE_SYNC_TABLE = ConfigOptions
      .key("hive_sync.table")
      .stringType()
      .defaultValue("unknown")
      .withDescription("Table name for hive sync, default 'unknown'");

  @AdvancedConfig
  public static final ConfigOption<String> HIVE_SYNC_FILE_FORMAT = ConfigOptions
      .key("hive_sync.file_format")
      .stringType()
      .defaultValue("PARQUET")
      .withDescription("File format for hive sync, default 'PARQUET'");

  public static final ConfigOption<String> HIVE_SYNC_MODE = ConfigOptions
      .key("hive_sync.mode")
      .stringType()
      .defaultValue(HiveSyncMode.HMS.name())
      .withDescription("Mode to choose for Hive ops. Valid values are hms, jdbc and hiveql, default 'hms'");

  @AdvancedConfig
  public static final ConfigOption<String> HIVE_SYNC_USERNAME = ConfigOptions
      .key("hive_sync.username")
      .stringType()
      .defaultValue("hive")
      .withDescription("Username for hive sync, default 'hive'");

  @AdvancedConfig
  public static final ConfigOption<String> HIVE_SYNC_PASSWORD = ConfigOptions
      .key("hive_sync.password")
      .stringType()
      .defaultValue("hive")
      .withDescription("Password for hive sync, default 'hive'");

  public static final ConfigOption<String> HIVE_SYNC_JDBC_URL = ConfigOptions
      .key("hive_sync.jdbc_url")
      .stringType()
      .defaultValue("jdbc:hive2://localhost:10000")
      .withDescription("Jdbc URL for hive sync, default 'jdbc:hive2://localhost:10000'");

  public static final ConfigOption<String> HIVE_SYNC_METASTORE_URIS = ConfigOptions
      .key("hive_sync.metastore.uris")
      .stringType()
      .defaultValue("")
      .withDescription("Metastore uris for hive sync, default ''");

  @AdvancedConfig
  public static final ConfigOption<String> HIVE_SYNC_PARTITION_FIELDS = ConfigOptions
      .key("hive_sync.partition_fields")
      .stringType()
      .defaultValue("")
      .withDescription("Partition fields for hive sync, default ''");

  @AdvancedConfig
  public static final ConfigOption<String> HIVE_SYNC_PARTITION_EXTRACTOR_CLASS_NAME = ConfigOptions
      .key("hive_sync.partition_extractor_class")
      .stringType()
      .defaultValue(MultiPartKeysValueExtractor.class.getName())
      .withDescription("Tool to extract the partition value from HDFS path, "
          + "default 'MultiPartKeysValueExtractor'");

  @AdvancedConfig
  public static final ConfigOption<Boolean> HIVE_SYNC_ASSUME_DATE_PARTITION = ConfigOptions
      .key("hive_sync.assume_date_partitioning")
      .booleanType()
      .defaultValue(false)
      .withDescription("Assume partitioning is yyyy/mm/dd, default false");

  @AdvancedConfig
  public static final ConfigOption<Boolean> HIVE_SYNC_USE_JDBC = ConfigOptions
      .key("hive_sync.use_jdbc")
      .booleanType()
      .defaultValue(true)
      .withDescription("Use JDBC when hive synchronization is enabled, default true");

  @AdvancedConfig
  public static final ConfigOption<Boolean> HIVE_SYNC_AUTO_CREATE_DB = ConfigOptions
      .key("hive_sync.auto_create_db")
      .booleanType()
      .defaultValue(true)
      .withDescription("Auto create hive database if it does not exists, default true");

  @AdvancedConfig
  public static final ConfigOption<Boolean> HIVE_SYNC_IGNORE_EXCEPTIONS = ConfigOptions
      .key("hive_sync.ignore_exceptions")
      .booleanType()
      .defaultValue(false)
      .withDescription("Ignore exceptions during hive synchronization, default false");

  @AdvancedConfig
  public static final ConfigOption<Boolean> HIVE_SYNC_SKIP_RO_SUFFIX = ConfigOptions
      .key("hive_sync.skip_ro_suffix")
      .booleanType()
      .defaultValue(false)
      .withDescription("Skip the _ro suffix for Read optimized table when registering, default false");

  @AdvancedConfig
  public static final ConfigOption<Boolean> HIVE_SYNC_SUPPORT_TIMESTAMP = ConfigOptions
      .key("hive_sync.support_timestamp")
      .booleanType()
      .defaultValue(true)
      .withDescription("INT64 with original type TIMESTAMP_MICROS is converted to hive timestamp type.\n"
          + "Disabled by default for backward compatibility.");

  @AdvancedConfig
  public static final ConfigOption<String> HIVE_SYNC_TABLE_PROPERTIES = ConfigOptions
      .key("hive_sync.table_properties")
      .stringType()
      .noDefaultValue()
      .withDescription("Additional properties to store with table, the data format is k1=v1\nk2=v2");

  @AdvancedConfig
  public static final ConfigOption<String> HIVE_SYNC_TABLE_SERDE_PROPERTIES = ConfigOptions
      .key("hive_sync.serde_properties")
      .stringType()
      .noDefaultValue()
      .withDescription("Serde properties to hive table, the data format is k1=v1\nk2=v2");

  @AdvancedConfig
  public static final ConfigOption<String> HIVE_SYNC_CONF_DIR = ConfigOptions
      .key("hive_sync.conf.dir")
      .stringType()
      .noDefaultValue()
      .withDescription("The hive configuration directory, where the hive-site.xml lies in, the file should be put on the client machine");

  @AdvancedConfig
  public static final ConfigOption<String> HIVE_SYNC_TABLE_STRATEGY = ConfigOptions
      .key("hive_sync.table.strategy")
      .stringType()
      .defaultValue(HoodieSyncTableStrategy.ALL.name())
      .withDescription("Hive table synchronization strategy. Available option: RO, RT, ALL.");

  // -------------------------------------------------------------------------
  //  Utilities
  // -------------------------------------------------------------------------

  // Prefix for Hoodie specific properties.
  private static final String PROPERTIES_PREFIX = "properties.";

  /**
   * Collects the config options that start with specified prefix {@code prefix} into a 'key'='value' list.
   */
  public static Map<String, String> getPropertiesWithPrefix(Map<String, String> options, String prefix) {
    final Map<String, String> hoodieProperties = new HashMap<>();
    if (hasPropertyOptions(options, prefix)) {
      options.keySet().stream()
          .filter(key -> key.startsWith(prefix))
          .forEach(key -> {
            final String value = options.get(key);
            final String subKey = key.substring(prefix.length());
            hoodieProperties.put(subKey, value);
          });
    }
    return hoodieProperties;
  }

  /**
   * Collects all the config options, the 'properties.' prefix would be removed if the option key starts with it.
   */
  public static Configuration flatOptions(Configuration conf) {
    final Map<String, String> propsMap = new HashMap<>();

    conf.toMap().forEach((key, value) -> {
      final String subKey = key.startsWith(PROPERTIES_PREFIX)
          ? key.substring((PROPERTIES_PREFIX).length())
          : key;
      propsMap.put(subKey, value);
    });
    return fromMap(propsMap);
  }

  private static boolean hasPropertyOptions(Map<String, String> options, String prefix) {
    return options.keySet().stream().anyMatch(k -> k.startsWith(prefix));
  }

  /**
   * Creates a new configuration that is initialized with the options of the given map.
   */
  public static Configuration fromMap(Map<String, String> map) {
    final Configuration configuration = new Configuration();
    for (Map.Entry<String, String> entry : map.entrySet()) {
      configuration.setString(entry.getKey().trim(), entry.getValue());
    }
    return configuration;
  }

  /**
   * Returns whether the given conf defines default value for the option {@code option}.
   */
  public static <T> boolean isDefaultValueDefined(Configuration conf, ConfigOption<T> option) {
    return !conf.getOptional(option).isPresent()
        || conf.get(option).equals(option.defaultValue());
  }

  /**
   * Returns all the optional config options.
   */
  public static Set<ConfigOption<?>> optionalOptions() {
    Set<ConfigOption<?>> options = new HashSet<>(allOptions());
    options.remove(PATH);
    return options;
  }

  /**
   * Returns all the config options.
   */
  public static List<ConfigOption<?>> allOptions() {
    return OptionsResolver.allOptions(FlinkOptions.class);
  }
}
