/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.apache.doris.kafka.connector.converter;

import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.mockStatic;
import static org.mockito.Mockito.when;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import org.apache.doris.kafka.connector.cfg.DorisOptions;
import org.apache.doris.kafka.connector.cfg.DorisSinkConnectorConfig;
import org.apache.doris.kafka.connector.converter.schema.SchemaChangeManager;
import org.apache.doris.kafka.connector.exception.DorisException;
import org.apache.doris.kafka.connector.model.doris.Schema;
import org.apache.doris.kafka.connector.service.DorisSystemService;
import org.apache.doris.kafka.connector.service.RestService;
import org.apache.doris.kafka.connector.writer.TestRecordBuffer;
import org.apache.kafka.connect.data.SchemaAndValue;
import org.apache.kafka.connect.json.JsonConverter;
import org.apache.kafka.connect.sink.SinkRecord;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.mockito.MockedStatic;
import org.mockito.Mockito;

public class TestRecordService {

    private final ObjectMapper objectMapper = new ObjectMapper();
    private RecordService recordService;
    private Properties props = new Properties();
    private JsonConverter jsonConverter = new JsonConverter();
    private MockedStatic<RestService> mockRestService;

    @Before
    public void init() throws IOException {
        InputStream stream =
                this.getClass()
                        .getClassLoader()
                        .getResourceAsStream("doris-connector-sink.properties");
        props.load(stream);
        DorisSinkConnectorConfig.setDefaultValues((Map) props);
        props.put("task_id", "1");
        props.put("converter.mode", "debezium_ingestion");
        props.put("debezium.schema.evolution", "basic");
        props.put(
                "doris.topic2table.map",
                "avro_schema.wdl_test.example_table:example_table,normal.wdl_test.test_sink_normal:test_sink_normal,mysql_test.doris_test.geo_table:geo_table,pg_test.doris_cdc.all_array_types:all_array_types,normal_type:normal_type");
        recordService = new RecordService(new DorisOptions((Map) props));
        HashMap<String, String> config = new HashMap<>();
        jsonConverter.configure(config, false);
        mockRestService = mockStatic(RestService.class);

        SchemaChangeManager mockSchemaChangeManager = Mockito.mock(SchemaChangeManager.class);
        DorisSystemService mockDorisSystemService = mock(DorisSystemService.class);
        doNothing().when(mockSchemaChangeManager).addColumnDDL(anyString(), any());
        when(mockDorisSystemService.tableExists(anyString(), anyString())).thenReturn(true);
        recordService.setDorisSystemService(mockDorisSystemService);
        recordService.setSchemaChangeManager(mockSchemaChangeManager);
    }

    /**
     * The mysql table schema is as follows.
     *
     * <p>CREATE TABLE example_table ( id INT AUTO_INCREMENT PRIMARY KEY, name VARCHAR(50), age INT,
     * email VARCHAR(100), birth_date DATE, integer_column INT, float_column FLOAT, decimal_column
     * DECIMAL(10,2), datetime_column DATETIME, date_column DATE, time_column TIME, text_column
     * TEXT, varchar_column VARCHAR(255), binary_column BINARY(10), blob_column BLOB, is_active
     * TINYINT(1) );
     */
    @Test
    public void processMysqlDebeziumStructRecord() throws IOException {
        String schemaStr =
                "{\"keysType\":\"UNIQUE_KEYS\",\"properties\":[{\"name\":\"id\",\"aggregation_type\":\"\",\"comment\":\"\",\"type\":\"BIGINT\"},{\"name\":\"name\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"VARCHAR\"},{\"name\":\"age\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"INT\"},{\"name\":\"email\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"VARCHAR\"},{\"name\":\"birth_date\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"DATEV2\"},{\"name\":\"integer_column\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"INT\"},{\"name\":\"float_column\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"FLOAT\"},{\"name\":\"decimal_column\",\"aggregation_type\":\"NONE\",\"scale\":\"2\",\"comment\":\"\",\"type\":\"DECIMAL64\",\"precision\":\"10\"},{\"name\":\"datetime_column\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"DATETIMEV2\"},{\"name\":\"date_column\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"DATEV2\"},{\"name\":\"text_column\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"VARCHAR\"},{\"name\":\"binary_column\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"VARCHAR\"},{\"name\":\"is_active\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"BOOLEAN\"},{\"name\":\"varchar_column\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"VARCHAR\"},{\"name\":\"blob_column\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"STRING\"},{\"name\":\"time_column\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"STRING\"}],\"status\":200}";
        Schema schema = null;
        try {
            schema = objectMapper.readValue(schemaStr, Schema.class);
        } catch (JsonProcessingException e) {
            throw new DorisException(e);
        }
        mockRestService
                .when(() -> RestService.getSchema(any(), any(), any(), any()))
                .thenReturn(schema);

        String topic = "avro_schema.wdl_test.example_table";
        // no delete value
        String noDeleteValue =
                "{\"schema\":{\"type\":\"struct\",\"fields\":[{\"type\":\"struct\",\"fields\":[{\"type\":\"int32\",\"optional\":false,\"field\":\"id\"},{\"type\":\"string\",\"optional\":true,\"field\":\"name\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"age\"},{\"type\":\"string\",\"optional\":true,\"field\":\"email\"},{\"type\":\"int32\",\"optional\":true,\"name\":\"io.debezium.time.Date\",\"version\":1,\"field\":\"birth_date\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"integer_column\"},{\"type\":\"float\",\"optional\":true,\"field\":\"float_column\"},{\"type\":\"bytes\",\"optional\":true,\"name\":\"org.apache.kafka.connect.data.Decimal\",\"version\":1,\"parameters\":{\"scale\":\"2\",\"connect.decimal.precision\":\"10\"},\"field\":\"decimal_column\"},{\"type\":\"int64\",\"optional\":true,\"name\":\"io.debezium.time.Timestamp\",\"version\":1,\"field\":\"datetime_column\"},{\"type\":\"int32\",\"optional\":true,\"name\":\"io.debezium.time.Date\",\"version\":1,\"field\":\"date_column\"},{\"type\":\"int64\",\"optional\":true,\"name\":\"io.debezium.time.MicroTime\",\"version\":1,\"field\":\"time_column\"},{\"type\":\"string\",\"optional\":true,\"field\":\"text_column\"},{\"type\":\"string\",\"optional\":true,\"field\":\"varchar_column\"},{\"type\":\"bytes\",\"optional\":true,\"field\":\"binary_column\"},{\"type\":\"bytes\",\"optional\":true,\"field\":\"blob_column\"},{\"type\":\"int16\",\"optional\":true,\"field\":\"is_active\"}],\"optional\":true,\"name\":\"normal.wdl_test.example_table.Value\",\"field\":\"before\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"int32\",\"optional\":false,\"field\":\"id\"},{\"type\":\"string\",\"optional\":true,\"field\":\"name\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"age\"},{\"type\":\"string\",\"optional\":true,\"field\":\"email\"},{\"type\":\"int32\",\"optional\":true,\"name\":\"io.debezium.time.Date\",\"version\":1,\"field\":\"birth_date\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"integer_column\"},{\"type\":\"float\",\"optional\":true,\"field\":\"float_column\"},{\"type\":\"bytes\",\"optional\":true,\"name\":\"org.apache.kafka.connect.data.Decimal\",\"version\":1,\"parameters\":{\"scale\":\"2\",\"connect.decimal.precision\":\"10\"},\"field\":\"decimal_column\"},{\"type\":\"int64\",\"optional\":true,\"name\":\"io.debezium.time.Timestamp\",\"version\":1,\"field\":\"datetime_column\"},{\"type\":\"int32\",\"optional\":true,\"name\":\"io.debezium.time.Date\",\"version\":1,\"field\":\"date_column\"},{\"type\":\"int64\",\"optional\":true,\"name\":\"io.debezium.time.MicroTime\",\"version\":1,\"field\":\"time_column\"},{\"type\":\"string\",\"optional\":true,\"field\":\"text_column\"},{\"type\":\"string\",\"optional\":true,\"field\":\"varchar_column\"},{\"type\":\"bytes\",\"optional\":true,\"field\":\"binary_column\"},{\"type\":\"bytes\",\"optional\":true,\"field\":\"blob_column\"},{\"type\":\"int16\",\"optional\":true,\"field\":\"is_active\"}],\"optional\":true,\"name\":\"normal.wdl_test.example_table.Value\",\"field\":\"after\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"string\",\"optional\":false,\"field\":\"version\"},{\"type\":\"string\",\"optional\":false,\"field\":\"connector\"},{\"type\":\"string\",\"optional\":false,\"field\":\"name\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"ts_ms\"},{\"type\":\"string\",\"optional\":true,\"name\":\"io.debezium.data.Enum\",\"version\":1,\"parameters\":{\"allowed\":\"true,last,false,incremental\"},\"default\":\"false\",\"field\":\"snapshot\"},{\"type\":\"string\",\"optional\":false,\"field\":\"db\"},{\"type\":\"string\",\"optional\":true,\"field\":\"sequence\"},{\"type\":\"string\",\"optional\":true,\"field\":\"table\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"server_id\"},{\"type\":\"string\",\"optional\":true,\"field\":\"gtid\"},{\"type\":\"string\",\"optional\":false,\"field\":\"file\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"pos\"},{\"type\":\"int32\",\"optional\":false,\"field\":\"row\"},{\"type\":\"int64\",\"optional\":true,\"field\":\"thread\"},{\"type\":\"string\",\"optional\":true,\"field\":\"query\"}],\"optional\":false,\"name\":\"io.debezium.connector.mysql.Source\",\"field\":\"source\"},{\"type\":\"string\",\"optional\":false,\"field\":\"op\"},{\"type\":\"int64\",\"optional\":true,\"field\":\"ts_ms\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"string\",\"optional\":false,\"field\":\"id\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"total_order\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"data_collection_order\"}],\"optional\":true,\"name\":\"event.block\",\"version\":1,\"field\":\"transaction\"}],\"optional\":false,\"name\":\"normal.wdl_test.example_table.Envelope\",\"version\":1},\"payload\":{\"before\":null,\"after\":{\"id\":8,\"name\":\"Jfohn Doe\",\"age\":430,\"email\":\"john@example.com\",\"birth_date\":8905,\"integer_column\":12323,\"float_column\":45.67,\"decimal_column\":\"MDk=\",\"datetime_column\":1712917800000,\"date_column\":19825,\"time_column\":37800000000,\"text_column\":\"Lorem ipsum dolor sit amet, consectetur adipiscing elit.\",\"varchar_column\":null,\"binary_column\":\"EjRWeJCrze8AAA==\",\"blob_column\":null,\"is_active\":2},\"source\":{\"version\":\"2.5.4.Final\",\"connector\":\"mysql\",\"name\":\"normal\",\"ts_ms\":1712915126000,\"snapshot\":\"false\",\"db\":\"wdl_test\",\"sequence\":null,\"table\":\"example_table\",\"server_id\":1,\"gtid\":null,\"file\":\"binlog.000063\",\"pos\":13454,\"row\":0,\"thread\":20,\"query\":null},\"op\":\"c\",\"ts_ms\":1712915126481,\"transaction\":null}}";
        String expectedNoDeleteValue =
                "{\"id\":8,\"name\":\"Jfohn Doe\",\"age\":430,\"email\":\"john@example.com\",\"birth_date\":\"1994-05-20\",\"integer_column\":12323,\"float_column\":45.67,\"decimal_column\":123.45,\"datetime_column\":\"2024-04-12T10:30\",\"date_column\":\"2024-04-12\",\"time_column\":\"10:30:00\",\"text_column\":\"Lorem ipsum dolor sit amet, consectetur adipiscing elit.\",\"varchar_column\":null,\"binary_column\":\"1234567890ABCDEF0000\",\"blob_column\":null,\"is_active\":2,\"__DORIS_DELETE_SIGN__\":\"0\"}";
        buildProcessStructRecord(topic, noDeleteValue, expectedNoDeleteValue);

        // delete value
        String deleteValue =
                "{\"schema\":{\"type\":\"struct\",\"fields\":[{\"type\":\"struct\",\"fields\":[{\"type\":\"int32\",\"optional\":false,\"field\":\"id\"},{\"type\":\"string\",\"optional\":true,\"field\":\"name\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"age\"},{\"type\":\"string\",\"optional\":true,\"field\":\"email\"},{\"type\":\"int32\",\"optional\":true,\"name\":\"io.debezium.time.Date\",\"version\":1,\"field\":\"birth_date\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"integer_column\"},{\"type\":\"float\",\"optional\":true,\"field\":\"float_column\"},{\"type\":\"bytes\",\"optional\":true,\"name\":\"org.apache.kafka.connect.data.Decimal\",\"version\":1,\"parameters\":{\"scale\":\"2\",\"connect.decimal.precision\":\"10\"},\"field\":\"decimal_column\"},{\"type\":\"int64\",\"optional\":true,\"name\":\"io.debezium.time.Timestamp\",\"version\":1,\"field\":\"datetime_column\"},{\"type\":\"int32\",\"optional\":true,\"name\":\"io.debezium.time.Date\",\"version\":1,\"field\":\"date_column\"},{\"type\":\"int64\",\"optional\":true,\"name\":\"io.debezium.time.MicroTime\",\"version\":1,\"field\":\"time_column\"},{\"type\":\"string\",\"optional\":true,\"field\":\"text_column\"},{\"type\":\"string\",\"optional\":true,\"field\":\"varchar_column\"},{\"type\":\"bytes\",\"optional\":true,\"field\":\"binary_column\"},{\"type\":\"bytes\",\"optional\":true,\"field\":\"blob_column\"},{\"type\":\"int16\",\"optional\":true,\"field\":\"is_active\"}],\"optional\":true,\"name\":\"normal.wdl_test.example_table.Value\",\"field\":\"before\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"int32\",\"optional\":false,\"field\":\"id\"},{\"type\":\"string\",\"optional\":true,\"field\":\"name\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"age\"},{\"type\":\"string\",\"optional\":true,\"field\":\"email\"},{\"type\":\"int32\",\"optional\":true,\"name\":\"io.debezium.time.Date\",\"version\":1,\"field\":\"birth_date\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"integer_column\"},{\"type\":\"float\",\"optional\":true,\"field\":\"float_column\"},{\"type\":\"bytes\",\"optional\":true,\"name\":\"org.apache.kafka.connect.data.Decimal\",\"version\":1,\"parameters\":{\"scale\":\"2\",\"connect.decimal.precision\":\"10\"},\"field\":\"decimal_column\"},{\"type\":\"int64\",\"optional\":true,\"name\":\"io.debezium.time.Timestamp\",\"version\":1,\"field\":\"datetime_column\"},{\"type\":\"int32\",\"optional\":true,\"name\":\"io.debezium.time.Date\",\"version\":1,\"field\":\"date_column\"},{\"type\":\"int64\",\"optional\":true,\"name\":\"io.debezium.time.MicroTime\",\"version\":1,\"field\":\"time_column\"},{\"type\":\"string\",\"optional\":true,\"field\":\"text_column\"},{\"type\":\"string\",\"optional\":true,\"field\":\"varchar_column\"},{\"type\":\"bytes\",\"optional\":true,\"field\":\"binary_column\"},{\"type\":\"bytes\",\"optional\":true,\"field\":\"blob_column\"},{\"type\":\"int16\",\"optional\":true,\"field\":\"is_active\"}],\"optional\":true,\"name\":\"normal.wdl_test.example_table.Value\",\"field\":\"after\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"string\",\"optional\":false,\"field\":\"version\"},{\"type\":\"string\",\"optional\":false,\"field\":\"connector\"},{\"type\":\"string\",\"optional\":false,\"field\":\"name\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"ts_ms\"},{\"type\":\"string\",\"optional\":true,\"name\":\"io.debezium.data.Enum\",\"version\":1,\"parameters\":{\"allowed\":\"true,last,false,incremental\"},\"default\":\"false\",\"field\":\"snapshot\"},{\"type\":\"string\",\"optional\":false,\"field\":\"db\"},{\"type\":\"string\",\"optional\":true,\"field\":\"sequence\"},{\"type\":\"string\",\"optional\":true,\"field\":\"table\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"server_id\"},{\"type\":\"string\",\"optional\":true,\"field\":\"gtid\"},{\"type\":\"string\",\"optional\":false,\"field\":\"file\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"pos\"},{\"type\":\"int32\",\"optional\":false,\"field\":\"row\"},{\"type\":\"int64\",\"optional\":true,\"field\":\"thread\"},{\"type\":\"string\",\"optional\":true,\"field\":\"query\"}],\"optional\":false,\"name\":\"io.debezium.connector.mysql.Source\",\"field\":\"source\"},{\"type\":\"string\",\"optional\":false,\"field\":\"op\"},{\"type\":\"int64\",\"optional\":true,\"field\":\"ts_ms\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"string\",\"optional\":false,\"field\":\"id\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"total_order\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"data_collection_order\"}],\"optional\":true,\"name\":\"event.block\",\"version\":1,\"field\":\"transaction\"}],\"optional\":false,\"name\":\"normal.wdl_test.example_table.Envelope\",\"version\":1},\"payload\":{\"before\":{\"id\":8,\"name\":\"Jfohn Doe\",\"age\":430,\"email\":\"john@example.com\",\"birth_date\":8905,\"integer_column\":12323,\"float_column\":45.67,\"decimal_column\":\"MDk=\",\"datetime_column\":1712917800000,\"date_column\":19825,\"time_column\":37800000000,\"text_column\":\"Lorem ipsum dolor sit amet, consectetur adipiscing elit.\",\"varchar_column\":null,\"binary_column\":\"EjRWeJCrze8AAA==\",\"blob_column\":null,\"is_active\":2},\"after\":null,\"source\":{\"version\":\"2.5.4.Final\",\"connector\":\"mysql\",\"name\":\"normal\",\"ts_ms\":1712915202000,\"snapshot\":\"false\",\"db\":\"wdl_test\",\"sequence\":null,\"table\":\"example_table\",\"server_id\":1,\"gtid\":null,\"file\":\"binlog.000063\",\"pos\":13915,\"row\":0,\"thread\":20,\"query\":null},\"op\":\"d\",\"ts_ms\":1712915202020,\"transaction\":null}}";
        String expectedDeleteValue =
                "{\"id\":8,\"name\":\"Jfohn Doe\",\"age\":430,\"email\":\"john@example.com\",\"birth_date\":\"1994-05-20\",\"integer_column\":12323,\"float_column\":45.67,\"decimal_column\":123.45,\"datetime_column\":\"2024-04-12T10:30\",\"date_column\":\"2024-04-12\",\"time_column\":\"10:30:00\",\"text_column\":\"Lorem ipsum dolor sit amet, consectetur adipiscing elit.\",\"varchar_column\":null,\"binary_column\":\"1234567890ABCDEF0000\",\"blob_column\":null,\"is_active\":2,\"__DORIS_DELETE_SIGN__\":\"1\"}";
        buildProcessStructRecord(topic, deleteValue, expectedDeleteValue);
    }

    @Test
    public void processMysqlDebeziumStructRecordAlter() throws IOException {
        String schemaStr =
                "{\"keysType\":\"UNIQUE_KEYS\",\"properties\":[{\"name\":\"id\",\"aggregation_type\":\"\",\"comment\":\"\",\"type\":\"BIGINT\"},{\"name\":\"name\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"VARCHAR\"},{\"name\":\"age\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"INT\"},{\"name\":\"email\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"VARCHAR\"},{\"name\":\"birth_date\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"DATEV2\"},{\"name\":\"integer_column\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"INT\"},{\"name\":\"float_column\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"FLOAT\"},{\"name\":\"decimal_column\",\"aggregation_type\":\"NONE\",\"scale\":\"2\",\"comment\":\"\",\"type\":\"DECIMAL64\",\"precision\":\"10\"},{\"name\":\"datetime_column\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"DATETIMEV2\"},{\"name\":\"date_column\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"DATEV2\"},{\"name\":\"text_column\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"VARCHAR\"},{\"name\":\"binary_column\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"VARCHAR\"},{\"name\":\"is_active\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"BOOLEAN\"},{\"name\":\"varchar_column\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"VARCHAR\"}],\"status\":200}";
        Schema schema = null;
        try {
            schema = objectMapper.readValue(schemaStr, Schema.class);
        } catch (JsonProcessingException e) {
            throw new DorisException(e);
        }
        mockRestService
                .when(() -> RestService.getSchema(any(), any(), any(), any()))
                .thenReturn(schema);

        String topic = "avro_schema.wdl_test.example_table";
        String topicMsg =
                "{\"schema\":{\"type\":\"struct\",\"fields\":[{\"type\":\"struct\",\"fields\":[{\"type\":\"int32\",\"optional\":false,\"field\":\"id\"},{\"type\":\"string\",\"optional\":true,\"field\":\"name\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"age\"},{\"type\":\"string\",\"optional\":true,\"field\":\"email\"},{\"type\":\"int32\",\"optional\":true,\"name\":\"io.debezium.time.Date\",\"version\":1,\"field\":\"birth_date\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"integer_column\"},{\"type\":\"float\",\"optional\":true,\"field\":\"float_column\"},{\"type\":\"bytes\",\"optional\":true,\"name\":\"org.apache.kafka.connect.data.Decimal\",\"version\":1,\"parameters\":{\"scale\":\"2\",\"connect.decimal.precision\":\"10\"},\"field\":\"decimal_column\"},{\"type\":\"int64\",\"optional\":true,\"name\":\"io.debezium.time.Timestamp\",\"version\":1,\"field\":\"datetime_column\"},{\"type\":\"int32\",\"optional\":true,\"name\":\"io.debezium.time.Date\",\"version\":1,\"field\":\"date_column\"},{\"type\":\"int64\",\"optional\":true,\"name\":\"io.debezium.time.MicroTime\",\"version\":1,\"field\":\"time_column\"},{\"type\":\"string\",\"optional\":true,\"field\":\"text_column\"},{\"type\":\"string\",\"optional\":true,\"field\":\"varchar_column\"},{\"type\":\"bytes\",\"optional\":true,\"field\":\"binary_column\"},{\"type\":\"bytes\",\"optional\":true,\"field\":\"blob_column\"},{\"type\":\"int16\",\"optional\":true,\"field\":\"is_active\"}],\"optional\":true,\"name\":\"normal.wdl_test.example_table.Value\",\"field\":\"before\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"int32\",\"optional\":false,\"field\":\"id\"},{\"type\":\"string\",\"optional\":true,\"field\":\"name\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"age\"},{\"type\":\"string\",\"optional\":true,\"field\":\"email\"},{\"type\":\"int32\",\"optional\":true,\"name\":\"io.debezium.time.Date\",\"version\":1,\"field\":\"birth_date\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"integer_column\"},{\"type\":\"float\",\"optional\":true,\"field\":\"float_column\"},{\"type\":\"bytes\",\"optional\":true,\"name\":\"org.apache.kafka.connect.data.Decimal\",\"version\":1,\"parameters\":{\"scale\":\"2\",\"connect.decimal.precision\":\"10\"},\"field\":\"decimal_column\"},{\"type\":\"int64\",\"optional\":true,\"name\":\"io.debezium.time.Timestamp\",\"version\":1,\"field\":\"datetime_column\"},{\"type\":\"int32\",\"optional\":true,\"name\":\"io.debezium.time.Date\",\"version\":1,\"field\":\"date_column\"},{\"type\":\"int64\",\"optional\":true,\"name\":\"io.debezium.time.MicroTime\",\"version\":1,\"field\":\"time_column\"},{\"type\":\"string\",\"optional\":true,\"field\":\"text_column\"},{\"type\":\"string\",\"optional\":true,\"field\":\"varchar_column\"},{\"type\":\"bytes\",\"optional\":true,\"field\":\"binary_column\"},{\"type\":\"bytes\",\"optional\":true,\"field\":\"blob_column\"},{\"type\":\"int16\",\"optional\":true,\"field\":\"is_active\"}],\"optional\":true,\"name\":\"normal.wdl_test.example_table.Value\",\"field\":\"after\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"string\",\"optional\":false,\"field\":\"version\"},{\"type\":\"string\",\"optional\":false,\"field\":\"connector\"},{\"type\":\"string\",\"optional\":false,\"field\":\"name\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"ts_ms\"},{\"type\":\"string\",\"optional\":true,\"name\":\"io.debezium.data.Enum\",\"version\":1,\"parameters\":{\"allowed\":\"true,last,false,incremental\"},\"default\":\"false\",\"field\":\"snapshot\"},{\"type\":\"string\",\"optional\":false,\"field\":\"db\"},{\"type\":\"string\",\"optional\":true,\"field\":\"sequence\"},{\"type\":\"string\",\"optional\":true,\"field\":\"table\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"server_id\"},{\"type\":\"string\",\"optional\":true,\"field\":\"gtid\"},{\"type\":\"string\",\"optional\":false,\"field\":\"file\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"pos\"},{\"type\":\"int32\",\"optional\":false,\"field\":\"row\"},{\"type\":\"int64\",\"optional\":true,\"field\":\"thread\"},{\"type\":\"string\",\"optional\":true,\"field\":\"query\"}],\"optional\":false,\"name\":\"io.debezium.connector.mysql.Source\",\"field\":\"source\"},{\"type\":\"string\",\"optional\":false,\"field\":\"op\"},{\"type\":\"int64\",\"optional\":true,\"field\":\"ts_ms\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"string\",\"optional\":false,\"field\":\"id\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"total_order\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"data_collection_order\"}],\"optional\":true,\"name\":\"event.block\",\"version\":1,\"field\":\"transaction\"}],\"optional\":false,\"name\":\"normal.wdl_test.example_table.Envelope\",\"version\":1},\"payload\":{\"before\":null,\"after\":{\"id\":8,\"name\":\"Jfohn Doe\",\"age\":430,\"email\":\"john@example.com\",\"birth_date\":8905,\"integer_column\":12323,\"float_column\":45.67,\"decimal_column\":\"MDk=\",\"datetime_column\":1712917800000,\"date_column\":19825,\"time_column\":37800000000,\"text_column\":\"Lorem ipsum dolor sit amet, consectetur adipiscing elit.\",\"varchar_column\":null,\"binary_column\":\"EjRWeJCrze8AAA==\",\"blob_column\":null,\"is_active\":2},\"source\":{\"version\":\"2.5.4.Final\",\"connector\":\"mysql\",\"name\":\"normal\",\"ts_ms\":1712915126000,\"snapshot\":\"false\",\"db\":\"wdl_test\",\"sequence\":null,\"table\":\"example_table\",\"server_id\":1,\"gtid\":null,\"file\":\"binlog.000063\",\"pos\":13454,\"row\":0,\"thread\":20,\"query\":null},\"op\":\"c\",\"ts_ms\":1712915126481,\"transaction\":null}}";
        SchemaAndValue schemaValue =
                jsonConverter.toConnectData(topic, topicMsg.getBytes(StandardCharsets.UTF_8));
        SinkRecord noDeleteSinkRecord =
                TestRecordBuffer.newSinkRecord(topic, schemaValue.value(), 8, schemaValue.schema());
        recordService.processStructRecord(noDeleteSinkRecord);

        // Compare the results of schema change
        Map<String, String> resultFields = new HashMap<>();
        resultFields.put("time_column", "DATETIME(0)");
        resultFields.put("blob_column", "STRING");
        Set<RecordDescriptor.FieldDescriptor> missingFields = recordService.getMissingFields();
        for (RecordDescriptor.FieldDescriptor missingField : missingFields) {
            Assert.assertTrue(resultFields.containsKey(missingField.getName()));
            Assert.assertEquals(
                    resultFields.get(missingField.getName()), missingField.getTypeName());
        }
    }

    private void buildProcessStructRecord(String topic, String sourceValue, String target)
            throws IOException {
        SchemaAndValue noDeleteSchemaValue =
                jsonConverter.toConnectData(topic, sourceValue.getBytes(StandardCharsets.UTF_8));
        SinkRecord noDeleteSinkRecord =
                TestRecordBuffer.newSinkRecord(
                        topic, noDeleteSchemaValue.value(), 8, noDeleteSchemaValue.schema());
        String processResult = recordService.processStructRecord(noDeleteSinkRecord);
        Assert.assertEquals(target, processResult);
    }

    @Test
    public void processStructRecord() throws IOException {
        props.put("converter.mode", "normal");
        recordService = new RecordService(new DorisOptions((Map) props));
        String topic = "normal.wdl_test.test_sink_normal";

        // no delete value
        String noDeleteValue =
                "{\"schema\":{\"type\":\"struct\",\"fields\":[{\"type\":\"struct\",\"fields\":[{\"type\":\"int32\",\"optional\":false,\"field\":\"id\"},{\"type\":\"string\",\"optional\":true,\"field\":\"name\"}],\"optional\":true,\"name\":\"normal.wdl_test.test_sink_normal.Value\",\"field\":\"before\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"int32\",\"optional\":false,\"field\":\"id\"},{\"type\":\"string\",\"optional\":true,\"field\":\"name\"}],\"optional\":true,\"name\":\"normal.wdl_test.test_sink_normal.Value\",\"field\":\"after\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"string\",\"optional\":false,\"field\":\"version\"},{\"type\":\"string\",\"optional\":false,\"field\":\"connector\"},{\"type\":\"string\",\"optional\":false,\"field\":\"name\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"ts_ms\"},{\"type\":\"string\",\"optional\":true,\"name\":\"io.debezium.data.Enum\",\"version\":1,\"parameters\":{\"allowed\":\"true,last,false,incremental\"},\"default\":\"false\",\"field\":\"snapshot\"},{\"type\":\"string\",\"optional\":false,\"field\":\"db\"},{\"type\":\"string\",\"optional\":true,\"field\":\"sequence\"},{\"type\":\"string\",\"optional\":true,\"field\":\"table\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"server_id\"},{\"type\":\"string\",\"optional\":true,\"field\":\"gtid\"},{\"type\":\"string\",\"optional\":false,\"field\":\"file\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"pos\"},{\"type\":\"int32\",\"optional\":false,\"field\":\"row\"},{\"type\":\"int64\",\"optional\":true,\"field\":\"thread\"},{\"type\":\"string\",\"optional\":true,\"field\":\"query\"}],\"optional\":false,\"name\":\"io.debezium.connector.mysql.Source\",\"field\":\"source\"},{\"type\":\"string\",\"optional\":false,\"field\":\"op\"},{\"type\":\"int64\",\"optional\":true,\"field\":\"ts_ms\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"string\",\"optional\":false,\"field\":\"id\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"total_order\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"data_collection_order\"}],\"optional\":true,\"name\":\"event.block\",\"version\":1,\"field\":\"transaction\"}],\"optional\":false,\"name\":\"normal.wdl_test.test_sink_normal.Envelope\",\"version\":1},\"payload\":{\"before\":null,\"after\":{\"id\":19,\"name\":\"fff\"},\"source\":{\"version\":\"2.5.4.Final\",\"connector\":\"mysql\",\"name\":\"normal\",\"ts_ms\":1712543697000,\"snapshot\":\"false\",\"db\":\"wdl_test\",\"sequence\":null,\"table\":\"test_sink_normal\",\"server_id\":1,\"gtid\":null,\"file\":\"binlog.000061\",\"pos\":5320,\"row\":0,\"thread\":260,\"query\":null},\"op\":\"c\",\"ts_ms\":1712543697062,\"transaction\":null}}";
        String expectedNoDeleteValue =
                "{\"before\":null,\"after\":{\"id\":19,\"name\":\"fff\"},\"source\":{\"version\":\"2.5.4.Final\",\"connector\":\"mysql\",\"name\":\"normal\",\"ts_ms\":1712543697000,\"snapshot\":\"false\",\"db\":\"wdl_test\",\"sequence\":null,\"table\":\"test_sink_normal\",\"server_id\":1,\"gtid\":null,\"file\":\"binlog.000061\",\"pos\":5320,\"row\":0,\"thread\":260,\"query\":null},\"op\":\"c\",\"ts_ms\":1712543697062,\"transaction\":null}";
        buildProcessStructRecord(topic, noDeleteValue, expectedNoDeleteValue);
    }

    @Test
    public void processStructRecordWithDebeziumSchema() throws IOException {
        String schemaStr =
                "{\"keysType\":\"UNIQUE_KEYS\",\"properties\":[{\"name\":\"id\",\"aggregation_type\":\"\",\"comment\":\"\",\"type\":\"INT\"},{\"name\":\"name\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"VARCHAR\"}],\"status\":200}";
        Schema schema = null;
        try {
            schema = objectMapper.readValue(schemaStr, Schema.class);
        } catch (JsonProcessingException e) {
            throw new DorisException(e);
        }
        mockRestService
                .when(() -> RestService.getSchema(any(), any(), any(), any()))
                .thenReturn(schema);

        String topic = "normal.wdl_test.test_sink_normal";

        // no delete value
        String noDeleteValue =
                "{\"schema\":{\"type\":\"struct\",\"fields\":[{\"type\":\"struct\",\"fields\":[{\"type\":\"int32\",\"optional\":false,\"field\":\"id\"},{\"type\":\"string\",\"optional\":true,\"field\":\"name\"}],\"optional\":true,\"name\":\"normal.wdl_test.test_sink_normal.Value\",\"field\":\"before\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"int32\",\"optional\":false,\"field\":\"id\"},{\"type\":\"string\",\"optional\":true,\"field\":\"name\"}],\"optional\":true,\"name\":\"normal.wdl_test.test_sink_normal.Value\",\"field\":\"after\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"string\",\"optional\":false,\"field\":\"version\"},{\"type\":\"string\",\"optional\":false,\"field\":\"connector\"},{\"type\":\"string\",\"optional\":false,\"field\":\"name\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"ts_ms\"},{\"type\":\"string\",\"optional\":true,\"name\":\"io.debezium.data.Enum\",\"version\":1,\"parameters\":{\"allowed\":\"true,last,false,incremental\"},\"default\":\"false\",\"field\":\"snapshot\"},{\"type\":\"string\",\"optional\":false,\"field\":\"db\"},{\"type\":\"string\",\"optional\":true,\"field\":\"sequence\"},{\"type\":\"string\",\"optional\":true,\"field\":\"table\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"server_id\"},{\"type\":\"string\",\"optional\":true,\"field\":\"gtid\"},{\"type\":\"string\",\"optional\":false,\"field\":\"file\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"pos\"},{\"type\":\"int32\",\"optional\":false,\"field\":\"row\"},{\"type\":\"int64\",\"optional\":true,\"field\":\"thread\"},{\"type\":\"string\",\"optional\":true,\"field\":\"query\"}],\"optional\":false,\"name\":\"io.debezium.connector.mysql.Source\",\"field\":\"source\"},{\"type\":\"string\",\"optional\":false,\"field\":\"op\"},{\"type\":\"int64\",\"optional\":true,\"field\":\"ts_ms\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"string\",\"optional\":false,\"field\":\"id\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"total_order\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"data_collection_order\"}],\"optional\":true,\"name\":\"event.block\",\"version\":1,\"field\":\"transaction\"}],\"optional\":false,\"name\":\"normal.wdl_test.test_sink_normal.Envelope\",\"version\":1},\"payload\":{\"before\":null,\"after\":{\"id\":19,\"name\":\"fff\"},\"source\":{\"version\":\"2.5.4.Final\",\"connector\":\"mysql\",\"name\":\"normal\",\"ts_ms\":1712543697000,\"snapshot\":\"false\",\"db\":\"wdl_test\",\"sequence\":null,\"table\":\"test_sink_normal\",\"server_id\":1,\"gtid\":null,\"file\":\"binlog.000061\",\"pos\":5320,\"row\":0,\"thread\":260,\"query\":null},\"op\":\"c\",\"ts_ms\":1712543697062,\"transaction\":null}}";
        String expectedNoDeleteValue =
                "{\"id\":19,\"name\":\"fff\",\"__DORIS_DELETE_SIGN__\":\"0\"}";
        buildProcessStructRecord(topic, noDeleteValue, expectedNoDeleteValue);

        // delete value
        String deleteValue =
                "{\"schema\":{\"type\":\"struct\",\"fields\":[{\"type\":\"struct\",\"fields\":[{\"type\":\"int32\",\"optional\":false,\"field\":\"id\"},{\"type\":\"string\",\"optional\":true,\"field\":\"name\"}],\"optional\":true,\"name\":\"normal.wdl_test.test_sink_normal.Value\",\"field\":\"before\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"int32\",\"optional\":false,\"field\":\"id\"},{\"type\":\"string\",\"optional\":true,\"field\":\"name\"}],\"optional\":true,\"name\":\"normal.wdl_test.test_sink_normal.Value\",\"field\":\"after\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"string\",\"optional\":false,\"field\":\"version\"},{\"type\":\"string\",\"optional\":false,\"field\":\"connector\"},{\"type\":\"string\",\"optional\":false,\"field\":\"name\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"ts_ms\"},{\"type\":\"string\",\"optional\":true,\"name\":\"io.debezium.data.Enum\",\"version\":1,\"parameters\":{\"allowed\":\"true,last,false,incremental\"},\"default\":\"false\",\"field\":\"snapshot\"},{\"type\":\"string\",\"optional\":false,\"field\":\"db\"},{\"type\":\"string\",\"optional\":true,\"field\":\"sequence\"},{\"type\":\"string\",\"optional\":true,\"field\":\"table\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"server_id\"},{\"type\":\"string\",\"optional\":true,\"field\":\"gtid\"},{\"type\":\"string\",\"optional\":false,\"field\":\"file\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"pos\"},{\"type\":\"int32\",\"optional\":false,\"field\":\"row\"},{\"type\":\"int64\",\"optional\":true,\"field\":\"thread\"},{\"type\":\"string\",\"optional\":true,\"field\":\"query\"}],\"optional\":false,\"name\":\"io.debezium.connector.mysql.Source\",\"field\":\"source\"},{\"type\":\"string\",\"optional\":false,\"field\":\"op\"},{\"type\":\"int64\",\"optional\":true,\"field\":\"ts_ms\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"string\",\"optional\":false,\"field\":\"id\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"total_order\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"data_collection_order\"}],\"optional\":true,\"name\":\"event.block\",\"version\":1,\"field\":\"transaction\"}],\"optional\":false,\"name\":\"normal.wdl_test.test_sink_normal.Envelope\",\"version\":1},\"payload\":{\"before\":{\"id\":24,\"name\":\"bb\"},\"after\":null,\"source\":{\"version\":\"2.5.4.Final\",\"connector\":\"mysql\",\"name\":\"normal\",\"ts_ms\":1712545844000,\"snapshot\":\"false\",\"db\":\"wdl_test\",\"sequence\":null,\"table\":\"test_sink_normal\",\"server_id\":1,\"gtid\":null,\"file\":\"binlog.000061\",\"pos\":5627,\"row\":0,\"thread\":260,\"query\":null},\"op\":\"d\",\"ts_ms\":1712545844948,\"transaction\":null}}";
        String expectedDeleteValue = "{\"id\":24,\"name\":\"bb\",\"__DORIS_DELETE_SIGN__\":\"1\"}";
        buildProcessStructRecord(topic, deleteValue, expectedDeleteValue);
    }

    @Test
    public void processListRecord() throws JsonProcessingException {
        List<Map<String, String>> list = new ArrayList<>();
        Map<String, String> mapValue = new HashMap<>();
        mapValue.put("name", "doris");
        mapValue.put("key", "1");

        Map<String, String> mapValue2 = new HashMap<>();
        mapValue2.put("name", "doris");
        mapValue2.put("key", "2");
        list.add(mapValue);
        list.add(mapValue2);

        SinkRecord record = TestRecordBuffer.newSinkRecord(list, 1);
        String s = recordService.processListRecord(record);
        Assert.assertEquals(
                "{\"name\":\"doris\",\"key\":\"1\"}\n{\"name\":\"doris\",\"key\":\"2\"}", s);
    }

    @Test
    public void processNormalAllTypes() throws IOException {
        props.put("converter.mode", "normal");
        recordService = new RecordService(new DorisOptions((Map) props));
        String data =
                "{\"schema\":{\"type\":\"struct\",\"fields\":[{\"type\":\"int64\",\"optional\":false,\"field\":\"id\"},{\"type\":\"int16\",\"optional\":true,\"field\":\"tiny_c\"},{\"type\":\"int16\",\"optional\":true,\"field\":\"tiny_un_c\"},{\"type\":\"int16\",\"optional\":true,\"field\":\"tiny_un_z_c\"},{\"type\":\"int16\",\"optional\":true,\"field\":\"small_c\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"small_un_c\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"small_un_z_c\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"medium_c\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"medium_un_c\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"medium_un_z_c\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"int_c\"},{\"type\":\"int64\",\"optional\":true,\"field\":\"int_un_c\"},{\"type\":\"int64\",\"optional\":true,\"field\":\"int_un_z_c\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"int11_c\"},{\"type\":\"int64\",\"optional\":true,\"field\":\"big_c\"},{\"type\":\"int64\",\"optional\":true,\"field\":\"big_un_c\"},{\"type\":\"int64\",\"optional\":true,\"field\":\"big_un_z_c\"},{\"type\":\"string\",\"optional\":true,\"field\":\"varchar_c\"},{\"type\":\"string\",\"optional\":true,\"field\":\"char_c\"},{\"type\":\"double\",\"optional\":true,\"field\":\"real_c\"},{\"type\":\"double\",\"optional\":true,\"field\":\"float_c\"},{\"type\":\"double\",\"optional\":true,\"field\":\"float_un_c\"},{\"type\":\"double\",\"optional\":true,\"field\":\"float_un_z_c\"},{\"type\":\"double\",\"optional\":true,\"field\":\"double_c\"},{\"type\":\"double\",\"optional\":true,\"field\":\"double_un_c\"},{\"type\":\"double\",\"optional\":true,\"field\":\"double_un_z_c\"},{\"type\":\"bytes\",\"optional\":true,\"name\":\"org.apache.kafka.connect.data.Decimal\",\"version\":1,\"parameters\":{\"scale\":\"4\",\"connect.decimal.precision\":\"8\"},\"field\":\"decimal_c\"},{\"type\":\"bytes\",\"optional\":true,\"name\":\"org.apache.kafka.connect.data.Decimal\",\"version\":1,\"parameters\":{\"scale\":\"4\",\"connect.decimal.precision\":\"8\"},\"field\":\"decimal_un_c\"},{\"type\":\"bytes\",\"optional\":true,\"name\":\"org.apache.kafka.connect.data.Decimal\",\"version\":1,\"parameters\":{\"scale\":\"4\",\"connect.decimal.precision\":\"8\"},\"field\":\"decimal_un_z_c\"},{\"type\":\"bytes\",\"optional\":true,\"name\":\"org.apache.kafka.connect.data.Decimal\",\"version\":1,\"parameters\":{\"scale\":\"0\",\"connect.decimal.precision\":\"6\"},\"field\":\"numeric_c\"},{\"type\":\"bytes\",\"optional\":true,\"name\":\"org.apache.kafka.connect.data.Decimal\",\"version\":1,\"parameters\":{\"scale\":\"1\",\"connect.decimal.precision\":\"65\"},\"field\":\"big_decimal_c\"},{\"type\":\"boolean\",\"optional\":true,\"field\":\"bit1_c\"},{\"type\":\"int16\",\"optional\":true,\"field\":\"tiny1_c\"},{\"type\":\"int16\",\"optional\":true,\"field\":\"boolean_c\"},{\"type\":\"string\",\"optional\":true,\"name\":\"io.debezium.time.Date\",\"version\":1,\"field\":\"date_c\"},{\"type\":\"string\",\"optional\":true,\"version\":1,\"field\":\"time_c\"},{\"type\":\"string\",\"optional\":true,\"version\":1,\"field\":\"datetime_c\"},{\"type\":\"string\",\"optional\":false,\"version\":1,\"default\":\"1970-01-01T00:00:00Z\",\"field\":\"timestamp_c\"},{\"type\":\"string\",\"optional\":true,\"field\":\"text_c\"},{\"type\":\"int32\",\"optional\":true,\"version\":1,\"field\":\"year_c\"},{\"type\":\"string\",\"optional\":true,\"version\":1,\"parameters\":{\"allowed\":\"red,white\"},\"default\":\"red\",\"field\":\"enum_c\"},{\"type\":\"string\",\"optional\":true,\"name\":\"io.debezium.data.EnumSet\",\"version\":1,\"parameters\":{\"allowed\":\"a,b\"},\"field\":\"set_c\"},{\"type\":\"string\",\"optional\":true,\"name\":\"io.debezium.data.Json\",\"version\":1,\"field\":\"json_c\"}],\"optional\":true,\"name\":\"mysql_test.doris_test.full_types.Value\",\"field\":\"after\"},\"payload\":{\"id\":1,\"tiny_c\":127,\"tiny_un_c\":255,\"tiny_un_z_c\":255,\"small_c\":32767,\"small_un_c\":65535,\"small_un_z_c\":65535,\"medium_c\":8388607,\"medium_un_c\":16777215,\"medium_un_z_c\":16777215,\"int_c\":2147483647,\"int_un_c\":4294967295,\"int_un_z_c\":4294967295,\"int11_c\":2147483647,\"big_c\":9223372036854775807,\"big_un_c\":-1,\"big_un_z_c\":-1,\"varchar_c\":\"Hello World\",\"char_c\":\"abc\",\"real_c\":123.102,\"float_c\":123.10199737548828,\"float_un_c\":123.10299682617188,\"float_un_z_c\":123.10399627685547,\"double_c\":404.4443,\"double_un_c\":404.4444,\"double_un_z_c\":404.4445,\"decimal_c\":\"EtaH\",\"decimal_un_c\":\"EtaI\",\"decimal_un_z_c\":\"EtaJ\",\"numeric_c\":\"AVo=\",\"big_decimal_c\":\"FJqkSQ==\",\"bit1_c\":false,\"tiny1_c\":1,\"boolean_c\":1,\"date_c\":\"2021-01-01\",\"time_c\":\"12:04:08\",\"datetime_c\":\"2021-01-01 12:04:08\",\"timestamp_c\":\"2021-01-01 12:04:08\",\"text_c\":\"text\",\"year_c\":2021,\"enum_c\":\"red\",\"set_c\":\"a,b\",\"json_c\":\"{\\\"key1\\\":\\\"value1\\\"}\"}}";
        String target =
                "{\"id\":1,\"tiny_c\":127,\"tiny_un_c\":255,\"tiny_un_z_c\":255,\"small_c\":32767,\"small_un_c\":65535,\"small_un_z_c\":65535,\"medium_c\":8388607,\"medium_un_c\":16777215,\"medium_un_z_c\":16777215,\"int_c\":2147483647,\"int_un_c\":4294967295,\"int_un_z_c\":4294967295,\"int11_c\":2147483647,\"big_c\":9223372036854775807,\"big_un_c\":-1,\"big_un_z_c\":-1,\"varchar_c\":\"Hello World\",\"char_c\":\"abc\",\"real_c\":123.102,\"float_c\":123.10199737548828,\"float_un_c\":123.10299682617188,\"float_un_z_c\":123.10399627685547,\"double_c\":404.4443,\"double_un_c\":404.4444,\"double_un_z_c\":404.4445,\"decimal_c\":123.4567,\"decimal_un_c\":123.4568,\"decimal_un_z_c\":123.4569,\"numeric_c\":346,\"big_decimal_c\":34567892.1,\"bit1_c\":false,\"tiny1_c\":1,\"boolean_c\":1,\"date_c\":\"2021-01-01\",\"time_c\":\"12:04:08\",\"datetime_c\":\"2021-01-01 12:04:08\",\"timestamp_c\":\"2021-01-01 12:04:08\",\"text_c\":\"text\",\"year_c\":2021,\"enum_c\":\"red\",\"set_c\":\"a,b\",\"json_c\":\"{\\\"key1\\\":\\\"value1\\\"}\"}";
        buildProcessStructRecord("normal_type", data, target);
    }

    @Test
    public void processMySQlGeoStructRecord() throws IOException {
        String schemaStr =
                "{\"keysType\":\"UNIQUE_KEYS\",\"properties\":[{\"name\":\"id\",\"aggregation_type\":\"\",\"comment\":\"\",\"type\":\"INT\"},{\"name\":\"geo_point\",\"aggregation_type\":\"REPLACE\",\"comment\":\"\",\"type\":\"STRING\"},{\"name\":\"geo_linestring\",\"aggregation_type\":\"REPLACE\",\"comment\":\"\",\"type\":\"STRING\"},{\"name\":\"geo_polygon\",\"aggregation_type\":\"REPLACE\",\"comment\":\"\",\"type\":\"STRING\"},{\"name\":\"geo_multipoint\",\"aggregation_type\":\"REPLACE\",\"comment\":\"\",\"type\":\"STRING\"},{\"name\":\"geo_multilinestring\",\"aggregation_type\":\"REPLACE\",\"comment\":\"\",\"type\":\"STRING\"},{\"name\":\"geo_multipolygon\",\"aggregation_type\":\"REPLACE\",\"comment\":\"\",\"type\":\"STRING\"},{\"name\":\"geo_geometry\",\"aggregation_type\":\"REPLACE\",\"comment\":\"\",\"type\":\"STRING\"},{\"name\":\"geo_geometrycollection\",\"aggregation_type\":\"REPLACE\",\"comment\":\"\",\"type\":\"STRING\"}],\"status\":200}";
        Schema schema = null;
        try {
            schema = objectMapper.readValue(schemaStr, Schema.class);
        } catch (JsonProcessingException e) {
            throw new DorisException(e);
        }
        mockRestService
                .when(() -> RestService.getSchema(any(), any(), any(), any()))
                .thenReturn(schema);

        String topic = "mysql_test.doris_test.geo_table";

        // no delete value
        String noDeleteValue =
                "{\"schema\":{\"type\":\"struct\",\"fields\":[{\"type\":\"struct\",\"fields\":[{\"type\":\"int32\",\"optional\":false,\"field\":\"id\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"double\",\"optional\":false,\"field\":\"x\"},{\"type\":\"double\",\"optional\":false,\"field\":\"y\"},{\"type\":\"bytes\",\"optional\":true,\"field\":\"wkb\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"srid\"}],\"optional\":true,\"name\":\"io.debezium.data.geometry.Point\",\"version\":1,\"doc\":\"Geometry (POINT)\",\"field\":\"geo_point\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"bytes\",\"optional\":false,\"field\":\"wkb\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"srid\"}],\"optional\":true,\"name\":\"io.debezium.data.geometry.Geometry\",\"version\":1,\"doc\":\"Geometry\",\"field\":\"geo_linestring\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"bytes\",\"optional\":false,\"field\":\"wkb\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"srid\"}],\"optional\":true,\"name\":\"io.debezium.data.geometry.Geometry\",\"version\":1,\"doc\":\"Geometry\",\"field\":\"geo_polygon\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"bytes\",\"optional\":false,\"field\":\"wkb\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"srid\"}],\"optional\":true,\"name\":\"io.debezium.data.geometry.Geometry\",\"version\":1,\"doc\":\"Geometry\",\"field\":\"geo_multipoint\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"bytes\",\"optional\":false,\"field\":\"wkb\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"srid\"}],\"optional\":true,\"name\":\"io.debezium.data.geometry.Geometry\",\"version\":1,\"doc\":\"Geometry\",\"field\":\"geo_multilinestring\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"bytes\",\"optional\":false,\"field\":\"wkb\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"srid\"}],\"optional\":true,\"name\":\"io.debezium.data.geometry.Geometry\",\"version\":1,\"doc\":\"Geometry\",\"field\":\"geo_multipolygon\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"bytes\",\"optional\":false,\"field\":\"wkb\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"srid\"}],\"optional\":true,\"name\":\"io.debezium.data.geometry.Geometry\",\"version\":1,\"doc\":\"Geometry\",\"field\":\"geo_geometry\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"bytes\",\"optional\":false,\"field\":\"wkb\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"srid\"}],\"optional\":true,\"name\":\"io.debezium.data.geometry.Geometry\",\"version\":1,\"doc\":\"Geometry\",\"field\":\"geo_geometrycollection\"}],\"optional\":true,\"name\":\"mysql_test.doris_test.geo_table.Value\",\"field\":\"before\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"int32\",\"optional\":false,\"field\":\"id\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"double\",\"optional\":false,\"field\":\"x\"},{\"type\":\"double\",\"optional\":false,\"field\":\"y\"},{\"type\":\"bytes\",\"optional\":true,\"field\":\"wkb\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"srid\"}],\"optional\":true,\"name\":\"io.debezium.data.geometry.Point\",\"version\":1,\"doc\":\"Geometry (POINT)\",\"field\":\"geo_point\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"bytes\",\"optional\":false,\"field\":\"wkb\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"srid\"}],\"optional\":true,\"name\":\"io.debezium.data.geometry.Geometry\",\"version\":1,\"doc\":\"Geometry\",\"field\":\"geo_linestring\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"bytes\",\"optional\":false,\"field\":\"wkb\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"srid\"}],\"optional\":true,\"name\":\"io.debezium.data.geometry.Geometry\",\"version\":1,\"doc\":\"Geometry\",\"field\":\"geo_polygon\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"bytes\",\"optional\":false,\"field\":\"wkb\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"srid\"}],\"optional\":true,\"name\":\"io.debezium.data.geometry.Geometry\",\"version\":1,\"doc\":\"Geometry\",\"field\":\"geo_multipoint\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"bytes\",\"optional\":false,\"field\":\"wkb\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"srid\"}],\"optional\":true,\"name\":\"io.debezium.data.geometry.Geometry\",\"version\":1,\"doc\":\"Geometry\",\"field\":\"geo_multilinestring\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"bytes\",\"optional\":false,\"field\":\"wkb\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"srid\"}],\"optional\":true,\"name\":\"io.debezium.data.geometry.Geometry\",\"version\":1,\"doc\":\"Geometry\",\"field\":\"geo_multipolygon\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"bytes\",\"optional\":false,\"field\":\"wkb\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"srid\"}],\"optional\":true,\"name\":\"io.debezium.data.geometry.Geometry\",\"version\":1,\"doc\":\"Geometry\",\"field\":\"geo_geometry\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"bytes\",\"optional\":false,\"field\":\"wkb\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"srid\"}],\"optional\":true,\"name\":\"io.debezium.data.geometry.Geometry\",\"version\":1,\"doc\":\"Geometry\",\"field\":\"geo_geometrycollection\"}],\"optional\":true,\"name\":\"mysql_test.doris_test.geo_table.Value\",\"field\":\"after\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"string\",\"optional\":false,\"field\":\"version\"},{\"type\":\"string\",\"optional\":false,\"field\":\"connector\"},{\"type\":\"string\",\"optional\":false,\"field\":\"name\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"ts_ms\"},{\"type\":\"string\",\"optional\":true,\"name\":\"io.debezium.data.Enum\",\"version\":1,\"parameters\":{\"allowed\":\"true,last,false,incremental\"},\"default\":\"false\",\"field\":\"snapshot\"},{\"type\":\"string\",\"optional\":false,\"field\":\"db\"},{\"type\":\"string\",\"optional\":true,\"field\":\"sequence\"},{\"type\":\"string\",\"optional\":true,\"field\":\"table\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"server_id\"},{\"type\":\"string\",\"optional\":true,\"field\":\"gtid\"},{\"type\":\"string\",\"optional\":false,\"field\":\"file\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"pos\"},{\"type\":\"int32\",\"optional\":false,\"field\":\"row\"},{\"type\":\"int64\",\"optional\":true,\"field\":\"thread\"},{\"type\":\"string\",\"optional\":true,\"field\":\"query\"}],\"optional\":false,\"name\":\"io.debezium.connector.mysql.Source\",\"field\":\"source\"},{\"type\":\"string\",\"optional\":false,\"field\":\"op\"},{\"type\":\"int64\",\"optional\":true,\"field\":\"ts_ms\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"string\",\"optional\":false,\"field\":\"id\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"total_order\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"data_collection_order\"}],\"optional\":true,\"field\":\"transaction\"}],\"optional\":false,\"name\":\"mysql_test.doris_test.geo_table.Envelope\"},\"payload\":{\"before\":null,\"after\":{\"id\":1,\"geo_point\":{\"x\":1.0,\"y\":1.0,\"wkb\":\"AQEAAAAAAAAAAADwPwAAAAAAAPA/\",\"srid\":null},\"geo_linestring\":{\"wkb\":\"AQIAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAABA\",\"srid\":null},\"geo_polygon\":{\"wkb\":\"AQMAAAABAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\",\"srid\":null},\"geo_multipoint\":{\"wkb\":\"AQQAAAACAAAAAQEAAAAAAAAAAAAAAAAAAAAAAAAAAQEAAAAAAAAAAADwPwAAAAAAAPA/\",\"srid\":null},\"geo_multilinestring\":{\"wkb\":\"AQUAAAACAAAAAQIAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPA/AAAAAAAA8D8BAgAAAAIAAAAAAAAAAAAAQAAAAAAAAABAAAAAAAAACEAAAAAAAAAIQA==\",\"srid\":null},\"geo_multipolygon\":{\"wkb\":\"AQYAAAACAAAAAQMAAAABAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAwAAAAEAAAAEAAAAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAAhAAAAAAAAACEAAAAAAAAAIQAAAAAAAAABAAAAAAAAAAEAAAAAAAAAAQA==\",\"srid\":null},\"geo_geometry\":{\"wkb\":\"AQcAAAACAAAAAQEAAAAAAAAAAADwPwAAAAAAAPA/AQIAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPA/AAAAAAAA8D8=\",\"srid\":null},\"geo_geometrycollection\":{\"wkb\":\"AQcAAAADAAAAAQEAAAAAAAAAAAAAQAAAAAAAAABAAQIAAAACAAAAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAAhAAAAAAAAACEABAwAAAAEAAAAEAAAAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAAhAAAAAAAAACEAAAAAAAAAIQAAAAAAAAABAAAAAAAAAAEAAAAAAAAAAQA==\",\"srid\":null}},\"source\":{\"version\":\"1.9.8.Final\",\"connector\":\"mysql\",\"name\":\"mysql_test\",\"ts_ms\":1717483867000,\"snapshot\":\"false\",\"db\":\"doris_test\",\"sequence\":null,\"table\":\"geo_table\",\"server_id\":1,\"gtid\":null,\"file\":\"mysql-bin.000366\",\"pos\":896,\"row\":0,\"thread\":6,\"query\":null},\"op\":\"c\",\"ts_ms\":1717483867044,\"transaction\":null}}";
        String expectedNoDeleteValue =
                "{\"id\":1,\"geo_point\":\"{coordinates=[1,1], type=Point, srid=0}\",\"geo_linestring\":\"{coordinates=[[0,0],[1,1],[2,2]], type=LineString, srid=0}\",\"geo_polygon\":\"{coordinates=[[[0,0],[1,0],[1,1],[0,0]]], type=Polygon, srid=0}\",\"geo_multipoint\":\"{coordinates=[[0,0],[1,1]], type=MultiPoint, srid=0}\",\"geo_multilinestring\":\"{coordinates=[[[0,0],[1,1]],[[2,2],[3,3]]], type=MultiLineString, srid=0}\",\"geo_multipolygon\":\"{coordinates=[[[[0,0],[1,0],[1,1],[0,0]]],[[[2,2],[3,2],[3,3],[2,2]]]], type=MultiPolygon, srid=0}\",\"geo_geometry\":\"{geometries=[{\\\"type\\\":\\\"Point\\\",\\\"coordinates\\\":[1,1]},{\\\"type\\\":\\\"LineString\\\",\\\"coordinates\\\":[[0,0],[1,1]]}], type=GeometryCollection, srid=0}\",\"geo_geometrycollection\":\"{geometries=[{\\\"type\\\":\\\"Point\\\",\\\"coordinates\\\":[2,2]},{\\\"type\\\":\\\"LineString\\\",\\\"coordinates\\\":[[2,2],[3,3]]},{\\\"type\\\":\\\"Polygon\\\",\\\"coordinates\\\":[[[2,2],[3,2],[3,3],[2,2]]]}], type=GeometryCollection, srid=0}\",\"__DORIS_DELETE_SIGN__\":\"0\"}";
        buildProcessStructRecord(topic, noDeleteValue, expectedNoDeleteValue);

        // delete value
        String deleteValue =
                "{\"schema\":{\"type\":\"struct\",\"fields\":[{\"type\":\"struct\",\"fields\":[{\"type\":\"int32\",\"optional\":false,\"field\":\"id\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"double\",\"optional\":false,\"field\":\"x\"},{\"type\":\"double\",\"optional\":false,\"field\":\"y\"},{\"type\":\"bytes\",\"optional\":true,\"field\":\"wkb\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"srid\"}],\"optional\":true,\"name\":\"io.debezium.data.geometry.Point\",\"version\":1,\"doc\":\"Geometry (POINT)\",\"field\":\"geo_point\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"bytes\",\"optional\":false,\"field\":\"wkb\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"srid\"}],\"optional\":true,\"name\":\"io.debezium.data.geometry.Geometry\",\"version\":1,\"doc\":\"Geometry\",\"field\":\"geo_linestring\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"bytes\",\"optional\":false,\"field\":\"wkb\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"srid\"}],\"optional\":true,\"name\":\"io.debezium.data.geometry.Geometry\",\"version\":1,\"doc\":\"Geometry\",\"field\":\"geo_polygon\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"bytes\",\"optional\":false,\"field\":\"wkb\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"srid\"}],\"optional\":true,\"name\":\"io.debezium.data.geometry.Geometry\",\"version\":1,\"doc\":\"Geometry\",\"field\":\"geo_multipoint\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"bytes\",\"optional\":false,\"field\":\"wkb\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"srid\"}],\"optional\":true,\"name\":\"io.debezium.data.geometry.Geometry\",\"version\":1,\"doc\":\"Geometry\",\"field\":\"geo_multilinestring\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"bytes\",\"optional\":false,\"field\":\"wkb\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"srid\"}],\"optional\":true,\"name\":\"io.debezium.data.geometry.Geometry\",\"version\":1,\"doc\":\"Geometry\",\"field\":\"geo_multipolygon\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"bytes\",\"optional\":false,\"field\":\"wkb\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"srid\"}],\"optional\":true,\"name\":\"io.debezium.data.geometry.Geometry\",\"version\":1,\"doc\":\"Geometry\",\"field\":\"geo_geometry\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"bytes\",\"optional\":false,\"field\":\"wkb\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"srid\"}],\"optional\":true,\"name\":\"io.debezium.data.geometry.Geometry\",\"version\":1,\"doc\":\"Geometry\",\"field\":\"geo_geometrycollection\"}],\"optional\":true,\"name\":\"mysql_test.doris_test.geo_table.Value\",\"field\":\"before\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"int32\",\"optional\":false,\"field\":\"id\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"double\",\"optional\":false,\"field\":\"x\"},{\"type\":\"double\",\"optional\":false,\"field\":\"y\"},{\"type\":\"bytes\",\"optional\":true,\"field\":\"wkb\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"srid\"}],\"optional\":true,\"name\":\"io.debezium.data.geometry.Point\",\"version\":1,\"doc\":\"Geometry (POINT)\",\"field\":\"geo_point\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"bytes\",\"optional\":false,\"field\":\"wkb\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"srid\"}],\"optional\":true,\"name\":\"io.debezium.data.geometry.Geometry\",\"version\":1,\"doc\":\"Geometry\",\"field\":\"geo_linestring\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"bytes\",\"optional\":false,\"field\":\"wkb\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"srid\"}],\"optional\":true,\"name\":\"io.debezium.data.geometry.Geometry\",\"version\":1,\"doc\":\"Geometry\",\"field\":\"geo_polygon\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"bytes\",\"optional\":false,\"field\":\"wkb\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"srid\"}],\"optional\":true,\"name\":\"io.debezium.data.geometry.Geometry\",\"version\":1,\"doc\":\"Geometry\",\"field\":\"geo_multipoint\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"bytes\",\"optional\":false,\"field\":\"wkb\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"srid\"}],\"optional\":true,\"name\":\"io.debezium.data.geometry.Geometry\",\"version\":1,\"doc\":\"Geometry\",\"field\":\"geo_multilinestring\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"bytes\",\"optional\":false,\"field\":\"wkb\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"srid\"}],\"optional\":true,\"name\":\"io.debezium.data.geometry.Geometry\",\"version\":1,\"doc\":\"Geometry\",\"field\":\"geo_multipolygon\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"bytes\",\"optional\":false,\"field\":\"wkb\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"srid\"}],\"optional\":true,\"name\":\"io.debezium.data.geometry.Geometry\",\"version\":1,\"doc\":\"Geometry\",\"field\":\"geo_geometry\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"bytes\",\"optional\":false,\"field\":\"wkb\"},{\"type\":\"int32\",\"optional\":true,\"field\":\"srid\"}],\"optional\":true,\"name\":\"io.debezium.data.geometry.Geometry\",\"version\":1,\"doc\":\"Geometry\",\"field\":\"geo_geometrycollection\"}],\"optional\":true,\"name\":\"mysql_test.doris_test.geo_table.Value\",\"field\":\"after\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"string\",\"optional\":false,\"field\":\"version\"},{\"type\":\"string\",\"optional\":false,\"field\":\"connector\"},{\"type\":\"string\",\"optional\":false,\"field\":\"name\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"ts_ms\"},{\"type\":\"string\",\"optional\":true,\"name\":\"io.debezium.data.Enum\",\"version\":1,\"parameters\":{\"allowed\":\"true,last,false,incremental\"},\"default\":\"false\",\"field\":\"snapshot\"},{\"type\":\"string\",\"optional\":false,\"field\":\"db\"},{\"type\":\"string\",\"optional\":true,\"field\":\"sequence\"},{\"type\":\"string\",\"optional\":true,\"field\":\"table\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"server_id\"},{\"type\":\"string\",\"optional\":true,\"field\":\"gtid\"},{\"type\":\"string\",\"optional\":false,\"field\":\"file\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"pos\"},{\"type\":\"int32\",\"optional\":false,\"field\":\"row\"},{\"type\":\"int64\",\"optional\":true,\"field\":\"thread\"},{\"type\":\"string\",\"optional\":true,\"field\":\"query\"}],\"optional\":false,\"name\":\"io.debezium.connector.mysql.Source\",\"field\":\"source\"},{\"type\":\"string\",\"optional\":false,\"field\":\"op\"},{\"type\":\"int64\",\"optional\":true,\"field\":\"ts_ms\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"string\",\"optional\":false,\"field\":\"id\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"total_order\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"data_collection_order\"}],\"optional\":true,\"field\":\"transaction\"}],\"optional\":false,\"name\":\"mysql_test.doris_test.geo_table.Envelope\"},\"payload\":{\"before\":{\"id\":1,\"geo_point\":{\"x\":1.0,\"y\":1.0,\"wkb\":\"AQEAAAAAAAAAAADwPwAAAAAAAPA/\",\"srid\":null},\"geo_linestring\":{\"wkb\":\"AQIAAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAQAAAAAAAAABA\",\"srid\":null},\"geo_polygon\":{\"wkb\":\"AQMAAAABAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\",\"srid\":null},\"geo_multipoint\":{\"wkb\":\"AQQAAAACAAAAAQEAAAAAAAAAAAAAAAAAAAAAAAAAAQEAAAAAAAAAAADwPwAAAAAAAPA/\",\"srid\":null},\"geo_multilinestring\":{\"wkb\":\"AQUAAAACAAAAAQIAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPA/AAAAAAAA8D8BAgAAAAIAAAAAAAAAAAAAQAAAAAAAAABAAAAAAAAACEAAAAAAAAAIQA==\",\"srid\":null},\"geo_multipolygon\":{\"wkb\":\"AQYAAAACAAAAAQMAAAABAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADwPwAAAAAAAPA/AAAAAAAA8D8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAwAAAAEAAAAEAAAAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAAhAAAAAAAAACEAAAAAAAAAIQAAAAAAAAABAAAAAAAAAAEAAAAAAAAAAQA==\",\"srid\":null},\"geo_geometry\":{\"wkb\":\"AQcAAAACAAAAAQEAAAAAAAAAAADwPwAAAAAAAPA/AQIAAAACAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAPA/AAAAAAAA8D8=\",\"srid\":null},\"geo_geometrycollection\":{\"wkb\":\"AQcAAAADAAAAAQEAAAAAAAAAAAAAQAAAAAAAAABAAQIAAAACAAAAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAAhAAAAAAAAACEABAwAAAAEAAAAEAAAAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAAhAAAAAAAAACEAAAAAAAAAIQAAAAAAAAABAAAAAAAAAAEAAAAAAAAAAQA==\",\"srid\":null}},\"after\":null,\"source\":{\"version\":\"1.9.8.Final\",\"connector\":\"mysql\",\"name\":\"mysql_test\",\"ts_ms\":1717489288000,\"snapshot\":\"false\",\"db\":\"doris_test\",\"sequence\":null,\"table\":\"geo_table\",\"server_id\":1,\"gtid\":null,\"file\":\"mysql-bin.000366\",\"pos\":1930,\"row\":0,\"thread\":10,\"query\":null},\"op\":\"d\",\"ts_ms\":1717489289115,\"transaction\":null}}";
        String expectedDeleteValue =
                "{\"id\":1,\"geo_point\":\"{coordinates=[1,1], type=Point, srid=0}\",\"geo_linestring\":\"{coordinates=[[0,0],[1,1],[2,2]], type=LineString, srid=0}\",\"geo_polygon\":\"{coordinates=[[[0,0],[1,0],[1,1],[0,0]]], type=Polygon, srid=0}\",\"geo_multipoint\":\"{coordinates=[[0,0],[1,1]], type=MultiPoint, srid=0}\",\"geo_multilinestring\":\"{coordinates=[[[0,0],[1,1]],[[2,2],[3,3]]], type=MultiLineString, srid=0}\",\"geo_multipolygon\":\"{coordinates=[[[[0,0],[1,0],[1,1],[0,0]]],[[[2,2],[3,2],[3,3],[2,2]]]], type=MultiPolygon, srid=0}\",\"geo_geometry\":\"{geometries=[{\\\"type\\\":\\\"Point\\\",\\\"coordinates\\\":[1,1]},{\\\"type\\\":\\\"LineString\\\",\\\"coordinates\\\":[[0,0],[1,1]]}], type=GeometryCollection, srid=0}\",\"geo_geometrycollection\":\"{geometries=[{\\\"type\\\":\\\"Point\\\",\\\"coordinates\\\":[2,2]},{\\\"type\\\":\\\"LineString\\\",\\\"coordinates\\\":[[2,2],[3,3]]},{\\\"type\\\":\\\"Polygon\\\",\\\"coordinates\\\":[[[2,2],[3,2],[3,3],[2,2]]]}], type=GeometryCollection, srid=0}\",\"__DORIS_DELETE_SIGN__\":\"1\"}";
        buildProcessStructRecord(topic, deleteValue, expectedDeleteValue);
    }

    @Test
    public void processPostgreArrayStructRecord() throws IOException {
        String schemaStr =
                "{\"keysType\":\"DUP_KEYS\",\"properties\":[{\"name\":\"int_array\",\"aggregation_type\":\"\",\"comment\":\"\",\"type\":\"INT\"},{\"name\":\"smallint_array\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"ARRAY\"},{\"name\":\"bigint_array\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"ARRAY\"},{\"name\":\"float_array\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"ARRAY\"},{\"name\":\"double_array\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"ARRAY\"},{\"name\":\"numeric_array\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"ARRAY\"},{\"name\":\"text_array\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"ARRAY\"},{\"name\":\"varchar_array\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"ARRAY\"},{\"name\":\"char_array\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"ARRAY\"},{\"name\":\"bool_array\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"ARRAY\"},{\"name\":\"date_array\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"ARRAY\"},{\"name\":\"timestamp_array\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"ARRAY\"},{\"name\":\"time_array\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"ARRAY\"},{\"name\":\"interval_array\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"ARRAY\"},{\"name\":\"uuid_array\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"ARRAY\"},{\"name\":\"json_array\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"STRING\"},{\"name\":\"jsonb_array\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"STRING\"},{\"name\":\"inet_array\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"ARRAY\"},{\"name\":\"cidr_array\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"ARRAY\"},{\"name\":\"macaddr_array\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"ARRAY\"},{\"name\":\"macaddr8_array\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"ARRAY\"}],\"status\":200}";
        Schema schema = null;
        try {
            schema = objectMapper.readValue(schemaStr, Schema.class);
        } catch (JsonProcessingException e) {
            throw new DorisException(e);
        }
        mockRestService
                .when(() -> RestService.getSchema(any(), any(), any(), any()))
                .thenReturn(schema);

        String topic = "pg_test.doris_cdc.all_array_types";
        String noDeleteArrayValue =
                "{\"schema\":{\"type\":\"struct\",\"fields\":[{\"type\":\"struct\",\"fields\":[{\"type\":\"int32\",\"optional\":true,\"field\":\"id\"},{\"type\":\"array\",\"items\":{\"type\":\"int16\",\"optional\":true},\"optional\":true,\"field\":\"smallint_array\"},{\"type\":\"array\",\"items\":{\"type\":\"int64\",\"optional\":true},\"optional\":true,\"field\":\"bigint_array\"},{\"type\":\"array\",\"items\":{\"type\":\"float\",\"optional\":true},\"optional\":true,\"field\":\"float_array\"},{\"type\":\"array\",\"items\":{\"type\":\"double\",\"optional\":true},\"optional\":true,\"field\":\"double_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true},\"optional\":true,\"field\":\"numeric_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true},\"optional\":true,\"field\":\"text_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true},\"optional\":true,\"field\":\"varchar_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true},\"optional\":true,\"field\":\"char_array\"},{\"type\":\"array\",\"items\":{\"type\":\"boolean\",\"optional\":true},\"optional\":true,\"field\":\"bool_array\"},{\"type\":\"array\",\"items\":{\"type\":\"int32\",\"optional\":true,\"name\":\"io.debezium.time.Date\",\"version\":1},\"optional\":true,\"field\":\"date_array\"},{\"type\":\"array\",\"items\":{\"type\":\"int64\",\"optional\":true,\"name\":\"io.debezium.time.MicroTimestamp\",\"version\":1},\"optional\":true,\"field\":\"timestamp_array\"},{\"type\":\"array\",\"items\":{\"type\":\"int64\",\"optional\":true,\"name\":\"io.debezium.time.MicroTime\",\"version\":1},\"optional\":true,\"field\":\"time_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true,\"name\":\"io.debezium.data.Uuid\",\"version\":1},\"optional\":true,\"field\":\"uuid_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true,\"name\":\"io.debezium.data.Json\",\"version\":1},\"optional\":true,\"field\":\"json_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true,\"name\":\"io.debezium.data.Json\",\"version\":1},\"optional\":true,\"field\":\"jsonb_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true},\"optional\":true,\"field\":\"inet_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true},\"optional\":true,\"field\":\"cidr_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true},\"optional\":true,\"field\":\"macaddr_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true},\"optional\":true,\"field\":\"macaddr8_array\"}],\"optional\":true,\"name\":\"pg_test.doris_cdc.all_array_types.Value\",\"field\":\"before\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"int32\",\"optional\":true,\"field\":\"id\"},{\"type\":\"array\",\"items\":{\"type\":\"int16\",\"optional\":true},\"optional\":true,\"field\":\"smallint_array\"},{\"type\":\"array\",\"items\":{\"type\":\"int64\",\"optional\":true},\"optional\":true,\"field\":\"bigint_array\"},{\"type\":\"array\",\"items\":{\"type\":\"float\",\"optional\":true},\"optional\":true,\"field\":\"float_array\"},{\"type\":\"array\",\"items\":{\"type\":\"double\",\"optional\":true},\"optional\":true,\"field\":\"double_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true},\"optional\":true,\"field\":\"numeric_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true},\"optional\":true,\"field\":\"text_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true},\"optional\":true,\"field\":\"varchar_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true},\"optional\":true,\"field\":\"char_array\"},{\"type\":\"array\",\"items\":{\"type\":\"boolean\",\"optional\":true},\"optional\":true,\"field\":\"bool_array\"},{\"type\":\"array\",\"items\":{\"type\":\"int32\",\"optional\":true,\"name\":\"io.debezium.time.Date\",\"version\":1},\"optional\":true,\"field\":\"date_array\"},{\"type\":\"array\",\"items\":{\"type\":\"int64\",\"optional\":true,\"name\":\"io.debezium.time.MicroTimestamp\",\"version\":1},\"optional\":true,\"field\":\"timestamp_array\"},{\"type\":\"array\",\"items\":{\"type\":\"int64\",\"optional\":true,\"name\":\"io.debezium.time.MicroTime\",\"version\":1},\"optional\":true,\"field\":\"time_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true,\"name\":\"io.debezium.data.Uuid\",\"version\":1},\"optional\":true,\"field\":\"uuid_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true,\"name\":\"io.debezium.data.Json\",\"version\":1},\"optional\":true,\"field\":\"json_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true,\"name\":\"io.debezium.data.Json\",\"version\":1},\"optional\":true,\"field\":\"jsonb_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true},\"optional\":true,\"field\":\"inet_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true},\"optional\":true,\"field\":\"cidr_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true},\"optional\":true,\"field\":\"macaddr_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true},\"optional\":true,\"field\":\"macaddr8_array\"}],\"optional\":true,\"name\":\"pg_test.doris_cdc.all_array_types.Value\",\"field\":\"after\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"string\",\"optional\":false,\"field\":\"version\"},{\"type\":\"string\",\"optional\":false,\"field\":\"connector\"},{\"type\":\"string\",\"optional\":false,\"field\":\"name\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"ts_ms\"},{\"type\":\"string\",\"optional\":true,\"name\":\"io.debezium.data.Enum\",\"version\":1,\"parameters\":{\"allowed\":\"true,last,false,incremental\"},\"default\":\"false\",\"field\":\"snapshot\"},{\"type\":\"string\",\"optional\":false,\"field\":\"db\"},{\"type\":\"string\",\"optional\":true,\"field\":\"sequence\"},{\"type\":\"string\",\"optional\":false,\"field\":\"schema\"},{\"type\":\"string\",\"optional\":false,\"field\":\"table\"},{\"type\":\"int64\",\"optional\":true,\"field\":\"txId\"},{\"type\":\"int64\",\"optional\":true,\"field\":\"lsn\"},{\"type\":\"int64\",\"optional\":true,\"field\":\"xmin\"}],\"optional\":false,\"name\":\"io.debezium.connector.postgresql.Source\",\"field\":\"source\"},{\"type\":\"string\",\"optional\":false,\"field\":\"op\"},{\"type\":\"int64\",\"optional\":true,\"field\":\"ts_ms\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"string\",\"optional\":false,\"field\":\"id\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"total_order\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"data_collection_order\"}],\"optional\":true,\"field\":\"transaction\"}],\"optional\":false,\"name\":\"pg_test.doris_cdc.all_array_types.Envelope\"},\"payload\":{\"before\":null,\"after\":{\"id\":1,\"smallint_array\":[100,200,300],\"bigint_array\":[10000000000,20000000000,30000000000],\"float_array\":[1.1,2.2,3.3],\"double_array\":[10.1,20.2,30.3],\"numeric_array\":[\"123.45\",\"678.90\"],\"text_array\":[\"text1\",\"text2\",\"text3\"],\"varchar_array\":[\"varchar1\",\"varchar2\",\"varchar3\"],\"char_array\":[\"c\",\"h\",\"a\"],\"bool_array\":[true,false,true],\"date_array\":[19723,19724],\"timestamp_array\":[1641031200000000,1672567200000000,1704103200000000],\"time_array\":[36000000000,39600000000,43200000000],\"uuid_array\":[\"550e8400-e29b-41d4-a716-446655440000\",\"550e8400-e29b-41d4-a716-446655440001\",\"550e8400-e29b-41d4-a716-446655440002\"],\"json_array\":[\"{\\\"key\\\": \\\"value1\\\"}\",\"{\\\"key\\\": \\\"value2\\\"}\",\"{\\\"key\\\": \\\"value3\\\"}\"],\"jsonb_array\":[\"{\\\"key\\\": \\\"value1\\\"}\",\"{\\\"key\\\": \\\"value2\\\"}\",\"{\\\"key\\\": \\\"value3\\\"}\"],\"inet_array\":[\"192.168.0.1\",\"192.168.0.2\",\"192.168.0.3\"],\"cidr_array\":[\"192.168.100.0/24\",\"192.168.101.0/24\",\"192.168.102.0/24\"],\"macaddr_array\":[\"08:00:2b:01:02:03\",\"08:00:2b:01:02:04\",\"08:00:2b:01:02:05\"],\"macaddr8_array\":[\"08:00:2b:01:02:03:04:05\",\"08:00:2b:01:02:03:04:06\",\"08:00:2b:01:02:03:04:07\"]},\"source\":{\"version\":\"1.9.8.Final\",\"connector\":\"postgresql\",\"name\":\"pg_test\",\"ts_ms\":1717491350397,\"snapshot\":\"false\",\"db\":\"test_cdc\",\"sequence\":\"[\\\"95025840\\\",\\\"95026320\\\"]\",\"schema\":\"doris_cdc\",\"table\":\"all_array_types\",\"txId\":2462,\"lsn\":95026320,\"xmin\":null},\"op\":\"c\",\"ts_ms\":1717491422530,\"transaction\":null}}";
        String expectedNoDeleteArrayValue =
                "{\"id\":1,\"smallint_array\":\"[100, 200, 300]\",\"bigint_array\":\"[10000000000, 20000000000, 30000000000]\",\"float_array\":\"[1.1, 2.2, 3.3]\",\"double_array\":\"[10.1, 20.2, 30.3]\",\"numeric_array\":\"[123.45, 678.90]\",\"text_array\":\"[text1, text2, text3]\",\"varchar_array\":\"[varchar1, varchar2, varchar3]\",\"char_array\":\"[c, h, a]\",\"bool_array\":\"[true, false, true]\",\"date_array\":\"[2024-01-01, 2024-01-02]\",\"timestamp_array\":\"[2022-01-01T10:00, 2023-01-01T10:00, 2024-01-01T10:00]\",\"time_array\":\"[10:00:00, 11:00:00, 12:00:00]\",\"uuid_array\":\"[550e8400-e29b-41d4-a716-446655440000, 550e8400-e29b-41d4-a716-446655440001, 550e8400-e29b-41d4-a716-446655440002]\",\"json_array\":\"[{\\\"key\\\": \\\"value1\\\"}, {\\\"key\\\": \\\"value2\\\"}, {\\\"key\\\": \\\"value3\\\"}]\",\"jsonb_array\":\"[{\\\"key\\\": \\\"value1\\\"}, {\\\"key\\\": \\\"value2\\\"}, {\\\"key\\\": \\\"value3\\\"}]\",\"inet_array\":\"[192.168.0.1, 192.168.0.2, 192.168.0.3]\",\"cidr_array\":\"[192.168.100.0/24, 192.168.101.0/24, 192.168.102.0/24]\",\"macaddr_array\":\"[08:00:2b:01:02:03, 08:00:2b:01:02:04, 08:00:2b:01:02:05]\",\"macaddr8_array\":\"[08:00:2b:01:02:03:04:05, 08:00:2b:01:02:03:04:06, 08:00:2b:01:02:03:04:07]\",\"__DORIS_DELETE_SIGN__\":\"0\"}";
        buildProcessStructRecord(topic, noDeleteArrayValue, expectedNoDeleteArrayValue);

        String deletedArrayValue =
                "{\"schema\":{\"type\":\"struct\",\"fields\":[{\"type\":\"struct\",\"fields\":[{\"type\":\"int32\",\"optional\":true,\"field\":\"id\"},{\"type\":\"array\",\"items\":{\"type\":\"int16\",\"optional\":true},\"optional\":true,\"field\":\"smallint_array\"},{\"type\":\"array\",\"items\":{\"type\":\"int64\",\"optional\":true},\"optional\":true,\"field\":\"bigint_array\"},{\"type\":\"array\",\"items\":{\"type\":\"float\",\"optional\":true},\"optional\":true,\"field\":\"float_array\"},{\"type\":\"array\",\"items\":{\"type\":\"double\",\"optional\":true},\"optional\":true,\"field\":\"double_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true},\"optional\":true,\"field\":\"numeric_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true},\"optional\":true,\"field\":\"text_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true},\"optional\":true,\"field\":\"varchar_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true},\"optional\":true,\"field\":\"char_array\"},{\"type\":\"array\",\"items\":{\"type\":\"boolean\",\"optional\":true},\"optional\":true,\"field\":\"bool_array\"},{\"type\":\"array\",\"items\":{\"type\":\"int32\",\"optional\":true,\"name\":\"io.debezium.time.Date\",\"version\":1},\"optional\":true,\"field\":\"date_array\"},{\"type\":\"array\",\"items\":{\"type\":\"int64\",\"optional\":true,\"name\":\"io.debezium.time.MicroTimestamp\",\"version\":1},\"optional\":true,\"field\":\"timestamp_array\"},{\"type\":\"array\",\"items\":{\"type\":\"int64\",\"optional\":true,\"name\":\"io.debezium.time.MicroTime\",\"version\":1},\"optional\":true,\"field\":\"time_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true,\"name\":\"io.debezium.data.Uuid\",\"version\":1},\"optional\":true,\"field\":\"uuid_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true,\"name\":\"io.debezium.data.Json\",\"version\":1},\"optional\":true,\"field\":\"json_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true,\"name\":\"io.debezium.data.Json\",\"version\":1},\"optional\":true,\"field\":\"jsonb_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true},\"optional\":true,\"field\":\"inet_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true},\"optional\":true,\"field\":\"cidr_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true},\"optional\":true,\"field\":\"macaddr_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true},\"optional\":true,\"field\":\"macaddr8_array\"}],\"optional\":true,\"name\":\"pg_test.doris_cdc.all_array_types.Value\",\"field\":\"before\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"int32\",\"optional\":true,\"field\":\"id\"},{\"type\":\"array\",\"items\":{\"type\":\"int16\",\"optional\":true},\"optional\":true,\"field\":\"smallint_array\"},{\"type\":\"array\",\"items\":{\"type\":\"int64\",\"optional\":true},\"optional\":true,\"field\":\"bigint_array\"},{\"type\":\"array\",\"items\":{\"type\":\"float\",\"optional\":true},\"optional\":true,\"field\":\"float_array\"},{\"type\":\"array\",\"items\":{\"type\":\"double\",\"optional\":true},\"optional\":true,\"field\":\"double_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true},\"optional\":true,\"field\":\"numeric_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true},\"optional\":true,\"field\":\"text_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true},\"optional\":true,\"field\":\"varchar_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true},\"optional\":true,\"field\":\"char_array\"},{\"type\":\"array\",\"items\":{\"type\":\"boolean\",\"optional\":true},\"optional\":true,\"field\":\"bool_array\"},{\"type\":\"array\",\"items\":{\"type\":\"int32\",\"optional\":true,\"name\":\"io.debezium.time.Date\",\"version\":1},\"optional\":true,\"field\":\"date_array\"},{\"type\":\"array\",\"items\":{\"type\":\"int64\",\"optional\":true,\"name\":\"io.debezium.time.MicroTimestamp\",\"version\":1},\"optional\":true,\"field\":\"timestamp_array\"},{\"type\":\"array\",\"items\":{\"type\":\"int64\",\"optional\":true,\"name\":\"io.debezium.time.MicroTime\",\"version\":1},\"optional\":true,\"field\":\"time_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true,\"name\":\"io.debezium.data.Uuid\",\"version\":1},\"optional\":true,\"field\":\"uuid_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true,\"name\":\"io.debezium.data.Json\",\"version\":1},\"optional\":true,\"field\":\"json_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true,\"name\":\"io.debezium.data.Json\",\"version\":1},\"optional\":true,\"field\":\"jsonb_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true},\"optional\":true,\"field\":\"inet_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true},\"optional\":true,\"field\":\"cidr_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true},\"optional\":true,\"field\":\"macaddr_array\"},{\"type\":\"array\",\"items\":{\"type\":\"string\",\"optional\":true},\"optional\":true,\"field\":\"macaddr8_array\"}],\"optional\":true,\"name\":\"pg_test.doris_cdc.all_array_types.Value\",\"field\":\"after\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"string\",\"optional\":false,\"field\":\"version\"},{\"type\":\"string\",\"optional\":false,\"field\":\"connector\"},{\"type\":\"string\",\"optional\":false,\"field\":\"name\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"ts_ms\"},{\"type\":\"string\",\"optional\":true,\"name\":\"io.debezium.data.Enum\",\"version\":1,\"parameters\":{\"allowed\":\"true,last,false,incremental\"},\"default\":\"false\",\"field\":\"snapshot\"},{\"type\":\"string\",\"optional\":false,\"field\":\"db\"},{\"type\":\"string\",\"optional\":true,\"field\":\"sequence\"},{\"type\":\"string\",\"optional\":false,\"field\":\"schema\"},{\"type\":\"string\",\"optional\":false,\"field\":\"table\"},{\"type\":\"int64\",\"optional\":true,\"field\":\"txId\"},{\"type\":\"int64\",\"optional\":true,\"field\":\"lsn\"},{\"type\":\"int64\",\"optional\":true,\"field\":\"xmin\"}],\"optional\":false,\"name\":\"io.debezium.connector.postgresql.Source\",\"field\":\"source\"},{\"type\":\"string\",\"optional\":false,\"field\":\"op\"},{\"type\":\"int64\",\"optional\":true,\"field\":\"ts_ms\"},{\"type\":\"struct\",\"fields\":[{\"type\":\"string\",\"optional\":false,\"field\":\"id\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"total_order\"},{\"type\":\"int64\",\"optional\":false,\"field\":\"data_collection_order\"}],\"optional\":true,\"field\":\"transaction\"}],\"optional\":false,\"name\":\"pg_test.doris_cdc.all_array_types.Envelope\"},\"payload\":{\"before\":{\"id\":1,\"smallint_array\":[100,200,300],\"bigint_array\":[10000000000,20000000000,30000000000],\"float_array\":[1.1,2.2,3.3],\"double_array\":[10.1,20.2,30.3],\"numeric_array\":[\"123.45\",\"678.90\"],\"text_array\":[\"text1\",\"text2\",\"text3\"],\"varchar_array\":[\"varchar1\",\"varchar2\",\"varchar3\"],\"char_array\":[\"c\",\"h\",\"a\"],\"bool_array\":[true,false,true],\"date_array\":[19723,19724],\"timestamp_array\":[1641031200000000,1672567200000000,1704103200000000],\"time_array\":[36000000000,39600000000,43200000000],\"uuid_array\":[\"550e8400-e29b-41d4-a716-446655440000\",\"550e8400-e29b-41d4-a716-446655440001\",\"550e8400-e29b-41d4-a716-446655440002\"],\"json_array\":[\"{\\\"key\\\": \\\"value1\\\"}\",\"{\\\"key\\\": \\\"value2\\\"}\",\"{\\\"key\\\": \\\"value3\\\"}\"],\"jsonb_array\":[\"{\\\"key\\\": \\\"value1\\\"}\",\"{\\\"key\\\": \\\"value2\\\"}\",\"{\\\"key\\\": \\\"value3\\\"}\"],\"inet_array\":[\"192.168.0.1\",\"192.168.0.2\",\"192.168.0.3\"],\"cidr_array\":[\"192.168.100.0/24\",\"192.168.101.0/24\",\"192.168.102.0/24\"],\"macaddr_array\":[\"08:00:2b:01:02:03\",\"08:00:2b:01:02:04\",\"08:00:2b:01:02:05\"],\"macaddr8_array\":[\"08:00:2b:01:02:03:04:05\",\"08:00:2b:01:02:03:04:06\",\"08:00:2b:01:02:03:04:07\"]},\"after\":null,\"source\":{\"version\":\"1.9.8.Final\",\"connector\":\"postgresql\",\"name\":\"pg_test\",\"ts_ms\":1717492866396,\"snapshot\":\"false\",\"db\":\"test_cdc\",\"sequence\":\"[\\\"95034728\\\",\\\"95035112\\\"]\",\"schema\":\"doris_cdc\",\"table\":\"all_array_types\",\"txId\":2466,\"lsn\":95035112,\"xmin\":null},\"op\":\"d\",\"ts_ms\":1717492866895,\"transaction\":null}}";
        String expectedDeleteArrayValue =
                "{\"id\":1,\"smallint_array\":\"[100, 200, 300]\",\"bigint_array\":\"[10000000000, 20000000000, 30000000000]\",\"float_array\":\"[1.1, 2.2, 3.3]\",\"double_array\":\"[10.1, 20.2, 30.3]\",\"numeric_array\":\"[123.45, 678.90]\",\"text_array\":\"[text1, text2, text3]\",\"varchar_array\":\"[varchar1, varchar2, varchar3]\",\"char_array\":\"[c, h, a]\",\"bool_array\":\"[true, false, true]\",\"date_array\":\"[2024-01-01, 2024-01-02]\",\"timestamp_array\":\"[2022-01-01T10:00, 2023-01-01T10:00, 2024-01-01T10:00]\",\"time_array\":\"[10:00:00, 11:00:00, 12:00:00]\",\"uuid_array\":\"[550e8400-e29b-41d4-a716-446655440000, 550e8400-e29b-41d4-a716-446655440001, 550e8400-e29b-41d4-a716-446655440002]\",\"json_array\":\"[{\\\"key\\\": \\\"value1\\\"}, {\\\"key\\\": \\\"value2\\\"}, {\\\"key\\\": \\\"value3\\\"}]\",\"jsonb_array\":\"[{\\\"key\\\": \\\"value1\\\"}, {\\\"key\\\": \\\"value2\\\"}, {\\\"key\\\": \\\"value3\\\"}]\",\"inet_array\":\"[192.168.0.1, 192.168.0.2, 192.168.0.3]\",\"cidr_array\":\"[192.168.100.0/24, 192.168.101.0/24, 192.168.102.0/24]\",\"macaddr_array\":\"[08:00:2b:01:02:03, 08:00:2b:01:02:04, 08:00:2b:01:02:05]\",\"macaddr8_array\":\"[08:00:2b:01:02:03:04:05, 08:00:2b:01:02:03:04:06, 08:00:2b:01:02:03:04:07]\",\"__DORIS_DELETE_SIGN__\":\"1\"}";
        buildProcessStructRecord(topic, deletedArrayValue, expectedDeleteArrayValue);
    }

    @Test
    public void processMapRecord() throws JsonProcessingException {
        Map<String, String> mapValue = new HashMap<>();
        mapValue.put("name", "doris");
        mapValue.put("key", "1");
        SinkRecord record = TestRecordBuffer.newSinkRecord(mapValue, 1);
        String s = recordService.processMapRecord(record);
        Assert.assertEquals("{\"name\":\"doris\",\"key\":\"1\"}", s);

        ObjectNode objectNode = new ObjectMapper().createObjectNode();
        objectNode.put("name", "doris");
        objectNode.put("key", "1");
        Assert.assertEquals(
                "{\"name\":\"doris\",\"key\":\"1\"}",
                new ObjectMapper().writeValueAsString(objectNode));
    }

    @Test
    public void testFeResponseToSchemaNotMap() throws Exception {
        String schemaStr =
                "{\"keysType_notExist\":\"UNIQUE_KEYS\",\"properties\":[{\"name\":\"id\", \"noExistField\":\"test\", \"aggregation_type\":\"\",\"comment\":\"\",\"type\":\"BIGINT\"},{\"name\":\"name\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"VARCHAR\"},{\"name\":\"age\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"INT\"},{\"name\":\"email\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"VARCHAR\"},{\"name\":\"birth_date\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"DATEV2\"},{\"name\":\"integer_column\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"INT\"},{\"name\":\"float_column\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"FLOAT\"},{\"name\":\"decimal_column\",\"aggregation_type\":\"NONE\",\"scale\":\"2\",\"comment\":\"\",\"type\":\"DECIMAL64\",\"precision\":\"10\"},{\"name\":\"datetime_column\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"DATETIMEV2\"},{\"name\":\"date_column\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"DATEV2\"},{\"name\":\"text_column\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"VARCHAR\"},{\"name\":\"binary_column\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"VARCHAR\"},{\"name\":\"is_active\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"BOOLEAN\"},{\"name\":\"varchar_column\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"VARCHAR\"},{\"name\":\"blob_column\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"STRING\"},{\"name\":\"time_column\",\"aggregation_type\":\"NONE\",\"comment\":\"\",\"type\":\"STRING\"}],\"status\":200}";
        Schema schema = null;
        try {
            schema = objectMapper.readValue(schemaStr, Schema.class);
        } catch (JsonProcessingException e) {
            throw new DorisException(e);
        }
        mockRestService
                .when(() -> RestService.getSchema(any(), any(), any(), any()))
                .thenReturn(schema);

        Assert.assertNull(schema.getKeysType());
        Assert.assertEquals(16, schema.getProperties().size());
    }

    @After
    public void close() {
        mockRestService.close();
    }
}
