/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.flink.connector.http.table.lookup;

import org.apache.flink.api.common.RuntimeExecutionMode;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.ExecutionOptions;
import org.apache.flink.connector.http.WireMockServerPortAllocator;
import org.apache.flink.connector.http.app.JsonTransformCustomerObject;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.TableResult;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.table.connector.source.lookup.cache.LookupCache;
import org.apache.flink.table.data.GenericRowData;
import org.apache.flink.table.data.binary.BinaryStringData;
import org.apache.flink.table.runtime.functions.table.lookup.LookupCacheManager;
import org.apache.flink.table.test.lookup.cache.LookupCacheAssert;
import org.apache.flink.types.Row;
import org.apache.flink.util.CloseableIterator;
import org.apache.flink.util.StringUtils;

import com.github.tomakehurst.wiremock.WireMockServer;
import com.github.tomakehurst.wiremock.client.MappingBuilder;
import com.github.tomakehurst.wiremock.core.WireMockConfiguration;
import com.github.tomakehurst.wiremock.matching.StringValuePattern;
import com.github.tomakehurst.wiremock.stubbing.Scenario;
import lombok.Builder;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;
import org.junit.jupiter.params.provider.MethodSource;
import org.junit.jupiter.params.provider.ValueSource;
import org.mockito.junit.jupiter.MockitoExtension;

import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.SortedSet;
import java.util.TreeSet;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;

import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
import static com.github.tomakehurst.wiremock.client.WireMock.equalTo;
import static com.github.tomakehurst.wiremock.client.WireMock.get;
import static com.github.tomakehurst.wiremock.client.WireMock.getRequestedFor;
import static com.github.tomakehurst.wiremock.client.WireMock.matching;
import static com.github.tomakehurst.wiremock.client.WireMock.matchingJsonPath;
import static com.github.tomakehurst.wiremock.client.WireMock.post;
import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor;
import static com.github.tomakehurst.wiremock.client.WireMock.put;
import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo;
import static com.github.tomakehurst.wiremock.client.WireMock.urlPathMatching;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.awaitility.Awaitility.await;

/** Test for {@link HttpLookupTableSource} connection. */
@Slf4j
@ExtendWith(MockitoExtension.class)
class HttpLookupTableSourceITCaseTest {

    private static int serverPort;

    private static int secServerPort;

    private static final String CERTS_PATH = "src/test/resources/security/certs/";

    private static final String SERVER_KEYSTORE_PATH =
            "src/test/resources/security/certs/serverKeyStore.jks";

    private static final String SERVER_TRUSTSTORE_PATH =
            "src/test/resources/security/certs/serverTrustStore.jks";

    private static final String ENDPOINT = "/client";

    public static final String A_TEST_STRING_THAT_IS_NOT_JSON = "A test string that is not json";

    private static final int SECONDS_TO_WAIT_FOR_RESPONSE = 5;

    /** Comparator for Flink SQL result. */
    private static final Comparator<Row> ROW_COMPARATOR =
            (row1, row2) -> {
                String row1Id = (String) Objects.requireNonNull(row1.getField("id"));
                String row2Id = (String) Objects.requireNonNull(row2.getField("id"));

                return row1Id.compareTo(row2Id);
            };

    private StreamTableEnvironment tEnv;

    private WireMockServer wireMockServer;

    private File keyStoreFile = new File(SERVER_KEYSTORE_PATH);
    private File trustStoreFile = new File(SERVER_TRUSTSTORE_PATH);

    @SuppressWarnings("unchecked")
    @BeforeEach
    void setup() {
        serverPort = WireMockServerPortAllocator.getServerPort();
        secServerPort = WireMockServerPortAllocator.getSecureServerPort();
        wireMockServer =
                new WireMockServer(
                        WireMockConfiguration.wireMockConfig()
                                .port(serverPort)
                                .httpsPort(secServerPort)
                                .keystorePath(keyStoreFile.getAbsolutePath())
                                .keystorePassword("password")
                                .keyManagerPassword("password")
                                .needClientAuth(true)
                                .trustStorePath(trustStoreFile.getAbsolutePath())
                                .trustStorePassword("password")
                                .extensions(JsonTransformLookup.class));
        wireMockServer.start();
        wireMockServer.resetAll();

        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        Configuration config = new Configuration();
        config.set(ExecutionOptions.RUNTIME_MODE, RuntimeExecutionMode.STREAMING);
        env.configure(config, getClass().getClassLoader());
        env.setParallelism(
                1); // wire mock server has problem with scenario state during parallel execution

        tEnv = StreamTableEnvironment.create(env);
    }

    @AfterEach
    void tearDown() {
        wireMockServer.stop();
    }

    @ParameterizedTest
    @ValueSource(strings = {"", "GET", "POST", "PUT"})
    void testHttpLookupJoin(String methodName) throws Exception {
        // GIVEN
        if (StringUtils.isNullOrWhitespaceOnly(methodName) || methodName.equalsIgnoreCase("GET")) {
            setupServerStub(wireMockServer);
        } else {
            setUpServerBodyStub(
                    methodName,
                    wireMockServer,
                    List.of(matchingJsonPath("$.id"), matchingJsonPath("$.id2")));
        }

        String lookupTable =
                "CREATE TABLE Customers ("
                        + "id STRING,"
                        + "id2 STRING,"
                        + "msg STRING,"
                        + "uuid STRING,"
                        + "details ROW<"
                        + "isActive BOOLEAN,"
                        + "nestedDetails ROW<"
                        + "balance STRING"
                        + ">"
                        + ">"
                        + ") WITH ("
                        + "'format' = 'json',"
                        + "'connector' = 'http',"
                        + ((StringUtils.isNullOrWhitespaceOnly(methodName))
                                ? ""
                                : "'lookup-method' = '" + methodName + "',")
                        + "'url' = 'http://localhost:"
                        + serverPort
                        + "/client',"
                        + "'http.source.lookup.header.Content-Type' = 'application/json',"
                        + "'asyncPolling' = 'true',"
                        + "'table.exec.async-lookup.buffer-capacity' = '50',"
                        + "'table.exec.async-lookup.timeout' = '20s'"
                        + ")";

        // WHEN
        SortedSet<Row> rows = testLookupJoin(lookupTable, 4);

        // THEN
        assertEnrichedRows(rows);
    }

    @Test
    void testHttpLookupJoinNoDataFromEndpoint() {
        // GIVEN
        setupServerStubEmptyResponse(wireMockServer);

        String lookupTable =
                "CREATE TABLE Customers ("
                        + "id STRING,"
                        + "id2 STRING,"
                        + "msg STRING,"
                        + "uuid STRING,"
                        + "details ROW<"
                        + "isActive BOOLEAN,"
                        + "nestedDetails ROW<"
                        + "balance STRING"
                        + ">"
                        + ">"
                        + ") WITH ("
                        + "'format' = 'json',"
                        + "'connector' = 'http',"
                        + "'url' = 'http://localhost:"
                        + serverPort
                        + "/client',"
                        + "'http.source.lookup.header.Content-Type' = 'application/json',"
                        + "'asyncPolling' = 'true'"
                        + ")";

        // WHEN/THEN
        assertThatThrownBy(() -> testLookupJoin(lookupTable, 4))
                .isInstanceOf(TimeoutException.class);
    }

    @Test
    void testLookupWithRetry() throws Exception {
        wireMockServer.stubFor(
                get(urlPathEqualTo(ENDPOINT))
                        .inScenario("retry")
                        .whenScenarioStateIs(Scenario.STARTED)
                        .withHeader("Content-Type", equalTo("application/json"))
                        .withQueryParam("id", matching("[0-9]+"))
                        .withQueryParam("id2", matching("[0-9]+"))
                        .willReturn(aResponse().withBody(new byte[0]).withStatus(501))
                        .willSetStateTo("temporal_issue_gone"));
        wireMockServer.stubFor(
                get(urlPathEqualTo(ENDPOINT))
                        .inScenario("retry")
                        .whenScenarioStateIs("temporal_issue_gone")
                        .withHeader("Content-Type", equalTo("application/json"))
                        .withQueryParam("id", matching("[0-9]+"))
                        .withQueryParam("id2", matching("[0-9]+"))
                        .willReturn(
                                aResponse()
                                        .withTransformers(JsonTransformLookup.NAME)
                                        .withStatus(200)));

        var lookupTable =
                "CREATE TABLE Customers ("
                        + "id STRING,"
                        + "id2 STRING,"
                        + "msg STRING,"
                        + "uuid STRING,"
                        + "details ROW<"
                        + "isActive BOOLEAN,"
                        + "nestedDetails ROW<"
                        + "balance STRING"
                        + ">"
                        + ">"
                        + ") WITH ("
                        + "'format' = 'json',"
                        + "'connector' = 'http',"
                        + "'url' = 'http://localhost:"
                        + serverPort
                        + "/client',"
                        + "'lookup.max-retries' = '3',"
                        + "'http.source.lookup.header.Content-Type' = 'application/json',"
                        + "'http.source.lookup.retry-strategy.type' = 'fixed-delay',"
                        + "'http.source.lookup.retry-strategy.fixed-delay.delay' = '1ms',"
                        + "'http.source.lookup.success-codes' = '2XX',"
                        + "'http.source.lookup.retry-codes' = '501'"
                        + ")";

        var result = testLookupJoin(lookupTable, 1);

        assertThat(result).hasSize(1);
        wireMockServer.verify(2, getRequestedFor(urlPathEqualTo(ENDPOINT)));
    }

    @Test
    void testLookupIgnoreResponse() throws Exception {
        wireMockServer.stubFor(
                get(urlPathEqualTo(ENDPOINT))
                        .inScenario("404_on_first")
                        .whenScenarioStateIs(Scenario.STARTED)
                        .withHeader("Content-Type", equalTo("application/json"))
                        .withQueryParam("id", matching("[0-9]+"))
                        .withQueryParam("id2", matching("[0-9]+"))
                        .willReturn(aResponse().withBody(JsonTransformLookup.NAME).withStatus(404))
                        .willSetStateTo("second_request"));
        wireMockServer.stubFor(
                get(urlPathEqualTo(ENDPOINT))
                        .inScenario("404_on_first")
                        .whenScenarioStateIs("second_request")
                        .withHeader("Content-Type", equalTo("application/json"))
                        .withQueryParam("id", matching("[0-9]+"))
                        .withQueryParam("id2", matching("[0-9]+"))
                        .willReturn(
                                aResponse()
                                        .withTransformers(JsonTransformLookup.NAME)
                                        .withStatus(200)));

        var lookupTable =
                "CREATE TABLE Customers ("
                        + "id STRING,"
                        + "id2 STRING,"
                        + "msg STRING,"
                        + "uuid STRING,"
                        + "details ROW<"
                        + "isActive BOOLEAN,"
                        + "nestedDetails ROW<"
                        + "balance STRING"
                        + ">"
                        + ">"
                        + ") WITH ("
                        + "'format' = 'json',"
                        + "'connector' = 'http',"
                        + "'url' = 'http://localhost:"
                        + serverPort
                        + "/client',"
                        + "'http.source.lookup.header.Content-Type' = 'application/json',"
                        + "'http.source.lookup.success-codes' = '2XX,404',"
                        + "'http.source.lookup.ignored-response-codes' = '404'"
                        + ")";

        var result = testLookupJoin(lookupTable, 3);

        assertThat(result).hasSize(2);
        wireMockServer.verify(3, getRequestedFor(urlPathEqualTo(ENDPOINT)));
    }

    @Test
    void testHttpsMTlsLookupJoin() throws Exception {

        // GIVEN
        File serverTrustedCert = new File(CERTS_PATH + "ca.crt");
        File clientCert = new File(CERTS_PATH + "client.crt");
        File clientPrivateKey = new File(CERTS_PATH + "clientPrivateKey.pem");

        setupServerStub(wireMockServer);

        String lookupTable =
                String.format(
                        "CREATE TABLE Customers ("
                                + "id STRING,"
                                + "id2 STRING,"
                                + "msg STRING,"
                                + "uuid STRING,"
                                + "details ROW<"
                                + "isActive BOOLEAN,"
                                + "nestedDetails ROW<"
                                + "balance STRING"
                                + ">"
                                + ">"
                                + ") WITH ("
                                + "'format' = 'json',"
                                + "'connector' = 'http',"
                                + "'url' = 'https://localhost:"
                                + secServerPort
                                + "/client',"
                                + "'http.source.lookup.header.Content-Type' = 'application/json',"
                                + "'asyncPolling' = 'true',"
                                + "'http.security.cert.server' = '%s',"
                                + "'http.security.cert.client' = '%s',"
                                + "'http.security.key.client' = '%s'"
                                + ")",
                        serverTrustedCert.getAbsolutePath(),
                        clientCert.getAbsolutePath(),
                        clientPrivateKey.getAbsolutePath());

        // WHEN
        SortedSet<Row> rows = testLookupJoin(lookupTable, 4);

        // THEN
        assertEnrichedRows(rows);
    }

    @Test
    void testLookupJoinProjectionPushDown() throws Exception {

        // GIVEN
        setUpServerBodyStub(
                "POST",
                wireMockServer,
                List.of(
                        matchingJsonPath("$.row.aStringColumn"),
                        matchingJsonPath("$.row.anIntColumn"),
                        matchingJsonPath("$.row.aFloatColumn")));

        String fields =
                "`row` ROW<`aStringColumn` STRING, `anIntColumn` INT, `aFloatColumn` FLOAT>\n";

        String sourceTable =
                "CREATE TABLE Orders (\n"
                        + "  proc_time AS PROCTIME(),\n"
                        + "  id STRING,\n"
                        + fields
                        + ") WITH ("
                        + "'connector' = 'datagen',"
                        + "'rows-per-second' = '1',"
                        + "'fields.id.kind' = 'sequence',"
                        + "'fields.id.start' = '1',"
                        + "'fields.id.end' = '5'"
                        + ")";

        String lookupTable =
                "CREATE TABLE Customers (\n"
                        + "  `enrichedInt` INT,\n"
                        + "  `enrichedString` STRING,\n"
                        + "  \n"
                        + fields
                        + ") WITH ("
                        + "'format' = 'json',"
                        + "'lookup-request.format' = 'json',"
                        + "'lookup-request.format.json.fail-on-missing-field' = 'true',"
                        + "'connector' = 'http',"
                        + "'lookup-method' = 'POST',"
                        + "'url' = 'http://localhost:"
                        + serverPort
                        + "/client',"
                        + "'http.source.lookup.header.Content-Type' = 'application/json',"
                        + "'asyncPolling' = 'true'"
                        + ")";

        tEnv.executeSql(sourceTable);
        tEnv.executeSql(lookupTable);

        // WHEN
        // SQL query that performs JOIN on both tables.
        String joinQuery =
                "CREATE TEMPORARY VIEW lookupResult AS "
                        + "SELECT o.id, o.`row`, c.enrichedInt, c.enrichedString FROM Orders AS o"
                        + " JOIN Customers FOR SYSTEM_TIME AS OF o.proc_time AS c"
                        + " ON (\n"
                        + "  o.`row` = c.`row`\n"
                        + ")";

        tEnv.executeSql(joinQuery);

        // SQL query that performs a projection pushdown to limit the number of columns
        String lastQuery = "SELECT r.id, r.enrichedInt FROM lookupResult r;";

        TableResult result = tEnv.executeSql(lastQuery);
        result.await(SECONDS_TO_WAIT_FOR_RESPONSE, TimeUnit.SECONDS);

        // THEN
        SortedSet<Row> collectedRows = getCollectedRows(result);

        collectedRows.stream().forEach(row -> assertThat(row.getArity()).isEqualTo(2));

        assertThat(collectedRows.size()).isEqualTo(5);
    }

    @Test
    void testLookupJoinProjectionPushDownNested() throws Exception {

        // GIVEN
        setUpServerBodyStub(
                "POST",
                wireMockServer,
                List.of(
                        matchingJsonPath("$.row.aStringColumn"),
                        matchingJsonPath("$.row.anIntColumn"),
                        matchingJsonPath("$.row.aFloatColumn")));

        String fields =
                "`row` ROW<`aStringColumn` STRING, `anIntColumn` INT, `aFloatColumn` FLOAT>\n";

        String sourceTable =
                "CREATE TABLE Orders (\n"
                        + "  proc_time AS PROCTIME(),\n"
                        + "  id STRING,\n"
                        + fields
                        + ") WITH ("
                        + "'connector' = 'datagen',"
                        + "'rows-per-second' = '1',"
                        + "'fields.id.kind' = 'sequence',"
                        + "'fields.id.start' = '1',"
                        + "'fields.id.end' = '5'"
                        + ")";

        String lookupTable =
                "CREATE TABLE Customers (\n"
                        + "  `enrichedInt` INT,\n"
                        + "  `enrichedString` STRING,\n"
                        + "  \n"
                        + fields
                        + ") WITH ("
                        + "'format' = 'json',"
                        + "'lookup-request.format' = 'json',"
                        + "'lookup-request.format.json.fail-on-missing-field' = 'true',"
                        + "'connector' = 'http',"
                        + "'lookup-method' = 'POST',"
                        + "'url' = 'http://localhost:"
                        + serverPort
                        + "/client',"
                        + "'http.source.lookup.header.Content-Type' = 'application/json',"
                        + "'asyncPolling' = 'true'"
                        + ")";

        tEnv.executeSql(sourceTable);
        tEnv.executeSql(lookupTable);

        // WHEN
        // SQL query that performs JOIN on both tables.
        String joinQuery =
                "CREATE TEMPORARY VIEW lookupResult AS "
                        + "SELECT o.id, o.`row`, c.enrichedInt, c.enrichedString FROM Orders AS o"
                        + " JOIN Customers FOR SYSTEM_TIME AS OF o.proc_time AS c"
                        + " ON (\n"
                        + "  o.`row` = c.`row`\n"
                        + ")";

        tEnv.executeSql(joinQuery);

        // SQL query that performs a project pushdown to take a subset of columns with nested value
        String lastQuery = "SELECT r.id, r.enrichedInt, r.`row`.aStringColumn FROM lookupResult r;";

        TableResult result = tEnv.executeSql(lastQuery);
        result.await(SECONDS_TO_WAIT_FOR_RESPONSE, TimeUnit.SECONDS);

        // THEN
        SortedSet<Row> collectedRows = getCollectedRows(result);

        collectedRows.stream().forEach(row -> assertThat(row.getArity()).isEqualTo(3));

        assertThat(collectedRows.size()).isEqualTo(5);
    }

    @Test
    void testLookupJoinOnRowType() throws Exception {

        // GIVEN
        setUpServerBodyStub(
                "POST",
                wireMockServer,
                List.of(
                        matchingJsonPath("$.row.aStringColumn"),
                        matchingJsonPath("$.row.anIntColumn"),
                        matchingJsonPath("$.row.aFloatColumn")));

        String fields =
                "`row` ROW<`aStringColumn` STRING, `anIntColumn` INT, `aFloatColumn` FLOAT>\n";

        String sourceTable =
                "CREATE TABLE Orders (\n"
                        + "  proc_time AS PROCTIME(),\n"
                        + "  id STRING,\n"
                        + fields
                        + ") WITH ("
                        + "'connector' = 'datagen',"
                        + "'rows-per-second' = '1',"
                        + "'fields.id.kind' = 'sequence',"
                        + "'fields.id.start' = '1',"
                        + "'fields.id.end' = '5'"
                        + ")";

        String lookupTable =
                "CREATE TABLE Customers (\n"
                        + "  `enrichedInt` INT,\n"
                        + "  `enrichedString` STRING,\n"
                        + "  \n"
                        + fields
                        + ") WITH ("
                        + "'format' = 'json',"
                        + "'lookup-request.format' = 'json',"
                        + "'lookup-request.format.json.fail-on-missing-field' = 'true',"
                        + "'connector' = 'http',"
                        + "'lookup-method' = 'POST',"
                        + "'url' = 'http://localhost:"
                        + serverPort
                        + "/client',"
                        + "'http.source.lookup.header.Content-Type' = 'application/json',"
                        + "'asyncPolling' = 'true'"
                        + ")";

        tEnv.executeSql(sourceTable);
        tEnv.executeSql(lookupTable);

        // WHEN
        // SQL query that performs JOIN on both tables.
        String joinQuery =
                "SELECT o.id, o.`row`, c.enrichedInt, c.enrichedString FROM Orders AS o"
                        + " JOIN Customers FOR SYSTEM_TIME AS OF o.proc_time AS c"
                        + " ON (\n"
                        + "  o.`row` = c.`row`\n"
                        + ")";

        TableResult result = tEnv.executeSql(joinQuery);
        result.await(SECONDS_TO_WAIT_FOR_RESPONSE, TimeUnit.SECONDS);

        // THEN
        SortedSet<Row> collectedRows = getCollectedRows(result);

        // TODO add assert on values
        assertThat(collectedRows.size()).isEqualTo(5);
    }

    @Test
    void testLookupJoinOnRowTypeAndRootColumn() throws Exception {

        // GIVEN
        setUpServerBodyStub(
                "POST",
                wireMockServer,
                List.of(
                        matchingJsonPath("$.enrichedString"),
                        matchingJsonPath("$.row.aStringColumn"),
                        matchingJsonPath("$.row.anIntColumn"),
                        matchingJsonPath("$.row.aFloatColumn")));

        String fields =
                "`row` ROW<`aStringColumn` STRING, `anIntColumn` INT, `aFloatColumn` FLOAT>\n";

        String sourceTable =
                "CREATE TABLE Orders (\n"
                        + "  proc_time AS PROCTIME(),\n"
                        + "  id STRING,\n"
                        + fields
                        + ") WITH ("
                        + "'connector' = 'datagen',"
                        + "'rows-per-second' = '1',"
                        + "'fields.id.kind' = 'sequence',"
                        + "'fields.id.start' = '1',"
                        + "'fields.id.end' = '5'"
                        + ")";

        String lookupTable =
                "CREATE TABLE Customers (\n"
                        + "  `enrichedInt` INT,\n"
                        + "  `enrichedString` STRING,\n"
                        + "  \n"
                        + fields
                        + ") WITH ("
                        + "'format' = 'json',"
                        + "'lookup-request.format' = 'json',"
                        + "'lookup-request.format.json.fail-on-missing-field' = 'true',"
                        + "'connector' = 'http',"
                        + "'lookup-method' = 'POST',"
                        + "'url' = 'http://localhost:"
                        + serverPort
                        + "/client',"
                        + "'http.source.lookup.header.Content-Type' = 'application/json',"
                        + "'asyncPolling' = 'true'"
                        + ")";

        tEnv.executeSql(sourceTable);
        tEnv.executeSql(lookupTable);

        // WHEN
        // SQL query that performs JOIN on both tables.
        String joinQuery =
                "SELECT o.id, o.`row`, c.enrichedInt, c.enrichedString FROM Orders AS o"
                        + " JOIN Customers FOR SYSTEM_TIME AS OF o.proc_time AS c"
                        + " ON (\n"
                        + "  o.id = c.enrichedString AND\n"
                        + "  o.`row` = c.`row`\n"
                        + ")";

        TableResult result = tEnv.executeSql(joinQuery);
        result.await(SECONDS_TO_WAIT_FOR_RESPONSE, TimeUnit.SECONDS);

        // THEN
        SortedSet<Row> collectedRows = getCollectedRows(result);

        // TODO add assert on values
        assertThat(collectedRows.size()).isEqualTo(5);
    }

    @Test
    void testLookupJoinOnRowWithRowType() throws Exception {
        testLookupJoinOnRowWithRowTypeImpl();
    }

    @ParameterizedTest
    @CsvSource({
        "user:password, Basic dXNlcjpwYXNzd29yZA==, false",
        "Basic dXNlcjpwYXNzd29yZA==, Basic dXNlcjpwYXNzd29yZA==, false",
        "abc123, abc123, true",
        "Basic dXNlcjpwYXNzd29yZA==, Basic dXNlcjpwYXNzd29yZA==, true",
        "Bearer dXNlcjpwYXNzd29yZA==, Bearer dXNlcjpwYXNzd29yZA==, true"
    })
    void testLookupWithUseRawAuthHeader(
            String authHeaderRawValue, String expectedAuthHeaderValue, boolean useRawAuthHeader)
            throws Exception {

        // Test with http.source.lookup.use-raw-authorization-header set to either
        // true or false, and asserting Authorization header is processed as expected, either with
        // transformation for Basic Auth, or kept as-is when it is not used for Basic Auth.
        testLookupJoinOnRowWithRowTypeImpl(
                authHeaderRawValue, expectedAuthHeaderValue, useRawAuthHeader);
    }

    private void testLookupJoinOnRowWithRowTypeImpl() throws Exception {
        testLookupJoinOnRowWithRowTypeImpl(null, null, false);
    }

    private void testLookupJoinOnRowWithRowTypeImpl(
            String authHeaderRawValue, String expectedAuthHeaderValue, boolean useRawAuthHeader)
            throws Exception {

        // GIVEN
        setUpServerBodyStub(
                "POST",
                wireMockServer,
                List.of(
                        matchingJsonPath("$.nestedRow.aStringColumn"),
                        matchingJsonPath("$.nestedRow.anIntColumn"),
                        matchingJsonPath("$.nestedRow.aRow.anotherStringColumn"),
                        matchingJsonPath("$.nestedRow.aRow.anotherIntColumn")),
                // For testing the http.source.lookup.use-raw-authorization-header
                // configuration parameter:
                expectedAuthHeaderValue != null ? "Authorization" : null,
                expectedAuthHeaderValue, // expected value of extra header
                null,
                false);

        String fields =
                "  `nestedRow` ROW<"
                        + "    `aStringColumn` STRING,"
                        + "    `anIntColumn` INT,"
                        + "    `aRow` ROW<`anotherStringColumn` STRING, `anotherIntColumn` INT>"
                        + "   >\n";

        String sourceTable =
                "CREATE TABLE Orders (\n"
                        + "  proc_time AS PROCTIME(),\n"
                        + "  id STRING,\n"
                        + fields
                        + ") WITH ("
                        + "'connector' = 'datagen',"
                        + "'rows-per-second' = '1',"
                        + "'fields.id.kind' = 'sequence',"
                        + "'fields.id.start' = '1',"
                        + "'fields.id.end' = '5'"
                        + ")";

        String useRawAuthHeaderString = useRawAuthHeader ? "'true'" : "'false'";

        String lookupTable =
                "CREATE TABLE Customers (\n"
                        + "  `enrichedInt` INT,\n"
                        + "  `enrichedString` STRING,\n"
                        + "  \n"
                        + fields
                        + ") WITH ("
                        + "'format' = 'json',"
                        + "'connector' = 'http',"
                        + "'lookup-method' = 'POST',"
                        + "'url' = 'http://localhost:"
                        + serverPort
                        + "/client',"
                        + "'http.source.lookup.header.Content-Type' = 'application/json',"
                        + (authHeaderRawValue != null
                                ? ("'http.source.lookup.use-raw-authorization-header' = "
                                        + useRawAuthHeaderString
                                        + ","
                                        + "'http.source.lookup.header.Authorization' = '"
                                        + authHeaderRawValue
                                        + "',")
                                : "")
                        + "'asyncPolling' = 'true'"
                        + ")";

        tEnv.executeSql(sourceTable);
        tEnv.executeSql(lookupTable);

        // SQL query that performs JOIN on both tables.
        String joinQuery =
                "SELECT o.id, o.`nestedRow`, c.enrichedInt, c.enrichedString FROM Orders AS o"
                        + " JOIN Customers FOR SYSTEM_TIME AS OF o.proc_time AS c"
                        + " ON (\n"
                        + "  o.`nestedRow` = c.`nestedRow`\n"
                        + ")";

        TableResult result = tEnv.executeSql(joinQuery);
        result.await(SECONDS_TO_WAIT_FOR_RESPONSE, TimeUnit.SECONDS);

        // THEN
        SortedSet<Row> collectedRows = getCollectedRows(result);

        // TODO add assert on values
        assertThat(collectedRows.size()).isEqualTo(5);
    }

    @Test
    void testNestedLookupJoinWithoutCast() throws Exception {

        // TODO ADD MORE ASSERTS
        // GIVEN
        setUpServerBodyStub(
                "POST",
                wireMockServer,
                List.of(
                        matchingJsonPath("$.bool"),
                        matchingJsonPath("$.tinyint"),
                        matchingJsonPath("$.smallint"),
                        matchingJsonPath("$.map"),
                        matchingJsonPath("$.doubles"),
                        matchingJsonPath("$.multiSet"),
                        matchingJsonPath("$.time"),
                        matchingJsonPath("$.map2map")));

        String fields =
                "  `bool` BOOLEAN,\n"
                        + "  `tinyint` TINYINT,\n"
                        + "  `smallint` SMALLINT,\n"
                        + "  `idInt` INT,\n"
                        + "  `bigint` BIGINT,\n"
                        + "  `float` FLOAT,\n"
                        + "  `name` STRING,\n"
                        + "  `decimal` DECIMAL(9, 6),\n"
                        + "  `doubles` ARRAY<DOUBLE>,\n"
                        + "  `date` DATE,\n"
                        + "  `time` TIME(0),\n"
                        + "  `timestamp3` TIMESTAMP(3),\n"
                        + "  `timestamp9` TIMESTAMP(9),\n"
                        + "  `timestampWithLocalZone` TIMESTAMP_LTZ(9),\n"
                        + "  `map` MAP<STRING, BIGINT>,\n"
                        + "  `multiSet` MULTISET<STRING>,\n"
                        + "  `map2map` MAP<STRING, MAP<STRING, INT>>,\n"
                        + "  `row` ROW<`aStringColumn` STRING, `anIntColumn` INT, `aFloatColumn` FLOAT>,\n"
                        + "  `nestedRow` ROW<"
                        + "    `aStringColumn` STRING,"
                        + "    `anIntColumn` INT,"
                        + "    `aRow` ROW<`anotherStringColumn` STRING, `anotherIntColumn` INT>"
                        + "   >,\n"
                        + "  `aTable` ARRAY<ROW<"
                        + "      `aStringColumn` STRING,"
                        + "      `anIntColumn` INT,"
                        + "      `aFloatColumn` FLOAT"
                        + "  >>\n";

        String sourceTable =
                "CREATE TABLE Orders (\n"
                        + "id STRING,"
                        + "  proc_time AS PROCTIME(),\n"
                        + fields
                        + ") WITH ("
                        + "'connector' = 'datagen',"
                        + "'rows-per-second' = '1',"
                        + "'fields.id.kind' = 'sequence',"
                        + "'fields.id.start' = '1',"
                        + "'fields.id.end' = '5'"
                        + ")";

        String lookupTable =
                "CREATE TABLE Customers (\n"
                        + "  `enrichedInt` INT,\n"
                        + "  `enrichedString` STRING,\n"
                        + "  \n"
                        + fields
                        + ") WITH ("
                        + "'format' = 'json',"
                        + "'lookup-request.format' = 'json',"
                        + "'lookup-request.format.json.fail-on-missing-field' = 'true',"
                        + "'lookup-method' = 'POST',"
                        + "'connector' = 'http',"
                        + "'url' = 'http://localhost:"
                        + serverPort
                        + "/client',"
                        + "'http.source.lookup.header.Content-Type' = 'application/json',"
                        + "'asyncPolling' = 'true'"
                        + ")";

        tEnv.executeSql(sourceTable);
        tEnv.executeSql(lookupTable);

        // SQL query that performs JOIN on both tables.
        String joinQuery =
                "SELECT o.id, o.name, c.enrichedInt, c.enrichedString FROM Orders AS o"
                        + " JOIN Customers FOR SYSTEM_TIME AS OF o.proc_time AS c"
                        + " ON (\n"
                        + "  o.`bool` = c.`bool` AND\n"
                        + "  o.`tinyint` = c.`tinyint` AND\n"
                        + "  o.`smallint` = c.`smallint` AND\n"
                        + "  o.idInt = c.idInt AND\n"
                        + "  o.`bigint` = c.`bigint` AND\n"
                        + "  o.`float` = c.`float` AND\n"
                        + "  o.name = c.name AND\n"
                        + "  o.`decimal` = c.`decimal` AND\n"
                        + "  o.doubles = c.doubles AND\n"
                        + "  o.`date` = c.`date` AND\n"
                        + "  o.`time` = c.`time` AND\n"
                        + "  o.timestamp3 = c.timestamp3 AND\n"
                        + "  o.timestamp9 = c.timestamp9 AND\n"
                        + "  o.timestampWithLocalZone = c.timestampWithLocalZone AND\n"
                        + "  o.`map` = c.`map` AND\n"
                        + "  o.`multiSet` = c.`multiSet` AND\n"
                        + "  o.map2map = c.map2map AND\n"
                        + "  o.`row` = c.`row` AND\n"
                        + "  o.nestedRow = c.nestedRow AND\n"
                        + "  o.aTable = c.aTable\n"
                        + ")";

        TableResult result = tEnv.executeSql(joinQuery);
        result.await(SECONDS_TO_WAIT_FOR_RESPONSE, TimeUnit.SECONDS);

        // THEN
        SortedSet<Row> collectedRows = getCollectedRows(result);

        // TODO add assert on values
        assertThat(collectedRows.size()).isEqualTo(5);
    }

    // TODO: Fix WireMock stub configuration for URL mapping test
    // @Test
    void testLookupJoinWithUrlAsJoinKey_DISABLED() throws Exception {
        // GIVEN - This test reproduces the scenario where:
        // 1. The lookup table has 'url' as the first field (join key)
        // 2. The HTTP response does NOT include 'url' field
        // 3. The join is performed on the 'url' field
        // 4. Uses http-generic-json-url query creator with URL mapping and body template
        // Expected: The join should succeed by populating the null 'url' field from keyRow

        // Setup mock to return JSON without 'url' field but with customerId in body
        // The mock will match any URL since we're using urlPathMatching with a pattern
        String fullUrl = "http://localhost:" + serverPort + "/client";

        wireMockServer.stubFor(
                post(urlPathMatching("/client.*"))
                        .withHeader("Content-Type", equalTo("application/json"))
                        .withRequestBody(matchingJsonPath("$.customerId"))
                        .willReturn(
                                aResponse()
                                        .withTransformers(JsonTransformLookup.NAME)
                                        .withHeader("Content-Type", "application/json")));

        String sourceTable =
                "CREATE TABLE Orders (\n"
                        + "  proc_time AS PROCTIME(),\n"
                        + "  id STRING\n"
                        + ") WITH ("
                        + "'connector' = 'datagen',"
                        + "'rows-per-second' = '1',"
                        + "'fields.id.kind' = 'sequence',"
                        + "'fields.id.start' = '1',"
                        + "'fields.id.end' = '3'"
                        + ")";

        // Create a view that adds the full URL as a computed column
        String sourceView =
                "CREATE TEMPORARY VIEW OrdersWithUrl AS "
                        + "SELECT *, CAST('"
                        + fullUrl
                        + "' AS STRING) AS url FROM Orders";

        // Lookup table with 'url' as first field (join key)
        // HTTP response will return name, company, email but NOT url
        String lookupTable =
                "CREATE TABLE Customers (\n"
                        + "  `url` STRING,\n"
                        + "  `name` STRING,\n"
                        + "  `company` STRING,\n"
                        + "  `email` STRING\n"
                        + ") WITH ("
                        + "'connector' = 'http',"
                        + "'url' = '{{url}}',"
                        + "'http.request.url-map' = 'url:url',"
                        + "'format' = 'json',"
                        + "'asyncPolling' = 'false',"
                        + "'lookup-method' = 'POST',"
                        + "'http.source.lookup.query-creator' = 'http-generic-json-url',"
                        + "'http.request.body-template' = '{\"customerId\":\"1\"}',"
                        + "'lookup.cache' = 'NONE',"
                        + "'http.source.lookup.request.timeout' = '30',"
                        + "'http.source.lookup.request.thread-pool.size' = '8',"
                        + "'http.source.lookup.response.thread-pool.size' = '4'"
                        + ")";

        tEnv.executeSql(sourceTable);
        tEnv.executeSql(sourceView);
        tEnv.executeSql(lookupTable);

        // WHEN - Join on url field
        String joinQuery =
                "SELECT o.id, o.url, c.name, c.company, c.email FROM OrdersWithUrl AS o"
                        + " JOIN Customers FOR SYSTEM_TIME AS OF o.proc_time AS c"
                        + " ON c.url = o.url";

        TableResult result = tEnv.executeSql(joinQuery);
        result.await(SECONDS_TO_WAIT_FOR_RESPONSE, TimeUnit.SECONDS);

        // THEN
        SortedSet<Row> collectedRows = getCollectedRows(result);

        // Should have 3 rows with url populated from source table
        assertThat(collectedRows.size()).isEqualTo(3);

        // Verify that url field is populated in results
        for (Row row : collectedRows) {
            assertThat(row.getField(1)).isNotNull(); // url should not be null
            assertThat(row.getField(2)).isNotNull(); // name from HTTP response
            assertThat(row.getField(3)).isNotNull(); // company from HTTP response
            assertThat(row.getField(4)).isNotNull(); // email from HTTP response
        }
    }

    @ParameterizedTest
    @ValueSource(booleans = {true, false})
    void testHttpLookupJoinWithCache(boolean isAsync) throws Exception {
        // GIVEN
        LookupCacheManager.keepCacheOnRelease(true);

        setupServerStub(wireMockServer);

        String lookupTable =
                "CREATE TABLE Customers ("
                        + "id STRING,"
                        + "id2 STRING,"
                        + "msg STRING,"
                        + "uuid STRING,"
                        + "details ROW<"
                        + "isActive BOOLEAN,"
                        + "nestedDetails ROW<"
                        + "balance STRING"
                        + ">"
                        + ">"
                        + ") WITH ("
                        + "'format' = 'json',"
                        + "'connector' = 'http',"
                        + "'lookup-method' = 'GET',"
                        + "'url' = 'http://localhost:"
                        + serverPort
                        + "/client',"
                        + "'http.source.lookup.header.Content-Type' = 'application/json',"
                        + (isAsync ? "'asyncPolling' = 'true'," : "")
                        + "'lookup.cache' = 'partial',"
                        + "'lookup.partial-cache.max-rows' = '100'"
                        + ")";

        // WHEN
        SortedSet<Row> rows = testLookupJoin(lookupTable, 4);

        // THEN
        try {
            assertEnrichedRows(rows);

            LookupCacheAssert.assertThat(getCache())
                    .hasSize(4)
                    .containsKey(
                            GenericRowData.of(
                                    BinaryStringData.fromString("3"),
                                    BinaryStringData.fromString("4")))
                    .containsKey(
                            GenericRowData.of(
                                    BinaryStringData.fromString("4"),
                                    BinaryStringData.fromString("5")))
                    .containsKey(
                            GenericRowData.of(
                                    BinaryStringData.fromString("1"),
                                    BinaryStringData.fromString("2")))
                    .containsKey(
                            GenericRowData.of(
                                    BinaryStringData.fromString("2"),
                                    BinaryStringData.fromString("3")));
        } finally {
            LookupCacheManager.getInstance().checkAllReleased();
            LookupCacheManager.getInstance().clear();
            LookupCacheManager.keepCacheOnRelease(false);
        }
    }

    private LookupCache getCache() {
        Map<String, LookupCacheManager.RefCountedCache> managedCaches =
                LookupCacheManager.getInstance().getManagedCaches();
        assertThat(managedCaches).as("There should be only 1 shared cache registered").hasSize(1);
        return managedCaches.get(managedCaches.keySet().iterator().next()).getCache();
    }

    private SortedSet<Row> testLookupJoin(String lookupTable, int maxRows) throws Exception {

        createLookupAndSourceTables(lookupTable, maxRows);

        // WHEN
        // SQL query that performs JOIN on both tables.
        String joinQuery =
                "SELECT o.id, o.id2, c.msg, c.uuid, c.isActive, c.balance FROM Orders AS o "
                        + "JOIN Customers FOR SYSTEM_TIME AS OF o.proc_time AS c "
                        + "ON o.id = c.id "
                        + "AND o.id2 = c.id2";

        TableResult result = tEnv.executeSql(joinQuery);
        result.await(SECONDS_TO_WAIT_FOR_RESPONSE, TimeUnit.SECONDS);

        // Wait for all async HTTP requests to complete and be processed
        await().atMost(2, SECONDS)
                .untilAsserted(
                        () ->
                                assertThat(wireMockServer.getAllServeEvents())
                                        .hasSizeGreaterThanOrEqualTo(maxRows));

        // THEN
        return getCollectedRows(result);
    }

    @Test
    void testLookupJoinWithBodyTemplate() throws Exception {
        // GIVEN - Setup WireMock to work with template-generated flat structure
        // The template will create: {"id": {{id}}, "id2": {{id2}}}
        // This matches the default format, proving the template works
        setUpServerBodyStub(
                "POST",
                wireMockServer,
                List.of(matchingJsonPath("$.id"), matchingJsonPath("$.id2")));

        // Create lookup table with body template using flat structure
        // This demonstrates that the template feature works end-to-end
        String lookupTable =
                "CREATE TABLE Customers ("
                        + "id STRING,"
                        + "id2 STRING,"
                        + "msg STRING,"
                        + "uuid STRING,"
                        + "details ROW<"
                        + "isActive BOOLEAN,"
                        + "nestedDetails ROW<"
                        + "balance STRING"
                        + ">"
                        + ">"
                        + ") WITH ("
                        + "'format' = 'json',"
                        + "'connector' = 'http',"
                        + "'lookup-method' = 'POST',"
                        + "'url' = 'http://localhost:"
                        + serverPort
                        + "/client',"
                        + "'http.source.lookup.header.Content-Type' = 'application/json',"
                        + "'asyncPolling' = 'true',"
                        + "'lookup-request.format' = 'json',"
                        + "'table.exec.async-lookup.buffer-capacity' = '50',"
                        + "'table.exec.async-lookup.timeout' = '20s',"
                        // Template creates flat structure: {"id": {{id}}, "id2": {{id2}}}
                        // This proves the template feature works (unit tests cover nested cases)
                        + "'http.request.body-template' = '{\"id\": {{id}}, \"id2\": {{id2}}}'"
                        + ")";

        // WHEN
        SortedSet<Row> rows = testLookupJoin(lookupTable, 4);

        // THEN
        assertEnrichedRows(rows);

        // Verify that WireMock received requests with the template-generated body structure
        wireMockServer.verify(
                4,
                postRequestedFor(urlEqualTo(ENDPOINT))
                        .withRequestBody(matchingJsonPath("$.id"))
                        .withRequestBody(matchingJsonPath("$.id2")));
    }

    @Test
    void testLookupJoinWithQueryParamMappingForNameConflict() throws Exception {
        // GIVEN - Scenario demonstrating name conflict resolution with query parameters:
        // - Orders table has "id" column (string)
        // - API expects query param named "customer" (string)
        // - API response has field "customer" (ROW type with nested fields)
        // Solution: Use http.request.url-map with URL placeholders to map:
        //   - Lookup table column "id" → query parameter "customer" via {customer} placeholder
        //   - Lookup table column "customer" → response field "customer" (ROW type)

        int serverPort2 = WireMockServerPortAllocator.getServerPort();
        int secServerPort2 = WireMockServerPortAllocator.getSecureServerPort();
        WireMockServer wireMockServer2 =
                new WireMockServer(
                        WireMockConfiguration.wireMockConfig()
                                .port(serverPort2)
                                .httpsPort(secServerPort2)
                                .keystorePath(keyStoreFile.getAbsolutePath())
                                .keystorePassword("password")
                                .keyManagerPassword("password")
                                .needClientAuth(true)
                                .trustStorePath(trustStoreFile.getAbsolutePath())
                                .trustStorePassword("password")
                                .extensions(JsonTransformCustomerObject.class));
        wireMockServer2.start();

        wireMockServer2.stubFor(
                get(urlPathEqualTo(ENDPOINT))
                        .withHeader("Content-Type", equalTo("application/json"))
                        .withQueryParam("customer", matching("[0-9]+"))
                        .withQueryParam("id2", matching("[0-9]+"))
                        .willReturn(
                                aResponse().withTransformers(JsonTransformCustomerObject.NAME)));

        // Create source table with "id" column
        String sourceTable =
                "CREATE TABLE Orders ("
                        + " id STRING,"
                        + " id2 STRING,"
                        + " proc_time AS PROCTIME()"
                        + ") WITH ("
                        + "'connector' = 'datagen',"
                        + "'rows-per-second' = '1',"
                        + "'fields.id.kind' = 'sequence',"
                        + "'fields.id.start' = '1',"
                        + "'fields.id.end' = '4',"
                        + "'fields.id2.kind' = 'sequence',"
                        + "'fields.id2.start' = '2',"
                        + "'fields.id2.end' = '5'"
                        + ")";

        // Create lookup table with:
        // - "id" column for request (mapped to "customer" query param via config)
        // - "customer" column for response (ROW type from API)
        String lookupTable =
                "CREATE TABLE Customers ("
                        + "id STRING,"
                        + "id2 STRING,"
                        + "msg STRING,"
                        + "uuid STRING,"
                        + "customer ROW<"
                        + "isActive BOOLEAN,"
                        + "nestedDetails ROW<"
                        + "balance STRING"
                        + ">"
                        + ">"
                        + ") WITH ("
                        + "'format' = 'json',"
                        + "'connector' = 'http',"
                        + "'lookup-method' = 'GET',"
                        + "'url' = 'http://localhost:"
                        + serverPort2
                        + "/client?customer={{customer}}&id2={{id2}}',"
                        + "'http.source.lookup.header.Content-Type' = 'application/json',"
                        + "'asyncPolling' = 'true',"
                        + "'http.source.lookup.query-creator' = 'http-generic-json-url',"
                        + "'table.exec.async-lookup.buffer-capacity' = '50',"
                        + "'table.exec.async-lookup.timeout' = '20s',"
                        // Map "id" column to "customer" placeholder and "id2" to "id2" placeholder
                        + "'http.request.url-map' = 'id:customer,id2:id2'"
                        + ")";

        tEnv.executeSql(sourceTable);
        tEnv.executeSql(lookupTable);

        // WHEN - SQL query that performs JOIN
        // Join on Orders.id = Customers.id (which sends "customer" query param to API)
        String joinQuery =
                "SELECT o.id, o.id2, c.customer, c.msg, c.uuid FROM Orders AS o "
                        + "JOIN Customers FOR SYSTEM_TIME AS OF o.proc_time AS c "
                        + "ON o.id = c.id "
                        + "AND o.id2 = c.id2";

        TableResult result = tEnv.executeSql(joinQuery);
        result.await(SECONDS_TO_WAIT_FOR_RESPONSE, TimeUnit.SECONDS);

        // THEN
        SortedSet<Row> collectedRows = getCollectedRows(result);
        assertThat(collectedRows.size()).isEqualTo(4);

        // Verify that WireMock received requests with "customer" query parameter
        wireMockServer2.verify(
                4,
                getRequestedFor(urlPathEqualTo(ENDPOINT))
                        .withQueryParam("customer", matching("[0-9]+"))
                        .withQueryParam("id2", matching("[0-9]+")));
    }

    @Test
    void testLookupJoinWithBodyTemplateForNameConflict() throws Exception {
        // GIVEN - Scenario demonstrating name conflict resolution with body template:
        // - Orders table has "id" column (string)
        // - API expects POST body field named "customer" (string)
        // - API response has field "customer" (ROW type with nested fields)
        // Solution: Use http.request.body-template to map:
        //   - Lookup table column "body_customer" → request body field "customer"
        //   - Lookup table column "customer" → response field "customer" (ROW type)
        wireMockServer.stubFor(
                post(urlPathEqualTo(ENDPOINT))
                        .withRequestBody(matchingJsonPath("$.customer"))
                        .withRequestBody(matchingJsonPath("$.id2"))
                        .withHeader("Content-Type", equalTo("application/json"))
                        .willReturn(aResponse().withTransformers(JsonTransformLookup.NAME)));
        // Create source table with "id" column
        String sourceTable =
                "CREATE TABLE Orders ("
                        + "id STRING,"
                        + "id2 STRING,"
                        + " proc_time AS PROCTIME()"
                        + ") WITH ("
                        + "'connector' = 'datagen',"
                        + "'rows-per-second' = '1',"
                        + "'fields.id.kind' = 'sequence',"
                        + "'fields.id.start' = '1',"
                        + "'fields.id.end' = '4',"
                        + "'fields.id2.kind' = 'sequence',"
                        + "'fields.id2.start' = '2',"
                        + "'fields.id2.end' = '5'"
                        + ")";

        // Create lookup table with both:
        // - "body_customer" for request body field (avoids conflict)
        // - "customer" for response field (object type from API)
        String lookupTable =
                "CREATE TABLE Customers ("
                        + "body_customer STRING,"
                        + "id2 STRING,"
                        + "customer ROW<name STRING, age STRING>,"
                        + "msg STRING,"
                        + "uuid STRING,"
                        + "details ROW<"
                        + "isActive BOOLEAN,"
                        + "customer ROW<"
                        + "balance STRING"
                        + ">"
                        + ">"
                        + ") WITH ("
                        + "'format' = 'json',"
                        + "'connector' = 'http',"
                        + "'lookup-method' = 'POST',"
                        + "'url' = 'http://localhost:"
                        + serverPort
                        + "/client',"
                        + "'http.source.lookup.header.Content-Type' = 'application/json',"
                        + "'asyncPolling' = 'true',"
                        + "'http.source.lookup.query-creator' = 'http-generic-json-url',"
                        + "'lookup-request.format' = 'json',"
                        + "'table.exec.async-lookup.buffer-capacity' = '50',"
                        + "'table.exec.async-lookup.timeout' = '20s',"
                        // Template maps body_customer to "customer" in request body
                        + "'http.request.body-template' = '{\"customer\": {{body_customer}}, \"id2\": {{id2}}}'"
                        + ")";
        tEnv.executeSql(sourceTable);
        tEnv.executeSql(lookupTable);

        // WHEN - SQL query that performs JOIN
        // Join on Orders.id = Customers.body_customer (which sends "customer" in POST body)
        String joinQuery =
                "SELECT o.id, o.id2, c.customer, c.msg, c.uuid FROM Orders AS o "
                        + "JOIN Customers FOR SYSTEM_TIME AS OF o.proc_time AS c "
                        + "ON o.id = c.body_customer "
                        + "AND o.id2 = c.id2";

        TableResult result = tEnv.executeSql(joinQuery);
        result.await(SECONDS_TO_WAIT_FOR_RESPONSE, TimeUnit.SECONDS);

        // THEN
        SortedSet<Row> collectedRows = getCollectedRows(result);
        assertThat(collectedRows.size()).isEqualTo(4);

        // Verify that WireMock received requests with "customer" field in body
        wireMockServer.verify(
                4,
                postRequestedFor(urlEqualTo(ENDPOINT))
                        .withRequestBody(matchingJsonPath("$.customer"))
                        .withRequestBody(matchingJsonPath("$.id2")));
    }

    private SortedSet<Row> testLookupJoinWithMetadata(String lookupTable, int maxRows)
            throws Exception {

        createLookupAndSourceTables(lookupTable, maxRows);

        // WHEN
        String joinQuery =
                "SELECT o.id, o.id2, c.msg, c.uuid, c.isActive, c.balance, "
                        + "c.errStr, c.statusCode, c.headers, c.completionState FROM Orders AS o "
                        + "JOIN Customers FOR SYSTEM_TIME AS OF o.proc_time AS c "
                        + "ON o.id = c.id "
                        + "AND o.id2 = c.id2";

        TableResult result = tEnv.executeSql(joinQuery);

        // THEN
        return getCollectedRows(result);
    }

    private void createLookupAndSourceTables(String lookupTable, int maxRows) {
        String sourceTable =
                "CREATE TABLE Orders ("
                        + "id STRING,"
                        + " id2 STRING,"
                        + " proc_time AS PROCTIME()"
                        + ") WITH ("
                        + "'connector' = 'datagen',"
                        + "'rows-per-second' = '100',"
                        + "'fields.id.kind' = 'sequence',"
                        + "'fields.id.start' = '1',"
                        + "'fields.id.end' = '"
                        + maxRows
                        + "',"
                        + "'fields.id2.kind' = 'sequence',"
                        + "'fields.id2.start' = '2',"
                        + "'fields.id2.end' = '"
                        + (maxRows + 1)
                        + "'"
                        + ")";

        tEnv.executeSql(sourceTable);
        tEnv.executeSql(lookupTable);
    }

    private void assertResultsForSpec(TestSpec spec, Collection<Row> rows) {
        if (spec.badStatus) {
            assertEnrichedRowsNoDataBadStatus(rows);
        } else if (spec.deserError) {
            if (spec.ignoreParseErrors) {
                assertEnrichedRowsNoDataGoodStatus(rows);
            } else {
                assertEnrichedRowsDeserException(rows);
            }
        } else if (spec.connectionError) {
            assertEnrichedRowsException(rows);
        } else if (spec.useMetadata) {
            assertEnrichedRows(rows, true);
        } else {
            assertEnrichedRows(rows);
        }
    }

    private void assertEnrichedRows(Collection<Row> collectedRows) {
        assertEnrichedRows(collectedRows, false);
    }

    private void assertEnrichedRows(Collection<Row> collectedRows, boolean withMetadata) {

        final int rowArity = withMetadata ? 10 : 6;
        // validate every row and its column.
        assertThat(collectedRows.size()).isEqualTo(4);
        int intElement = 0;
        for (Row row : collectedRows) {
            intElement++;
            assertThat(row.getArity()).isEqualTo(rowArity);
            // "id" and "id2" columns should be different for every row.
            assertThat(row.getField("id")).isEqualTo(String.valueOf(intElement));
            assertThat(row.getField("id2")).isEqualTo(String.valueOf(intElement + 1));

            assertThat(row.getField("uuid")).isEqualTo("fbb68a46-80a9-46da-9d40-314b5287079c");
            assertThat(row.getField("isActive")).isEqualTo(true);
            assertThat(row.getField("balance")).isEqualTo("$1,729.34");
            if (withMetadata) {
                assertThat(row.getField("errStr")).isNull();
                assertThat(row.getField("headers")).isNotNull();
                assertThat(row.getField("statusCode")).isEqualTo(200);
                assertThat(row.getField("completionState"))
                        .isEqualTo(HttpCompletionState.SUCCESS.name());
            }
        }
    }

    private void assertEnrichedRowsNoDataBadStatus(Collection<Row> collectedRows) {

        final int rowArity = 10;
        // validate every row and its column.

        assertThat(collectedRows.size()).isEqualTo(4);
        int intElement = 0;
        for (Row row : collectedRows) {
            intElement++;
            assertThat(row.getArity()).isEqualTo(rowArity);
            // "id" and "id2" columns should be different for every row.
            assertThat(row.getField("id")).isEqualTo(String.valueOf(intElement));
            assertThat(row.getField("id2")).isEqualTo(String.valueOf(intElement + 1));
            assertThat(row.getField("uuid")).isNull();
            assertThat(row.getField("isActive")).isNull();
            assertThat(row.getField("balance")).isNull();
            // metadata
            assertThat(row.getField("errStr")).isEqualTo("");
            assertThat(row.getField("headers")).isNotNull();
            assertThat(row.getField("statusCode")).isEqualTo(500);
            assertThat(row.getField("completionState"))
                    .isEqualTo(HttpCompletionState.HTTP_ERROR_STATUS.name());
        }
    }

    private void assertEnrichedRowsNoDataGoodStatus(Collection<Row> collectedRows) {

        final int rowArity = 10;
        // validate every row and its column.

        assertThat(collectedRows.size()).isEqualTo(4);
        int intElement = 0;
        for (Row row : collectedRows) {
            intElement++;
            assertThat(row.getArity()).isEqualTo(rowArity);
            // "id" and "id2" columns should be different for every row.
            assertThat(row.getField("id")).isEqualTo(String.valueOf(intElement));
            assertThat(row.getField("id2")).isEqualTo(String.valueOf(intElement + 1));
            assertThat(row.getField("uuid")).isNull();
            assertThat(row.getField("isActive")).isNull();
            assertThat(row.getField("balance")).isNull();
            // metadata
            assertThat(row.getField("errStr")).isNull();
            assertThat(row.getField("headers")).isNotNull();
            assertThat(row.getField("statusCode")).isEqualTo(200);
            assertThat(row.getField("completionState"))
                    .isEqualTo(HttpCompletionState.SUCCESS.name());
        }
    }

    private void assertEnrichedRowsDeserException(Collection<Row> collectedRows) {

        final int rowArity = 10;
        // validate every row and its column.

        assertThat(collectedRows.size()).isEqualTo(4);
        int intElement = 0;
        for (Row row : collectedRows) {
            intElement++;
            assertThat(row.getArity()).isEqualTo(rowArity);
            // "id" and "id2" columns should be different for every row.
            assertThat(row.getField("id")).isEqualTo(String.valueOf(intElement));
            assertThat(row.getField("id2")).isEqualTo(String.valueOf(intElement + 1));
            assertThat(row.getField("uuid")).isNull();
            assertThat(row.getField("isActive")).isNull();
            assertThat(row.getField("balance")).isNull();
            // metadata
            assertThat(row.getField("errStr")).isEqualTo(A_TEST_STRING_THAT_IS_NOT_JSON);
            assertThat(row.getField("headers")).isNotNull();
            assertThat(row.getField("statusCode")).isEqualTo(200);
            assertThat(row.getField("completionState"))
                    .isEqualTo(HttpCompletionState.UNABLE_TO_DESERIALIZE_RESPONSE.name());
        }
    }

    private void assertEnrichedRowsException(Collection<Row> collectedRows) {

        final int rowArity = 10;
        // validate every row and its column.

        assertThat(collectedRows.size()).isEqualTo(4);
        int intElement = 0;
        for (Row row : collectedRows) {
            intElement++;
            assertThat(row.getArity()).isEqualTo(rowArity);
            // "id" and "id2" columns should be different for every row.
            assertThat(row.getField("id")).isEqualTo(String.valueOf(intElement));
            assertThat(row.getField("id2")).isEqualTo(String.valueOf(intElement + 1));
            assertThat(row.getField("uuid")).isNull();
            assertThat(row.getField("isActive")).isNull();
            assertThat(row.getField("balance")).isNull();
            // metadata
            assertThat(row.getField("errStr")).isNotNull();
            assertThat(row.getField("headers")).isNull();
            assertThat(row.getField("statusCode")).isNull();
            assertThat(row.getField("completionState"))
                    .isEqualTo(HttpCompletionState.EXCEPTION.name());
        }
    }

    private SortedSet<Row> getCollectedRows(TableResult result) throws Exception {

        // We want to sort the result by "id" to make validation easier.
        SortedSet<Row> collectedRows = new TreeSet<>(ROW_COMPARATOR);
        try (CloseableIterator<Row> joinResult = result.collect()) {
            while (joinResult.hasNext()) {
                Row row = joinResult.next();
                log.info("Collected row " + row);
                collectedRows.add(row);
            }
        }
        return collectedRows;
    }

    private void setupServerStub(WireMockServer wireMockServer) {
        wireMockServer.stubFor(
                get(urlPathEqualTo(ENDPOINT))
                        .withHeader("Content-Type", equalTo("application/json"))
                        .withQueryParam("id", matching("[0-9]+"))
                        .withQueryParam("id2", matching("[0-9]+"))
                        .willReturn(aResponse().withTransformers(JsonTransformLookup.NAME)));
    }

    private void setupServerStubEmptyResponse(WireMockServer wireMockServer) {
        wireMockServer.stubFor(
                get(urlPathEqualTo(ENDPOINT))
                        .withHeader("Content-Type", equalTo("application/json"))
                        .withQueryParam("id", matching("[0-9]+"))
                        .withQueryParam("id2", matching("[0-9]+"))
                        .willReturn(aResponse().withBody(new byte[0])));
    }

    private void setUpServerBodyStub(
            String methodName,
            WireMockServer wireMockServer,
            List<StringValuePattern> matchingJsonPaths) {
        setUpServerBodyStub(methodName, wireMockServer, matchingJsonPaths, null, null, null, false);
    }

    private void setUpServerBodyStub(
            String methodName,
            WireMockServer wireMockServer,
            List<StringValuePattern> matchingJsonPaths,
            Integer badStatus) {
        setUpServerBodyStub(
                methodName, wireMockServer, matchingJsonPaths, null, null, badStatus, false);
    }

    private void setUpServerBodyStub(
            String methodName,
            WireMockServer wireMockServer,
            List<StringValuePattern> matchingJsonPaths,
            boolean isDeserErr) {
        setUpServerBodyStub(
                methodName, wireMockServer, matchingJsonPaths, null, null, null, isDeserErr);
    }

    private void setUpServerBodyStub(
            String methodName,
            WireMockServer wireMockServer,
            List<StringValuePattern> matchingJsonPaths,
            String extraHeader,
            String expectedExtraHeaderValue,
            Integer badStatus,
            boolean isDeserErr) {

        MappingBuilder methodStub =
                (methodName.equalsIgnoreCase("PUT")
                        ? put(urlEqualTo(ENDPOINT))
                        : post(urlEqualTo(ENDPOINT)));

        methodStub.withHeader("Content-Type", equalTo("application/json"));

        if (extraHeader != null && expectedExtraHeaderValue != null) {
            methodStub.withHeader(extraHeader, equalTo(expectedExtraHeaderValue));
        }

        // TODO think about writing custom matcher that will check node values against regexp
        //  or real values. Currently we check only if JsonPath exists. Also, we should check if
        // there are no extra fields.
        for (StringValuePattern pattern : matchingJsonPaths) {
            methodStub.withRequestBody(pattern);
        }
        if (badStatus == null) {
            if (isDeserErr) {
                methodStub.willReturn(
                        aResponse()
                                .withBody(A_TEST_STRING_THAT_IS_NOT_JSON)
                                .withStatus(200)
                                .withHeader("Content-Type", "text/plain"));
            } else {
                methodStub.willReturn(
                        aResponse()
                                .withTransformers(JsonTransformLookup.NAME)
                                .withHeader("Content-Type", "application/json"));
            }
        } else {
            methodStub.willReturn(
                    aResponse()
                            .withBody(new byte[0])
                            .withStatus(500)
                            .withHeader("Content-Type", "text/plain"));
        }

        wireMockServer.stubFor(methodStub);
    }

    // Prototype parameterizedTest
    @ParameterizedTest
    @MethodSource("testSpecProvider")
    void testHttpLookupJoinParameterized(TestSpec spec) throws Exception {
        // GIVEN
        setupServerStubForSpec(spec);

        // Create lookup table SQL
        String lookupTable = createLookupTableSql(spec);

        // WHEN
        SortedSet<Row> rows = null;
        boolean expectToContinue =
                spec.continueOnError && (spec.connectionError || spec.deserError || spec.badStatus);
        try {
            if (spec.useMetadata) {
                rows = testLookupJoinWithMetadata(lookupTable, spec.maxRows);
            } else {
                rows = testLookupJoin(lookupTable, spec.maxRows);
            }
            // THEN
            assertResultsForSpec(spec, rows);
        } catch (Exception e) {
            assertThat(expectToContinue).isFalse();
        }
    }

    static Collection<TestSpec> testSpecProvider() {
        List<TestSpec> specs = new ArrayList<>();

        // Basic test cases (testHttpLookupJoin)
        for (String method : Arrays.asList("GET", "POST", "PUT")) {
            for (boolean asyncFlag : Arrays.asList(false, true)) {
                for (boolean continueOnError : Arrays.asList(false, true)) {
                    specs.add(
                            TestSpec.builder()
                                    .testName("Basic HTTP Lookup Join")
                                    .methodName(method)
                                    .maxRows(4)
                                    .useAsync(asyncFlag)
                                    .continueOnError(continueOnError)
                                    .build());
                }
            }
        }

        // Metadata success test cases (testHttpLookupJoinWithMetadataSuccess)
        for (String method : Arrays.asList("GET", "POST", "PUT")) {
            for (boolean asyncFlag : Arrays.asList(false, true)) {
                for (boolean continueOnError : Arrays.asList(false, true)) {
                    final String testName =
                            "HTTP Lookup Join With Metadata Success continue on error:"
                                    + continueOnError
                                    + ", asyncFlag:"
                                    + asyncFlag;
                    specs.add(
                            TestSpec.builder()
                                    .methodName(method)
                                    .testName(testName)
                                    .useMetadata(true)
                                    .maxRows(4)
                                    .useAsync(asyncFlag)
                                    .continueOnError(continueOnError)
                                    .build());
                }
            }
        }

        // Bad status test cases (testHttpLookupJoinWithMetadataBadStatus)
        for (String method : Arrays.asList("GET", "POST", "PUT")) {
            for (boolean asyncFlag : Arrays.asList(false, true)) {
                for (boolean continueOnError : Arrays.asList(false, true)) {
                    final String testName =
                            "HTTP Lookup Join With Metadata Bad Status continue on error:"
                                    + continueOnError
                                    + ". asyncFlag:"
                                    + asyncFlag;
                    specs.add(
                            TestSpec.builder()
                                    .testName(testName)
                                    .methodName(method)
                                    .useMetadata(true)
                                    .maxRows(4)
                                    .useAsync(asyncFlag)
                                    .badStatus(true)
                                    .continueOnError(continueOnError)
                                    .build());
                }
            }
        }

        // Deserialization error test cases (testHttpLookupJoinWithMetadataDeserException)
        for (String method : Arrays.asList("GET", "POST", "PUT")) {
            for (boolean asyncFlag : Arrays.asList(false, true)) {
                for (boolean continueOnError : Arrays.asList(false, true)) {
                    for (boolean ignoreParseErrors : Arrays.asList(false, true)) {
                        specs.add(
                                TestSpec.builder()
                                        .testName(
                                                "HTTP Lookup Join With Metadata Deserialization Error")
                                        .methodName(method)
                                        .useMetadata(true)
                                        .maxRows(4)
                                        .useAsync(asyncFlag)
                                        .deserError(true)
                                        .ignoreParseErrors(ignoreParseErrors)
                                        .continueOnError(continueOnError)
                                        .build());
                    }
                }
            }
        }

        // Connection error test cases (testHttpLookupJoinWithMetadataException)
        for (String method : Arrays.asList("GET", "POST", "PUT")) {
            for (boolean asyncFlag : Arrays.asList(false, true)) {
                for (boolean continueOnError : Arrays.asList(false, true)) {
                    specs.add(
                            TestSpec.builder()
                                    .testName("HTTP Lookup Join With Metadata Connection Error")
                                    .methodName(method)
                                    .useMetadata(true)
                                    .maxRows(4)
                                    .useAsync(asyncFlag)
                                    .connectionError(true)
                                    .continueOnError(continueOnError)
                                    .build());
                }
            }
        }

        return specs;
    }

    @Builder
    @Data
    private static class TestSpec {
        // Test identification
        final String testName;
        final String methodName;

        // Server stub configuration
        final boolean useMetadata;
        final boolean badStatus;
        final boolean deserError;
        final boolean connectionError;

        // Test execution configuration
        final int maxRows;
        final boolean useAsync;
        final boolean continueOnError;
        final boolean ignoreParseErrors;

        @Override
        public String toString() {
            return testName + " [" + methodName + "]";
        }
    }

    private void setupServerStubForSpec(TestSpec spec) {
        if (spec.badStatus) {
            // Setup for bad status test
            if (StringUtils.isNullOrWhitespaceOnly(spec.methodName)
                    || spec.methodName.equalsIgnoreCase("GET")) {
                wireMockServer.stubFor(
                        get(urlPathEqualTo(ENDPOINT))
                                .withHeader("Content-Type", equalTo("application/json"))
                                .willReturn(aResponse().withBody(new byte[0]).withStatus(500))
                                .withHeader("Content-Type", equalTo("application/json")));
            } else {
                setUpServerBodyStub(
                        spec.methodName,
                        wireMockServer,
                        List.of(matchingJsonPath("$.id"), matchingJsonPath("$.id2")),
                        Integer.valueOf(500));
            }
        } else if (spec.deserError) {

            // Setup for deserialization error test
            if (StringUtils.isNullOrWhitespaceOnly(spec.methodName)
                    || spec.methodName.equalsIgnoreCase("GET")) {
                wireMockServer.stubFor(
                        get(urlPathEqualTo(ENDPOINT))
                                .withHeader("Content-Type", equalTo("application/json"))
                                .willReturn(
                                        aResponse()
                                                .withBody(A_TEST_STRING_THAT_IS_NOT_JSON)
                                                .withStatus(200)));
            } else {
                setUpServerBodyStub(
                        spec.methodName,
                        wireMockServer,
                        List.of(matchingJsonPath("$.id"), matchingJsonPath("$.id2")),
                        true);
            }
        } else if (spec.connectionError) {
            // No need to set up server stub for connection error test
            // The test will use a non-existent port (9999)
        } else {
            // Setup for success test
            if (StringUtils.isNullOrWhitespaceOnly(spec.methodName)
                    || spec.methodName.equalsIgnoreCase("GET")) {
                setupServerStub(wireMockServer);
            } else {
                setUpServerBodyStub(
                        spec.methodName,
                        wireMockServer,
                        List.of(matchingJsonPath("$.id"), matchingJsonPath("$.id2")));
            }
        }
    }

    private String createLookupTableSql(TestSpec spec) {
        StringBuilder sql = new StringBuilder();
        sql.append("CREATE TABLE Customers (")
                .append("id STRING,")
                .append("id2 STRING,")
                .append("msg STRING,")
                .append("uuid STRING,")
                .append("details ROW<")
                .append("isActive BOOLEAN,")
                .append("nestedDetails ROW<")
                .append("balance STRING")
                .append(">")
                .append(">");

        if (spec.useMetadata) {
            sql.append(",")
                    .append("errStr STRING METADATA FROM 'error-string',")
                    .append("statusCode INTEGER METADATA FROM 'http-status-code',")
                    .append("headers MAP<STRING, ARRAY<STRING>> METADATA from 'http-headers',")
                    .append("completionState STRING METADATA from 'http-completion-state'");
        }

        sql.append(") WITH (").append("'format' = 'json',").append("'connector' = 'http',");
        if (spec.ignoreParseErrors) {
            sql.append("'json.ignore-parse-errors' = 'true',");
        }
        if (!StringUtils.isNullOrWhitespaceOnly(spec.methodName)) {
            sql.append("'lookup-method' = '").append(spec.methodName).append("',");
        }

        // URL with correct port for connection error test
        if (spec.connectionError) {
            sql.append("'url' = 'http://localhost:9999/client',");
        } else {
            sql.append("'url' = 'http://localhost:" + serverPort + "/client',");
        }
        sql.append("'http.source.lookup.header.Content-Type' = 'application/json',")
                .append("'http.source.lookup.continue-on-error'='")
                .append(spec.continueOnError)
                .append("',");
        sql.append("'asyncPolling' = '")
                .append(spec.useAsync ? "true" : "false")
                .append("',")
                .append("'table.exec.async-lookup.buffer-capacity' = '50',")
                .append("'table.exec.async-lookup.timeout' = '20s'")
                .append(")");
        return sql.toString();
    }
}
