/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.flink.table.planner.analyze;

import org.apache.flink.table.api.ExplainDetail;
import org.apache.flink.table.api.StatementSet;
import org.apache.flink.table.api.TableConfig;
import org.apache.flink.table.api.TableEnvironment;
import org.apache.flink.table.api.TableException;
import org.apache.flink.table.api.config.OptimizerConfigOptions;
import org.apache.flink.table.api.config.OptimizerConfigOptions.NonDeterministicUpdateStrategy;
import org.apache.flink.table.planner.expressions.utils.TestNonDeterministicUdf;
import org.apache.flink.table.planner.utils.PlanKind;
import org.apache.flink.table.planner.utils.StreamTableTestUtil;
import org.apache.flink.table.planner.utils.TableTestBase;

import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;

import scala.Enumeration;

import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static scala.runtime.BoxedUnit.UNIT;

/** Test for {@link NonDeterministicUpdateAnalyzer}. */
class NonDeterministicUpdateAnalyzerTest extends TableTestBase {

    private TableEnvironment tEnv;
    private final StreamTableTestUtil util = streamTestUtil(TableConfig.getDefault());

    private static final String expectedOverAggNduErrorMsg =
            "The column(s): $3(generated by non-deterministic function: UUID ) can not satisfy the determinism requirement for correctly processing update message('UB'/'UA'/'D' in changelogMode, not 'I' only), this usually happens when input node has no upsertKey(upsertKeys=[{}]) or current node outputs non-deterministic update messages. Please consider removing these non-deterministic columns or making them deterministic by using deterministic functions.\n"
                    + "\n"
                    + "related rel plan:\n"
                    + "Calc(select=[key, val, ts, UUID() AS $3], changelogMode=[I,UB,UA])\n"
                    + "+- TableSourceScan(table=[[default_catalog, default_database, source_t]], fields=[key, val, ts], changelogMode=[I,UB,UA])\n";

    @BeforeEach
    void before() {
        tEnv = util.getTableEnv();
        tEnv.executeSql(
                "create temporary table cdc (\n"
                        + "  a int,\n"
                        + "  b bigint,\n"
                        + "  c string,\n"
                        + "  d bigint,\n"
                        + "  primary key (a) not enforced\n"
                        + ") with (\n"
                        + "  'connector' = 'values',\n"
                        + "  'changelog-mode' = 'I,UA,UB,D'\n"
                        + ")");
        tEnv.executeSql(
                "create temporary table cdc_with_meta (\n"
                        + "  a int,\n"
                        + "  b bigint,\n"
                        + "  c string,\n"
                        + "  d boolean,\n"
                        + "  metadata_1 int metadata,\n"
                        + "  metadata_2 string metadata,\n"
                        + "  metadata_3 bigint metadata,\n"
                        + "  primary key (a) not enforced\n"
                        + ") with (\n"
                        + "  'connector' = 'values',\n"
                        + "  'changelog-mode' = 'I,UA,UB,D',\n"
                        + "  'readable-metadata' = 'metadata_1:INT, metadata_2:STRING, metadata_3:BIGINT'\n"
                        + ")");
        tEnv.executeSql(
                "create temporary table sink_with_pk (\n"
                        + "  a int,\n"
                        + "  b bigint,\n"
                        + "  c string,\n"
                        + "  primary key (a) not enforced\n"
                        + ") with (\n"
                        + "  'connector' = 'values',\n"
                        + "  'sink-insert-only' = 'false'\n"
                        + ")");
        tEnv.executeSql(
                "create temporary table sink_without_pk (\n"
                        + "  a int,\n"
                        + "  b bigint,\n"
                        + "  c string\n"
                        + ") with (\n"
                        + "  'connector' = 'values',\n"
                        + "  'sink-insert-only' = 'false'\n"
                        + ")");
        tEnv.executeSql(
                "create temporary table dim_with_pk (\n"
                        + " a int,\n"
                        + " b bigint,\n"
                        + " c string,\n"
                        + " primary key (a) not enforced\n"
                        + ") with (\n"
                        + " 'connector' = 'values'\n"
                        + ")");
        // custom ND function
        tEnv.createTemporaryFunction("ndFunc", new TestNonDeterministicUdf());

        String sourceTable =
                "CREATE TABLE source_t(\n"
                        + "  key STRING,\n"
                        + "  val BIGINT,\n"
                        + "  ts BIGINT\n"
                        + ") with (\n"
                        + "  'connector' = 'values',\n"
                        + "  'bounded' = 'false',\n"
                        + "  'changelog-mode' = 'I,UB,UA')";
        tEnv.executeSql(sourceTable);

        String sinkTable =
                "CREATE TABLE sink_t(\n"
                        + "  key STRING,\n"
                        + "  val BIGINT,\n"
                        + "  ts BIGINT,\n"
                        + "  sum_val BIGINT\n"
                        + ") with (\n"
                        + "  'connector' = 'values',\n"
                        + "  'bounded' = 'false',\n"
                        + "  'sink-insert-only' = 'false',\n"
                        + "  'changelog-mode' = 'I,UB,UA')";
        tEnv.executeSql(sinkTable);
    }

    @Test
    void testCdcWithMetaRenameSinkWithCompositePk() {
        // from NonDeterministicDagTest#testCdcWithMetaRenameSinkWithCompositePk
        tEnv.executeSql(
                "create temporary table cdc_with_meta_rename (\n"
                        + "  a int,\n"
                        + "  b bigint,\n"
                        + "  c string,\n"
                        + "  d boolean,\n"
                        + "  metadata_3 bigint metadata,\n"
                        + "  e as metadata_3,\n"
                        + "  primary key (a) not enforced\n"
                        + ") with (\n"
                        + "  'connector' = 'values',\n"
                        + "  'changelog-mode' = 'I,UA,UB,D',\n"
                        + "  'readable-metadata' = 'metadata_3:BIGINT'\n"
                        + ")");
        tEnv.executeSql(
                "create temporary table sink_with_composite_pk (\n"
                        + "  a int,\n"
                        + "  b bigint,\n"
                        + "  c string,\n"
                        + "  d bigint,\n"
                        + "  primary key (a,d) not enforced\n"
                        + ") with (\n"
                        + "  'connector' = 'values',\n"
                        + "  'sink-insert-only' = 'false'\n"
                        + ")");
        util.doVerifyPlanInsert(
                "insert into sink_with_composite_pk\n"
                        + "select a, b, c, e from cdc_with_meta_rename",
                new ExplainDetail[] {ExplainDetail.PLAN_ADVICE},
                false,
                new Enumeration.Value[] {PlanKind.OPT_REL_WITH_ADVICE()});
    }

    @Test
    void testCdcSourceWithoutPkSinkWithoutPk() {
        // from NonDeterministicDagTest#testCdcSourceWithoutPkSinkWithoutPk
        tEnv.executeSql(
                "create temporary table cdc_without_pk (\n"
                        + " a int,\n"
                        + " b bigint,\n"
                        + " c string,\n"
                        + " d boolean,\n"
                        + " metadata_1 int metadata,\n"
                        + " metadata_2 string metadata\n"
                        + ") with (\n"
                        + " 'connector' = 'values',\n"
                        + " 'changelog-mode' = 'I,UA,UB,D',\n"
                        + " 'readable-metadata' = 'metadata_1:INT, metadata_2:STRING'\n"
                        + ")");

        util.doVerifyPlanInsert(
                "insert into sink_without_pk\n"
                        + "select metadata_1, b, metadata_2\n"
                        + "from cdc_without_pk",
                new ExplainDetail[] {ExplainDetail.PLAN_ADVICE},
                false,
                new Enumeration.Value[] {PlanKind.OPT_REL_WITH_ADVICE()});
    }

    @Test
    void testSourceWithComputedColumnSinkWithPk() {
        // from NonDeterministicDagTest#testSourceWithComputedColumnSinkWithPk
        tEnv.executeSql(
                "create temporary table cdc_with_computed_col (\n"
                        + "  a int,\n"
                        + "  b bigint,\n"
                        + "  c string,\n"
                        + "  d int,\n"
                        + "  `day` as DATE_FORMAT(CURRENT_TIMESTAMP, 'yyMMdd'),\n"
                        + "  primary key(a, c) not enforced\n"
                        + ") with (\n"
                        + " 'connector' = 'values',\n"
                        + " 'changelog-mode' = 'I,UA,UB,D'\n"
                        + ")");
        util.doVerifyPlanInsert(
                "insert into sink_with_pk\n"
                        + "select a, b, `day`\n"
                        + "from cdc_with_computed_col\n"
                        + "where b > 100\n"
                        + "on conflict do deduplicate",
                new ExplainDetail[] {ExplainDetail.PLAN_ADVICE},
                false,
                new Enumeration.Value[] {PlanKind.OPT_REL_WITH_ADVICE()});
    }

    @Test
    void testCdcJoinDimWithPkNonDeterministicLocalCondition() {
        // from NonDeterministicDagTest#testCdcJoinDimWithPkNonDeterministicLocalCondition
        util.doVerifyPlanInsert(
                "insert into sink_without_pk\n"
                        + "select t1.a, t1.b, t1.c\n"
                        + "from (\n"
                        + "  select *, proctime() proctime from cdc\n"
                        + ") t1 join dim_with_pk for system_time as of t1.proctime as t2\n"
                        + "on t1.a = t2.a and ndFunc(t2.b) > 100",
                new ExplainDetail[] {ExplainDetail.PLAN_ADVICE},
                false,
                new Enumeration.Value[] {PlanKind.OPT_REL_WITH_ADVICE()});
    }

    @Test
    void testCdcWithMetaSinkWithPk() {
        // from NonDeterministicDagTest#testCdcWithMetaSinkWithPk
        util.doVerifyPlanInsert(
                "insert into sink_with_pk\n" + "select a, metadata_3, c\n" + "from cdc_with_meta",
                new ExplainDetail[] {ExplainDetail.PLAN_ADVICE},
                false,
                new Enumeration.Value[] {PlanKind.OPT_REL_WITH_ADVICE()});
    }

    @Test
    void testGroupByNonDeterministicFuncWithCdcSource() {
        // from NonDeterministicDagTest#testGroupByNonDeterministicFuncWithCdcSource
        util.doVerifyPlanInsert(
                "insert into sink_with_pk\n"
                        + "select\n"
                        + "  a, count(*) cnt, `day`\n"
                        + "from (\n"
                        + "  select *, DATE_FORMAT(CURRENT_TIMESTAMP, 'yyMMdd') `day` from cdc\n"
                        + ") t\n"
                        + "group by `day`, a\n"
                        + "on conflict do deduplicate",
                new ExplainDetail[] {ExplainDetail.PLAN_ADVICE},
                false,
                new Enumeration.Value[] {PlanKind.OPT_REL_WITH_ADVICE()});
    }

    @Test
    void testMultiSinkOnJoinedView() {
        // from NonDeterministicDagTest#testMultiSinkOnJoinedView
        tEnv.executeSql(
                "create temporary table src1 (\n"
                        + "  a int,\n"
                        + "  b bigint,\n"
                        + "  c string,\n"
                        + "  d int,\n"
                        + "  primary key(a, c) not enforced\n"
                        + ") with (\n"
                        + " 'connector' = 'values',\n"
                        + " 'changelog-mode' = 'I,UA,UB,D'\n"
                        + ")");
        tEnv.executeSql(
                "create temporary table src2 (\n"
                        + "  a int,\n"
                        + "  b bigint,\n"
                        + "  c string,\n"
                        + "  d int,\n"
                        + "  primary key(a, c) not enforced\n"
                        + ") with (\n"
                        + " 'connector' = 'values',\n"
                        + " 'changelog-mode' = 'I,UA,UB,D'\n"
                        + ")");
        tEnv.executeSql(
                "create temporary table sink1 (\n"
                        + "  a int,\n"
                        + "  b string,\n"
                        + "  c bigint,\n"
                        + "  d bigint\n"
                        + ") with (\n"
                        + " 'connector' = 'values',\n"
                        + " 'sink-insert-only' = 'false'\n"
                        + ")");
        tEnv.executeSql(
                "create temporary table sink2 (\n"
                        + "  a int,\n"
                        + "  b string,\n"
                        + "  c bigint,\n"
                        + "  d string\n"
                        + ") with (\n"
                        + " 'connector' = 'values',\n"
                        + " 'sink-insert-only' = 'false'\n"
                        + ")");
        tEnv.executeSql(
                "create temporary view v1 as\n"
                        + "select\n"
                        + "  t1.a as a, t1.`day` as `day`, t2.b as b, t2.c as c\n"
                        + "from (\n"
                        + "  select a, b, DATE_FORMAT(CURRENT_TIMESTAMP, 'yyMMdd') as `day`\n"
                        + "  from src1\n"
                        + " ) t1\n"
                        + "join (\n"
                        + "  select b, CONCAT(c, DATE_FORMAT(CURRENT_TIMESTAMP, 'yyMMdd')) as `day`, c, d\n"
                        + "  from src2\n"
                        + ") t2\n"
                        + " on t1.a = t2.d");

        StatementSet stmtSet = tEnv.createStatementSet();
        stmtSet.addInsertSql(
                "insert into sink1\n"
                        + "  select a, `day`, sum(b), count(distinct c)\n"
                        + "  from v1\n"
                        + "  group by a, `day`");
        stmtSet.addInsertSql(
                "insert into sink2\n"
                        + "  select a, `day`, b, c\n"
                        + "  from v1\n"
                        + "  where b > 100");
        util.doVerifyPlan(
                stmtSet,
                new ExplainDetail[] {ExplainDetail.PLAN_ADVICE},
                false,
                new Enumeration.Value[] {PlanKind.OPT_REL_WITH_ADVICE()},
                () -> UNIT,
                false,
                false);
    }

    @Test
    void testCdcJoinDimWithPkOutputNoPkSinkWithoutPk() {
        // from NonDeterministicDagTest#testCdcJoinDimWithPkOutputNoPkSinkWithoutPk
        util.doVerifyPlanInsert(
                "insert into sink_without_pk\n"
                        + "  select t1.a, t2.b, t1.c\n"
                        + "  from (\n"
                        + "    select *, proctime() proctime from cdc\n"
                        + "  ) t1 join dim_with_pk for system_time as of t1.proctime as t2\n"
                        + "  on t1.a = t2.a",
                new ExplainDetail[] {ExplainDetail.PLAN_ADVICE},
                false,
                new Enumeration.Value[] {PlanKind.OPT_REL_WITH_ADVICE()});
    }

    @Test
    void testOverAggregateWithNonDeterminismInPartitionBy() {
        tEnv.getConfig()
                .set(
                        OptimizerConfigOptions.TABLE_OPTIMIZER_NONDETERMINISTIC_UPDATE_STRATEGY,
                        NonDeterministicUpdateStrategy.TRY_RESOLVE);

        String sql =
                "INSERT INTO sink_t SELECT key, val, ts, SUM(val) OVER ("
                        + "PARTITION BY UUID() "
                        + "ORDER BY val "
                        + "RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) "
                        + "AS sum_val "
                        + "FROM source_t";

        TableException tableException =
                assertThrows(TableException.class, () -> util.verifyJsonPlan(sql));

        assertEquals(expectedOverAggNduErrorMsg, tableException.getMessage());
    }

    @Test
    void testOverAggregateWithNonDeterminismInOrderBy() {
        tEnv.getConfig()
                .set(
                        OptimizerConfigOptions.TABLE_OPTIMIZER_NONDETERMINISTIC_UPDATE_STRATEGY,
                        NonDeterministicUpdateStrategy.TRY_RESOLVE);

        String sql =
                "INSERT INTO sink_t SELECT key, val, ts, SUM(val) OVER ("
                        + "PARTITION BY key "
                        + "ORDER BY UUID() "
                        + "RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) "
                        + "AS sum_val "
                        + "FROM source_t";

        TableException tableException =
                assertThrows(TableException.class, () -> util.verifyJsonPlan(sql));

        assertEquals(expectedOverAggNduErrorMsg, tableException.getMessage());
    }

    @Test
    void testOverAggregateWithNonDeterminismInProjection() {
        tEnv.getConfig()
                .set(
                        OptimizerConfigOptions.TABLE_OPTIMIZER_NONDETERMINISTIC_UPDATE_STRATEGY,
                        NonDeterministicUpdateStrategy.TRY_RESOLVE);

        String sql =
                "INSERT INTO sink_t SELECT UUID(), val, ts, SUM(val) OVER ("
                        + "PARTITION BY key "
                        + "ORDER BY val "
                        + "RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) "
                        + "AS sum_val "
                        + "FROM source_t";

        TableException tableException =
                assertThrows(TableException.class, () -> util.verifyJsonPlan(sql));

        assertEquals(expectedOverAggNduErrorMsg, tableException.getMessage());
    }

    @Test
    void testNowFilterPushedIntoChangelogNormalizeOnUpsertSource() {
        tEnv.getConfig()
                .set(
                        OptimizerConfigOptions.TABLE_OPTIMIZER_NONDETERMINISTIC_UPDATE_STRATEGY,
                        NonDeterministicUpdateStrategy.TRY_RESOLVE);

        tEnv.executeSql(
                "CREATE TEMPORARY TABLE upsert_src (\n"
                        + "  a INT,\n"
                        + "  b BIGINT,\n"
                        + "  c STRING,\n"
                        + "  ts TIMESTAMP(3),\n"
                        + "  PRIMARY KEY (a) NOT ENFORCED\n"
                        + ") WITH (\n"
                        + "  'connector' = 'values',\n"
                        + "  'changelog-mode' = 'I,UA,D'\n"
                        + ")");

        final String sql =
                "INSERT INTO sink_with_pk SELECT a, b, c FROM upsert_src"
                        + " WHERE ts >= NOW() - INTERVAL '90' DAY";

        assertThatThrownBy(() -> util.verifyJsonPlan(sql))
                .isInstanceOf(TableException.class)
                .hasMessageContaining("non deterministic function")
                .hasMessageContaining("NOW");
    }

    @Test
    void testCurrentTimestampFilterPushedIntoChangelogNormalizeOnUpsertSource() {
        tEnv.getConfig()
                .set(
                        OptimizerConfigOptions.TABLE_OPTIMIZER_NONDETERMINISTIC_UPDATE_STRATEGY,
                        NonDeterministicUpdateStrategy.TRY_RESOLVE);

        tEnv.executeSql(
                "CREATE TEMPORARY TABLE upsert_src2 (\n"
                        + "  a INT,\n"
                        + "  b BIGINT,\n"
                        + "  c STRING,\n"
                        + "  ts TIMESTAMP(3),\n"
                        + "  PRIMARY KEY (a) NOT ENFORCED\n"
                        + ") WITH (\n"
                        + "  'connector' = 'values',\n"
                        + "  'changelog-mode' = 'I,UA,D'\n"
                        + ")");

        final String sql =
                "INSERT INTO sink_with_pk SELECT a, b, c FROM upsert_src2"
                        + " WHERE ts >= CURRENT_TIMESTAMP - INTERVAL '90' DAY";

        assertThatThrownBy(() -> util.verifyJsonPlan(sql))
                .isInstanceOf(TableException.class)
                .hasMessageContaining("non deterministic function")
                .hasMessageContaining("CURRENT_TIMESTAMP");
    }

    @Test
    void testDeterministicFilterOnUpsertSourceIsAllowed() {
        tEnv.getConfig()
                .set(
                        OptimizerConfigOptions.TABLE_OPTIMIZER_NONDETERMINISTIC_UPDATE_STRATEGY,
                        NonDeterministicUpdateStrategy.TRY_RESOLVE);

        tEnv.executeSql(
                "CREATE TEMPORARY TABLE upsert_src3 (\n"
                        + "  a INT,\n"
                        + "  b BIGINT,\n"
                        + "  c STRING,\n"
                        + "  PRIMARY KEY (a) NOT ENFORCED\n"
                        + ") WITH (\n"
                        + "  'connector' = 'values',\n"
                        + "  'changelog-mode' = 'I,UA,D'\n"
                        + ")");

        // Deterministic filter should compile without error
        final String sql = "INSERT INTO sink_with_pk SELECT a, b, c FROM upsert_src3 WHERE b > 100";

        util.verifyJsonPlan(sql);
    }
}
