/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.kafka.streams.integration;

import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.common.serialization.IntegerDeserializer;
import org.apache.kafka.common.serialization.IntegerSerializer;
import org.apache.kafka.common.serialization.Serdes;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.apache.kafka.common.serialization.StringSerializer;
import org.apache.kafka.common.utils.LogCaptureAppender;
import org.apache.kafka.common.utils.MockTime;
import org.apache.kafka.streams.GroupProtocol;
import org.apache.kafka.streams.KafkaStreams;
import org.apache.kafka.streams.KeyValue;
import org.apache.kafka.streams.StreamsBuilder;
import org.apache.kafka.streams.StreamsConfig;
import org.apache.kafka.streams.Topology;
import org.apache.kafka.streams.errors.StreamsException;
import org.apache.kafka.streams.errors.StreamsUncaughtExceptionHandler.StreamThreadExceptionResponse;
import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster;
import org.apache.kafka.streams.integration.utils.IntegrationTestUtils;
import org.apache.kafka.streams.kstream.Consumed;
import org.apache.kafka.streams.kstream.KStream;
import org.apache.kafka.streams.kstream.Materialized;
import org.apache.kafka.streams.kstream.Named;
import org.apache.kafka.streams.processor.api.Processor;
import org.apache.kafka.streams.processor.api.ProcessorContext;
import org.apache.kafka.streams.processor.api.Record;
import org.apache.kafka.streams.processor.internals.StreamThread;
import org.apache.kafka.streams.state.Stores;
import org.apache.kafka.streams.state.internals.KeyValueStoreBuilder;
import org.apache.kafka.test.TestUtils;

import org.apache.logging.log4j.Level;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.TestInfo;
import org.junit.jupiter.api.Timeout;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;

import java.io.IOException;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;

import static java.util.Arrays.asList;
import static org.apache.kafka.common.utils.Utils.mkEntry;
import static org.apache.kafka.common.utils.Utils.mkMap;
import static org.apache.kafka.common.utils.Utils.mkObjectProperties;
import static org.apache.kafka.streams.errors.StreamsUncaughtExceptionHandler.StreamThreadExceptionResponse.REPLACE_THREAD;
import static org.apache.kafka.streams.errors.StreamsUncaughtExceptionHandler.StreamThreadExceptionResponse.SHUTDOWN_APPLICATION;
import static org.apache.kafka.streams.errors.StreamsUncaughtExceptionHandler.StreamThreadExceptionResponse.SHUTDOWN_CLIENT;
import static org.apache.kafka.streams.integration.utils.IntegrationTestUtils.purgeLocalStreamsState;
import static org.apache.kafka.streams.integration.utils.IntegrationTestUtils.startApplicationAndWaitUntilRunning;
import static org.apache.kafka.streams.utils.TestUtils.safeUniqueTestName;
import static org.apache.kafka.streams.utils.TestUtils.waitForApplicationState;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.jupiter.api.Assertions.fail;

@Tag("integration")
@Timeout(600)
public class StreamsUncaughtExceptionHandlerIntegrationTest {
    private static final long NOW = Instant.now().toEpochMilli();

    public static final EmbeddedKafkaCluster CLUSTER = new EmbeddedKafkaCluster(1);

    @BeforeAll
    public static void startCluster() throws IOException {
        CLUSTER.start();
    }

    @AfterAll
    public static void closeCluster() {
        CLUSTER.stop();
    }

    public static final Duration DEFAULT_DURATION = Duration.ofSeconds(30);

    private String appId;
    private String inputTopic;
    private String inputTopic2;
    private String outputTopic;
    private String outputTopic2;
    private final StreamsBuilder builder = new StreamsBuilder();
    private final List<String> processorValueCollector = new ArrayList<>();
    private static final AtomicBoolean THROW_ERROR = new AtomicBoolean(true);

    private Properties properties;

    private Properties basicProps(final boolean streamsRebalanceProtocolEnabled) {
        final String protocol;
        if (streamsRebalanceProtocolEnabled) {
            protocol = GroupProtocol.STREAMS.name().toLowerCase(Locale.getDefault());
        } else {
            protocol = GroupProtocol.CLASSIC.name().toLowerCase(Locale.getDefault());
        }
        return mkObjectProperties(
            mkMap(
                mkEntry(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers()),
                mkEntry(StreamsConfig.APPLICATION_ID_CONFIG, appId),
                mkEntry(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath()),
                mkEntry(StreamsConfig.NUM_STREAM_THREADS_CONFIG, 2),
                mkEntry(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.StringSerde.class),
                mkEntry(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.StringSerde.class),
                mkEntry(StreamsConfig.consumerPrefix(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG), 10000),
                mkEntry(StreamsConfig.GROUP_PROTOCOL_CONFIG, protocol)
            )
        );
    }

    @BeforeEach
    public void setup(final TestInfo testInfo) {
        final String testId = safeUniqueTestName(testInfo);
        appId = "appId_" + testId;
        inputTopic = "input" + testId;
        inputTopic2 = "input2" + testId;
        outputTopic = "output" + testId;
        outputTopic2 = "output2" + testId;
        IntegrationTestUtils.cleanStateBeforeTest(CLUSTER, inputTopic, inputTopic2, outputTopic, outputTopic2);
        final KStream<String, String> stream = builder.stream(inputTopic);
        stream.process(() -> new ShutdownProcessor<>(processorValueCollector), Named.as("process"));
    }

    @AfterEach
    public void teardown() throws IOException {
        purgeLocalStreamsState(properties);
    }

    @ParameterizedTest
    @ValueSource(booleans = {false, true})
    public void shouldShutdownClient(final boolean streamsRebalanceProtocolEnabled) throws Exception {
        properties = basicProps(streamsRebalanceProtocolEnabled);
        try (final KafkaStreams kafkaStreams = new KafkaStreams(builder.build(), properties)) {

            kafkaStreams.setUncaughtExceptionHandler(exception -> SHUTDOWN_CLIENT);

            startApplicationAndWaitUntilRunning(kafkaStreams);

            produceMessages(NOW, inputTopic, "A");
            waitForApplicationState(Collections.singletonList(kafkaStreams), KafkaStreams.State.ERROR, DEFAULT_DURATION);

            assertThat(processorValueCollector.size(), equalTo(1));
        }
    }

    @ParameterizedTest
    @ValueSource(booleans = {false, true})
    public void shouldReplaceThreads(final boolean streamsRebalanceProtocolEnabled) throws Exception {
        properties = basicProps(streamsRebalanceProtocolEnabled);
        testReplaceThreads(2);
    }

    @ParameterizedTest
    @ValueSource(booleans = {false, true})
    public void shouldReplaceThreadsWithoutJavaHandler(final boolean streamsRebalanceProtocolEnabled) throws Exception {
        properties = basicProps(streamsRebalanceProtocolEnabled);
        Thread.setDefaultUncaughtExceptionHandler((t, e) -> fail("exception thrown"));
        testReplaceThreads(2);
    }

    @ParameterizedTest
    @ValueSource(booleans = {false, true})
    public void shouldReplaceSingleThread(final boolean streamsRebalanceProtocolEnabled) throws Exception {
        properties = basicProps(streamsRebalanceProtocolEnabled);
        testReplaceThreads(1);
    }

    @ParameterizedTest
    @ValueSource(booleans = {false, true})
    public void shouldShutdownMultipleThreadApplication(final boolean streamsRebalanceProtocolEnabled) throws Exception {
        properties = basicProps(streamsRebalanceProtocolEnabled);
        testShutdownApplication(2);
    }

    @ParameterizedTest
    @ValueSource(booleans = {false, true})
    public void shouldShutdownSingleThreadApplication(final boolean streamsRebalanceProtocolEnabled) throws Exception {
        properties = basicProps(streamsRebalanceProtocolEnabled);
        testShutdownApplication(1);
    }

    private static class ShutdownProcessor<KIn, VIn, KOut, VOut> implements Processor<KIn, VIn, KOut, VOut> {

        final List<String> valueList;

        ShutdownProcessor(final List<String> valueList) {
            this.valueList = valueList;
        }

        @Override
        public void init(final ProcessorContext<KOut, VOut> context) {
        }

        @Override
        public void close() {
            // do nothing
        }

        @Override
        public void process(final Record<KIn, VIn> record) {
            valueList.add(record.value().toString());
            if (THROW_ERROR.get()) {
                throw new StreamsException(Thread.currentThread().getName());
            }
            THROW_ERROR.set(true);
        }
    }

    @ParameterizedTest
    @ValueSource(booleans = {false, true})
    public void shouldShutDownClientIfGlobalStreamThreadWantsToReplaceThread(final boolean streamsRebalanceProtocolEnabled) throws Exception {
        properties = basicProps(streamsRebalanceProtocolEnabled);
        builder.addGlobalStore(
                new KeyValueStoreBuilder<>(
                        Stores.persistentKeyValueStore("globalStore"),
                        Serdes.String(),
                        Serdes.String(),
                        CLUSTER.time
                ),
                inputTopic2,
                Consumed.with(Serdes.String(), Serdes.String()),
                () -> new ShutdownProcessor<String, String, Void, Void>(processorValueCollector)
        );
        properties.put(StreamsConfig.NUM_STREAM_THREADS_CONFIG, 0);

        try (final KafkaStreams kafkaStreams = new KafkaStreams(builder.build(), properties)) {
            kafkaStreams.setUncaughtExceptionHandler(exception -> REPLACE_THREAD);

            startApplicationAndWaitUntilRunning(kafkaStreams);

            produceMessages(NOW, inputTopic2, "A");
            waitForApplicationState(Collections.singletonList(kafkaStreams), KafkaStreams.State.ERROR, DEFAULT_DURATION);

            assertThat(processorValueCollector.size(), equalTo(1));
        }
    }

    @ParameterizedTest
    @ValueSource(booleans = {false, true})
    public void shouldEmitSameRecordAfterFailover(final boolean streamsRebalanceProtocolEnabled) throws Exception {
        properties = basicProps(streamsRebalanceProtocolEnabled);
        properties.put(StreamsConfig.NUM_STREAM_THREADS_CONFIG, 1);
        properties.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 300000L);
        properties.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 0);
        properties.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.IntegerSerde.class);
        properties.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.StringSerde.class);
        properties.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, 10000);

        final AtomicBoolean shouldThrow = new AtomicBoolean(true);
        final StreamsBuilder builder = new StreamsBuilder();
        builder.table(inputTopic, Materialized.as("test-store"))
            .toStream()
            .map((key, value) -> {
                if (shouldThrow.compareAndSet(true, false)) {
                    throw new RuntimeException("Kaboom");
                } else {
                    return new KeyValue<>(key, value);
                }
            })
            .to(outputTopic);
        builder.stream(inputTopic2).to(outputTopic2);

        try (final KafkaStreams kafkaStreams = new KafkaStreams(builder.build(), properties)) {
            kafkaStreams.setUncaughtExceptionHandler(exception -> StreamThreadExceptionResponse.REPLACE_THREAD);
            startApplicationAndWaitUntilRunning(kafkaStreams);

            IntegrationTestUtils.produceKeyValuesSynchronouslyWithTimestamp(
                inputTopic,
                asList(
                    new KeyValue<>(1, "A"),
                    new KeyValue<>(1, "B")
                ),
                TestUtils.producerConfig(
                    CLUSTER.bootstrapServers(),
                    IntegerSerializer.class,
                    StringSerializer.class,
                    new Properties()),
                    NOW);

            IntegrationTestUtils.produceKeyValuesSynchronouslyWithTimestamp(
                inputTopic2,
                asList(
                    new KeyValue<>(1, "A"),
                    new KeyValue<>(1, "B")
                ),
                TestUtils.producerConfig(
                    CLUSTER.bootstrapServers(),
                    IntegerSerializer.class,
                    StringSerializer.class,
                    new Properties()),
                    NOW);

            IntegrationTestUtils.waitUntilFinalKeyValueRecordsReceived(
                TestUtils.consumerConfig(
                    CLUSTER.bootstrapServers(),
                    IntegerDeserializer.class,
                    StringDeserializer.class
                ),
                outputTopic,
                asList(
                    new KeyValue<>(1, "A"),
                    new KeyValue<>(1, "B")
                )
            );
            IntegrationTestUtils.waitUntilFinalKeyValueRecordsReceived(
                TestUtils.consumerConfig(
                    CLUSTER.bootstrapServers(),
                    IntegerDeserializer.class,
                    StringDeserializer.class
                ),
                outputTopic2,
                asList(
                    new KeyValue<>(1, "A"),
                    new KeyValue<>(1, "B")
                )
            );
        }
    }

    private void produceMessages(final long timestamp, final String streamOneInput, final String msg) {
        IntegrationTestUtils.produceKeyValuesSynchronouslyWithTimestamp(
            streamOneInput,
            Collections.singletonList(new KeyValue<>("1", msg)),
            TestUtils.producerConfig(
                CLUSTER.bootstrapServers(),
                StringSerializer.class,
                StringSerializer.class,
                new Properties()),
            timestamp);
    }

    private void testShutdownApplication(final int numThreads) throws Exception {
        properties.put(StreamsConfig.NUM_STREAM_THREADS_CONFIG, numThreads);

        final Topology topology = builder.build();
        final MockTime time = new MockTime(0L);
        
        try (final KafkaStreams kafkaStreams1 = new KafkaStreams(topology, properties, time);
             final KafkaStreams kafkaStreams2 = new KafkaStreams(topology, properties, time);
             final LogCaptureAppender logCaptureAppender = LogCaptureAppender.createAndRegister()) {
            kafkaStreams1.setUncaughtExceptionHandler(exception -> SHUTDOWN_APPLICATION);
            kafkaStreams2.setUncaughtExceptionHandler(exception -> SHUTDOWN_APPLICATION);
            logCaptureAppender.setClassLogger(StreamThread.class, Level.WARN);

            startApplicationAndWaitUntilRunning(asList(kafkaStreams1, kafkaStreams2));

            produceMessages(NOW, inputTopic, "A");
            waitForApplicationState(asList(kafkaStreams1, kafkaStreams2), KafkaStreams.State.ERROR, DEFAULT_DURATION);

            assertThat(processorValueCollector.size(), equalTo(1));
            assertThat("Shutdown warning log message should be exported exactly once",
                    logCaptureAppender.getMessages("WARN").stream().filter(msg -> msg.contains("Detected that shutdown was requested")).count(), equalTo(1L));
        }
    }

    private void testReplaceThreads(final int numThreads) throws Exception {
        properties.put(StreamsConfig.NUM_STREAM_THREADS_CONFIG, numThreads);
        try (final KafkaStreams kafkaStreams = new KafkaStreams(builder.build(), properties)) {
            final AtomicInteger count = new AtomicInteger();
            kafkaStreams.setUncaughtExceptionHandler(exception -> {
                if (count.incrementAndGet() == numThreads) {
                    THROW_ERROR.set(false);
                }
                return REPLACE_THREAD;
            });
            startApplicationAndWaitUntilRunning(kafkaStreams);

            produceMessages(NOW, inputTopic, "A");
            TestUtils.waitForCondition(() -> count.get() == numThreads, "finished replacing threads");
            TestUtils.waitForCondition(THROW_ERROR::get, "finished replacing threads");
            kafkaStreams.close();
            waitForApplicationState(Collections.singletonList(kafkaStreams), KafkaStreams.State.NOT_RUNNING, DEFAULT_DURATION);

            assertThat("All initial threads have failed and the replacement thread had processed on record",
                processorValueCollector.size(), equalTo(numThreads + 1));
        }
    }
}
