/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements. See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.kafka.streams.kstream.internals;

import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.common.header.internals.RecordHeaders;
import org.apache.kafka.common.metrics.Sensor;
import org.apache.kafka.common.utils.Time;
import org.apache.kafka.streams.KeyValue;
import org.apache.kafka.streams.StreamsConfig;
import org.apache.kafka.streams.errors.InvalidStateStoreException;
import org.apache.kafka.streams.kstream.Aggregator;
import org.apache.kafka.streams.kstream.EmitStrategy;
import org.apache.kafka.streams.kstream.Initializer;
import org.apache.kafka.streams.kstream.Merger;
import org.apache.kafka.streams.kstream.SessionWindows;
import org.apache.kafka.streams.kstream.Windowed;
import org.apache.kafka.streams.processor.StateStore;
import org.apache.kafka.streams.processor.api.ContextualProcessor;
import org.apache.kafka.streams.processor.api.Processor;
import org.apache.kafka.streams.processor.api.ProcessorContext;
import org.apache.kafka.streams.processor.api.Record;
import org.apache.kafka.streams.processor.api.RecordMetadata;
import org.apache.kafka.streams.processor.internals.InternalProcessorContext;
import org.apache.kafka.streams.processor.internals.StoreFactory;
import org.apache.kafka.streams.processor.internals.StoreFactory.FactoryWrappingStoreBuilder;
import org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl;
import org.apache.kafka.streams.state.AggregationWithHeaders;
import org.apache.kafka.streams.state.KeyValueIterator;
import org.apache.kafka.streams.state.SessionStore;
import org.apache.kafka.streams.state.StoreBuilder;
import org.apache.kafka.streams.state.ValueTimestampHeaders;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;

import static org.apache.kafka.streams.StreamsConfig.InternalConfig.EMIT_INTERVAL_MS_KSTREAMS_WINDOWED_AGGREGATION;
import static org.apache.kafka.streams.processor.internals.metrics.ProcessorNodeMetrics.emitFinalLatencySensor;
import static org.apache.kafka.streams.processor.internals.metrics.ProcessorNodeMetrics.emittedRecordsSensor;
import static org.apache.kafka.streams.processor.internals.metrics.TaskMetrics.droppedRecordsSensor;

public class KStreamSessionWindowAggregate<KIn, VIn, VAgg> implements KStreamAggProcessorSupplier<KIn, VIn, Windowed<KIn>, VAgg> {

    private static final Logger LOG = LoggerFactory.getLogger(KStreamSessionWindowAggregate.class);

    private final String storeName;
    private final StoreFactory storeFactory;
    private final SessionWindows windows;
    private final Initializer<VAgg> initializer;
    private final Aggregator<? super KIn, ? super VIn, VAgg> aggregator;
    private final Merger<? super KIn, VAgg> sessionMerger;
    private final EmitStrategy emitStrategy;

    private boolean sendOldValues = false;

    public KStreamSessionWindowAggregate(final SessionWindows windows,
                                         final StoreFactory storeFactory,
                                         final EmitStrategy emitStrategy,
                                         final Initializer<VAgg> initializer,
                                         final Aggregator<? super KIn, ? super VIn, VAgg> aggregator,
                                         final Merger<? super KIn, VAgg> sessionMerger) {
        this.windows = windows;
        this.storeName = storeFactory.storeName();
        this.storeFactory = storeFactory;
        this.emitStrategy = emitStrategy;
        this.initializer = initializer;
        this.aggregator = aggregator;
        this.sessionMerger = sessionMerger;
    }

    @Override
    public Set<StoreBuilder<?>> stores() {
        return Collections.singleton(new FactoryWrappingStoreBuilder<>(storeFactory));
    }

    @Override
    public Processor<KIn, VIn, Windowed<KIn>, Change<VAgg>> get() {
        return new KStreamSessionWindowAggregateProcessor();
    }

    public SessionWindows windows() {
        return windows;
    }

    @Override
    public void enableSendingOldValues() {
        sendOldValues = true;
    }

    private class KStreamSessionWindowAggregateProcessor extends
        ContextualProcessor<KIn, VIn, Windowed<KIn>, Change<VAgg>> {

        private SessionStore<KIn, AggregationWithHeaders<VAgg>> store;
        private TimestampedTupleForwarder<Windowed<KIn>, VAgg> tupleForwarder;
        private Sensor droppedRecordsSensor;
        private Sensor emittedRecordsSensor;
        private Sensor emitFinalLatencySensor;
        private long lastEmitWindowCloseTime = ConsumerRecord.NO_TIMESTAMP;
        private long observedStreamTime = ConsumerRecord.NO_TIMESTAMP;
        private InternalProcessorContext<Windowed<KIn>, Change<VAgg>> internalProcessorContext;

        private final Time time = Time.SYSTEM;
        protected final KStreamImplJoin.TimeTracker timeTracker = new KStreamImplJoin.TimeTracker();

        @Override
        public void init(final ProcessorContext<Windowed<KIn>, Change<VAgg>> context) {
            super.init(context);
            internalProcessorContext = (InternalProcessorContext<Windowed<KIn>, Change<VAgg>>) context;
            final StreamsMetricsImpl metrics = (StreamsMetricsImpl) context.metrics();
            final String threadId = Thread.currentThread().getName();
            final String processorName = internalProcessorContext.currentNode().name();
            droppedRecordsSensor = droppedRecordsSensor(threadId, context.taskId().toString(), metrics);
            emittedRecordsSensor = emittedRecordsSensor(threadId, context.taskId().toString(), processorName, metrics);
            emitFinalLatencySensor = emitFinalLatencySensor(threadId, context.taskId().toString(), processorName, metrics);

            boolean isHeadersStore = false;
            // Try plain session store
            try {
                store = new SessionHeadersStoreToSessionStoreAdapter<>(context.getStateStore(storeName));
            } catch (final ClassCastException swallow) {
                // not plain session store

                // Try headers-aware session store
                try {
                    store = context.getStateStore(storeName);
                    isHeadersStore = true;
                } catch (final ClassCastException fatal) {
                    final StateStore store = context.getStateStore(storeName);
                    final String storeType = store == null ? "null" : store.getClass().getName();
                    throw new InvalidStateStoreException("Session-KTable state store must implement either "
                        + "SessionStore, or SessionStoreWithHeaders. Got: " + storeType);
                }
            }
            if (emitStrategy.type() == EmitStrategy.StrategyType.ON_WINDOW_CLOSE) {
                // Restore last emit close time for ON_WINDOW_CLOSE strategy
                final Long lastEmitWindowCloseTime = internalProcessorContext.processorMetadataForKey(storeName);
                if (lastEmitWindowCloseTime != null) {
                    this.lastEmitWindowCloseTime = lastEmitWindowCloseTime;
                }
                final long emitInterval = StreamsConfig.InternalConfig.getLong(
                        context.appConfigs(),
                        EMIT_INTERVAL_MS_KSTREAMS_WINDOWED_AGGREGATION,
                        1000L
                );
                timeTracker.setEmitInterval(emitInterval);

                tupleForwarder = new TimestampedTupleForwarder<>(context, sendOldValues);
            } else {
                tupleForwarder = new TimestampedTupleForwarder<>(
                    store,
                    context,
                    isHeadersStore ? new SessionCacheFlushListenerWithHeader<>(context) : new SessionCacheFlushListener<>(context),
                    sendOldValues);
            }
        }

        @Override
        public void process(final Record<KIn, VIn> record) {
            // if the key is null, we do not need proceed aggregating
            // the record with the table
            if (record.key() == null) {
                logSkippedRecordForNullKey();
                return;
            }

            final long timestamp = record.timestamp();
            observedStreamTime = Math.max(observedStreamTime, timestamp);
            final long windowCloseTime = observedStreamTime - windows.gracePeriodMs() - windows.inactivityGap();

            final List<KeyValue<Windowed<KIn>, AggregationWithHeaders<VAgg>>> merged = new ArrayList<>();
            final SessionWindow newSessionWindow = new SessionWindow(timestamp, timestamp);
            SessionWindow mergedWindow = newSessionWindow;
            VAgg agg = initializer.apply();

            try (
                final KeyValueIterator<Windowed<KIn>, AggregationWithHeaders<VAgg>> iterator = store.findSessions(
                    record.key(),
                    timestamp - windows.inactivityGap(),
                    timestamp + windows.inactivityGap()
                )
            ) {
                while (iterator.hasNext()) {
                    final KeyValue<Windowed<KIn>, AggregationWithHeaders<VAgg>> next = iterator.next();
                    merged.add(next);
                    agg = sessionMerger.apply(record.key(), agg, AggregationWithHeaders.getAggregationOrNull(next.value));
                    mergedWindow = mergeSessionWindow(mergedWindow, (SessionWindow) next.key.window());
                }
            }

            if (mergedWindow.end() < windowCloseTime) {
                logSkippedRecordForExpiredWindow(timestamp, windowCloseTime, mergedWindow);
            } else {
                if (!mergedWindow.equals(newSessionWindow)) {
                    for (final KeyValue<Windowed<KIn>, AggregationWithHeaders<VAgg>> session : merged) {
                        store.remove(session.key);
                        maybeForwardUpdate(session.key, AggregationWithHeaders.getAggregationOrNull(session.value), null);
                    }
                }

                agg = aggregator.apply(record.key(), record.value(), agg);
                final Windowed<KIn> sessionKey = new Windowed<>(record.key(), mergedWindow);
                store.put(sessionKey, AggregationWithHeaders.make(agg, new RecordHeaders()));

                maybeForwardUpdate(sessionKey, null, agg);
            }

            maybeForwardFinalResult(record, windowCloseTime);
        }

        private void maybeForwardUpdate(final Windowed<KIn> windowedkey,
                                        final VAgg oldAgg,
                                        final VAgg newAgg) {
            if (emitStrategy.type() == EmitStrategy.StrategyType.ON_WINDOW_CLOSE) {
                return;
            }

            // Update the sent record timestamp to the window end time if possible
            final long newTimestamp = windowedkey.window().end();
            tupleForwarder.maybeForward(new Record<>(windowedkey, new Change<>(newAgg, sendOldValues ? oldAgg : null), newTimestamp));
        }

        // TODO: consolidate SessionWindow with TimeWindow to merge common functions
        private void maybeForwardFinalResult(final Record<KIn, VIn> record, final long windowCloseTime) {
            if (shouldEmitFinal(windowCloseTime)) {
                final long emitRangeUpperBound = emitRangeUpperBound(windowCloseTime);

                // if the upper bound is smaller than 0, then there's no window closed ever;
                // and we can skip range fetching
                if (emitRangeUpperBound >= 0) {
                    final long emitRangeLowerBound = emitRangeLowerBound();

                    if (shouldRangeFetch(emitRangeLowerBound, emitRangeUpperBound)) {
                        fetchAndEmit(record, windowCloseTime, emitRangeLowerBound, emitRangeUpperBound);
                    }
                }
            }
        }

        private boolean shouldEmitFinal(final long windowCloseTime) {
            if (emitStrategy.type() != EmitStrategy.StrategyType.ON_WINDOW_CLOSE) {
                return false;
            }

            final long now = internalProcessorContext.currentSystemTimeMs();
            // Throttle emit frequency
            if (now < timeTracker.nextTimeToEmit) {
                return false;
            }

            // Schedule next emit time based on now to avoid the case that if system time jumps a lot,
            // this can be triggered every time
            timeTracker.nextTimeToEmit = now;
            timeTracker.advanceNextTimeToEmit();

            // Only EMIT if the window close time does progress
            return lastEmitWindowCloseTime == ConsumerRecord.NO_TIMESTAMP || lastEmitWindowCloseTime < windowCloseTime;
        }

        private long emitRangeLowerBound() {
            return Math.max(0L, lastEmitWindowCloseTime);
        }

        private long emitRangeUpperBound(final long windowCloseTime) {
            // Session window's start and end timestamps are inclusive, so
            // we should minus 1 for the inclusive closed window-end upper bound
            return windowCloseTime - 1;
        }

        private boolean shouldRangeFetch(final long emitRangeLowerBound, final long emitRangeUpperBound) {
            // since a session window could be a single point (i.e. [t, t]),
            // we need to range fetch and emit even if the upper and lower bound are the same
            return emitRangeUpperBound >= emitRangeLowerBound;
        }

        private void fetchAndEmit(final Record<KIn, VIn> record,
                                  final long windowCloseTime,
                                  final long emitRangeLowerBound,
                                  final long emitRangeUpperBound) {
            final long startMs = time.milliseconds();

            int emittedCount = 0;

            // Only time ordered (indexed) session store should have implemented
            // this function, otherwise a not-supported exception would throw
            try (final KeyValueIterator<Windowed<KIn>, AggregationWithHeaders<VAgg>> windowToEmit =
                     store.findSessions(emitRangeLowerBound, emitRangeUpperBound)) {

                while (windowToEmit.hasNext()) {
                    emittedCount++;
                    final KeyValue<Windowed<KIn>, AggregationWithHeaders<VAgg>> kv = windowToEmit.next();

                    tupleForwarder.maybeForward(
                        record.withKey(kv.key)
                            .withValue(new Change<>(AggregationWithHeaders.getAggregationOrNull(kv.value), null))
                            // set the timestamp as the window end timestamp
                            .withTimestamp(kv.key.window().end())
                            .withHeaders(record.headers()));
                }
            }
            emittedRecordsSensor.record(emittedCount);
            emitFinalLatencySensor.record(time.milliseconds() - startMs);

            lastEmitWindowCloseTime = windowCloseTime;
            internalProcessorContext.addProcessorMetadataKeyValue(storeName, windowCloseTime);
        }

        private void logSkippedRecordForNullKey() {
            if (context().recordMetadata().isPresent()) {
                final RecordMetadata recordMetadata = context().recordMetadata().get();
                LOG.warn(
                        "Skipping record due to null key. "
                                + "topic=[{}] partition=[{}] offset=[{}]",
                        recordMetadata.topic(), recordMetadata.partition(), recordMetadata.offset()
                );
            } else {
                LOG.warn(
                        "Skipping record due to null key. Topic, partition, and offset not known."
                );
            }
            droppedRecordsSensor.record();
        }

        private void logSkippedRecordForExpiredWindow(final long timestamp,
                                                      final long windowExpire,
                                                      final SessionWindow window) {
            final String windowString = "[" + window.start() + "," + window.end() + "]";

            if (context().recordMetadata().isPresent()) {
                final RecordMetadata recordMetadata = context().recordMetadata().get();
                LOG.warn("Skipping record for expired window. " +
                                "topic=[{}] " +
                                "partition=[{}] " +
                                "offset=[{}] " +
                                "timestamp=[{}] " +
                                "window={} " +
                                "expiration=[{}] " +
                                "streamTime=[{}]",
                        recordMetadata.topic(),
                        recordMetadata.partition(),
                        recordMetadata.offset(),
                        timestamp,
                        windowString,
                        windowExpire,
                        observedStreamTime
                );
            } else {
                LOG.warn("Skipping record for expired window. Topic, partition, and offset not known. " +
                                "timestamp=[{}] " +
                                "window={} " +
                                "expiration=[{}] " +
                                "streamTime=[{}]",
                        timestamp,
                        windowString,
                        windowExpire,
                        observedStreamTime
                );
            }
            droppedRecordsSensor.record();
        }
    }

    private SessionWindow mergeSessionWindow(final SessionWindow one, final SessionWindow two) {
        final long start = Math.min(one.start(), two.start());
        final long end = Math.max(one.end(), two.end());
        return new SessionWindow(start, end);
    }

    @Override
    public KTableValueGetterSupplier<Windowed<KIn>, VAgg> view() {
        return new KTableValueGetterSupplier<>() {
            @Override
            public KTableValueGetter<Windowed<KIn>, VAgg> get() {
                return new KTableSessionWindowValueGetter();
            }

            @Override
            public String[] storeNames() {
                return new String[]{storeName};
            }
        };
    }

    private class KTableSessionWindowValueGetter implements KTableValueGetter<Windowed<KIn>, VAgg> {

        private SessionStore<KIn, AggregationWithHeaders<VAgg>> store;
        
        @Override
        public void init(final ProcessorContext<?, ?> context) {
            try {
                store = new SessionHeadersStoreToSessionStoreAdapter<>(context.getStateStore(storeName));
            } catch (final ClassCastException swallow) {
                // not plain session store

                // Try headers-aware session store
                try {
                    store = context.getStateStore(storeName);
                } catch (final ClassCastException fatal) {
                    final StateStore store = context.getStateStore(storeName);
                    final String storeType = store == null ? "null" : store.getClass().getName();
                    throw new InvalidStateStoreException("Session-KTable state store must implement either "
                        + "SessionStore, or SessionStoreWithHeaders. Got: " + storeType);
                }
            }
        }

        @Override
        public ValueTimestampHeaders<VAgg> get(final Windowed<KIn> key) {
            final AggregationWithHeaders<VAgg> result =
                store.fetchSession(key.key(), key.window().start(), key.window().end());
            return ValueTimestampHeaders.make(
                AggregationWithHeaders.getAggregationOrNull(result),
                key.window().end(),
                new RecordHeaders());
        }

        @Override
        public boolean isVersioned() {
            return false;
        }
    }
}
