// Licensed to Apache Software Foundation (ASF) under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Apache Software Foundation (ASF) licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

package lifecycle

import (
	"context"
	"encoding/json"
	"fmt"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"sync"
	"time"

	"github.com/benbjohnson/clock"
	"github.com/go-chi/chi/v5"
	grpc_validator "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/validator"
	"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
	"github.com/pkg/errors"
	"github.com/robfig/cron/v3"
	grpclib "google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/credentials/insecure"
	"google.golang.org/grpc/health"
	"google.golang.org/grpc/health/grpc_health_v1"
	"google.golang.org/protobuf/types/known/timestamppb"

	"github.com/apache/skywalking-banyandb/api/common"
	commonv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/common/v1"
	databasev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/database/v1"
	measurev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/measure/v1"
	streamv1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/stream/v1"
	tracev1 "github.com/apache/skywalking-banyandb/api/proto/banyandb/trace/v1"
	"github.com/apache/skywalking-banyandb/banyand/backup/snapshot"
	"github.com/apache/skywalking-banyandb/banyand/internal/storage"
	"github.com/apache/skywalking-banyandb/banyand/metadata"
	"github.com/apache/skywalking-banyandb/banyand/observability"
	"github.com/apache/skywalking-banyandb/banyand/protector"
	"github.com/apache/skywalking-banyandb/pkg/healthcheck"
	"github.com/apache/skywalking-banyandb/pkg/logger"
	"github.com/apache/skywalking-banyandb/pkg/run"
	"github.com/apache/skywalking-banyandb/pkg/timestamp"
	pkgtls "github.com/apache/skywalking-banyandb/pkg/tls"
)

type service interface {
	run.Config
	run.PreRunner
	run.Service
}

var _ service = (*lifecycleService)(nil)

type lifecycleService struct {
	databasev1.UnimplementedClusterStateServiceServer
	databasev1.UnimplementedNodeQueryServiceServer
	metadata          metadata.Repo
	omr               observability.MetricsRegistry
	pm                protector.Memory
	clusterStateMgr   *clusterStateManager
	l                 *logger.Logger
	sch               *timestamp.Scheduler
	grpcServer        *grpclib.Server
	httpSrv           *http.Server
	tlsReloader       *pkgtls.Reloader
	currentNode       *databasev1.Node
	clientCloser      context.CancelFunc
	stopCh            chan struct{}
	measureRoot       string
	streamRoot        string
	traceRoot         string
	progressFilePath  string
	reportDir         string
	schedule          string
	cert              string
	gRPCAddr          string
	lifecycleHost     string
	lifecycleGRPCAddr string
	lifecycleHTTPAddr string
	lifecycleCertFile string
	lifecycleKeyFile  string
	lifecycleGRPCPort uint32
	lifecycleHTTPPort uint32
	maxExecutionTimes int
	enableTLS         bool
	insecure          bool
	lifecycleTLS      bool
	chunkSize         run.Bytes
}

// NewService creates a new lifecycle service.
func NewService(meta metadata.Repo) run.Unit {
	ls := &lifecycleService{
		metadata:        meta,
		omr:             observability.BypassRegistry,
		clusterStateMgr: &clusterStateManager{},
	}
	ls.pm = protector.NewMemory(ls.omr)
	return ls
}

func (l *lifecycleService) FlagSet() *run.FlagSet {
	flagS := run.NewFlagSet(l.Name())
	flagS.StringSliceVar(&common.FlagNodeLabels, "node-labels", nil, "the node labels. e.g. key1=value1,key2=value2")
	flagS.StringVar(&l.gRPCAddr, "grpc-addr", "127.0.0.1:17912", "gRPC address of the data node")
	flagS.BoolVar(&l.enableTLS, "enable-tls", false, "Enable TLS for gRPC connection")
	flagS.BoolVar(&l.insecure, "insecure", false, "Skip server certificate verification")
	flagS.StringVar(&l.cert, "cert", "", "Path to the gRPC server certificate")
	flagS.StringVar(&l.streamRoot, "stream-root-path", "/tmp", "Root directory for stream catalog")
	flagS.StringVar(&l.traceRoot, "trace-root-path", "/tmp", "Root directory for trace catalog")
	flagS.StringVar(&l.measureRoot, "measure-root-path", "/tmp", "Root directory for measure catalog")
	flagS.StringVar(&l.progressFilePath, "progress-file", "/tmp/lifecycle-progress.json", "Path to store progress for crash recovery")
	flagS.StringVar(&l.reportDir, "report-dir", "/tmp/lifecycle-reports", "Directory to store migration reports")
	flagS.StringVar(
		&l.schedule,
		"schedule",
		"",
		"Schedule expression for periodic backup. Options: @yearly, @monthly, @weekly, @daily, @hourly or @every <duration>",
	)
	flagS.IntVar(&l.maxExecutionTimes, "max-execution-times", 0, "Maximum number of times to execute the lifecycle migration. 0 means no limit.")
	l.chunkSize = run.Bytes(1024 * 1024)
	flagS.VarP(&l.chunkSize, "chunk-size", "", "Chunk size in bytes for streaming data during migration (default: 1MB)")

	// Lifecycle server flags
	flagS.BoolVar(&l.lifecycleTLS, "lifecycle-tls", false, "connection uses TLS if true, else plain TCP")
	flagS.StringVar(&l.lifecycleCertFile, "lifecycle-cert-file", "", "the TLS cert file")
	flagS.StringVar(&l.lifecycleKeyFile, "lifecycle-key-file", "", "the TLS key file")
	flagS.StringVar(&l.lifecycleHost, "lifecycle-grpc-host", "", "the host of lifecycle server listens")
	flagS.Uint32Var(&l.lifecycleGRPCPort, "lifecycle-grpc-port", 17914, "the port of lifecycle server listens")
	flagS.Uint32Var(&l.lifecycleHTTPPort, "lifecycle-http-port", 17915, "the port of lifecycle http api listens")

	return flagS
}

func (l *lifecycleService) Validate() error {
	if l.schedule != "" {
		l.lifecycleGRPCAddr = net.JoinHostPort(l.lifecycleHost, strconv.FormatUint(uint64(l.lifecycleGRPCPort), 10))
		if l.lifecycleGRPCAddr == ":" {
			return errors.New("no gRPC address")
		}
		l.lifecycleHTTPAddr = net.JoinHostPort(l.lifecycleHost, strconv.FormatUint(uint64(l.lifecycleHTTPPort), 10))
		if l.lifecycleHTTPAddr == ":" {
			return errors.New("no HTTP address")
		}
		if l.lifecycleTLS {
			if l.lifecycleCertFile == "" {
				return errors.New("missing cert file when TLS is enabled")
			}
			if l.lifecycleKeyFile == "" {
				return errors.New("missing key file when TLS is enabled")
			}
		}
		l.currentNode = &databasev1.Node{
			Metadata: &commonv1.Metadata{
				Name: l.lifecycleGRPCAddr,
			},
			GrpcAddress: l.lifecycleGRPCAddr,
			HttpAddress: l.lifecycleHTTPAddr,
			Roles:       make([]databasev1.Role, 0),
			Labels:      common.ParseNodeFlags(),
			CreatedAt:   timestamppb.Now(),
		}
	}
	return nil
}

// PreRun initializes the lifecycle service and its embedded server.
func (l *lifecycleService) PreRun(_ context.Context) error {
	l.l = logger.GetLogger("lifecycle")

	if l.schedule != "" && l.lifecycleTLS {
		var err error
		l.tlsReloader, err = pkgtls.NewReloader(l.lifecycleCertFile, l.lifecycleKeyFile, l.l)
		if err != nil {
			return errors.Wrap(err, "failed to initialize TLS reloader")
		}
	}

	return nil
}

func (l *lifecycleService) GracefulStop() {
	if l.sch != nil {
		l.sch.Close()
	}

	l.l.Info().Msg("Stopping lifecycle server")

	if l.tlsReloader != nil {
		l.tlsReloader.Stop()
	}

	if l.clientCloser != nil {
		l.clientCloser()
	}

	// Stop HTTP server
	if l.httpSrv != nil {
		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
		defer cancel()
		if shutdownErr := l.httpSrv.Shutdown(ctx); shutdownErr != nil {
			l.l.Warn().Err(shutdownErr).Msg("HTTP server shutdown error")
		}
	}

	// Stop gRPC server
	if l.grpcServer != nil {
		stopped := make(chan struct{})
		go func() {
			l.grpcServer.GracefulStop()
			close(stopped)
		}()

		t := time.NewTimer(10 * time.Second)
		select {
		case <-t.C:
			l.grpcServer.Stop()
			l.l.Info().Msg("Lifecycle server force stopped")
		case <-stopped:
			t.Stop()
			l.l.Info().Msg("Lifecycle server stopped gracefully")
		}
	}
}

func (l *lifecycleService) Name() string {
	return "lifecycle"
}

func (l *lifecycleService) Serve() run.StopNotify {
	l.l = logger.GetLogger("lifecycle")
	l.stopCh = make(chan struct{})

	// Start gRPC/HTTP servers when schedule is set
	if l.schedule != "" {
		l.startServers()
	}

	done := make(chan struct{})
	if l.schedule == "" {
		defer close(done)
		l.l.Info().Msg("starting lifecycle migration without schedule")
		if err := l.action(); err != nil {
			logger.Panicf("failed to run lifecycle migration: %v", err)
		}
		return done
	}
	l.l.Info().Msgf("lifecycle migration will run with schedule: %s", l.schedule)
	clockInstance := clock.New()
	l.sch = timestamp.NewScheduler(l.l, clockInstance)
	var executionCount int
	err := l.sch.Register("lifecycle", cron.Descriptor, l.schedule, func(triggerTime time.Time, _ *logger.Logger) bool {
		l.l.Info().Msgf("lifecycle migration triggered at %s", triggerTime)
		if err := l.action(); err != nil {
			l.l.Error().Err(err).Msg("failed to run lifecycle migration action")
		}
		executionCount++
		if l.maxExecutionTimes > 0 && executionCount >= l.maxExecutionTimes {
			l.l.Info().Msgf("lifecycle migration reached max execution times: %d, stopping scheduler", l.maxExecutionTimes)
			close(done)
			return false
		}
		return true
	})
	if err != nil {
		l.l.Error().Err(err).Msg("failed to register lifecycle migration schedule")
		close(done)
		return done
	}

	// Wait for either migration completion or server stop
	go func() {
		select {
		case <-done:
			// Migration completed
		case <-l.stopCh:
			// Server stopped
			close(done)
		}
	}()

	return done
}

func (l *lifecycleService) startServers() {
	// Setup gRPC server
	var opts []grpclib.ServerOption
	if l.lifecycleTLS && l.tlsReloader != nil {
		if startErr := l.tlsReloader.Start(); startErr != nil {
			l.l.Error().Err(startErr).Msg("Failed to start TLS reloader")
			close(l.stopCh)
			return
		}
		tlsConfig := l.tlsReloader.GetTLSConfig()
		creds := credentials.NewTLS(tlsConfig)
		opts = append(opts, grpclib.Creds(creds))
	}

	opts = append(opts,
		grpclib.ChainUnaryInterceptor(
			grpc_validator.UnaryServerInterceptor(),
		),
	)

	l.grpcServer = grpclib.NewServer(opts...)
	databasev1.RegisterClusterStateServiceServer(l.grpcServer, l)
	databasev1.RegisterNodeQueryServiceServer(l.grpcServer, l)
	grpc_health_v1.RegisterHealthServer(l.grpcServer, health.NewServer())

	// Setup HTTP server
	var ctx context.Context
	ctx, l.clientCloser = context.WithCancel(context.Background())

	clientOpts := make([]grpclib.DialOption, 0, 1)
	if l.lifecycleTLS && l.tlsReloader != nil {
		tlsConfig := l.tlsReloader.GetTLSConfig()
		creds := credentials.NewTLS(tlsConfig)
		clientOpts = append(clientOpts, grpclib.WithTransportCredentials(creds))
	} else {
		clientOpts = append(clientOpts, grpclib.WithTransportCredentials(insecure.NewCredentials()))
	}

	client, err := healthcheck.NewClient(ctx, l.l, l.lifecycleGRPCAddr, clientOpts)
	if err != nil {
		l.l.Error().Err(err).Msg("Failed to create health check client")
		close(l.stopCh)
		return
	}

	gwMux := runtime.NewServeMux(runtime.WithHealthzEndpoint(client))
	if registerErr := databasev1.RegisterClusterStateServiceHandlerFromEndpoint(ctx, gwMux, l.lifecycleGRPCAddr, clientOpts); registerErr != nil {
		l.l.Error().Err(registerErr).Msg("Failed to register cluster state service")
		close(l.stopCh)
		return
	}

	mux := chi.NewRouter()
	mux.Mount("/api", http.StripPrefix("/api", gwMux))

	l.httpSrv = &http.Server{
		Addr:              l.lifecycleHTTPAddr,
		Handler:           mux,
		ReadHeaderTimeout: 3 * time.Second,
	}

	// Start both servers
	var wg sync.WaitGroup
	wg.Add(2)

	// gRPC server goroutine
	go func() {
		defer wg.Done()
		lis, listenErr := net.Listen("tcp", l.lifecycleGRPCAddr)
		if listenErr != nil {
			l.l.Error().Err(listenErr).Msg("Failed to listen on gRPC addr")
			return
		}
		l.l.Info().Str("addr", l.lifecycleGRPCAddr).Msg("Lifecycle gRPC server listening")
		if serveErr := l.grpcServer.Serve(lis); serveErr != nil {
			l.l.Error().Err(serveErr).Msg("gRPC server error")
		}
	}()

	// HTTP server goroutine
	go func() {
		defer wg.Done()
		l.l.Info().Str("addr", l.lifecycleHTTPAddr).Msg("Lifecycle HTTP server listening")
		var serveErr error
		if l.lifecycleTLS && l.tlsReloader != nil {
			l.httpSrv.TLSConfig = l.tlsReloader.GetTLSConfig()
			serveErr = l.httpSrv.ListenAndServeTLS("", "")
		} else {
			serveErr = l.httpSrv.ListenAndServe()
		}
		if serveErr != nil && serveErr != http.ErrServerClosed {
			l.l.Error().Err(serveErr).Msg("HTTP server error")
		}
	}()

	// Wait for both servers to stop
	go func() {
		wg.Wait()
		close(l.stopCh)
	}()
}

func (l *lifecycleService) action() error {
	ctx := context.Background()
	progress := LoadProgress(l.progressFilePath, l.l)
	progress.ClearErrors()

	groups, err := l.getGroupsToProcess(ctx, progress)
	if err != nil {
		l.l.Error().Err(err).Msg("failed to get groups to process")
		return err
	}

	l.l.Info().Msgf("starting migration for %d groups: %v", len(groups), getGroupNames(groups))

	if len(groups) == 0 {
		l.l.Info().Msg("no groups to process, all groups already completed")
		progress.Remove(l.progressFilePath, l.l)
		return err
	}

	// Pass progress to getSnapshots
	streamDir, measureDir, traceDir, err := l.getSnapshots(groups, progress)
	if err != nil {
		l.l.Error().Err(err).Msg("failed to get snapshots")
		return err
	}
	if streamDir == "" && measureDir == "" && traceDir == "" {
		l.l.Warn().Msg("no snapshots found, skipping lifecycle migration")
		l.generateReport(progress)
		return nil
	}
	l.l.Info().
		Str("stream_snapshot", streamDir).
		Str("measure_snapshot", measureDir).
		Str("trace_snapshot", traceDir).
		Msg("created snapshots")
	progress.Save(l.progressFilePath, l.l)

	nodes, err := l.metadata.NodeRegistry().ListNode(ctx, databasev1.Role_ROLE_DATA)
	if err != nil {
		l.l.Error().Err(err).Msg("failed to list data nodes")
		return err
	}
	labels := common.ParseNodeFlags()

	allGroupsCompleted := true
	for _, g := range groups {
		switch g.Catalog {
		case commonv1.Catalog_CATALOG_STREAM:
			if streamDir == "" {
				l.l.Warn().Msgf("stream snapshot directory is not available, skipping group: %s", g.Metadata.Name)
				progress.MarkGroupCompleted(g.Metadata.Name)
				continue
			}
			l.processStreamGroup(ctx, g, streamDir, nodes, labels, progress)
		case commonv1.Catalog_CATALOG_MEASURE:
			if measureDir == "" {
				l.l.Warn().Msgf("measure snapshot directory is not available, skipping group: %s", g.Metadata.Name)
				progress.MarkGroupCompleted(g.Metadata.Name)
				continue
			}
			l.processMeasureGroup(ctx, g, measureDir, nodes, labels, progress)
		case commonv1.Catalog_CATALOG_TRACE:
			if traceDir == "" {
				l.l.Warn().Msgf("trace snapshot directory is not available, skipping group: %s", g.Metadata.Name)
				progress.MarkGroupCompleted(g.Metadata.Name)
				continue
			}
			l.processTraceGroup(ctx, g, traceDir, nodes, labels, progress)
		default:
			l.l.Info().Msgf("group catalog: %s doesn't support lifecycle management", g.Catalog)
		}
		progress.Save(l.progressFilePath, l.l)
	}

	// Only remove progress file if ALL groups are fully completed
	notCompleteGroups := progress.AllGroupsNotFullyCompleted(groups)
	if allGroupsCompleted && len(notCompleteGroups) == 0 {
		progress.Remove(l.progressFilePath, l.l)
		l.l.Info().Msg("lifecycle migration completed successfully")
		l.generateReport(progress)
		return nil
	}
	l.l.Info().Msg("lifecycle migration partially completed, progress file retained")
	return fmt.Errorf("lifecycle migration partially completed, progress file retained; %v groups not fully completed", notCompleteGroups)
}

// generateReport gathers detailed counts & errors from Progress, writes comprehensive JSON file per run, and keeps only 5 latest.
func (l *lifecycleService) generateReport(p *Progress) {
	reportDir := l.reportDir
	if err := os.MkdirAll(reportDir, 0o755); err != nil {
		l.l.Error().Err(err).Msgf("failed to create report dir %s", reportDir)
		return
	}

	// Build comprehensive migration report
	report := l.buildMigrationReport(p)

	// write file
	fname := time.Now().Format("20060102_150405") + ".json"
	fpath := filepath.Join(reportDir, fname)
	data, err := json.MarshalIndent(report, "", "  ")
	if err != nil {
		l.l.Error().Err(err).Msg("failed to marshal migration report")
		return
	}
	if err = os.WriteFile(fpath, data, 0o600); err != nil {
		l.l.Error().Err(err).Msgf("failed to write report file %s", fpath)
		return
	}
	l.l.Info().Msgf("wrote comprehensive migration report %s", fpath)

	// rotate: keep only 5 latest
	l.rotateReportFiles(reportDir)
}

// buildMigrationReport creates a comprehensive migration report from progress data.
func (l *lifecycleService) buildMigrationReport(p *Progress) map[string]interface{} {
	p.mu.Lock()
	defer p.mu.Unlock()

	now := time.Now()
	report := map[string]interface{}{
		"generated_at":   now,
		"report_version": "2.0",
		"summary":        l.buildSummaryStats(p),
		"errors":         l.buildErrorSummary(p),
		"snapshot_info": map[string]interface{}{
			"stream_dir":  p.SnapshotStreamDir,
			"measure_dir": p.SnapshotMeasureDir,
		},
	}

	return report
}

// buildSummaryStats creates overall migration statistics.
func (l *lifecycleService) buildSummaryStats(p *Progress) map[string]interface{} {
	totalGroups := len(p.CompletedGroups) + len(p.DeletedStreamGroups) + len(p.DeletedMeasureGroups)
	completedGroups := len(p.CompletedGroups)

	// Calculate total parts and series across all groups
	totalStreamParts, completedStreamParts := l.calculateTotalCounts(p.StreamPartCounts, p.StreamPartProgress)
	totalStreamSeries, completedStreamSeries := l.calculateTotalCounts(p.StreamSeriesCounts, p.StreamSeriesProgress)
	totalStreamElementIndex, completedStreamElementIndex := l.calculateTotalCounts(p.StreamElementIndexCounts, p.StreamElementIndexProgress)
	totalMeasureParts, completedMeasureParts := l.calculateTotalCounts(p.MeasurePartCounts, p.MeasurePartProgress)
	totalMeasureSeries, completedMeasureSeries := l.calculateTotalCounts(p.MeasureSeriesCounts, p.MeasureSeriesProgress)

	// Calculate error counts
	streamPartErrors := l.countErrors(p.StreamPartErrors)
	streamSeriesErrors := l.countErrors(p.StreamSeriesErrors)
	streamElementIndexErrors := l.countErrors(p.StreamElementIndexErrors)
	measurePartErrors := l.countErrors(p.MeasurePartErrors)
	measureSeriesErrors := l.countErrors(p.MeasureSeriesErrors)

	return map[string]interface{}{
		"migration_status": map[string]interface{}{
			"total_groups":     totalGroups,
			"completed_groups": completedGroups,
			"completion_rate":  l.calculatePercentage(completedGroups, totalGroups),
		},
		"stream_migration": map[string]interface{}{
			"parts": map[string]interface{}{
				"total":           totalStreamParts,
				"completed":       completedStreamParts,
				"errors":          streamPartErrors,
				"completion_rate": l.calculatePercentage(completedStreamParts, totalStreamParts),
			},
			"series": map[string]interface{}{
				"total":           totalStreamSeries,
				"completed":       completedStreamSeries,
				"errors":          streamSeriesErrors,
				"completion_rate": l.calculatePercentage(completedStreamSeries, totalStreamSeries),
			},
			"element_index": map[string]interface{}{
				"total":           totalStreamElementIndex,
				"completed":       completedStreamElementIndex,
				"errors":          streamElementIndexErrors,
				"completion_rate": l.calculatePercentage(completedStreamElementIndex, totalStreamElementIndex),
			},
		},
		"measure_migration": map[string]interface{}{
			"parts": map[string]interface{}{
				"total":           totalMeasureParts,
				"completed":       completedMeasureParts,
				"errors":          measurePartErrors,
				"completion_rate": l.calculatePercentage(completedMeasureParts, totalMeasureParts),
			},
			"series": map[string]interface{}{
				"total":           totalMeasureSeries,
				"completed":       completedMeasureSeries,
				"errors":          measureSeriesErrors,
				"completion_rate": l.calculatePercentage(completedMeasureSeries, totalMeasureSeries),
			},
		},
	}
}

// buildErrorSummary creates detailed error information.
func (l *lifecycleService) buildErrorSummary(p *Progress) map[string]interface{} {
	errors := map[string]interface{}{
		"stream_parts":         l.buildErrorDetails(p.StreamPartErrors),
		"stream_series":        l.buildErrorDetails(p.StreamSeriesErrors),
		"stream_element_index": l.buildErrorDetails(p.StreamElementIndexErrors),
		"measure_parts":        l.buildErrorDetails(p.MeasurePartErrors),
		"measure_series":       l.buildErrorDetails(p.MeasureSeriesErrors),
	}

	return errors
}

// Helper functions.
func (l *lifecycleService) calculateTotalCounts(counts, progress map[string]int) (int, int) {
	totalCount, totalProgress := 0, 0
	for _, count := range counts {
		totalCount += count
	}
	for _, prog := range progress {
		totalProgress += prog
	}
	return totalCount, totalProgress
}

func (l *lifecycleService) countErrors(errorMaps interface{}) int {
	switch v := errorMaps.(type) {
	// New four-level structure: map[group]map[segmentID]map[shardID]map[partID]error
	case map[string]map[string]map[common.ShardID]map[uint64]string:
		total := 0
		for _, segments := range v {
			for _, shards := range segments {
				for _, parts := range shards {
					total += len(parts)
				}
			}
		}
		return total
	// New three-level structure: map[group]map[segmentID]map[shardID]error
	case map[string]map[string]map[common.ShardID]string:
		total := 0
		for _, segments := range v {
			for _, shards := range segments {
				total += len(shards)
			}
		}
		return total
	// Legacy two-level structure: map[group]map[partID]error
	case map[string]map[uint64]string:
		total := 0
		for _, groupErrors := range v {
			total += len(groupErrors)
		}
		return total
	// Legacy two-level structure: map[group]map[shardID]error
	case map[string]map[string]string:
		total := 0
		for _, groupErrors := range v {
			total += len(groupErrors)
		}
		return total
	default:
		return 0
	}
}

func (l *lifecycleService) calculatePercentage(completed, total int) float64 {
	if total == 0 {
		return 0.0
	}
	return float64(completed) / float64(total) * 100.0
}

func (l *lifecycleService) buildErrorDetails(errorMaps interface{}) map[string]interface{} {
	result := make(map[string]interface{})

	switch v := errorMaps.(type) {
	// New four-level structure: map[group]map[segmentID]map[shardID]map[partID]error
	case map[string]map[string]map[common.ShardID]map[uint64]string:
		for group, segments := range v {
			groupDetails := make(map[string]interface{})
			for segmentID, shards := range segments {
				segmentDetails := make(map[string]interface{})
				for shardID, parts := range shards {
					if len(parts) > 0 {
						segmentDetails[fmt.Sprintf("shard_%d", shardID)] = parts
					}
				}
				if len(segmentDetails) > 0 {
					groupDetails[segmentID] = segmentDetails
				}
			}
			if len(groupDetails) > 0 {
				result[group] = groupDetails
			}
		}
	// New three-level structure: map[group]map[segmentID]map[shardID]error
	case map[string]map[string]map[common.ShardID]string:
		for group, segments := range v {
			groupDetails := make(map[string]interface{})
			for segmentID, shards := range segments {
				if len(shards) > 0 {
					// Convert ShardID keys to strings for JSON serialization
					shardDetails := make(map[string]string)
					for shardID, errorMsg := range shards {
						shardDetails[fmt.Sprintf("shard_%d", shardID)] = errorMsg
					}
					groupDetails[segmentID] = shardDetails
				}
			}
			if len(groupDetails) > 0 {
				result[group] = groupDetails
			}
		}
	// Legacy two-level structure: map[group]map[partID]error
	case map[string]map[uint64]string:
		for group, groupErrors := range v {
			if len(groupErrors) > 0 {
				result[group] = groupErrors
			}
		}
	// Legacy two-level structure: map[group]map[shardID]error
	case map[string]map[string]string:
		for group, groupErrors := range v {
			if len(groupErrors) > 0 {
				result[group] = groupErrors
			}
		}
	}

	return result
}

// rotateReportFiles keeps only the 5 most recent report files.
func (l *lifecycleService) rotateReportFiles(reportDir string) {
	entries, err := os.ReadDir(reportDir)
	if err != nil {
		return
	}

	type fileInfo struct {
		t    time.Time
		name string
	}

	var list []fileInfo
	for _, e := range entries {
		if e.IsDir() {
			continue
		}
		info, err := e.Info()
		if err != nil {
			continue
		}
		list = append(list, fileInfo{name: e.Name(), t: info.ModTime()})
	}

	sort.Slice(list, func(i, j int) bool {
		return list[i].t.After(list[j].t)
	})

	for i := 5; i < len(list); i++ {
		_ = os.Remove(filepath.Join(reportDir, list[i].name))
	}

	if len(list) > 5 {
		l.l.Info().Msgf("rotated %d old migration reports", len(list)-5)
	}
}

func getGroupNames(groups []*commonv1.Group) []string {
	names := make([]string, 0, len(groups))
	for _, g := range groups {
		names = append(names, g.Metadata.Name)
	}
	return names
}

func (l *lifecycleService) getGroupsToProcess(ctx context.Context, progress *Progress) ([]*commonv1.Group, error) {
	gg, err := l.metadata.GroupRegistry().ListGroup(ctx)
	if err != nil {
		l.l.Error().Err(err).Msg("failed to list groups")
		return nil, err
	}
	allGroupNames := getGroupNames(gg)

	groups := make([]*commonv1.Group, 0, len(gg))
	for _, g := range gg {
		if g.ResourceOpts == nil {
			l.l.Debug().Msgf("skipping group %s because resource opts is nil", g.Metadata.Name)
			continue
		}
		if len(g.ResourceOpts.Stages) == 0 {
			l.l.Debug().Msgf("skipping group %s because stages is empty", g.Metadata.Name)
			continue
		}
		if progress.IsGroupCompleted(g.Metadata.Name) {
			l.l.Debug().Msgf("skipping already completed group: %s", g.Metadata.Name)
			continue
		}
		groups = append(groups, g)
	}
	l.l.Info().Msgf("found groups needs to do lifecycle processing: %v, all groups: %v", getGroupNames(groups), allGroupNames)

	return groups, nil
}

func (l *lifecycleService) processStreamGroup(ctx context.Context, g *commonv1.Group,
	streamDir string, nodes []*databasev1.Node, labels map[string]string, progress *Progress,
) {
	group, err := parseGroup(g, labels, nodes, l.l, l.metadata, l.clusterStateMgr)
	if err != nil {
		l.l.Error().Err(err).Msgf("failed to parse group %s", g.Metadata.Name)
		return
	}
	defer group.Close()
	tr := l.getRemovalSegmentsTimeRange(group)
	if tr.Start.IsZero() && tr.End.IsZero() {
		l.l.Info().Msgf("no removal segments time range for group %s, skipping stream migration", g.Metadata.Name)
		progress.MarkGroupCompleted(g.Metadata.Name)
		return
	}

	segmentSuffixes, err := l.processStreamGroupFileBased(ctx, group, streamDir, tr, progress)
	if err != nil {
		l.l.Error().Err(err).Msgf("failed to migrate stream group %s using file-based approach", g.Metadata.Name)
		return
	}

	l.l.Info().Msgf("deleting expired stream segments for group: %s, time range: %s", g.Metadata.Name, tr.String())
	l.deleteExpiredStreamSegments(ctx, g, segmentSuffixes, progress)
	progress.MarkGroupCompleted(g.Metadata.Name)
}

// processStreamGroupFileBased uses file-based migration instead of element-based queries.
func (l *lifecycleService) processStreamGroupFileBased(_ context.Context, g *GroupConfig, streamDir string,
	tr *timestamp.TimeRange, progress *Progress,
) ([]string, error) {
	if progress.IsStreamGroupDeleted(g.Metadata.Name) {
		l.l.Info().Msgf("skipping already completed file-based migration for group: %s", g.Metadata.Name)
		return nil, nil
	}

	l.l.Info().Msgf("starting file-based stream migration for group: %s, time range: %s", g.Metadata.Name, tr.String())

	rootDir := filepath.Join(streamDir, g.Metadata.Name)
	// skip the counting if the tsdb root path does not exist
	// may no data found in the snapshot
	if _, err := os.Stat(rootDir); err != nil && errors.Is(err, os.ErrNotExist) {
		l.l.Info().Msgf("skipping file-based stream migration for group because is empty in the snapshot dir: %s", g.Metadata.Name)
		return nil, nil
	}

	// Use the file-based migration with existing visitor pattern
	segmentSuffixes, err := migrateStreamWithFileBasedAndProgress(rootDir, *tr, g, l.l, progress, int(l.chunkSize))
	if err != nil {
		return nil, fmt.Errorf("file-based stream migration failed: %w", err)
	}

	l.l.Info().Msgf("completed file-based stream migration for group: %s", g.Metadata.Name)
	return segmentSuffixes, nil
}

// getRemovalSegmentsTimeRange calculates the time range for segments that should be migrated
// based on the group's TTL configuration, similar to storage.segmentController.getExpiredSegmentsTimeRange.
func (l *lifecycleService) getRemovalSegmentsTimeRange(g *GroupConfig) *timestamp.TimeRange {
	// Convert TTL to storage.IntervalRule
	ttlRule := storage.MustToIntervalRule(g.AccumulatedTTL)

	// Calculate deadline based on TTL (same logic as segmentController.getExpiredSegmentsTimeRange)
	deadline := time.Now().Local().Add(-l.calculateTTLDuration(ttlRule))

	// Create time range for segments before the deadline
	timeRange := &timestamp.TimeRange{
		Start:        time.Time{}, // Will be set to earliest segment start time
		End:          deadline,    // All segments before this time should be migrated
		IncludeStart: true,
		IncludeEnd:   false,
	}

	l.l.Info().
		Str("group", g.Metadata.Name).
		Time("deadline", deadline).
		Str("ttl", fmt.Sprintf("%d %s", g.AccumulatedTTL.Num, g.AccumulatedTTL.Unit)).
		Msg("calculated removal segments time range based on TTL")

	return timeRange
}

// calculateTTLDuration calculates the duration for a TTL interval rule.
// This implements the same logic as storage.IntervalRule.estimatedDuration().
func (l *lifecycleService) calculateTTLDuration(ttl storage.IntervalRule) time.Duration {
	switch ttl.Unit {
	case storage.HOUR:
		return time.Hour * time.Duration(ttl.Num)
	case storage.DAY:
		return 24 * time.Hour * time.Duration(ttl.Num)
	default:
		l.l.Warn().Msgf("unknown TTL unit %v, defaulting to 1 day", ttl.Unit)
		return 24 * time.Hour
	}
}

func (l *lifecycleService) deleteExpiredStreamSegments(ctx context.Context, g *commonv1.Group, segmentSuffixes []string, progress *Progress) {
	if progress.IsStreamGroupDeleted(g.Metadata.Name) {
		l.l.Info().Msgf("skipping already deleted stream group segments: %s", g.Metadata.Name)
		return
	}

	resp, err := snapshot.Conn(l.gRPCAddr, l.enableTLS, l.insecure, l.cert, func(conn *grpclib.ClientConn) (*streamv1.DeleteExpiredSegmentsResponse, error) {
		client := streamv1.NewStreamServiceClient(conn)
		return client.DeleteExpiredSegments(ctx, &streamv1.DeleteExpiredSegmentsRequest{
			Group:           g.Metadata.Name,
			SegmentSuffixes: segmentSuffixes,
		})
	})
	if err != nil {
		l.l.Error().Err(err).Msgf("failed to delete expired segments in group %s, segments: %s", g.Metadata.Name, segmentSuffixes)
		return
	}

	l.l.Info().Msgf("deleted %d expired segments in group %s, segments: %s", resp.Deleted, g.Metadata.Name, segmentSuffixes)
	progress.MarkStreamGroupDeleted(g.Metadata.Name)
	progress.Save(l.progressFilePath, l.l)
}

func (l *lifecycleService) processMeasureGroup(ctx context.Context, g *commonv1.Group, measureDir string,
	nodes []*databasev1.Node, labels map[string]string, progress *Progress,
) {
	group, err := parseGroup(g, labels, nodes, l.l, l.metadata, l.clusterStateMgr)
	if err != nil {
		l.l.Error().Err(err).Msgf("failed to parse group %s", g.Metadata.Name)
		return
	}
	defer group.Close()

	tr := l.getRemovalSegmentsTimeRange(group)
	if tr.Start.IsZero() && tr.End.IsZero() {
		l.l.Info().Msgf("no removal segments time range for group %s, skipping measure migration", g.Metadata.Name)
		progress.MarkGroupCompleted(g.Metadata.Name)
		progress.Save(l.progressFilePath, l.l)
		return
	}

	// Try file-based migration first
	segmentSuffixes, err := l.processMeasureGroupFileBased(ctx, group, measureDir, tr, progress)
	if err != nil {
		l.l.Error().Err(err).Msgf("failed to migrate measure group %s using file-based approach", g.Metadata.Name)
		return
	}

	l.l.Info().Msgf("deleting expired measure segments for group: %s", g.Metadata.Name)
	l.deleteExpiredMeasureSegments(ctx, g, segmentSuffixes, progress)
	progress.MarkGroupCompleted(g.Metadata.Name)
	progress.Save(l.progressFilePath, l.l)
}

// processMeasureGroupFileBased uses file-based migration instead of query-based migration.
func (l *lifecycleService) processMeasureGroupFileBased(_ context.Context, g *GroupConfig, measureDir string,
	tr *timestamp.TimeRange, progress *Progress,
) ([]string, error) {
	if progress.IsMeasureGroupDeleted(g.Metadata.Name) {
		l.l.Info().Msgf("skipping already completed file-based measure migration for group: %s", g.Metadata.Name)
		return nil, nil
	}

	l.l.Info().Msgf("starting file-based measure migration for group: %s", g.Metadata.Name)

	rootDir := filepath.Join(measureDir, g.Metadata.Name)
	// skip the counting if the tsdb root path does not exist
	// may no data found in the snapshot
	if _, err := os.Stat(rootDir); err != nil && errors.Is(err, os.ErrNotExist) {
		l.l.Info().Msgf("skipping file-based measure migration for group because is empty in the snapshot dir: %s", g.Metadata.Name)
		return nil, nil
	}

	// Use the file-based migration with existing visitor pattern
	segmentSuffixes, err := migrateMeasureWithFileBasedAndProgress(rootDir, *tr, g, l.l, progress, int(l.chunkSize))
	if err != nil {
		return nil, fmt.Errorf("file-based measure migration failed: %w", err)
	}

	l.l.Info().Msgf("completed file-based measure migration for group: %s", g.Metadata.Name)
	return segmentSuffixes, nil
}

func (l *lifecycleService) deleteExpiredMeasureSegments(ctx context.Context, g *commonv1.Group, segmentSuffixes []string, progress *Progress) {
	if progress.IsMeasureGroupDeleted(g.Metadata.Name) {
		l.l.Info().Msgf("skipping already deleted measure group segments: %s", g.Metadata.Name)
		return
	}

	resp, err := snapshot.Conn(l.gRPCAddr, l.enableTLS, l.insecure, l.cert, func(conn *grpclib.ClientConn) (*measurev1.DeleteExpiredSegmentsResponse, error) {
		client := measurev1.NewMeasureServiceClient(conn)
		return client.DeleteExpiredSegments(ctx, &measurev1.DeleteExpiredSegmentsRequest{
			Group:           g.Metadata.Name,
			SegmentSuffixes: segmentSuffixes,
		})
	})
	if err != nil {
		l.l.Error().Err(err).Msgf("failed to delete expired segments in group %s, suffixes: %s", g.Metadata.Name, segmentSuffixes)
		return
	}

	l.l.Info().Msgf("deleted %d expired segments in group %s, suffixes: %s", resp.Deleted, g.Metadata.Name, segmentSuffixes)
	progress.MarkMeasureGroupDeleted(g.Metadata.Name)
	progress.Save(l.progressFilePath, l.l)
}

func (l *lifecycleService) deleteExpiredTraceSegments(ctx context.Context, g *commonv1.Group, segmentSuffixes []string, progress *Progress) {
	if progress.IsTraceGroupDeleted(g.Metadata.Name) {
		l.l.Info().Msgf("skipping already deleted trace group segments: %s", g.Metadata.Name)
		return
	}

	resp, err := snapshot.Conn(l.gRPCAddr, l.enableTLS, l.insecure, l.cert, func(conn *grpclib.ClientConn) (*tracev1.DeleteExpiredSegmentsResponse, error) {
		client := tracev1.NewTraceServiceClient(conn)
		return client.DeleteExpiredSegments(ctx, &tracev1.DeleteExpiredSegmentsRequest{
			Group:           g.Metadata.Name,
			SegmentSuffixes: segmentSuffixes,
		})
	})
	if err != nil {
		l.l.Error().Err(err).Msgf("failed to delete expired segments in group %s, suffixes: %s", g.Metadata.Name, segmentSuffixes)
		return
	}

	l.l.Warn().Msgf("deleted %d expired segments in group %s, suffixes: %s", resp.Deleted, g.Metadata.Name, segmentSuffixes)
	progress.MarkTraceGroupDeleted(g.Metadata.Name)
	progress.Save(l.progressFilePath, l.l)
}

func (l *lifecycleService) processTraceGroup(ctx context.Context, g *commonv1.Group, traceDir string,
	nodes []*databasev1.Node, labels map[string]string, progress *Progress,
) {
	group, err := parseGroup(g, labels, nodes, l.l, l.metadata, l.clusterStateMgr)
	if err != nil {
		l.l.Error().Err(err).Msgf("failed to parse group %s", g.Metadata.Name)
		return
	}
	defer group.Close()

	tr := l.getRemovalSegmentsTimeRange(group)
	if tr.Start.IsZero() && tr.End.IsZero() {
		l.l.Info().Msgf("no removal segments time range for group %s, skipping trace migration", g.Metadata.Name)
		progress.MarkGroupCompleted(g.Metadata.Name)
		progress.Save(l.progressFilePath, l.l)
		return
	}

	// Try file-based migration first
	segmentSuffixes, err := l.processTraceGroupFileBased(ctx, group, traceDir, tr, progress)
	if err != nil {
		l.l.Error().Err(err).Msgf("failed to migrate trace group %s using file-based approach", g.Metadata.Name)
		return
	}

	l.l.Info().Msgf("deleting expired trace segments for group: %s", g.Metadata.Name)
	l.deleteExpiredTraceSegments(ctx, g, segmentSuffixes, progress)
	progress.MarkGroupCompleted(g.Metadata.Name)
	progress.Save(l.progressFilePath, l.l)
}

func (l *lifecycleService) processTraceGroupFileBased(_ context.Context, g *GroupConfig, traceDir string,
	tr *timestamp.TimeRange, progress *Progress,
) ([]string, error) {
	if progress.IsTraceGroupDeleted(g.Metadata.Name) {
		l.l.Info().Msgf("skipping already completed file-based trace migration for group: %s", g.Metadata.Name)
		return nil, nil
	}

	l.l.Info().Msgf("starting file-based trace migration for group: %s", g.Metadata.Name)

	rootDir := filepath.Join(traceDir, g.Metadata.Name)
	// skip the counting if the tsdb root path does not exist
	// may no data found in the snapshot
	if _, err := os.Stat(rootDir); err != nil && errors.Is(err, os.ErrNotExist) {
		l.l.Info().Msgf("skipping file-based trace migration for group because is empty in the snapshot dir: %s", g.Metadata.Name)
		return nil, nil
	}

	// Use the file-based migration with existing visitor pattern
	segmentSuffixes, err := migrateTraceWithFileBasedAndProgress(rootDir, *tr, g, l.l, progress, int(l.chunkSize))
	if err != nil {
		return nil, fmt.Errorf("file-based trace migration failed: %w", err)
	}

	l.l.Info().Msgf("completed file-based trace migration for group: %s", g.Metadata.Name)
	return segmentSuffixes, nil
}
