// Licensed to Apache Software Foundation (ASF) under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Apache Software Foundation (ASF) licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.

package sidx

import (
	"encoding/json"
	"fmt"
	"path"
	"path/filepath"
	"sort"
	"strings"

	"github.com/apache/skywalking-banyandb/api/common"
	internalencoding "github.com/apache/skywalking-banyandb/banyand/internal/encoding"
	"github.com/apache/skywalking-banyandb/banyand/internal/storage"
	"github.com/apache/skywalking-banyandb/banyand/queue"
	"github.com/apache/skywalking-banyandb/pkg/bytes"
	"github.com/apache/skywalking-banyandb/pkg/compress/zstd"
	"github.com/apache/skywalking-banyandb/pkg/encoding"
	"github.com/apache/skywalking-banyandb/pkg/fs"
	"github.com/apache/skywalking-banyandb/pkg/logger"
	"github.com/apache/skywalking-banyandb/pkg/pool"
)

const (
	// SidxPrimaryName is the name for primary files.
	SidxPrimaryName = "primary"
	// SidxDataName is the name for data files.
	SidxDataName = "data"
	// SidxKeysName is the name for keys files.
	SidxKeysName = "keys"
	// SidxMetaName is the name for meta files.
	SidxMetaName = "meta"

	// primaryFilename is the filename for primary files.
	primaryFilename = SidxPrimaryName + ".bin"
	// dataFilename is the filename for data files.
	dataFilename = SidxDataName + ".bin"
	// keysFilename is the filename for keys files.
	keysFilename = SidxKeysName + ".bin"
	// metaFilename is the filename for meta files.
	metaFilename = SidxMetaName + ".bin"
	// manifestFilename is the filename for manifest files.
	manifestFilename = "manifest.json"

	// tagDataExtension is the extension for tag data files.
	tagDataExtension = ".td"
	// tagMetadataExtension is the extension for tag metadata files.
	tagMetadataExtension = ".tm"
	// tagFilterExtension is the extension for tag filter files.
	tagFilterExtension = ".tf"

	// TagDataPrefix is the prefix for tag data files.
	TagDataPrefix = "td:"
	// TagMetadataPrefix is the prefix for tag metadata files.
	TagMetadataPrefix = "tm:"
	// TagFilterPrefix is the prefix for tag filter files.
	TagFilterPrefix = "tf:"
)

// Part is a type alias for part.
type Part = part

// part represents a collection of files containing sidx data.
// Each part contains multiple files organized by type:
// - primary.bin: Block metadata and structure information
// - data.bin: User payload data (compressed)
// - keys.bin: User-provided int64 keys (encoded but not compressed)
// - meta.bin: part metadata
// - <name>.td: Tag data files (one per tag)
// - <name>.tm: Tag metadata files (one per tag)
// - <name>.tf: Tag filter files (bloom filters, one per tag).
type part struct {
	primary              fs.Reader
	data                 fs.Reader
	keys                 fs.Reader
	meta                 fs.Reader
	fileSystem           fs.FileSystem
	tagData              map[string]fs.Reader
	tagMetadata          map[string]fs.Reader
	tagFilters           map[string]fs.Reader
	partMetadata         *partMetadata
	path                 string
	primaryBlockMetadata []primaryBlockMetadata
}

func (p *part) ID() uint64 {
	return p.partMetadata.ID
}

// mustOpenPart opens a part from the specified path using the given file system.
// It opens all standard files and discovers tag files automatically.
// Panics if any required file cannot be opened.
func mustOpenPart(partID uint64, path string, fileSystem fs.FileSystem) *part {
	p := &part{
		path:       path,
		fileSystem: fileSystem,
	}

	// Open standard files.
	p.primary = mustOpenReader(filepath.Join(path, primaryFilename), fileSystem)
	p.data = mustOpenReader(filepath.Join(path, dataFilename), fileSystem)
	p.keys = mustOpenReader(filepath.Join(path, keysFilename), fileSystem)
	p.meta = mustOpenReader(filepath.Join(path, metaFilename), fileSystem)

	// Load part metadata from meta.bin.
	if err := p.loadPartMetadata(); err != nil {
		p.close()
		logger.GetLogger().Panic().Err(err).Str("path", path).Msg("failed to load part metadata")
	}
	p.partMetadata.ID = partID

	// Load primary block metadata from primary.bin.
	p.loadPrimaryBlockMetadata()

	// Discover and open tag files.
	p.openTagFiles()

	return p
}

// loadPartMetadata reads and parses the part metadata from manifest.json.
func (p *part) loadPartMetadata() error {
	// First try to read from manifest.json (new format)
	manifestPath := filepath.Join(p.path, manifestFilename)
	manifestData, err := p.fileSystem.Read(manifestPath)
	if err == nil {
		// Parse JSON manifest
		pm := &partMetadata{}
		if unmarshalErr := json.Unmarshal(manifestData, pm); unmarshalErr != nil {
			return fmt.Errorf("failed to unmarshal manifest.json: %w", unmarshalErr)
		}
		p.partMetadata = pm
		return nil
	}

	// Fallback to meta.bin for backward compatibility
	metaData, err := p.fileSystem.Read(filepath.Join(p.path, metaFilename))
	if err != nil {
		return fmt.Errorf("failed to read meta.bin: %w", err)
	}

	// Parse the metadata.
	pm, err := unmarshalPartMetadata(metaData)
	if err != nil {
		return fmt.Errorf("failed to unmarshal part metadata: %w", err)
	}

	p.partMetadata = pm
	return nil
}

// ParsePartMetadata reads and parses part metadata from manifest.json in the specified parent path.
func ParsePartMetadata(fs fs.FileSystem, parentPath string) (*queue.StreamingPartData, error) {
	manifestData, err := fs.Read(filepath.Join(parentPath, manifestFilename))
	if err != nil {
		return nil, fmt.Errorf("failed to read manifest.json: %w", err)
	}
	// Parse JSON manifest
	pm := &partMetadata{}
	if unmarshalErr := json.Unmarshal(manifestData, pm); unmarshalErr != nil {
		return nil, fmt.Errorf("failed to unmarshal manifest.json: %w", unmarshalErr)
	}
	return &queue.StreamingPartData{
		ID:                    pm.ID,
		CompressedSizeBytes:   pm.CompressedSizeBytes,
		UncompressedSizeBytes: pm.UncompressedSizeBytes,
		TotalCount:            pm.TotalCount,
		BlocksCount:           pm.BlocksCount,
		MinKey:                pm.MinKey,
		MaxKey:                pm.MaxKey,
	}, nil
}

// CreatePartFileReaderFromPath opens all files in a part directory and returns their FileInfo and a cleanup function.
func CreatePartFileReaderFromPath(partPath string, lfs fs.FileSystem) ([]queue.FileInfo, func()) {
	var files []queue.FileInfo
	var readers []fs.Reader

	// Core trace files (required files)
	metaPath := path.Join(partPath, metaFilename)
	metaReader, err := lfs.OpenFile(metaPath)
	if err != nil {
		logger.Panicf("cannot open trace meta file %q: %s", metaPath, err)
	}
	readers = append(readers, metaReader)
	files = append(files, queue.FileInfo{
		Name:   SidxMetaName,
		Reader: metaReader.SequentialRead(),
	})

	primaryPath := path.Join(partPath, primaryFilename)
	primaryReader, err := lfs.OpenFile(primaryPath)
	if err != nil {
		logger.Panicf("cannot open trace primary file %q: %s", primaryPath, err)
	}
	readers = append(readers, primaryReader)
	files = append(files, queue.FileInfo{
		Name:   SidxPrimaryName,
		Reader: primaryReader.SequentialRead(),
	})

	keysPath := path.Join(partPath, keysFilename)
	if keysReader, err := lfs.OpenFile(keysPath); err == nil {
		readers = append(readers, keysReader)
		files = append(files, queue.FileInfo{
			Name:   SidxKeysName,
			Reader: keysReader.SequentialRead(),
		})
	}

	dataPath := path.Join(partPath, dataFilename)
	if dataReader, err := lfs.OpenFile(dataPath); err == nil {
		readers = append(readers, dataReader)
		files = append(files, queue.FileInfo{
			Name:   SidxDataName,
			Reader: dataReader.SequentialRead(),
		})
	}

	// Dynamic tag files (*.t and *.tm)
	ee := lfs.ReadDir(partPath)
	for _, e := range ee {
		if e.IsDir() {
			continue
		}

		// Tag metadata files (.tm)
		if filepath.Ext(e.Name()) == tagMetadataExtension {
			tmPath := path.Join(partPath, e.Name())
			tmReader, err := lfs.OpenFile(tmPath)
			if err != nil {
				logger.Panicf("cannot open trace tag metadata file %q: %s", tmPath, err)
			}
			readers = append(readers, tmReader)
			tagName := removeExt(e.Name(), tagMetadataExtension)
			files = append(files, queue.FileInfo{
				Name:   TagMetadataPrefix + tagName,
				Reader: tmReader.SequentialRead(),
			})
		}

		// Tag family files (.tf)
		if filepath.Ext(e.Name()) == tagFilterExtension {
			tPath := path.Join(partPath, e.Name())
			tReader, err := lfs.OpenFile(tPath)
			if err != nil {
				logger.Panicf("cannot open trace tag file %q: %s", tPath, err)
			}
			readers = append(readers, tReader)
			tagName := removeExt(e.Name(), tagFilterExtension)
			files = append(files, queue.FileInfo{
				Name:   TagFilterPrefix + tagName,
				Reader: tReader.SequentialRead(),
			})
		}

		// Tag data files (.td)
		if filepath.Ext(e.Name()) == tagDataExtension {
			tdPath := path.Join(partPath, e.Name())
			tdReader, err := lfs.OpenFile(tdPath)
			if err != nil {
				logger.Panicf("cannot open trace tag data file %q: %s", tdPath, err)
			}
			readers = append(readers, tdReader)
			tagName := removeExt(e.Name(), tagDataExtension)
			files = append(files, queue.FileInfo{
				Name:   TagDataPrefix + tagName,
				Reader: tdReader.SequentialRead(),
			})
		}
	}

	cleanup := func() {
		for _, reader := range readers {
			fs.MustClose(reader)
		}
	}

	return files, cleanup
}

// loadPrimaryBlockMetadata reads and parses primary block metadata from meta.bin.
func (p *part) loadPrimaryBlockMetadata() {
	// Load primary block metadata from meta.bin file (compressed primaryBlockMetadata)
	p.primaryBlockMetadata = mustReadPrimaryBlockMetadata(p.primaryBlockMetadata[:0], p.meta)
}

// openTagFiles discovers and opens all tag files in the part directory.
// Tag files follow the pattern: <name>.<extension>
// where extension is .td (data), .tm (metadata), or .tf (filter).
func (p *part) openTagFiles() {
	// Read directory entries.
	entries := p.fileSystem.ReadDir(p.path)

	// Initialize maps.
	p.tagData = make(map[string]fs.Reader)
	p.tagMetadata = make(map[string]fs.Reader)
	p.tagFilters = make(map[string]fs.Reader)

	// Process each file in the directory.
	for _, entry := range entries {
		if entry.IsDir() {
			continue
		}

		fileName := entry.Name()
		// Check if this is a tag file by checking for tag extensions
		if !strings.HasSuffix(fileName, tagDataExtension) &&
			!strings.HasSuffix(fileName, tagMetadataExtension) &&
			!strings.HasSuffix(fileName, tagFilterExtension) {
			continue
		}

		// Extract tag name and extension.
		tagName, extension, found := extractTagNameAndExtension(fileName)
		if !found {
			continue
		}

		// Open the appropriate reader based on extension.
		filePath := filepath.Join(p.path, fileName)
		reader := mustOpenReader(filePath, p.fileSystem)

		switch extension {
		case tagDataExtension:
			p.tagData[tagName] = reader
		case tagMetadataExtension:
			p.tagMetadata[tagName] = reader
		case tagFilterExtension:
			p.tagFilters[tagName] = reader
		default:
			// Unknown extension, close the reader.
			fs.MustClose(reader)
		}
	}
}

// extractTagNameAndExtension parses a tag filename to extract the tag name and extension.
// Expected format: <name>.<extension>
// Returns the tag name, extension, and whether parsing was successful.
func extractTagNameAndExtension(fileName string) (tagName, extension string, found bool) {
	// Find the extension.
	extIndex := strings.LastIndex(fileName, ".")
	if extIndex == -1 {
		return "", "", false
	}

	tagName = fileName[:extIndex]
	extension = fileName[extIndex:]

	// Validate extension.
	switch extension {
	case tagDataExtension, tagMetadataExtension, tagFilterExtension:
		return tagName, extension, true
	default:
		return "", "", false
	}
}

// close closes all open file readers and releases resources.
func (p *part) close() {
	if p == nil {
		return
	}
	// Close standard files.
	if p.primary != nil {
		fs.MustClose(p.primary)
	}
	if p.data != nil {
		fs.MustClose(p.data)
	}
	if p.keys != nil {
		fs.MustClose(p.keys)
	}
	if p.meta != nil {
		fs.MustClose(p.meta)
	}

	// Close tag files.
	for _, reader := range p.tagData {
		fs.MustClose(reader)
	}
	for _, reader := range p.tagMetadata {
		fs.MustClose(reader)
	}
	for _, reader := range p.tagFilters {
		fs.MustClose(reader)
	}

	// Release metadata.
	if p.partMetadata != nil {
		p.partMetadata = nil
	}

	// No block metadata to release since it's now passed as parameter
}

// mustOpenReader opens a file reader and panics if it fails.
func mustOpenReader(filePath string, fileSystem fs.FileSystem) fs.Reader {
	file, err := fileSystem.OpenFile(filePath)
	if err != nil {
		logger.GetLogger().Panic().Err(err).Str("path", filePath).Msg("cannot open file")
	}
	return file
}

// readAll reads all blocks from the part and returns them as separate elements.
// Each elements collection represents the data from a single block.
func (p *part) readAll() ([]*elements, error) {
	if len(p.primaryBlockMetadata) == 0 {
		return nil, nil
	}

	result := make([]*elements, 0, len(p.primaryBlockMetadata))
	compressedPrimaryBuf := make([]byte, 0, 1024)
	primaryBuf := make([]byte, 0, 1024)
	dataBuf := make([]byte, 0, 1024)
	compressedKeysBuf := make([]byte, 0, 1024)

	for _, pbm := range p.primaryBlockMetadata {
		// Read and decompress primary block metadata
		compressedPrimaryBuf = bytes.ResizeOver(compressedPrimaryBuf, int(pbm.size))
		fs.MustReadData(p.primary, int64(pbm.offset), compressedPrimaryBuf)

		var err error
		primaryBuf, err = zstd.Decompress(primaryBuf[:0], compressedPrimaryBuf)
		if err != nil {
			// Clean up any elements created so far
			for _, e := range result {
				releaseElements(e)
			}
			return nil, fmt.Errorf("cannot decompress primary block: %w", err)
		}

		// Unmarshal all block metadata entries in this primary block
		blockMetadataArray, err := unmarshalBlockMetadata(nil, primaryBuf)
		if err != nil {
			// Clean up any elements created so far
			for _, e := range result {
				releaseElements(e)
			}
			return nil, fmt.Errorf("cannot unmarshal block metadata: %w", err)
		}

		// Process each block metadata
		for i := range blockMetadataArray {
			bm := &blockMetadataArray[i]

			// Create elements for this block
			elems := generateElements()

			// Read user keys
			compressedKeysBuf = bytes.ResizeOver(compressedKeysBuf, int(bm.keysBlock.size))
			fs.MustReadData(p.keys, int64(bm.keysBlock.offset), compressedKeysBuf)

			// Decode user keys directly using the stored encoding information
			elems.userKeys, err = encoding.BytesToInt64List(elems.userKeys[:0], compressedKeysBuf, bm.keysEncodeType, bm.minKey, int(bm.count))
			if err != nil {
				releaseElements(elems)
				for _, e := range result {
					releaseElements(e)
				}
				return nil, fmt.Errorf("cannot decode user keys: %w", err)
			}

			// Read data payloads
			dataBuf = bytes.ResizeOver(dataBuf, int(bm.dataBlock.size))
			fs.MustReadData(p.data, int64(bm.dataBlock.offset), dataBuf)

			// Decode data payloads - create a new decoder for each block to avoid state corruption
			blockBytesDecoder := &encoding.BytesBlockDecoder{}
			elems.data, err = blockBytesDecoder.Decode(elems.data[:0], dataBuf, bm.count)
			if err != nil {
				releaseElements(elems)
				for _, e := range result {
					releaseElements(e)
				}
				return nil, fmt.Errorf("cannot decode data payloads: %w", err)
			}

			// Initialize seriesIDs and tags slices
			elems.seriesIDs = make([]common.SeriesID, int(bm.count))
			elems.tags = make([][]*tag, int(bm.count))

			// Fill seriesIDs - all elements in this block have the same seriesID
			for j := range elems.seriesIDs {
				elems.seriesIDs[j] = bm.seriesID
			}

			// Read tags for each tag name
			for tagName := range bm.tagsBlocks {
				err = p.readBlockTags(tagName, bm, elems, blockBytesDecoder)
				if err != nil {
					releaseElements(elems)
					for _, e := range result {
						releaseElements(e)
					}
					return nil, fmt.Errorf("cannot read tags for %s: %w", tagName, err)
				}
			}

			result = append(result, elems)
		}
	}

	return result, nil
}

// readBlockTags reads and decodes tag data for a specific tag in a block.
func (p *part) readBlockTags(tagName string, bm *blockMetadata, elems *elements, decoder *encoding.BytesBlockDecoder) error {
	tagBlockInfo, exists := bm.tagsBlocks[tagName]
	if !exists {
		return fmt.Errorf("tag block info not found for tag: %s", tagName)
	}

	// Get tag metadata reader
	tmReader, tmExists := p.getTagMetadataReader(tagName)
	if !tmExists {
		return fmt.Errorf("tag metadata reader not found for tag: %s", tagName)
	}

	// Get tag data reader
	tdReader, tdExists := p.getTagDataReader(tagName)
	if !tdExists {
		return fmt.Errorf("tag data reader not found for tag: %s", tagName)
	}

	// Read tag metadata
	tmData := make([]byte, tagBlockInfo.size)
	fs.MustReadData(tmReader, int64(tagBlockInfo.offset), tmData)

	tm, err := unmarshalTagMetadata(tmData)
	if err != nil {
		return fmt.Errorf("cannot unmarshal tag metadata: %w", err)
	}
	defer releaseTagMetadata(tm)

	// Read tag data
	tdData := make([]byte, tm.dataBlock.size)
	fs.MustReadData(tdReader, int64(tm.dataBlock.offset), tdData)

	// Decode tag values directly (no compression)
	tagValues, err := internalencoding.DecodeTagValues(nil, decoder, &bytes.Buffer{Buf: tdData}, tm.valueType, int(bm.count))
	if err != nil {
		return fmt.Errorf("cannot decode tag values: %w", err)
	}

	// Add tag values to elements (only for non-nil values)
	for i, value := range tagValues {
		if i >= len(elems.tags) {
			break
		}
		// Skip nil values - they represent missing tags for this element
		if value == nil {
			continue
		}
		if elems.tags[i] == nil {
			elems.tags[i] = make([]*tag, 0, 1)
		}
		newTag := generateTag()
		newTag.name = tagName
		newTag.value = value
		newTag.valueType = tm.valueType
		elems.tags[i] = append(elems.tags[i], newTag)
	}

	return nil
}

// String returns a string representation of the part.
func (p *part) String() string {
	if p.partMetadata != nil {
		return fmt.Sprintf("sidx part %d at %s", p.partMetadata.ID, p.path)
	}
	return fmt.Sprintf("sidx part at %s", p.path)
}

// getTagDataReader returns the tag data reader for the specified tag name.
func (p *part) getTagDataReader(tagName string) (fs.Reader, bool) {
	reader, exists := p.tagData[tagName]
	return reader, exists
}

// getTagMetadataReader returns the tag metadata reader for the specified tag name.
func (p *part) getTagMetadataReader(tagName string) (fs.Reader, bool) {
	reader, exists := p.tagMetadata[tagName]
	return reader, exists
}

// getTagFilterReader returns the tag filter reader for the specified tag name.
func (p *part) getTagFilterReader(tagName string) (fs.Reader, bool) {
	reader, exists := p.tagFilters[tagName]
	return reader, exists
}

// Path returns the part's directory path.
func (p *part) Path() string {
	return p.path
}

// MemPart is a type alias for memPart.
type MemPart = memPart

// memPart represents an in-memory part for SIDX with tag-based file design.
// This structure mirrors the stream module's memPart but uses user keys instead of timestamps.
type memPart struct {
	tagMetadata  map[string]*bytes.Buffer
	tagData      map[string]*bytes.Buffer
	tagFilters   map[string]*bytes.Buffer
	partMetadata *partMetadata
	meta         bytes.Buffer
	primary      bytes.Buffer
	data         bytes.Buffer
	keys         bytes.Buffer
}

func (mp *memPart) ID() uint64 {
	return mp.partMetadata.ID
}

// SetPartMetadata sets the part metadata for the MemPart.
func (mp *memPart) SetPartMetadata(compressedSizeBytes, uncompressedSizeBytes, totalCount, blocksCount uint64, minKey, maxKey int64, id uint64) {
	if mp.partMetadata == nil {
		mp.partMetadata = &partMetadata{}
	}
	mp.partMetadata.CompressedSizeBytes = compressedSizeBytes
	mp.partMetadata.UncompressedSizeBytes = uncompressedSizeBytes
	mp.partMetadata.TotalCount = totalCount
	mp.partMetadata.BlocksCount = blocksCount
	mp.partMetadata.MinKey = minKey
	mp.partMetadata.MaxKey = maxKey
	mp.partMetadata.ID = id
}

// mustCreateTagWriters creates writers for individual tag files.
// Returns metadata writer, data writer, and filter writer for the specified tag.
func (mp *memPart) mustCreateTagWriters(tagName string) (fs.Writer, fs.Writer, fs.Writer) {
	if mp.tagData == nil {
		mp.tagData = make(map[string]*bytes.Buffer)
		mp.tagMetadata = make(map[string]*bytes.Buffer)
		mp.tagFilters = make(map[string]*bytes.Buffer)
	}

	// Get or create buffers for this tag
	td, tdExists := mp.tagData[tagName]
	tm := mp.tagMetadata[tagName]
	tf := mp.tagFilters[tagName]

	if tdExists {
		td.Reset()
		tm.Reset()
		tf.Reset()
		return tm, td, tf
	}

	// Create new buffers for this tag
	mp.tagData[tagName] = &bytes.Buffer{}
	mp.tagMetadata[tagName] = &bytes.Buffer{}
	mp.tagFilters[tagName] = &bytes.Buffer{}

	return mp.tagMetadata[tagName], mp.tagData[tagName], mp.tagFilters[tagName]
}

// reset clears the memory part for reuse.
func (mp *memPart) reset() {
	if mp.partMetadata != nil {
		mp.partMetadata.reset()
	}
	mp.meta.Reset()
	mp.primary.Reset()
	mp.data.Reset()
	mp.keys.Reset()

	if mp.tagData != nil {
		for k, td := range mp.tagData {
			td.Reset()
			delete(mp.tagData, k)
		}
	}
	if mp.tagMetadata != nil {
		for k, tm := range mp.tagMetadata {
			tm.Reset()
			delete(mp.tagMetadata, k)
		}
	}
	if mp.tagFilters != nil {
		for k, tf := range mp.tagFilters {
			tf.Reset()
			delete(mp.tagFilters, k)
		}
	}
}

// mustInitFromElements initializes the memory part from sorted elements using blockWriter.
func (mp *memPart) mustInitFromElements(es *elements) {
	mp.reset()

	if len(es.userKeys) == 0 {
		return
	}

	// Sort elements by seriesID first, then by user key
	sort.Sort(es)

	// Initialize part metadata
	if mp.partMetadata == nil {
		mp.partMetadata = &partMetadata{}
	}

	// Initialize block writer for memory part
	bw := generateBlockWriter()
	defer releaseBlockWriter(bw)

	bw.MustInitForMemPart(mp)

	// Group elements by seriesID and write to blocks
	currentSeriesID := es.seriesIDs[0]
	blockStart := 0
	var accumulatedSize uint64

	for i := 1; i <= len(es.seriesIDs); i++ {
		// Calculate size of current element if not at end
		if i < len(es.seriesIDs) {
			// Add userKey size (8 bytes)
			accumulatedSize += 8
			// Add data payload size
			accumulatedSize += uint64(len(es.data[i-1]))
			// Add tag sizes
			for _, tag := range es.tags[i-1] {
				if tag != nil {
					accumulatedSize += uint64(tag.size())
				}
			}
		}

		// Process block when series changes, size limit reached, count limit reached, or at end
		seriesChanged := i < len(es.seriesIDs) && es.seriesIDs[i] != currentSeriesID
		if (i-blockStart) > maxBlockLength || accumulatedSize >= maxUncompressedBlockSize || i == len(es.seriesIDs) || seriesChanged {
			// Extract elements for current series
			seriesUserKeys := es.userKeys[blockStart:i]
			seriesData := es.data[blockStart:i]
			seriesTags := es.tags[blockStart:i]

			// Write elements for this series
			bw.MustWriteElements(currentSeriesID, seriesUserKeys, seriesData, seriesTags)

			if i < len(es.seriesIDs) {
				currentSeriesID = es.seriesIDs[i]
				blockStart = i
				accumulatedSize = 0
			}
		}
	}

	// Flush the block writer to finalize metadata
	bw.Flush(mp.partMetadata)
}

// mustFlush flushes the memory part to disk with tag-based file organization.
func (mp *memPart) mustFlush(fileSystem fs.FileSystem, partPath string) {
	fileSystem.MkdirPanicIfExist(partPath, storage.DirPerm)

	// Write core files
	fs.MustFlush(fileSystem, mp.primary.Buf, filepath.Join(partPath, primaryFilename), storage.FilePerm)
	fs.MustFlush(fileSystem, mp.data.Buf, filepath.Join(partPath, dataFilename), storage.FilePerm)
	fs.MustFlush(fileSystem, mp.keys.Buf, filepath.Join(partPath, keysFilename), storage.FilePerm)
	fs.MustFlush(fileSystem, mp.meta.Buf, filepath.Join(partPath, metaFilename), storage.FilePerm)

	// Write individual tag files
	for tagName, td := range mp.tagData {
		fs.MustFlush(fileSystem, td.Buf, filepath.Join(partPath, tagName+tagDataExtension), storage.FilePerm)
	}
	for tagName, tm := range mp.tagMetadata {
		fs.MustFlush(fileSystem, tm.Buf, filepath.Join(partPath, tagName+tagMetadataExtension), storage.FilePerm)
	}
	for tagName, tf := range mp.tagFilters {
		fs.MustFlush(fileSystem, tf.Buf, filepath.Join(partPath, tagName+tagFilterExtension), storage.FilePerm)
	}

	// Write part metadata manifest
	if mp.partMetadata != nil {
		mp.partMetadata.mustWriteMetadata(fileSystem, partPath)
	}

	fileSystem.SyncPath(partPath)
}

// GenerateMemPart gets memPart from pool or creates new.
func GenerateMemPart() *MemPart {
	v := memPartPool.Get()
	if v == nil {
		return &MemPart{}
	}
	return v
}

// ReleaseMemPart returns memPart to pool after reset.
func ReleaseMemPart(mp *memPart) {
	mp.reset()
	memPartPool.Put(mp)
}

// MustFlush flushes the memory part to disk (exported version).
func (mp *memPart) MustFlush(fileSystem fs.FileSystem, partPath string) {
	mp.mustFlush(fileSystem, partPath)
}

var memPartPool = pool.Register[*memPart]("sidx-memPart")

// openMemPart creates a part from a memory part.
func openMemPart(mp *memPart) *part {
	p := &part{}
	p.partMetadata = mp.partMetadata

	// Load primary block metadata from meta buffer
	if len(mp.meta.Buf) > 0 {
		p.primaryBlockMetadata = mustReadPrimaryBlockMetadata(nil, &mp.meta)
	}

	// Open data files as readers from memory buffers
	p.primary = &mp.primary
	p.data = &mp.data
	p.keys = &mp.keys
	p.meta = &mp.meta

	// Open individual tag files if they exist
	if mp.tagData != nil {
		p.tagData = make(map[string]fs.Reader)
		p.tagMetadata = make(map[string]fs.Reader)
		p.tagFilters = make(map[string]fs.Reader)

		for tagName, td := range mp.tagData {
			p.tagData[tagName] = td
			if tm, exists := mp.tagMetadata[tagName]; exists {
				p.tagMetadata[tagName] = tm
			}
			if tf, exists := mp.tagFilters[tagName]; exists {
				p.tagFilters[tagName] = tf
			}
		}
	}
	return p
}

// partPath returns the path for a part with the given epoch.
func partPath(root string, epoch uint64) string {
	return filepath.Join(root, partName(epoch))
}

// partName returns the directory name for a part with the given epoch.
// Uses 16-character hex format consistent with stream module.
func partName(epoch uint64) string {
	return fmt.Sprintf("%016x", epoch)
}

// SyncPartContext manages a file-backed sidx part during streaming sync.
type SyncPartContext struct {
	fileSystem fs.FileSystem
	writers    *writers
	name       string
	partPath   string
	partMeta   partMetadata
}

// NewSyncPartContext creates a new sidx part context.
func NewSyncPartContext() *SyncPartContext {
	return &SyncPartContext{}
}

// SetForFile initializes file-backed writers for direct file writing.
// The metadata fields are stored and written as manifest.json during Finish().
func (spc *SyncPartContext) SetForFile(name string, fileSystem fs.FileSystem, partPath string, ctx *queue.ChunkedSyncPartContext, shouldCache bool) {
	spc.name = name
	spc.fileSystem = fileSystem
	spc.partPath = partPath
	spc.partMeta.fillFromSyncContext(ctx)

	w := GenerateWriters()
	w.MustInitForFilePart(fileSystem, partPath, shouldCache)
	spc.writers = w
}

// Finish closes writers, writes manifest.json, and returns the on-disk part path.
// It clears spc.partPath so a subsequent Close will not remove the files.
func (spc *SyncPartContext) Finish() string {
	if spc.writers != nil {
		spc.writers.MustClose()
		ReleaseWriters(spc.writers)
		spc.writers = nil
	}
	if spc.fileSystem != nil && spc.partPath != "" {
		spc.partMeta.mustWriteMetadata(spc.fileSystem, spc.partPath)
		spc.fileSystem.SyncPath(spc.partPath)
	}
	p := spc.partPath
	spc.partPath = ""
	spc.fileSystem = nil
	return p
}

// Name gets the name.
func (spc *SyncPartContext) Name() string {
	return spc.name
}

// GetWriters gets the writers.
func (spc *SyncPartContext) GetWriters() *Writers {
	return spc.writers
}

// Close closes the sidx part context and cleans up any incomplete file parts on error.
func (spc *SyncPartContext) Close() {
	if spc.writers != nil {
		spc.writers.MustClose()
		ReleaseWriters(spc.writers)
		spc.writers = nil
	}
	if spc.partPath != "" && spc.fileSystem != nil {
		spc.fileSystem.MustRMAll(spc.partPath)
		spc.partPath = ""
	}
}

func removeExt(nameWithExt, ext string) string {
	return nameWithExt[:len(nameWithExt)-len(ext)]
}
