/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.paimon.append;

import org.apache.paimon.CoreOptions;
import org.apache.paimon.compact.NoopCompactManager;
import org.apache.paimon.compression.CompressOptions;
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.data.BinaryRowWriter;
import org.apache.paimon.data.BinaryString;
import org.apache.paimon.data.BinaryVector;
import org.apache.paimon.data.BlobData;
import org.apache.paimon.data.GenericMap;
import org.apache.paimon.data.GenericRow;
import org.apache.paimon.data.InternalMap;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.data.shredding.MapSharedShreddingFieldMeta;
import org.apache.paimon.data.shredding.MapSharedShreddingUtils;
import org.apache.paimon.disk.ChannelWithMeta;
import org.apache.paimon.disk.ExternalBuffer;
import org.apache.paimon.disk.IOManager;
import org.apache.paimon.disk.RowBuffer;
import org.apache.paimon.fileindex.FileIndexOptions;
import org.apache.paimon.format.FileFormat;
import org.apache.paimon.format.FormatReaderContext;
import org.apache.paimon.format.SimpleColStats;
import org.apache.paimon.format.SupportsFieldMetadata;
import org.apache.paimon.fs.Path;
import org.apache.paimon.fs.local.LocalFileIO;
import org.apache.paimon.io.DataFileMeta;
import org.apache.paimon.io.DataFilePathFactory;
import org.apache.paimon.manifest.FileSource;
import org.apache.paimon.memory.HeapMemorySegmentPool;
import org.apache.paimon.memory.MemoryPoolFactory;
import org.apache.paimon.operation.BaseAppendFileStoreWrite;
import org.apache.paimon.operation.BlobFileContext;
import org.apache.paimon.options.MemorySize;
import org.apache.paimon.options.Options;
import org.apache.paimon.schema.Schema;
import org.apache.paimon.schema.TableSchema;
import org.apache.paimon.stats.SimpleStatsConverter;
import org.apache.paimon.table.AppendOnlyFileStoreTable;
import org.apache.paimon.table.FileStoreTableFactory;
import org.apache.paimon.types.BlobType;
import org.apache.paimon.types.DataType;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.types.IntType;
import org.apache.paimon.types.RowType;
import org.apache.paimon.types.VarBinaryType;
import org.apache.paimon.types.VarCharType;
import org.apache.paimon.utils.CommitIncrement;
import org.apache.paimon.utils.ExecutorThreadFactory;
import org.apache.paimon.utils.Pair;
import org.apache.paimon.utils.RecordWriter;
import org.apache.paimon.utils.StatsCollectorFactories;

import org.assertj.core.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;

import javax.annotation.Nullable;

import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Executors;
import java.util.stream.Collectors;

import static org.apache.paimon.io.DataFileMeta.getMaxSequenceNumber;
import static org.apache.paimon.stats.SimpleStats.EMPTY_STATS;
import static org.assertj.core.api.Assertions.assertThat;

/** Test the correctness for {@link AppendOnlyWriter}. */
public class AppendOnlyWriterTest {

    private static final InternalRow EMPTY_ROW = BinaryRow.EMPTY_ROW;
    private static final RowType SCHEMA =
            RowType.builder()
                    .fields(
                            new DataType[] {new IntType(), new VarCharType(), new VarCharType()},
                            new String[] {"id", "name", "dt"})
                    .build();
    private static final SimpleStatsConverter STATS_SERIALIZER = new SimpleStatsConverter(SCHEMA);

    @TempDir public java.nio.file.Path tempDir;
    public DataFilePathFactory pathFactory;

    private static final String AVRO = "avro";
    private static final String PART = "2022-05-01";
    private static final long SCHEMA_ID = 0L;
    private static final int MIN_FILE_NUM = 3;

    @BeforeEach
    public void before() {
        pathFactory = createPathFactory();
    }

    @Test
    public void testEmptyCommits() throws Exception {
        RecordWriter<InternalRow> writer = createEmptyWriter(1024 * 1024L);

        for (int i = 0; i < 3; i++) {
            writer.sync();
            CommitIncrement inc = writer.prepareCommit(true);

            assertThat(inc.newFilesIncrement().isEmpty()).isTrue();
            assertThat(inc.compactIncrement().isEmpty()).isTrue();
        }
    }

    @Test
    public void testSingleWrite() throws Exception {
        RecordWriter<InternalRow> writer = createEmptyWriter(1024 * 1024L);
        writer.write(row(1, "AAA", PART));
        CommitIncrement increment = writer.prepareCommit(true);
        writer.close();

        assertThat(increment.newFilesIncrement().newFiles().size()).isEqualTo(1);
        DataFileMeta meta = increment.newFilesIncrement().newFiles().get(0);
        assertThat(meta).isNotNull();

        Path path = pathFactory.toPath(meta);
        assertThat(LocalFileIO.create().exists(path)).isTrue();

        assertThat(meta.rowCount()).isEqualTo(1L);
        assertThat(meta.minKey()).isEqualTo(EMPTY_ROW);
        assertThat(meta.maxKey()).isEqualTo(EMPTY_ROW);
        assertThat(meta.keyStats()).isEqualTo(EMPTY_STATS);

        SimpleColStats[] expected =
                new SimpleColStats[] {
                    initStats(1, 1, 0), initStats("AAA", "AAA", 0), initStats(PART, PART, 0)
                };
        assertThat(meta.valueStats()).isEqualTo(STATS_SERIALIZER.toBinaryAllMode(expected));

        assertThat(meta.minSequenceNumber()).isEqualTo(0);
        assertThat(meta.maxSequenceNumber()).isEqualTo(0);
        assertThat(meta.level()).isEqualTo(DataFileMeta.DUMMY_LEVEL);
        assertThat(meta.fileName().substring(meta.fileName().lastIndexOf(".") + 1))
                .isEqualTo(CoreOptions.FILE_FORMAT_AVRO);
    }

    @Test
    public void testBinaryColumnStatsRoundTrip() throws Exception {
        RowType binarySchema =
                RowType.builder()
                        .fields(
                                new DataType[] {new IntType(), new VarBinaryType()},
                                new String[] {"id", "data"})
                        .build();
        Map<String, String> options = new HashMap<>();
        options.put("metadata.stats-mode", "full");

        AppendOnlyWriter writer =
                createWriterBase(
                                1024 * 1024L,
                                null,
                                binarySchema,
                                false,
                                true,
                                true,
                                true,
                                Collections.emptyList(),
                                compactBefore -> Collections.emptyList(),
                                options)
                        .getKey();

        writer.write(GenericRow.of(1, new byte[] {1, 2, 3}));
        writer.write(GenericRow.of(2, new byte[] {(byte) 200}));
        writer.write(GenericRow.of(3, new byte[] {0}));
        CommitIncrement increment = writer.prepareCommit(true);
        writer.close();

        DataFileMeta meta = increment.newFilesIncrement().newFiles().get(0);
        assertThat(LocalFileIO.create().exists(pathFactory.toPath(meta))).isTrue();

        assertThat(meta.valueStats().minValues().isNullAt(1))
                .as("binary column min must be present in file stats")
                .isFalse();
        assertThat(meta.valueStats().minValues().getBinary(1)).isEqualTo(new byte[] {0});
        assertThat(meta.valueStats().maxValues().getBinary(1)).isEqualTo(new byte[] {(byte) 200});
        assertThat(meta.valueStats().nullCounts().getLong(1)).isEqualTo(0L);
    }

    @Test
    public void testMultipleCommits() throws Exception {
        RecordWriter<InternalRow> writer =
                createWriter(1024 * 1024L, true, Collections.emptyList()).getLeft();

        // Commit 5 continues txn.
        for (int txn = 0; txn < 5; txn += 1) {

            // Write the records with range [ txn*100, (txn+1)*100 ).
            int start = txn * 100;
            int end = txn * 100 + 100;
            for (int i = start; i < end; i++) {
                writer.write(row(i, String.format("%03d", i), PART));
            }

            writer.sync();
            CommitIncrement inc = writer.prepareCommit(true);
            if (txn > 0 && txn % 2 == 0) {
                assertThat(inc.compactIncrement().compactBefore()).hasSize(3);
                assertThat(inc.compactIncrement().compactAfter()).hasSize(1);
                DataFileMeta compactAfter = inc.compactIncrement().compactAfter().get(0);
                assertThat(compactAfter.fileName()).startsWith("compact-");
                assertThat(compactAfter.fileSize())
                        .isEqualTo(
                                inc.compactIncrement().compactBefore().stream()
                                        .mapToLong(DataFileMeta::fileSize)
                                        .sum());
                assertThat(compactAfter.rowCount())
                        .isEqualTo(
                                inc.compactIncrement().compactBefore().stream()
                                        .mapToLong(DataFileMeta::rowCount)
                                        .sum());
            } else {
                assertThat(inc.compactIncrement().compactBefore())
                        .isEqualTo(Collections.emptyList());
                assertThat(inc.compactIncrement().compactAfter())
                        .isEqualTo(Collections.emptyList());
            }

            assertThat(inc.newFilesIncrement().newFiles().size()).isEqualTo(1);
            DataFileMeta meta = inc.newFilesIncrement().newFiles().get(0);

            Path path = pathFactory.toPath(meta);
            assertThat(LocalFileIO.create().exists(path)).isTrue();

            assertThat(meta.rowCount()).isEqualTo(100L);
            assertThat(meta.minKey()).isEqualTo(EMPTY_ROW);
            assertThat(meta.maxKey()).isEqualTo(EMPTY_ROW);
            assertThat(meta.keyStats()).isEqualTo(EMPTY_STATS);

            SimpleColStats[] expected =
                    new SimpleColStats[] {
                        initStats(start, end - 1, 0),
                        initStats(String.format("%03d", start), String.format("%03d", end - 1), 0),
                        initStats(PART, PART, 0)
                    };
            assertThat(meta.valueStats()).isEqualTo(STATS_SERIALIZER.toBinaryAllMode(expected));

            assertThat(meta.minSequenceNumber()).isEqualTo(start);
            assertThat(meta.maxSequenceNumber()).isEqualTo(end - 1);
            assertThat(meta.level()).isEqualTo(DataFileMeta.DUMMY_LEVEL);
        }
    }

    @Test
    public void testRollingWrite() throws Exception {
        // Set a very small target file size, so the threshold to trigger rolling becomes record
        // count instead of file size, because we check rolling per 1000 records.
        AppendOnlyWriter writer = createEmptyWriter(10L);

        for (int i = 0; i < 10 * 1000; i++) {
            writer.write(row(i, String.format("%03d", i), PART));
        }

        writer.sync();
        CommitIncrement firstInc = writer.prepareCommit(true);
        assertThat(firstInc.compactIncrement().compactBefore()).isEqualTo(Collections.emptyList());
        assertThat(firstInc.compactIncrement().compactAfter()).isEqualTo(Collections.emptyList());

        assertThat(firstInc.newFilesIncrement().newFiles().size()).isEqualTo(10);

        int id = 0;
        for (DataFileMeta meta : firstInc.newFilesIncrement().newFiles()) {
            Path path = pathFactory.toPath(meta);
            assertThat(LocalFileIO.create().exists(path)).isTrue();

            assertThat(meta.rowCount()).isEqualTo(1000L);
            assertThat(meta.minKey()).isEqualTo(EMPTY_ROW);
            assertThat(meta.maxKey()).isEqualTo(EMPTY_ROW);
            assertThat(meta.keyStats()).isEqualTo(EMPTY_STATS);

            int min = id * 1000;
            int max = id * 1000 + 999;
            SimpleColStats[] expected =
                    new SimpleColStats[] {
                        initStats(min, max, 0),
                        initStats(String.format("%03d", min), String.format("%03d", max), 0),
                        initStats(PART, PART, 0)
                    };
            assertThat(meta.valueStats()).isEqualTo(STATS_SERIALIZER.toBinaryAllMode(expected));

            assertThat(meta.minSequenceNumber()).isEqualTo(min);
            assertThat(meta.maxSequenceNumber()).isEqualTo(max);
            assertThat(meta.level()).isEqualTo(DataFileMeta.DUMMY_LEVEL);

            id += 1;
        }

        // increase target file size to test compaction
        long targetFileSize = 1024 * 1024L;
        Pair<AppendOnlyWriter, List<DataFileMeta>> writerAndToCompact =
                createWriter(targetFileSize, true, firstInc.newFilesIncrement().newFiles());
        writer = writerAndToCompact.getLeft();
        List<DataFileMeta> toCompact = writerAndToCompact.getRight();
        assertThat(toCompact).containsExactlyElementsOf(firstInc.newFilesIncrement().newFiles());
        writer.write(row(id, String.format("%03d", id), PART));
        writer.sync();
        CommitIncrement secInc = writer.prepareCommit(true);

        // check compact before and after
        List<DataFileMeta> compactBefore = secInc.compactIncrement().compactBefore();
        List<DataFileMeta> compactAfter = secInc.compactIncrement().compactAfter();
        assertThat(compactBefore)
                .containsExactlyInAnyOrderElementsOf(
                        firstInc.newFilesIncrement().newFiles().subList(0, 3));
        assertThat(compactAfter).hasSize(1);
        assertThat(compactBefore.stream().mapToLong(DataFileMeta::fileSize).sum())
                .isEqualTo(compactAfter.stream().mapToLong(DataFileMeta::fileSize).sum());
        assertThat(compactBefore.stream().mapToLong(DataFileMeta::rowCount).sum())
                .isEqualTo(compactAfter.stream().mapToLong(DataFileMeta::rowCount).sum());
        // check seq number
        assertThat(compactBefore.get(0).minSequenceNumber())
                .isEqualTo(compactAfter.get(0).minSequenceNumber());
        assertThat(compactBefore.get(compactBefore.size() - 1).maxSequenceNumber())
                .isEqualTo(compactAfter.get(compactAfter.size() - 1).maxSequenceNumber());
        assertThat(secInc.newFilesIncrement().newFiles()).hasSize(1);
    }

    @Test
    public void testCloseUnexpectedly() throws Exception {
        final CountDownLatch latch = new CountDownLatch(1);
        AppendOnlyWriter writer =
                createWriter(20 * 1024, false, Collections.emptyList(), latch).getLeft();

        for (int i = 0; i < 10; i++) { // create 10 small files.
            for (int j = 0; j < 10; j++) {
                writer.write(row(j, String.format("%03d", j), PART));
            }
            CommitIncrement increment = writer.prepareCommit(false);
            assertThat(increment.compactIncrement().isEmpty()).isTrue();
        }
        Set<Path> committedFiles =
                Files.walk(tempDir)
                        .filter(Files::isRegularFile)
                        .map(p -> new Path(p.toString()))
                        .collect(Collectors.toSet());

        // start compaction and write more records
        latch.countDown();
        for (int j = 0; j < 10; j++) {
            writer.write(row(j, String.format("%03d", j), PART));
        }
        writer.sync();

        // writer closed unexpectedly
        writer.close();

        Set<Path> afterClosedUnexpectedly =
                Files.walk(tempDir)
                        .filter(Files::isRegularFile)
                        .map(p -> new Path(p.toString()))
                        .collect(Collectors.toSet());
        assertThat(afterClosedUnexpectedly).containsExactlyInAnyOrderElementsOf(committedFiles);
    }

    @Test
    public void testExternalBufferWorks() throws Exception {
        AppendOnlyWriter writer = createEmptyWriter(Long.MAX_VALUE, true);

        // we give it a small Memory Pool, force it to spill
        writer.setMemoryPool(new HeapMemorySegmentPool(16384L, 1024));

        char[] s = new char[990];
        Arrays.fill(s, 'a');

        // set the record that much larger than the maxMemory
        for (int j = 0; j < 100; j++) {
            writer.write(row(j, String.valueOf(s), PART));
        }

        RowBuffer buffer = writer.getWriteBuffer();
        Assertions.assertThat(buffer.size()).isEqualTo(100);
        Assertions.assertThat(buffer.memoryOccupancy()).isLessThanOrEqualTo(16384L);

        writer.close();
    }

    @Test
    public void testSpillWorksAndMoreSmallFilesGenerated() throws Exception {
        List<AppendOnlyWriter> writers = new ArrayList<>();
        HeapMemorySegmentPool heapMemorySegmentPool = new HeapMemorySegmentPool(2501024L, 1024);
        MemoryPoolFactory memoryPoolFactory = new MemoryPoolFactory(heapMemorySegmentPool);
        for (int i = 0; i < 1000; i++) {
            AppendOnlyWriter writer = createEmptyWriter(Long.MAX_VALUE, true);
            memoryPoolFactory.addOwners(Arrays.asList(writer));
            memoryPoolFactory.notifyNewOwner(writer);
            writers.add(writer);
        }

        char[] s = new char[1024];
        Arrays.fill(s, 'a');

        for (AppendOnlyWriter writer : writers) {
            writer.write(row(0, String.valueOf("a"), PART));
        }

        for (AppendOnlyWriter writer : writers) {
            writer.write(row(0, String.valueOf(s), PART));
        }

        for (int j = 0; j < 100; j++) {
            for (AppendOnlyWriter writer : writers) {
                writer.write(row(j, String.valueOf(s), PART));
                writer.write(row(j, String.valueOf(s), PART));
                writer.write(row(j, String.valueOf(s), PART));
                writer.write(row(j, String.valueOf(s), PART));
                writer.write(row(j, String.valueOf(s), PART));
            }
        }

        writers.forEach(
                writer -> {
                    try {
                        List<DataFileMeta> fileMetas =
                                writer.prepareCommit(false).newFilesIncrement().newFiles();
                        assertThat(fileMetas.size()).isEqualTo(1);
                    } catch (Exception e) {
                        throw new RuntimeException(e);
                    }
                });

        writers.forEach(
                writer -> {
                    try {
                        writer.close();
                    } catch (Exception e) {
                        throw new RuntimeException(e);
                    }
                });
    }

    @Test
    public void testNoSpillWhenMeetBlobType() throws Exception {
        // Create a schema with BLOB type
        RowType blobSchema =
                RowType.builder()
                        .fields(
                                new DataType[] {new IntType(), new VarCharType(), new BlobType()},
                                new String[] {"id", "name", "data"})
                        .build();

        AppendOnlyFileStoreTable table =
                (AppendOnlyFileStoreTable)
                        FileStoreTableFactory.create(
                                LocalFileIO.create(),
                                pathFactory.newPath(),
                                TableSchema.create(
                                        0,
                                        new Schema(
                                                blobSchema.getFields(),
                                                Collections.singletonList("id"),
                                                Collections.emptyList(),
                                                new HashMap<String, String>() {
                                                    {
                                                        put(
                                                                CoreOptions.DATA_EVOLUTION_ENABLED
                                                                        .key(),
                                                                "true");
                                                        put(
                                                                CoreOptions.ROW_TRACKING_ENABLED
                                                                        .key(),
                                                                "true");
                                                    }
                                                },
                                                "")));
        BaseAppendFileStoreWrite writer = table.store().newWrite("test");
        writer.withIOManager(IOManager.create(tempDir.toString()));
        writer.withMemoryPoolFactory(
                new MemoryPoolFactory(new HeapMemorySegmentPool(16384L, 1024)));

        char[] largeString = new char[990];
        Arrays.fill(largeString, 'a');
        byte[] largeBlobData = new byte[1024];
        Arrays.fill(largeBlobData, (byte) 'b');

        BinaryRow binaryRow = new BinaryRow(1);
        BinaryRowWriter binaryRowWriter = new BinaryRowWriter(binaryRow);
        for (int j = 0; j < 100; j++) {
            binaryRowWriter.reset();
            binaryRowWriter.writeInt(0, j);
            binaryRowWriter.complete();
            writer.write(
                    binaryRow, 0, createBlobRow(j, String.valueOf(largeString), largeBlobData));
        }

        binaryRowWriter.reset();
        binaryRowWriter.writeInt(0, 1000);
        binaryRowWriter.complete();
        AppendOnlyWriter appendOnlyWriter = (AppendOnlyWriter) writer.createWriter(binaryRow, 0);
        RowBuffer buffer = appendOnlyWriter.getWriteBuffer();
        assertThat(buffer).isNull();
        writer.close();
    }

    @ParameterizedTest(name = "{0}")
    @ValueSource(strings = {CoreOptions.FILE_FORMAT_PARQUET, CoreOptions.FILE_FORMAT_ORC})
    public void testWriteSharedShreddingWithFileIndex(String fileFormat) throws Exception {
        RowType writeType = sharedShreddingTagsWriteType();
        Options rawOptions = sharedShreddingOptions("tags", 3);
        rawOptions.setString("file-index.bloom-filter.columns", "id");
        SharedShreddingAppendContext context =
                createSharedShreddingAppendContext(fileFormat, rawOptions);
        AppendOnlyWriter writer =
                createSharedShreddingAppendWriter(
                        writeType, context, SCHEMA_ID, -1L, new FileIndexOptions(context.options));

        writer.write(sharedShreddingRow(1, "a", 10L));
        writer.write(sharedShreddingRow(2, "b", 20L));
        CommitIncrement increment = writer.prepareCommit(true);
        writer.close();

        DataFileMeta file = assertSingleNewFile(increment);
        assertThat(file.embeddedIndex() != null || !file.extraFiles().isEmpty()).isTrue();
        assertThat(readSharedShreddingFieldMeta(context, file, "tags"))
                .isEqualTo(
                        sharedShreddingMeta(
                                nameToId("a", 0, "b", 1),
                                fieldToColumns(0, columns(0), 1, columns(0)),
                                overflowFields(),
                                3,
                                1));
    }

    @ParameterizedTest(name = "{0}")
    @ValueSource(strings = {CoreOptions.FILE_FORMAT_PARQUET, CoreOptions.FILE_FORMAT_ORC})
    public void testSharedShreddingMapAllEmptyFirstFile(String fileFormat) throws Exception {
        RowType writeType = sharedShreddingTagsWriteType();
        Options rawOptions = sharedShreddingOptions("tags", 3);
        SharedShreddingAppendContext context =
                createSharedShreddingAppendContext(fileFormat, rawOptions);
        AppendOnlyWriter writer = createSharedShreddingAppendWriter(writeType, context);

        DataFileMeta file =
                writeSharedShreddingFile(writer, sharedShreddingRow(1), sharedShreddingRow(2));
        writer.close();

        assertThat(readSharedShreddingFieldMeta(context, file, "tags"))
                .isEqualTo(
                        sharedShreddingMeta(nameToId(), fieldToColumns(), overflowFields(), 3, 0));
    }

    @ParameterizedTest(name = "{0}")
    @ValueSource(strings = {CoreOptions.FILE_FORMAT_PARQUET, CoreOptions.FILE_FORMAT_ORC})
    public void testSharedShreddingMapAllNullThenAllEmptyFiles(String fileFormat) throws Exception {
        RowType writeType = sharedShreddingTagsWriteType();
        Options rawOptions = sharedShreddingOptions("tags", 3);
        SharedShreddingAppendContext context =
                createSharedShreddingAppendContext(fileFormat, rawOptions);
        AppendOnlyWriter writer = createSharedShreddingAppendWriter(writeType, context);

        DataFileMeta nullFile =
                writeSharedShreddingFile(
                        writer,
                        sharedShreddingLogicalRow(1, (Object) null),
                        sharedShreddingLogicalRow(2, (Object) null));
        DataFileMeta emptyFile =
                writeSharedShreddingFile(writer, sharedShreddingRow(3), sharedShreddingRow(4));
        DataFileMeta valuesFile =
                writeSharedShreddingFile(
                        writer,
                        sharedShreddingRow(5, "a", null),
                        sharedShreddingRow(6, "b", null),
                        sharedShreddingRow(7, "c", 7L, "d", null));
        writer.close();

        assertThat(readSharedShreddingFieldMeta(context, nullFile, "tags"))
                .isEqualTo(
                        sharedShreddingMeta(nameToId(), fieldToColumns(), overflowFields(), 3, 0));
        assertThat(readSharedShreddingFieldMeta(context, emptyFile, "tags"))
                .isEqualTo(
                        sharedShreddingMeta(nameToId(), fieldToColumns(), overflowFields(), 3, 0));
        assertThat(readSharedShreddingFieldMeta(context, valuesFile, "tags"))
                .isEqualTo(
                        sharedShreddingMeta(
                                nameToId("a", 0, "b", 1, "c", 2, "d", 3),
                                fieldToColumns(
                                        0, columns(0), 1, columns(0), 2, columns(0), 3, columns(1)),
                                overflowFields(),
                                3,
                                2));
    }

    @ParameterizedTest(name = "{0}")
    @ValueSource(strings = {CoreOptions.FILE_FORMAT_PARQUET, CoreOptions.FILE_FORMAT_ORC})
    public void testSharedShreddingMapDataFileMetaInfo(String fileFormat) throws Exception {
        RowType writeType = sharedShreddingTagsWriteType();
        Options rawOptions = sharedShreddingOptions("tags", 3);
        rawOptions.setString("metadata.stats-mode", "full");
        SharedShreddingAppendContext context =
                createSharedShreddingAppendContext(fileFormat, rawOptions);
        AppendOnlyWriter writer = createSharedShreddingAppendWriter(writeType, context, 5L, 9L);

        DataFileMeta file =
                writeSharedShreddingFile(
                        writer,
                        sharedShreddingRow(1, "a", 10L, "b", 20L),
                        sharedShreddingLogicalRow(2, (Object) null),
                        sharedShreddingRow(3, "a", 40L, "b", 50L, "c", 60L));
        writer.close();

        assertThat(file.rowCount()).isEqualTo(3L);
        assertThat(file.schemaId()).isEqualTo(5L);
        assertThat(file.minSequenceNumber()).isEqualTo(10L);
        assertThat(file.maxSequenceNumber()).isEqualTo(12L);
        assertThat(file.level()).isEqualTo(DataFileMeta.DUMMY_LEVEL);
        assertThat(file.minKey()).isEqualTo(EMPTY_ROW);
        assertThat(file.maxKey()).isEqualTo(EMPTY_ROW);
        assertThat(file.keyStats()).isEqualTo(EMPTY_STATS);
        assertThat(file.valueStats().minValues().getInt(0)).isEqualTo(1);
        assertThat(file.valueStats().maxValues().getInt(0)).isEqualTo(3);
        assertThat(file.valueStats().nullCounts().getLong(0)).isEqualTo(0L);
        assertThat(file.valueStats().minValues().isNullAt(1)).isTrue();
        assertThat(file.valueStats().maxValues().isNullAt(1)).isTrue();
        if (CoreOptions.FILE_FORMAT_ORC.equals(fileFormat)) {
            assertThat(file.valueStats().nullCounts().getLong(1)).isEqualTo(1L);
        } else {
            assertThat(file.valueStats().nullCounts().isNullAt(1)).isTrue();
        }
        assertThat(file.fileSource()).hasValue(FileSource.APPEND);
        assertThat(file.fileFormat()).isEqualTo(fileFormat);
        assertThat(readSharedShreddingFieldMeta(context, file, "tags"))
                .isEqualTo(
                        sharedShreddingMeta(
                                nameToId("a", 0, "b", 1, "c", 2),
                                fieldToColumns(0, columns(0), 1, columns(1), 2, columns(2)),
                                overflowFields(),
                                3,
                                3));
    }

    @ParameterizedTest(name = "{0}")
    @ValueSource(strings = {CoreOptions.FILE_FORMAT_PARQUET, CoreOptions.FILE_FORMAT_ORC})
    public void testSharedShreddingMapWithBlob(String fileFormat) throws Exception {
        RowType writeType =
                DataTypes.ROW(
                        DataTypes.FIELD(0, "id", DataTypes.INT()),
                        DataTypes.FIELD(
                                1,
                                "tags",
                                DataTypes.MAP(DataTypes.STRING().notNull(), DataTypes.BIGINT())),
                        DataTypes.FIELD(2, "payload", new BlobType()));
        Options rawOptions = sharedShreddingOptions("tags", 3);
        SharedShreddingAppendContext context =
                createSharedShreddingAppendContext(fileFormat, rawOptions);
        BlobFileContext blobContext =
                BlobFileContext.create(writeType, new CoreOptions(rawOptions));
        assertThat(blobContext).isNotNull();

        AppendOnlyWriter writer =
                createSharedShreddingAppendWriter(
                        writeType, context, SCHEMA_ID, -1L, new FileIndexOptions(), blobContext);

        writer.write(
                sharedShreddingLogicalRow(1, map("a", 10L), new BlobData(new byte[] {1, 2, 3})));
        CommitIncrement increment = writer.prepareCommit(true);
        writer.close();

        assertThat(increment.newFilesIncrement().newFiles()).hasSize(2);
        DataFileMeta mainFile =
                increment.newFilesIncrement().newFiles().stream()
                        .filter(file -> fileFormat.equals(file.fileFormat()))
                        .findFirst()
                        .orElseThrow(AssertionError::new);
        assertThat(readSharedShreddingFieldMeta(context, mainFile, "tags"))
                .isEqualTo(
                        sharedShreddingMeta(
                                nameToId("a", 0),
                                fieldToColumns(0, columns(0)),
                                overflowFields(),
                                3,
                                1));
    }

    @ParameterizedTest(name = "{0}")
    @ValueSource(strings = {CoreOptions.FILE_FORMAT_PARQUET, CoreOptions.FILE_FORMAT_ORC})
    public void testSharedShreddingMapAllowsForceBufferSpill(String fileFormat) throws Exception {
        RowType writeType = sharedShreddingTagsWriteType();
        Options rawOptions = sharedShreddingOptions("tags", 3);
        SharedShreddingAppendContext context =
                createSharedShreddingAppendContext(fileFormat, rawOptions);
        AppendOnlyWriter writer = createSharedShreddingAppendWriter(writeType, context);

        Assertions.assertThatCode(writer::toBufferedWriter).doesNotThrowAnyException();
        writer.close();
    }

    @Test
    public void testNoBuffer() throws Exception {
        AppendOnlyWriter writer = createEmptyWriter(Long.MAX_VALUE);

        // we give it a small Memory Pool, force it to spill
        writer.setMemoryPool(new HeapMemorySegmentPool(16384L, 1024));

        RowBuffer buffer = writer.getWriteBuffer();
        Assertions.assertThat(buffer).isNull();

        writer.close();
    }

    @Test
    public void testMultipleFlush() throws Exception {
        AppendOnlyWriter writer = createEmptyWriter(Long.MAX_VALUE, true);

        // we give it a small Memory Pool, force it to spill
        writer.setMemoryPool(new HeapMemorySegmentPool(16384L, 1024));

        char[] s = new char[990];
        Arrays.fill(s, 'a');

        // set the record that much larger than the maxMemory
        for (int j = 0; j < 100; j++) {
            writer.write(row(j, String.valueOf(s), PART));
        }

        writer.flush(false, false);
        Assertions.assertThat(writer.memoryOccupancy()).isEqualTo(0L);
        Assertions.assertThat(writer.getWriteBuffer().size()).isEqualTo(0);
        Assertions.assertThat(writer.getNewFiles().size()).isGreaterThan(0);
        long rowCount =
                writer.getNewFiles().stream().map(DataFileMeta::rowCount).reduce(0L, Long::sum);
        Assertions.assertThat(rowCount).isEqualTo(100);

        for (int j = 0; j < 100; j++) {
            writer.write(row(j, String.valueOf(s), PART));
        }
        writer.flush(false, false);

        Assertions.assertThat(writer.memoryOccupancy()).isEqualTo(0L);
        Assertions.assertThat(writer.getWriteBuffer().size()).isEqualTo(0);
        Assertions.assertThat(writer.getNewFiles().size()).isGreaterThan(1);
        rowCount = writer.getNewFiles().stream().map(DataFileMeta::rowCount).reduce(0L, Long::sum);
        Assertions.assertThat(rowCount).isEqualTo(200);
    }

    @Test
    public void testClose() throws Exception {
        AppendOnlyWriter writer = createEmptyWriter(Long.MAX_VALUE, true);

        // we give it a small Memory Pool, force it to spill
        writer.setMemoryPool(new HeapMemorySegmentPool(16384L, 1024));

        char[] s = new char[990];
        Arrays.fill(s, 'a');

        // set the record that much larger than the maxMemory
        for (int j = 0; j < 100; j++) {
            writer.write(row(j, String.valueOf(s), PART));
        }

        ExternalBuffer externalBuffer = (ExternalBuffer) writer.getWriteBuffer();
        List<ChannelWithMeta> channel = externalBuffer.getSpillChannels();

        writer.close();

        for (ChannelWithMeta meta : channel) {
            File file = new File(meta.getChannel().getPath());
            Assertions.assertThat(file.exists()).isEqualTo(false);
        }
        Assertions.assertThat(writer.getWriteBuffer()).isEqualTo(null);
    }

    @Test
    public void testNonSpillable() throws Exception {
        AppendOnlyWriter writer = createEmptyWriter(Long.MAX_VALUE, true);

        // we give it a small Memory Pool, force it to spill
        writer.setMemoryPool(new HeapMemorySegmentPool(2048L, 1024));

        char[] s = new char[990];
        Arrays.fill(s, 'a');

        // set the record that much larger than the maxMemory
        for (int j = 0; j < 100; j++) {
            writer.write(row(j, String.valueOf(s), PART));
        }
        // we got only one file after commit
        Assertions.assertThat(writer.prepareCommit(true).newFilesIncrement().newFiles().size())
                .isEqualTo(1);
        writer.close();

        writer = createEmptyWriter(Long.MAX_VALUE, false);

        // we give it a small Memory Pool, force it to spill
        writer.setMemoryPool(new HeapMemorySegmentPool(2048L, 1024));

        // set the record that much larger than the maxMemory
        for (int j = 0; j < 100; j++) {
            writer.write(row(j, String.valueOf(s), PART));
        }
        // we got 100 files
        Assertions.assertThat(writer.prepareCommit(true).newFilesIncrement().newFiles().size())
                .isEqualTo(100);
        writer.close();
    }

    @Test
    public void testVectorUsesRowDataWriterWithNoVectorFormat() throws Exception {
        RowType vectorStoreSchema =
                RowType.builder()
                        .fields(
                                new DataType[] {
                                    new IntType(),
                                    new VarCharType(),
                                    DataTypes.VECTOR(3, DataTypes.FLOAT())
                                },
                                new String[] {"id", "name", "embed"})
                        .build();
        FileFormat format = null;
        AppendOnlyWriter writer = createVectorStoreWriter(1024 * 1024L, format, vectorStoreSchema);
        writer.write(rowWithVectors(1, "AAA", new float[] {1.0f, 2.0f, 3.0f}));
        CommitIncrement increment = writer.prepareCommit(true);
        writer.close();

        assertThat(increment.newFilesIncrement().newFiles()).hasSize(1);
        DataFileMeta meta = increment.newFilesIncrement().newFiles().get(0);
        assertThat(meta.fileName()).doesNotContain(".vector");
    }

    @Test
    public void testVectorUsesNewWriterWithSameVectorFormat() throws Exception {
        RowType vectorStoreSchema =
                RowType.builder()
                        .fields(
                                new DataType[] {
                                    new IntType(),
                                    new VarCharType(),
                                    DataTypes.VECTOR(3, DataTypes.FLOAT())
                                },
                                new String[] {"id", "name", "embed"})
                        .build();
        FileFormat format = FileFormat.fromIdentifier(AVRO, new Options());
        AppendOnlyWriter writer = createVectorStoreWriter(1024 * 1024L, format, vectorStoreSchema);
        writer.write(rowWithVectors(1, "AAA", new float[] {1.0f, 2.0f, 3.0f}));
        CommitIncrement increment = writer.prepareCommit(true);
        writer.close();

        assertThat(increment.newFilesIncrement().newFiles()).hasSize(2);
        List<DataFileMeta> metaFiles = increment.newFilesIncrement().newFiles();
        int countWithVector = 0, countWithoutVector = 0;
        for (DataFileMeta meta : metaFiles) {
            if (meta.fileName().contains(".vector")) {
                ++countWithVector;
            } else {
                ++countWithoutVector;
            }
        }
        assertThat(countWithVector).isEqualTo(1);
        assertThat(countWithoutVector).isEqualTo(1);
    }

    private SimpleColStats initStats(Integer min, Integer max, long nullCount) {
        return new SimpleColStats(min, max, nullCount);
    }

    private SimpleColStats initStats(String min, String max, long nullCount) {
        return new SimpleColStats(
                BinaryString.fromString(min), BinaryString.fromString(max), nullCount);
    }

    private InternalRow row(int id, String name, String dt) {
        return GenericRow.of(id, BinaryString.fromString(name), BinaryString.fromString(dt));
    }

    private RowType sharedShreddingTagsWriteType() {
        return DataTypes.ROW(
                DataTypes.FIELD(0, "id", DataTypes.INT()),
                DataTypes.FIELD(
                        1,
                        "tags",
                        DataTypes.MAP(DataTypes.STRING().notNull(), DataTypes.BIGINT())));
    }

    private Options sharedShreddingOptions(Object... fieldToMaxColumns) {
        Options options = new Options();
        for (int i = 0; i < fieldToMaxColumns.length; i += 2) {
            String fieldName = (String) fieldToMaxColumns[i];
            options.setString("fields." + fieldName + ".map.storage-layout", "shared-shredding");
            options.setString(
                    "fields." + fieldName + ".map.shared-shredding.max-columns",
                    String.valueOf(fieldToMaxColumns[i + 1]));
            options.setString(
                    "fields." + fieldName + ".map.shared-shredding.column-placement-policy",
                    "plain");
        }
        options.setString("metadata.stats-mode", "none");
        return options;
    }

    private InternalRow sharedShreddingRow(int id, Object... keyValues) {
        return sharedShreddingLogicalRow(id, map(keyValues));
    }

    private InternalRow sharedShreddingLogicalRow(int id, Object... fields) {
        GenericRow row = new GenericRow(fields.length + 1);
        row.setField(0, id);
        for (int i = 0; i < fields.length; i++) {
            row.setField(i + 1, fields[i]);
        }
        return row;
    }

    private InternalMap map(Object... keyValues) {
        Map<BinaryString, Object> map = new LinkedHashMap<>();
        for (int i = 0; i < keyValues.length; i += 2) {
            map.put(
                    BinaryString.fromString((String) keyValues[i]),
                    internalValue(keyValues[i + 1]));
        }
        return new GenericMap(map);
    }

    private Object internalValue(Object value) {
        if (value instanceof String) {
            return BinaryString.fromString((String) value);
        }
        return value;
    }

    private MapSharedShreddingFieldMeta sharedShreddingMeta(
            Map<String, Integer> nameToId,
            Map<Integer, List<Integer>> fieldToColumns,
            Set<Integer> overflowFields,
            int numColumns,
            int maxRowWidth) {
        return new MapSharedShreddingFieldMeta(
                new TreeMap<>(nameToId),
                new TreeMap<>(fieldToColumns),
                new TreeSet<>(overflowFields),
                numColumns,
                maxRowWidth);
    }

    private Map<String, Integer> nameToId(Object... pairs) {
        Map<String, Integer> result = new TreeMap<>();
        for (int i = 0; i < pairs.length; i += 2) {
            result.put((String) pairs[i], (Integer) pairs[i + 1]);
        }
        return result;
    }

    @SuppressWarnings("unchecked")
    private Map<Integer, List<Integer>> fieldToColumns(Object... pairs) {
        Map<Integer, List<Integer>> result = new TreeMap<>();
        for (int i = 0; i < pairs.length; i += 2) {
            result.put((Integer) pairs[i], (List<Integer>) pairs[i + 1]);
        }
        return result;
    }

    private List<Integer> columns(int... columns) {
        List<Integer> result = new ArrayList<>();
        for (int column : columns) {
            result.add(column);
        }
        return result;
    }

    private Set<Integer> overflowFields(int... fieldIds) {
        Set<Integer> result = new TreeSet<>();
        for (int fieldId : fieldIds) {
            result.add(fieldId);
        }
        return result;
    }

    private InternalRow rowWithVectors(int id, String name, float[]... vectors) {
        GenericRow row = new GenericRow(vectors.length + 2);
        row.setField(0, id);
        row.setField(1, BinaryString.fromString(name));
        for (int i = 0; i < vectors.length; ++i) {
            row.setField(i + 2, BinaryVector.fromPrimitiveArray(vectors[i]));
        }
        return row;
    }

    private DataFilePathFactory createPathFactory() {
        return createPathFactory(CoreOptions.FILE_FORMAT_AVRO);
    }

    private DataFilePathFactory createPathFactory(String fileFormat) {
        return new DataFilePathFactory(
                new Path(tempDir + "/dt=" + PART + "/bucket-0"),
                fileFormat,
                CoreOptions.DATA_FILE_PREFIX.defaultValue(),
                CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue(),
                CoreOptions.FILE_SUFFIX_INCLUDE_COMPRESSION.defaultValue(),
                CoreOptions.FILE_COMPRESSION.defaultValue(),
                null);
    }

    private SharedShreddingAppendContext createSharedShreddingAppendContext(
            String fileFormat, Options rawOptions) {
        CoreOptions options = new CoreOptions(rawOptions);
        DataFilePathFactory pathFactory = createPathFactory(fileFormat);
        LocalFileIO fileIO = LocalFileIO.create();
        return new SharedShreddingAppendContext(
                options, pathFactory, fileIO, FileFormat.fromIdentifier(fileFormat, rawOptions));
    }

    private AppendOnlyWriter createSharedShreddingAppendWriter(
            RowType writeType, SharedShreddingAppendContext context) {
        return createSharedShreddingAppendWriter(writeType, context, SCHEMA_ID, -1L);
    }

    private AppendOnlyWriter createSharedShreddingAppendWriter(
            RowType writeType,
            SharedShreddingAppendContext context,
            long schemaId,
            long maxSequenceNumber) {
        return createSharedShreddingAppendWriter(
                writeType, context, schemaId, maxSequenceNumber, new FileIndexOptions());
    }

    private AppendOnlyWriter createSharedShreddingAppendWriter(
            RowType writeType,
            SharedShreddingAppendContext context,
            long schemaId,
            long maxSequenceNumber,
            FileIndexOptions fileIndexOptions) {
        return createSharedShreddingAppendWriter(
                writeType, context, schemaId, maxSequenceNumber, fileIndexOptions, null);
    }

    private AppendOnlyWriter createSharedShreddingAppendWriter(
            RowType writeType,
            SharedShreddingAppendContext context,
            long schemaId,
            long maxSequenceNumber,
            FileIndexOptions fileIndexOptions,
            @Nullable BlobFileContext blobContext) {
        return new AppendOnlyWriter(
                context.fileIO,
                null,
                schemaId,
                context.format,
                null,
                1024 * 1024L,
                1024 * 1024L,
                1024 * 1024L,
                Long.MAX_VALUE,
                writeType,
                null,
                maxSequenceNumber,
                new NoopCompactManager(),
                null,
                false,
                context.pathFactory,
                null,
                false,
                false,
                CoreOptions.FILE_COMPRESSION.defaultValue(),
                CompressOptions.defaultOptions(),
                new StatsCollectorFactories(context.options),
                MemorySize.MAX_VALUE,
                fileIndexOptions,
                true,
                false,
                context.options.dataEvolutionEnabled(),
                null,
                blobContext);
    }

    private DataFileMeta writeSharedShreddingFile(AppendOnlyWriter writer, InternalRow... rows)
            throws Exception {
        for (InternalRow row : rows) {
            writer.write(row);
        }
        return assertSingleNewFile(writer.prepareCommit(true));
    }

    private DataFileMeta assertSingleNewFile(CommitIncrement increment) {
        assertThat(increment.newFilesIncrement().newFiles()).hasSize(1);
        assertThat(increment.compactIncrement().isEmpty()).isTrue();
        return increment.newFilesIncrement().newFiles().get(0);
    }

    private MapSharedShreddingFieldMeta readSharedShreddingFieldMeta(
            SharedShreddingAppendContext context, DataFileMeta file, String fieldName)
            throws IOException {
        assertThat(context.format).isInstanceOf(SupportsFieldMetadata.class);
        Map<String, Map<String, String>> fieldMetadata =
                ((SupportsFieldMetadata) context.format)
                        .readFieldMetadata(
                                new FormatReaderContext(
                                        context.fileIO,
                                        context.pathFactory.toPath(file),
                                        file.fileSize()));
        return MapSharedShreddingUtils.deserializeMetadata(fieldMetadata.get(fieldName));
    }

    private static class SharedShreddingAppendContext {

        private final CoreOptions options;
        private final DataFilePathFactory pathFactory;
        private final LocalFileIO fileIO;
        private final FileFormat format;

        private SharedShreddingAppendContext(
                CoreOptions options,
                DataFilePathFactory pathFactory,
                LocalFileIO fileIO,
                FileFormat format) {
            this.options = options;
            this.pathFactory = pathFactory;
            this.fileIO = fileIO;
            this.format = format;
        }
    }

    private AppendOnlyWriter createEmptyWriter(long targetFileSize) {
        return createWriter(targetFileSize, false, false, false, Collections.emptyList()).getLeft();
    }

    private AppendOnlyWriter createEmptyWriter(long targetFileSize, boolean spillable) {
        return createWriter(targetFileSize, false, true, spillable, Collections.emptyList())
                .getLeft();
    }

    private AppendOnlyWriter createEmptyWriterWithoutIoManager(
            long targetFileSize, boolean spillable) {
        return createWriter(
                        targetFileSize,
                        false,
                        true,
                        spillable,
                        false,
                        Collections.emptyList(),
                        new CountDownLatch(0))
                .getLeft();
    }

    private Pair<AppendOnlyWriter, List<DataFileMeta>> createWriter(
            long targetFileSize, boolean forceCompact, List<DataFileMeta> scannedFiles) {
        return createWriter(
                targetFileSize,
                forceCompact,
                true,
                true,
                true,
                scannedFiles,
                new CountDownLatch(0));
    }

    private Pair<AppendOnlyWriter, List<DataFileMeta>> createWriter(
            long targetFileSize,
            boolean forceCompact,
            boolean useWriteBuffer,
            boolean spillable,
            List<DataFileMeta> scannedFiles) {
        return createWriter(
                targetFileSize,
                forceCompact,
                useWriteBuffer,
                spillable,
                true,
                scannedFiles,
                new CountDownLatch(0));
    }

    private Pair<AppendOnlyWriter, List<DataFileMeta>> createWriter(
            long targetFileSize,
            boolean forceCompact,
            List<DataFileMeta> scannedFiles,
            CountDownLatch latch) {
        return createWriter(targetFileSize, forceCompact, false, false, true, scannedFiles, latch);
    }

    private Pair<AppendOnlyWriter, List<DataFileMeta>> createWriter(
            long targetFileSize,
            boolean forceCompact,
            boolean useWriteBuffer,
            boolean spillable,
            boolean hasIoManager,
            List<DataFileMeta> scannedFiles,
            CountDownLatch latch) {
        Map<String, String> options = new HashMap<>();
        options.put("metadata.stats-mode", "truncate(16)");
        return createWriterBase(
                targetFileSize,
                null,
                AppendOnlyWriterTest.SCHEMA,
                forceCompact,
                useWriteBuffer,
                spillable,
                hasIoManager,
                scannedFiles,
                compactBefore -> {
                    latch.await();
                    return compactBefore.isEmpty()
                            ? Collections.emptyList()
                            : Collections.singletonList(generateCompactAfter(compactBefore));
                },
                options);
    }

    private AppendOnlyWriter createVectorStoreWriter(
            long targetFileSize, FileFormat vectorFileFormat, RowType writeSchema) {
        Map<String, String> options = new HashMap<>();
        options.put("metadata.stats-mode", "truncate(16)");
        options.put(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true");
        return createWriterBase(
                        targetFileSize,
                        vectorFileFormat,
                        writeSchema,
                        false,
                        true,
                        false,
                        true,
                        Collections.emptyList(),
                        compactBefore -> Collections.emptyList(),
                        options)
                .getKey();
    }

    private Pair<AppendOnlyWriter, List<DataFileMeta>> createWriterBase(
            long targetFileSize,
            FileFormat vectorFileFormat,
            RowType writeSchema,
            boolean forceCompact,
            boolean useWriteBuffer,
            boolean spillable,
            boolean hasIoManager,
            List<DataFileMeta> scannedFiles,
            BucketedAppendCompactManager.CompactRewriter rewriter,
            Map<String, String> optionsMap) {
        FileFormat fileFormat = FileFormat.fromIdentifier(AVRO, new Options());
        LinkedList<DataFileMeta> toCompact = new LinkedList<>(scannedFiles);
        BucketedAppendCompactManager compactManager =
                new BucketedAppendCompactManager(
                        Executors.newSingleThreadScheduledExecutor(
                                new ExecutorThreadFactory("compaction-thread")),
                        toCompact,
                        null,
                        MIN_FILE_NUM,
                        targetFileSize,
                        targetFileSize / 10 * 7,
                        false,
                        rewriter,
                        null);
        CoreOptions options = new CoreOptions(optionsMap);
        AppendOnlyWriter writer =
                new AppendOnlyWriter(
                        LocalFileIO.create(),
                        hasIoManager ? IOManager.create(tempDir.toString()) : null,
                        SCHEMA_ID,
                        fileFormat,
                        vectorFileFormat,
                        targetFileSize,
                        targetFileSize,
                        targetFileSize,
                        Long.MAX_VALUE,
                        writeSchema,
                        null,
                        getMaxSequenceNumber(toCompact),
                        compactManager,
                        null,
                        forceCompact,
                        pathFactory,
                        null,
                        useWriteBuffer,
                        spillable,
                        CoreOptions.FILE_COMPRESSION.defaultValue(),
                        CompressOptions.defaultOptions(),
                        new StatsCollectorFactories(options),
                        MemorySize.MAX_VALUE,
                        new FileIndexOptions(),
                        true,
                        false,
                        options.dataEvolutionEnabled(),
                        null,
                        BlobFileContext.create(writeSchema, options));
        writer.setMemoryPool(
                new HeapMemorySegmentPool(options.writeBufferSize(), options.pageSize()));
        return Pair.of(writer, compactManager.allFiles());
    }

    private DataFileMeta generateCompactAfter(List<DataFileMeta> toCompact) throws IOException {
        int size = toCompact.size();
        long minSeq = toCompact.get(0).minSequenceNumber();
        long maxSeq = toCompact.get(size - 1).maxSequenceNumber();
        Path path = pathFactory.newPath("compact-");
        LocalFileIO.create().newOutputStream(path, false).close();
        return DataFileMeta.forAppend(
                path.getName(),
                toCompact.stream().mapToLong(DataFileMeta::fileSize).sum(),
                toCompact.stream().mapToLong(DataFileMeta::rowCount).sum(),
                STATS_SERIALIZER.toBinaryAllMode(
                        new SimpleColStats[] {
                            initStats(
                                    toCompact.get(0).valueStats().minValues().getInt(0),
                                    toCompact.get(size - 1).valueStats().maxValues().getInt(0),
                                    0),
                            initStats(
                                    toCompact
                                            .get(0)
                                            .valueStats()
                                            .minValues()
                                            .getString(1)
                                            .toString(),
                                    toCompact
                                            .get(size - 1)
                                            .valueStats()
                                            .maxValues()
                                            .getString(1)
                                            .toString(),
                                    0),
                            initStats(PART, PART, 0)
                        }),
                minSeq,
                maxSeq,
                toCompact.get(0).schemaId(),
                Collections.emptyList(),
                null,
                FileSource.APPEND,
                null,
                null,
                null,
                null);
    }

    private InternalRow createBlobRow(int id, String name, byte[] blobData) {
        return GenericRow.of(id, BinaryString.fromString(name), new BlobData(blobData));
    }
}
