/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.paimon.table.format;

import org.apache.paimon.catalog.Identifier;
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.data.BinaryString;
import org.apache.paimon.data.GenericRow;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.data.serializer.InternalRowSerializer;
import org.apache.paimon.format.csv.CsvOptions;
import org.apache.paimon.format.json.JsonOptions;
import org.apache.paimon.fs.FileStatus;
import org.apache.paimon.fs.Path;
import org.apache.paimon.fs.local.LocalFileIO;
import org.apache.paimon.partition.PartitionPredicate;
import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.predicate.PredicateBuilder;
import org.apache.paimon.reader.RecordReader;
import org.apache.paimon.table.FormatTable;
import org.apache.paimon.table.source.ReadBuilder;
import org.apache.paimon.table.source.Split;
import org.apache.paimon.table.source.TableScan;
import org.apache.paimon.testutils.junit.parameterized.ParameterizedTestExtension;
import org.apache.paimon.testutils.junit.parameterized.Parameters;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.Pair;

import org.junit.jupiter.api.TestTemplate;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.io.TempDir;

import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicInteger;

import static org.apache.paimon.CoreOptions.FILE_FORMAT;
import static org.apache.paimon.CoreOptions.FORMAT_TABLE_PARTITION_ONLY_VALUE_IN_PATH;
import static org.apache.paimon.CoreOptions.PARTITION_DEFAULT_NAME;
import static org.apache.paimon.CoreOptions.SOURCE_SPLIT_OPEN_FILE_COST;
import static org.apache.paimon.CoreOptions.SOURCE_SPLIT_TARGET_SIZE;
import static org.apache.paimon.utils.PartitionPathUtils.searchPartSpecAndPaths;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;

/** Test for {@link FormatTableScan}. */
@ExtendWith(ParameterizedTestExtension.class)
public class FormatTableScanTest {

    @TempDir java.nio.file.Path tmpPath;

    private final boolean enablePartitionValueOnly;
    private final Path defaultTableLocation = new Path("/test/table_scan");
    private final RowType partitionType =
            RowType.builder()
                    .field("year", DataTypes.INT())
                    .field("month", DataTypes.INT())
                    .build();
    private final BinaryRow partition =
            new InternalRowSerializer(partitionType).toBinaryRow(GenericRow.of(2023, 2));
    private final String partitionPath;
    private final List<String> partitionKeys = partitionType.getFieldNames();

    public FormatTableScanTest(boolean enablePartitionValueOnly, String partitionPath) {
        this.enablePartitionValueOnly = enablePartitionValueOnly;
        this.partitionPath = defaultTableLocation + partitionPath;
    }

    @Parameters(name = "enablePartitionValueOnly={0},partitionPath={1}")
    public static List<Object[]> parameters() {
        return Arrays.asList(
                new Object[] {false, "/year=2023/month=2"}, new Object[] {true, "/2023/2"});
    }

    @TestTemplate
    void testComputeScanPathAndLevelNoPartitionKeys() {
        List<String> partitionKeys = Collections.emptyList();
        RowType partitionType = RowType.of();
        PartitionPredicate partitionFilter = PartitionPredicate.ALWAYS_TRUE;

        Pair<Path, Integer> result =
                FormatTableScan.computeScanPathAndLevel(
                        defaultTableLocation,
                        partitionKeys,
                        FormatTableScan.extractPartitionPredicate(partitionFilter),
                        partitionType,
                        false);

        assertThat(result.getLeft()).isEqualTo(defaultTableLocation);
        assertThat(result.getRight()).isEqualTo(0);
    }

    @TestTemplate
    void testGeneratePartitions() {
        List<Pair<LinkedHashMap<String, String>, Path>> result =
                FormatTableScan.generatePartitions(
                        partitionKeys,
                        partitionType,
                        "",
                        defaultTableLocation,
                        Collections.singleton(partition),
                        enablePartitionValueOnly);
        assertThat(result.size()).isEqualTo(1);
        assertThat(result.get(0).getLeft().toString()).isEqualTo("{year=2023, month=2}");
        assertThat(result.get(0).getRight().toString()).isEqualTo(partitionPath);
    }

    @TestTemplate
    void testGetScanPathAndLevelNullFilter() {
        Pair<Path, Integer> result =
                FormatTableScan.computeScanPathAndLevel(
                        defaultTableLocation,
                        partitionType.getFieldNames(),
                        Optional.empty(),
                        partitionType,
                        false);

        assertThat(result.getLeft()).isEqualTo(defaultTableLocation);
        assertThat(result.getRight()).isEqualTo(2);
    }

    @TestTemplate
    void testComputeScanPathWithoutFilter() throws IOException {
        Path tableLocation = new Path(tmpPath.toUri());
        PredicateBuilder builder = new PredicateBuilder(partitionType);
        Predicate predicate = PredicateBuilder.and(builder.greaterThan(0, 2022));
        PartitionPredicate partitionFilter =
                PartitionPredicate.fromPredicate(partitionType, predicate);
        Pair<Path, Integer> result =
                FormatTableScan.computeScanPathAndLevel(
                        tableLocation,
                        partitionKeys,
                        FormatTableScan.extractPartitionPredicate(partitionFilter),
                        partitionType,
                        enablePartitionValueOnly);

        // Should not be optimized because of greater than
        assertThat(result.getLeft()).isEqualTo(tableLocation);
        assertThat(result.getRight()).isEqualTo(2);

        // test searchPartSpecAndPaths
        LocalFileIO fileIO = LocalFileIO.create();
        String partitionPath = enablePartitionValueOnly ? "2023/12" : "year=2023/month=12";
        fileIO.mkdirs(new Path(tableLocation, partitionPath));
        List<Pair<LinkedHashMap<String, String>, Path>> searched =
                searchPartSpecAndPaths(
                        fileIO,
                        result.getLeft(),
                        result.getRight(),
                        partitionKeys,
                        enablePartitionValueOnly);
        LinkedHashMap<String, String> expectPartitionSpec =
                new LinkedHashMap<>(partitionKeys.size());
        expectPartitionSpec.put("year", "2023");
        expectPartitionSpec.put("month", "12");
        assertThat(searched.get(0).getLeft()).isEqualTo(expectPartitionSpec);
        assertThat(searched.size()).isEqualTo(1);
    }

    @TestTemplate
    void testGetScanPathAndLevelWithEqualityFilter() throws IOException {
        Path tableLocation = new Path(tmpPath.toUri());
        // Create equality predicate for all partition keys
        PredicateBuilder builder = new PredicateBuilder(partitionType);
        Predicate equalityPredicate =
                PredicateBuilder.and(builder.equal(0, 2023), builder.equal(1, 12));
        PartitionPredicate partitionFilter =
                PartitionPredicate.fromPredicate(partitionType, equalityPredicate);

        Pair<Path, Integer> result =
                FormatTableScan.computeScanPathAndLevel(
                        tableLocation,
                        partitionKeys,
                        FormatTableScan.extractPartitionPredicate(partitionFilter),
                        partitionType,
                        enablePartitionValueOnly);
        String partitionPath = enablePartitionValueOnly ? "2023/12" : "year=2023/month=12";

        // Should optimize to specific partition path
        assertThat(result.getLeft().toString()).isEqualTo(tableLocation + partitionPath);
        assertThat(result.getRight()).isEqualTo(0);

        // test searchPartSpecAndPaths
        LocalFileIO fileIO = LocalFileIO.create();
        fileIO.mkdirs(new Path(tableLocation, partitionPath));
        List<Pair<LinkedHashMap<String, String>, Path>> searched =
                searchPartSpecAndPaths(
                        fileIO,
                        result.getLeft(),
                        result.getRight(),
                        partitionKeys,
                        enablePartitionValueOnly);
        LinkedHashMap<String, String> expectPartitionSpec =
                new LinkedHashMap<>(partitionKeys.size());
        expectPartitionSpec.put("year", "2023");
        expectPartitionSpec.put("month", "12");
        assertThat(searched.get(0).getLeft()).isEqualTo(expectPartitionSpec);
        assertThat(searched.size()).isEqualTo(1);
    }

    @TestTemplate
    void testComputeScanPathWithDateEqualityFilter() {
        Path tableLocation = new Path(tmpPath.toUri());
        RowType datePartitionType = RowType.builder().field("dt", DataTypes.DATE()).build();
        List<String> datePartitionKeys = datePartitionType.getFieldNames();

        PredicateBuilder builder = new PredicateBuilder(datePartitionType);
        Predicate equalityPredicate =
                builder.equal(0, (int) java.time.LocalDate.parse("2026-05-01").toEpochDay());
        PartitionPredicate partitionFilter =
                PartitionPredicate.fromPredicate(datePartitionType, equalityPredicate);

        Pair<Path, Integer> result =
                FormatTableScan.computeScanPathAndLevel(
                        tableLocation,
                        datePartitionKeys,
                        FormatTableScan.extractPartitionPredicate(partitionFilter),
                        datePartitionType,
                        enablePartitionValueOnly);
        String partitionPath = enablePartitionValueOnly ? "2026-05-01" : "dt=2026-05-01";

        assertThat(result.getLeft().toString()).isEqualTo(tableLocation + partitionPath);
        assertThat(result.getRight()).isEqualTo(0);
    }

    @TestTemplate
    void testComputeScanPathWithFirstLevel() throws IOException {
        Path tableLocation = new Path(tmpPath.toUri());
        // Create equality predicate for only the first partition key
        PredicateBuilder builder = new PredicateBuilder(partitionType);
        Predicate firstKeyEqualityPredicate = builder.equal(0, 2023);
        PartitionPredicate partitionFilter =
                PartitionPredicate.fromPredicate(partitionType, firstKeyEqualityPredicate);

        Pair<Path, Integer> result =
                FormatTableScan.computeScanPathAndLevel(
                        tableLocation,
                        partitionKeys,
                        FormatTableScan.extractPartitionPredicate(partitionFilter),
                        partitionType,
                        enablePartitionValueOnly);

        // Should optimize to specific partition path for first key
        String partitionPath = enablePartitionValueOnly ? "2023" : "year=2023";
        assertThat(result.getLeft().toString()).isEqualTo(tableLocation + partitionPath);
        assertThat(result.getRight()).isEqualTo(1);

        // test searchPartSpecAndPaths
        LocalFileIO fileIO = LocalFileIO.create();
        partitionPath = enablePartitionValueOnly ? "2023/12" : "year=2023/month=12";
        fileIO.mkdirs(new Path(tableLocation, partitionPath));
        List<Pair<LinkedHashMap<String, String>, Path>> searched =
                searchPartSpecAndPaths(
                        fileIO,
                        result.getLeft(),
                        result.getRight(),
                        partitionKeys,
                        enablePartitionValueOnly);
        LinkedHashMap<String, String> expectPartitionSpec =
                new LinkedHashMap<>(partitionKeys.size());
        expectPartitionSpec.put("year", "2023");
        expectPartitionSpec.put("month", "12");
        assertThat(searched.get(0).getLeft()).isEqualTo(expectPartitionSpec);
        assertThat(searched.size()).isEqualTo(1);
    }

    @TestTemplate
    void testNoOptimizationWithSecondEquality() throws IOException {
        Path tableLocation = new Path(tmpPath.toUri());
        // Create equality predicate for only the second partition key
        PredicateBuilder builder = new PredicateBuilder(partitionType);
        Predicate predicate =
                PredicateBuilder.and(builder.greaterOrEqual(0, 2023), builder.equal(1, 12));
        PartitionPredicate partitionFilter =
                PartitionPredicate.fromPredicate(partitionType, predicate);

        Pair<Path, Integer> result =
                FormatTableScan.computeScanPathAndLevel(
                        tableLocation,
                        partitionKeys,
                        FormatTableScan.extractPartitionPredicate(partitionFilter),
                        partitionType,
                        enablePartitionValueOnly);

        // Should not optimize with second equality filter
        assertThat(result.getLeft()).isEqualTo(tableLocation);
        assertThat(result.getRight()).isEqualTo(2);

        // test searchPartSpecAndPaths
        LocalFileIO fileIO = LocalFileIO.create();
        String partitionPath = enablePartitionValueOnly ? "2023/12" : "year=2023/month=12";
        fileIO.mkdirs(new Path(tableLocation, partitionPath));
        List<Pair<LinkedHashMap<String, String>, Path>> searched =
                searchPartSpecAndPaths(
                        fileIO,
                        result.getLeft(),
                        result.getRight(),
                        partitionKeys,
                        enablePartitionValueOnly);
        LinkedHashMap<String, String> expectPartitionSpec =
                new LinkedHashMap<>(partitionKeys.size());
        expectPartitionSpec.put("year", "2023");
        expectPartitionSpec.put("month", "12");
        assertThat(searched.get(0).getLeft()).isEqualTo(expectPartitionSpec);
        assertThat(searched.size()).isEqualTo(1);
    }

    @TestTemplate
    void testSkipIllegalPath() throws IOException {
        Path tableLocation = new Path(tmpPath.toUri());
        PartitionPredicate partitionFilter = PartitionPredicate.fromPredicate(partitionType, null);
        Pair<Path, Integer> result =
                FormatTableScan.computeScanPathAndLevel(
                        tableLocation,
                        partitionKeys,
                        FormatTableScan.extractPartitionPredicate(partitionFilter),
                        partitionType,
                        enablePartitionValueOnly);

        LocalFileIO fileIO = LocalFileIO.create();
        String illegalPath =
                enablePartitionValueOnly
                        ? "_unknown-year/unknown-month"
                        : "unknown-year/unknown-month";
        fileIO.mkdirs(new Path(tableLocation, illegalPath));
        String partitionPath = enablePartitionValueOnly ? "2023/12" : "year=2023/month=12";
        fileIO.mkdirs(new Path(tableLocation, partitionPath));
        List<Pair<LinkedHashMap<String, String>, Path>> searched =
                searchPartSpecAndPaths(
                        fileIO,
                        result.getLeft(),
                        result.getRight(),
                        partitionKeys,
                        enablePartitionValueOnly);
        LinkedHashMap<String, String> expectPartitionSpec =
                new LinkedHashMap<>(partitionKeys.size());
        expectPartitionSpec.put("year", "2023");
        expectPartitionSpec.put("month", "12");
        assertThat(searched.get(0).getLeft()).isEqualTo(expectPartitionSpec);
        assertThat(searched.size()).isEqualTo(1);
    }

    @TestTemplate
    void testComputeScanPathAndLevel() {
        Path tableLocation = new Path(tmpPath.toUri());
        // Create non-equality predicate
        PredicateBuilder builder = new PredicateBuilder(partitionType);
        Predicate nonEqualityPredicate = builder.greaterThan(0, 2022);
        PartitionPredicate partitionFilter =
                PartitionPredicate.fromPredicate(partitionType, nonEqualityPredicate);

        Pair<Path, Integer> result =
                FormatTableScan.computeScanPathAndLevel(
                        tableLocation,
                        partitionKeys,
                        FormatTableScan.extractPartitionPredicate(partitionFilter),
                        partitionType,
                        enablePartitionValueOnly);

        // Should not optimize, keep original path and level
        assertThat(result.getLeft()).isEqualTo(tableLocation);
        assertThat(result.getRight()).isEqualTo(2);
    }

    @TestTemplate
    void testComputeScanPathAndLevelWithOrPredicate() {
        Path tableLocation = new Path(tmpPath.toUri());

        // Create OR predicate (not equality-only)
        PredicateBuilder builder = new PredicateBuilder(partitionType);
        Predicate orPredicate = PredicateBuilder.or(builder.equal(0, 2023), builder.equal(0, 2024));
        PartitionPredicate partitionFilter =
                PartitionPredicate.fromPredicate(partitionType, orPredicate);

        Pair<Path, Integer> result =
                FormatTableScan.computeScanPathAndLevel(
                        tableLocation,
                        partitionKeys,
                        FormatTableScan.extractPartitionPredicate(partitionFilter),
                        partitionType,
                        enablePartitionValueOnly);

        // Should not optimize, keep original path and level
        assertThat(result.getLeft()).isEqualTo(tableLocation);
        assertThat(result.getRight()).isEqualTo(2);
    }

    @TestTemplate
    void testExtractEqualityPartitionSpecWithLeadingConsecutiveEquality() {
        List<String> partitionKeys = Arrays.asList("year", "month", "day");
        RowType partitionType =
                RowType.builder()
                        .field("year", DataTypes.INT())
                        .field("month", DataTypes.INT())
                        .field("day", DataTypes.INT())
                        .build();

        // Create predicate: year = 2023 AND month = 12 AND day > 15
        PredicateBuilder builder = new PredicateBuilder(partitionType);
        Predicate mixedPredicate =
                PredicateBuilder.and(
                        PredicateBuilder.and(builder.equal(0, 2023), builder.equal(1, 12)),
                        builder.greaterThan(2, 15));
        PartitionPredicate partitionFilter =
                PartitionPredicate.fromPredicate(partitionType, mixedPredicate);

        Path tableLocation = new Path("/test/table");
        Pair<Path, Integer> result =
                FormatTableScan.computeScanPathAndLevel(
                        tableLocation,
                        partitionKeys,
                        FormatTableScan.extractPartitionPredicate(partitionFilter),
                        partitionType,
                        enablePartitionValueOnly);

        // Should optimize to year and month path (leading consecutive equality)
        String partitionPath = enablePartitionValueOnly ? "/2023/12" : "/year=2023/month=12";
        assertThat(result.getLeft().toString()).isEqualTo(tableLocation + partitionPath);
        assertThat(result.getRight()).isEqualTo(1);
    }

    @TestTemplate
    void testExtractEqualityPartitionSpecWithNonConsecutiveEquality() {
        List<String> partitionKeys = Arrays.asList("year", "month", "day");
        RowType partitionType =
                RowType.builder()
                        .field("year", DataTypes.INT())
                        .field("month", DataTypes.INT())
                        .field("day", DataTypes.INT())
                        .build();

        // Create predicate: year = 2023 AND month > 6 AND day = 15
        PredicateBuilder builder = new PredicateBuilder(partitionType);
        Predicate mixedPredicate =
                PredicateBuilder.and(
                        PredicateBuilder.and(builder.equal(0, 2023), builder.greaterThan(1, 6)),
                        builder.equal(2, 15));
        PartitionPredicate partitionFilter =
                PartitionPredicate.fromPredicate(partitionType, mixedPredicate);

        Path tableLocation = new Path("/test/table");
        Pair<Path, Integer> result =
                FormatTableScan.computeScanPathAndLevel(
                        tableLocation,
                        partitionKeys,
                        FormatTableScan.extractPartitionPredicate(partitionFilter),
                        partitionType,
                        enablePartitionValueOnly);

        // Should optimize only to year path (first equality, then stop at non-equality)
        String partitionPath = enablePartitionValueOnly ? "/2023" : "/year=2023";
        assertThat(result.getLeft().toString()).isEqualTo(tableLocation + partitionPath);
        assertThat(result.getRight()).isEqualTo(2);
    }

    @TestTemplate
    void testExtractEqualityPartitionSpecWithSecondPartitionKeyEqualityOnly() {
        List<String> partitionKeys = Arrays.asList("year", "month", "day");
        RowType partitionType =
                RowType.builder()
                        .field("year", DataTypes.INT())
                        .field("month", DataTypes.INT())
                        .field("day", DataTypes.INT())
                        .build();

        // Create predicate: year > 2020 AND month = 12 AND day = 15
        PredicateBuilder builder = new PredicateBuilder(partitionType);
        Predicate mixedPredicate =
                PredicateBuilder.and(
                        PredicateBuilder.and(builder.greaterThan(0, 2020), builder.equal(1, 12)),
                        builder.equal(2, 15));
        PartitionPredicate partitionFilter =
                PartitionPredicate.fromPredicate(partitionType, mixedPredicate);

        Path tableLocation = new Path("/test/table");
        Pair<Path, Integer> result =
                FormatTableScan.computeScanPathAndLevel(
                        tableLocation,
                        partitionKeys,
                        FormatTableScan.extractPartitionPredicate(partitionFilter),
                        partitionType,
                        enablePartitionValueOnly);

        // Should not optimize because first partition key is not equality
        assertThat(result.getLeft()).isEqualTo(tableLocation);
        assertThat(result.getRight()).isEqualTo(3);
    }

    @TestTemplate
    void testExtractEqualityPartitionSpecWithAllEqualityConditions() {
        List<String> partitionKeys = Arrays.asList("year", "month", "day");
        RowType partitionType =
                RowType.builder()
                        .field("year", DataTypes.INT())
                        .field("month", DataTypes.INT())
                        .field("day", DataTypes.INT())
                        .build();

        // Create predicate: year = 2023 AND month = 12 AND day = 25
        PredicateBuilder builder = new PredicateBuilder(partitionType);
        Predicate allEqualityPredicate =
                PredicateBuilder.and(
                        PredicateBuilder.and(builder.equal(0, 2023), builder.equal(1, 12)),
                        builder.equal(2, 25));
        PartitionPredicate partitionFilter =
                PartitionPredicate.fromPredicate(partitionType, allEqualityPredicate);

        Path tableLocation = new Path("/test/table");
        Pair<Path, Integer> result =
                FormatTableScan.computeScanPathAndLevel(
                        tableLocation,
                        partitionKeys,
                        FormatTableScan.extractPartitionPredicate(partitionFilter),
                        partitionType,
                        enablePartitionValueOnly);

        // Should optimize to full partition path and no further filtering needed
        String partitionPath =
                enablePartitionValueOnly ? "/2023/12/25" : "/year=2023/month=12/day=25";
        assertThat(result.getLeft().toString()).isEqualTo(tableLocation + partitionPath);
        assertThat(result.getRight()).isEqualTo(0);
    }

    @TestTemplate
    public void testExtractEqualityPartitionSpecWithAllEqualityWhenAllIsAnd() {
        RowType type =
                RowType.builder()
                        .field("year", DataTypes.INT())
                        .field("month", DataTypes.INT())
                        .field("day", DataTypes.INT())
                        .build();
        List<String> partitionKeys = Arrays.asList("year", "month", "day");

        // Create predicate: year = 2023 AND month = 12 AND day = 25
        PredicateBuilder builder = new PredicateBuilder(type);
        Predicate equalityPredicate =
                PredicateBuilder.and(
                        PredicateBuilder.and(builder.equal(0, 2023), builder.equal(1, 12)),
                        builder.equal(2, 25));

        Map<String, String> result =
                FormatTableScan.extractLeadingEqualityPartitionSpecWhenOnlyAnd(
                        partitionKeys, equalityPredicate, type);

        assertThat(result).hasSize(3);
        assertThat(result.get("year")).isEqualTo("2023");
        assertThat(result.get("month")).isEqualTo("12");
        assertThat(result.get("day")).isEqualTo("25");
    }

    @TestTemplate
    public void testExtractEqualityPartitionSpecWithLeadingConsecutiveEqualityWhenAllIsAnd() {
        RowType type =
                RowType.builder()
                        .field("year", DataTypes.INT())
                        .field("month", DataTypes.INT())
                        .field("day", DataTypes.INT())
                        .build();
        List<String> partitionKeys = Arrays.asList("year", "month", "day");

        // Create predicate: year = 2023 AND month = 12 AND day > 15
        PredicateBuilder builder = new PredicateBuilder(type);
        Predicate mixedPredicate =
                PredicateBuilder.and(
                        PredicateBuilder.and(builder.equal(0, 2023), builder.equal(1, 12)),
                        builder.greaterThan(2, 15));

        Map<String, String> result =
                FormatTableScan.extractLeadingEqualityPartitionSpecWhenOnlyAnd(
                        partitionKeys, mixedPredicate, type);

        assertThat(result).isNotNull();
        assertThat(result).hasSize(2);
        assertThat(result.get("year")).isEqualTo("2023");
        assertThat(result.get("month")).isEqualTo("12");
        assertThat(result.containsKey("day")).isFalse();
    }

    @TestTemplate
    public void testExtractEqualityPartitionSpecWithFirstPartitionKeyEqualityWhenAllIsAnd() {
        RowType type =
                RowType.builder()
                        .field("year", DataTypes.INT())
                        .field("month", DataTypes.INT())
                        .field("day", DataTypes.INT())
                        .build();
        List<String> partitionKeys = Arrays.asList("year", "month", "day");

        // Create predicate: year = 2023 AND month > 6 AND day = 15
        PredicateBuilder builder = new PredicateBuilder(type);
        Predicate mixedPredicate =
                PredicateBuilder.and(
                        PredicateBuilder.and(builder.equal(0, 2023), builder.greaterThan(1, 6)),
                        builder.equal(2, 15));

        Map<String, String> result =
                FormatTableScan.extractLeadingEqualityPartitionSpecWhenOnlyAnd(
                        partitionKeys, mixedPredicate, type);
        assertThat(result).hasSize(1);
        assertThat(result.get("year")).isEqualTo("2023");
        assertThat(result.containsKey("month")).isFalse();
        assertThat(result.containsKey("day")).isFalse();
    }

    @TestTemplate
    public void testExtractEqualityPartitionSpecWithNoLeadingEqualityWhenAllIsAnd() {
        RowType type =
                RowType.builder()
                        .field("year", DataTypes.INT())
                        .field("month", DataTypes.INT())
                        .field("day", DataTypes.INT())
                        .build();
        List<String> partitionKeys = Arrays.asList("year", "month", "day");

        // Create predicate: year > 2020 AND month = 12 AND day = 15
        PredicateBuilder builder = new PredicateBuilder(type);
        Predicate mixedPredicate =
                PredicateBuilder.and(
                        PredicateBuilder.and(builder.greaterThan(0, 2020), builder.equal(1, 12)),
                        builder.equal(2, 15));

        Map<String, String> result =
                FormatTableScan.extractLeadingEqualityPartitionSpecWhenOnlyAnd(
                        partitionKeys, mixedPredicate, type);

        assertThat(result).isEmpty();
    }

    @TestTemplate
    public void testExtractEqualityPartitionSpecWithNonEqualityPredicateWhenAllIsAnd() {
        RowType type =
                RowType.builder()
                        .field("year", DataTypes.INT())
                        .field("month", DataTypes.INT())
                        .build();
        List<String> partitionKeys = Arrays.asList("year", "month");

        // Create predicate: year > 2020 AND month > 6
        PredicateBuilder builder = new PredicateBuilder(type);
        Predicate nonEqualityPredicate =
                PredicateBuilder.and(builder.greaterThan(0, 2020), builder.greaterThan(1, 6));

        Map<String, String> result =
                FormatTableScan.extractLeadingEqualityPartitionSpecWhenOnlyAnd(
                        partitionKeys, nonEqualityPredicate, type);

        assertThat(result).isEmpty();
    }

    @TestTemplate
    public void testExtractLeadingEqualityPartitionSpecWhenOnlyAndWithOrPredicate() {
        RowType type =
                RowType.builder()
                        .field("year", DataTypes.INT())
                        .field("month", DataTypes.INT())
                        .build();
        List<String> partitionKeys = Arrays.asList("year", "month");

        // Create predicate: year = 2023 OR year = 2024
        PredicateBuilder builder = new PredicateBuilder(type);
        Predicate orPredicate = PredicateBuilder.or(builder.equal(0, 2023), builder.equal(0, 2024));

        Map<String, String> result =
                FormatTableScan.extractLeadingEqualityPartitionSpecWhenOnlyAnd(
                        partitionKeys, orPredicate, type);

        assertThat(result).isEmpty();
    }

    @TestTemplate
    public void testCreateSplitsWithMultipleFiles() throws IOException {
        for (String format : Arrays.asList("csv", "json")) {
            Path tableLocation = new Path(new Path(tmpPath.toUri()), format);
            LocalFileIO fileIO = LocalFileIO.create();
            Path file1 = new Path(tableLocation, "data1." + format);
            Path file2 = new Path(tableLocation, "data2." + format);
            Path file3 = new Path(tableLocation, "data3." + format);
            writeTestFile(fileIO, file1, 80);
            writeTestFile(fileIO, file2, 120);
            writeTestFile(fileIO, file3, 200);

            Map<String, String> options = new HashMap<>();
            options.put(SOURCE_SPLIT_TARGET_SIZE.key(), "100b");

            FormatTable formatTable =
                    createFormatTableWithOptions(
                            tableLocation,
                            FormatTable.Format.valueOf(format.toUpperCase()),
                            options);
            FormatTableScan scan = new FormatTableScan(formatTable, null, null);
            List<Split> splits = scan.plan().splits();
            assertThat(splits).hasSize(5);
        }
    }

    @TestTemplate
    void testPositiveLimitDoesNotPruneUnknownRowCountSplits() throws IOException {
        Path tableLocation = new Path(tmpPath.toUri());
        LocalFileIO fileIO = LocalFileIO.create();
        Path headerOnlyFile = new Path(tableLocation, "00-header-only.csv");
        Path dataFile = new Path(tableLocation, "01-data.csv");
        fileIO.mkdirs(tableLocation);
        fileIO.writeFile(headerOnlyFile, "id,name\n", false);
        fileIO.writeFile(dataFile, "id,name\n42,later\n", false);

        Map<String, String> options = new HashMap<>();
        options.put(FILE_FORMAT.key(), "csv");
        options.put(CsvOptions.INCLUDE_HEADER.key(), "true");
        options.put(SOURCE_SPLIT_TARGET_SIZE.key(), "32b");
        options.put(SOURCE_SPLIT_OPEN_FILE_COST.key(), "32b");
        FormatTable formatTable =
                createFormatTableWithOptions(tableLocation, FormatTable.Format.CSV, options);
        assertThat(formatTable.newReadBuilder().newScan().plan().splits()).hasSize(2);

        ReadBuilder readBuilder = formatTable.newReadBuilder().withLimit(1);
        TableScan.Plan plan = readBuilder.newScan().plan();
        List<String> rows = new ArrayList<>();
        try (RecordReader<InternalRow> reader = readBuilder.newRead().createReader(plan)) {
            reader.forEachRemaining(
                    row -> rows.add(row.getInt(0) + "," + row.getString(1).toString()));
        }

        assertThat(rows).containsExactly("42,later");
    }

    @TestTemplate
    public void testCreateSplitsWhenDefineLineDelimiter() throws IOException {
        for (String format : Arrays.asList("csv", "json")) {
            Path tableLocation = new Path(new Path(tmpPath.toUri()), format);
            LocalFileIO fileIO = LocalFileIO.create();
            Path file1 = new Path(tableLocation, "data1." + format);
            Path file2 = new Path(tableLocation, "data2." + format);
            Path file3 = new Path(tableLocation, "data3." + format);
            writeTestFile(fileIO, file1, 80);
            writeTestFile(fileIO, file2, 120);
            writeTestFile(fileIO, file3, 200);

            Map<String, String> options = new HashMap<>();
            options.put(SOURCE_SPLIT_TARGET_SIZE.key(), "100b");
            if ("csv".equals(format)) {
                options.put(CsvOptions.LINE_DELIMITER.key(), "\001");
            } else {
                options.put(JsonOptions.LINE_DELIMITER.key(), "\001");
            }

            FormatTable formatTable =
                    createFormatTableWithOptions(
                            tableLocation,
                            FormatTable.Format.valueOf(format.toUpperCase()),
                            options);
            FormatTableScan scan = new FormatTableScan(formatTable, null, null);
            List<Split> splits = scan.plan().splits();
            assertThat(splits).hasSize(3);
        }
    }

    @TestTemplate
    public void testCreateSplitsWithParquetFile() throws IOException {
        Path tableLocation = new Path(tmpPath.toUri());
        LocalFileIO fileIO = LocalFileIO.create();

        // Create a large Parquet file (non-splittable)
        Path parquetFile = new Path(tableLocation, "data.parquet");
        long fileSize = 300; // 300 bytes
        writeTestFile(fileIO, parquetFile, fileSize);

        // Set split max size to 100 bytes
        Map<String, String> options = new HashMap<>();
        options.put(SOURCE_SPLIT_TARGET_SIZE.key(), "100b");

        FormatTable formatTable =
                createFormatTableWithOptions(tableLocation, FormatTable.Format.PARQUET, options);
        FormatTableScan scan = new FormatTableScan(formatTable, null, null);
        List<Split> splits = scan.plan().splits();

        // Parquet files should NOT be split, should be a single split
        assertThat(splits).hasSize(1);
        FormatDataSplit split = (FormatDataSplit) splits.get(0);
        assertThat(split.files()).hasSize(1);
        assertThat(split.files().get(0).filePath()).isEqualTo(parquetFile);
        assertThat(split.files().get(0).offset()).isEqualTo(0);
    }

    @TestTemplate
    public void testCreateSplitsSkipsCommitterStagingFiles() throws IOException {
        Path tableLocation = new Path(tmpPath.toUri());
        LocalFileIO fileIO = LocalFileIO.create();
        Path dataFile = new Path(tableLocation, "data.csv");
        writeTestFile(fileIO, dataFile, 100);
        // Committer staging trees hold files whose names look exactly like data files; only the
        // directories above them say they are uncommitted. A recursive listing sees them all.
        for (String staging :
                Arrays.asList(
                        "_temporary/0/_temporary/attempt_202607271200_0001_m_000010_15",
                        "__magic/job-6e7f/tasks/attempt_202607271200_0001_m_000010_15/__base",
                        "__magic_job-6e7f/tasks/attempt_202607271200_0001_m_000010_15/__base",
                        ".hive-staging_hive_2026-07-27_12-00-00_000_1/-ext-10000")) {
            writeTestFile(
                    fileIO, new Path(new Path(tableLocation, staging), "part-00010.csv"), 100);
        }

        FormatTable formatTable =
                createFormatTableWithOptions(
                        tableLocation, FormatTable.Format.CSV, Collections.emptyMap());
        List<Split> splits = new FormatTableScan(formatTable, null, null).plan().splits();

        assertThat(splits).hasSize(1);
        assertThat(((FormatDataSplit) splits.get(0)).files())
                .extracting(FormatDataSplit.FileMeta::filePath)
                .containsExactly(dataFile);
    }

    @TestTemplate
    public void testStagingTreesAreNotEnumerated() throws IOException {
        Path tableLocation = new Path(tmpPath.toUri());
        LocalFileIO setupFileIO = LocalFileIO.create();
        writeTestFile(setupFileIO, new Path(tableLocation, "data.csv"), 100);
        // A staging tree with many task attempts, which is what a large job leaves behind. A
        // recursive listing walks all of it and then throws it away.
        for (int attempt = 0; attempt < 20; attempt++) {
            writeTestFile(
                    setupFileIO,
                    new Path(
                            tableLocation,
                            String.format(
                                    "__magic_job-6e7f/tasks/attempt_202607271200_0001_m_%06d_15"
                                            + "/__base/part-%05d.csv",
                                    attempt, attempt)),
                    100);
        }

        AtomicInteger listCount = new AtomicInteger(0);
        AtomicInteger enumerated = new AtomicInteger(0);
        LocalFileIO fileIO =
                new LocalFileIO() {
                    @Override
                    public FileStatus[] listStatus(Path path) throws IOException {
                        listCount.getAndIncrement();
                        FileStatus[] statuses = super.listStatus(path);
                        enumerated.addAndGet(statuses.length);
                        return statuses;
                    }
                };
        FormatTable formatTable =
                FormatTable.builder()
                        .fileIO(fileIO)
                        .identifier(Identifier.create("test_db", "test_table"))
                        .rowType(
                                RowType.builder()
                                        .field("id", DataTypes.INT())
                                        .field("name", DataTypes.STRING())
                                        .build())
                        .partitionKeys(Collections.emptyList())
                        .location(tableLocation.toString())
                        .format(FormatTable.Format.CSV)
                        .options(Collections.emptyMap())
                        .build();

        // What a recursive listing costs on the same tree, for comparison.
        FileStatus[] recursive = fileIO.listFiles(tableLocation, true);
        int recursiveListings = listCount.getAndSet(0);
        int recursiveEntries = enumerated.getAndSet(0);

        List<Split> splits = new FormatTableScan(formatTable, null, null).plan().splits();

        assertThat(splits).hasSize(1);
        // The recursive listing walks the whole staging tree and returns all of it, leaving the
        // caller to throw 20 of the 21 files away.
        assertEquals(21, recursive.length);
        // Planning skips the staging tree at its root: one listing of the table directory, and the
        // two entries in it. The recursive baseline is only compared against, not pinned: its
        // exact cost belongs to FileIO, which this test does not exercise.
        assertEquals(1, listCount.get());
        assertEquals(2, enumerated.get());
        assertThat(recursiveListings).isGreaterThan(listCount.get());
        assertThat(recursiveEntries).isGreaterThan(enumerated.get());
    }

    @TestTemplate
    public void testCreateSplitsKeepsFilesUnderAStagingLikeTableLocation() throws IOException {
        // The '_' rule applies below the listed directory only: a warehouse path with a leading
        // underscore must not make the whole table read as empty.
        Path tableLocation = new Path(new Path(tmpPath.toUri()), "_warehouse/db/t");
        LocalFileIO fileIO = LocalFileIO.create();
        Path dataFile = new Path(tableLocation, "data.csv");
        writeTestFile(fileIO, dataFile, 100);

        FormatTable formatTable =
                createFormatTableWithOptions(
                        tableLocation, FormatTable.Format.CSV, Collections.emptyMap());
        List<Split> splits = new FormatTableScan(formatTable, null, null).plan().splits();

        assertThat(splits).hasSize(1);
        assertThat(((FormatDataSplit) splits.get(0)).files())
                .extracting(FormatDataSplit.FileMeta::filePath)
                .containsExactly(dataFile);
    }

    @TestTemplate
    public void testCreateSplitsSkipsStagingFilesInsidePartitions() throws IOException {
        Path tableLocation = new Path(tmpPath.toUri());
        LocalFileIO fileIO = LocalFileIO.create();
        String partition = enablePartitionValueOnly ? "2024/1" : "year=2024/month=1";
        Path partitionPath = new Path(tableLocation, partition);
        Path dataFile = new Path(partitionPath, "data.csv");
        writeTestFile(fileIO, dataFile, 100);
        writeTestFile(
                fileIO,
                new Path(
                        partitionPath,
                        "__magic_job-6e7f/tasks/attempt_202607271200_0001_m_000010_15"
                                + "/__base/part-00010.csv"),
                100);

        FormatTable formatTable = createYearMonthFormatTable(LocalFileIO.create(), tableLocation);
        List<Split> splits = new FormatTableScan(formatTable, null, null).plan().splits();

        assertThat(splits).hasSize(1);
        assertThat(((FormatDataSplit) splits.get(0)).files())
                .extracting(FormatDataSplit.FileMeta::filePath)
                .containsExactly(dataFile);
    }

    @TestTemplate
    public void testCreateSplitsReadsTheDefaultPartitionDirectory() throws IOException {
        // In the value-only layout the directory of a null partition value is the default
        // partition name, which starts with '_'. It is the listed root here, and the root is
        // never judged, so its files are read while what is staged inside it is not.
        Path tableLocation = new Path(tmpPath.toUri());
        LocalFileIO fileIO = LocalFileIO.create();
        String defaultPartName = PARTITION_DEFAULT_NAME.defaultValue();
        String partition =
                enablePartitionValueOnly
                        ? "2024/" + defaultPartName
                        : "year=2024/month=" + defaultPartName;
        Path partitionPath = new Path(tableLocation, partition);
        Path dataFile = new Path(partitionPath, "data.csv");
        writeTestFile(fileIO, dataFile, 100);
        writeTestFile(fileIO, new Path(partitionPath, "_temporary/attempt/part-00010.csv"), 100);

        FormatTable formatTable = createYearMonthFormatTable(LocalFileIO.create(), tableLocation);
        List<Split> splits = new FormatTableScan(formatTable, null, null).plan().splits();

        assertThat(splits).hasSize(1);
        assertThat(((FormatDataSplit) splits.get(0)).files())
                .extracting(FormatDataSplit.FileMeta::filePath)
                .containsExactly(dataFile);
    }

    @TestTemplate
    public void testCreateSplitsWithEmptyDirectory() throws IOException {
        Path tableLocation = new Path(tmpPath.toUri());
        LocalFileIO fileIO = LocalFileIO.create();
        fileIO.mkdirs(tableLocation);

        FormatTable formatTable =
                createFormatTableWithOptions(
                        tableLocation, FormatTable.Format.CSV, Collections.emptyMap());
        FormatTableScan scan = new FormatTableScan(formatTable, null, null);
        List<Split> splits = scan.plan().splits();

        assertThat(splits).isEmpty();
    }

    private void writeTestFile(LocalFileIO fileIO, Path filePath, long size) throws IOException {
        fileIO.mkdirs(filePath.getParent());
        try (OutputStream out = fileIO.newOutputStream(filePath, false)) {
            byte[] data = new byte[(int) size];
            Arrays.fill(data, (byte) 'a');
            out.write(data);
        }
    }

    private FormatTable createFormatTableWithOptions(
            Path tableLocation, FormatTable.Format format, Map<String, String> options) {
        RowType rowType =
                RowType.builder()
                        .field("id", DataTypes.INT())
                        .field("name", DataTypes.STRING())
                        .build();

        return FormatTable.builder()
                .fileIO(LocalFileIO.create())
                .identifier(Identifier.create("test_db", "test_table"))
                .rowType(rowType)
                .partitionKeys(Collections.emptyList())
                .location(tableLocation.toString())
                .format(format)
                .options(options)
                .build();
    }

    @TestTemplate
    void testSearchPartSpecAndPathsWithRangeFilter() throws IOException {
        Path tableLocation = new Path(tmpPath.toUri());
        LocalFileIO fileIO = LocalFileIO.create();

        // Create partition directories for years 2022 to 2026
        for (int year = 2022; year <= 2026; year++) {
            String partPath = enablePartitionValueOnly ? String.valueOf(year) : "year=" + year;
            for (int month = 1; month <= 12; month++) {
                String monthPart =
                        enablePartitionValueOnly
                                ? partPath + "/" + month
                                : partPath + "/month=" + month;
                fileIO.mkdirs(new Path(tableLocation, monthPart));
            }
        }

        AtomicInteger atomicInteger = new AtomicInteger(0);
        LocalFileIO localFileIO =
                new LocalFileIO() {
                    @Override
                    public FileStatus[] listStatus(Path path) throws IOException {
                        atomicInteger.getAndIncrement();
                        return super.listStatus(path);
                    }
                };

        RowType rowType =
                RowType.builder()
                        .field("year", DataTypes.INT())
                        .field("month", DataTypes.INT())
                        .field("a", DataTypes.INT())
                        .build();

        FormatTable formatTable =
                FormatTable.builder()
                        .fileIO(localFileIO)
                        .identifier(Identifier.create("test_db", "test_table"))
                        .rowType(rowType)
                        .partitionKeys(Arrays.asList("year", "month"))
                        .location(tableLocation.toString())
                        .format(FormatTable.Format.CSV)
                        .options(
                                Collections.singletonMap(
                                        FORMAT_TABLE_PARTITION_ONLY_VALUE_IN_PATH.key(),
                                        String.valueOf(enablePartitionValueOnly)))
                        .build();

        // Create predicate: year > 2022 AND year < 2026
        PredicateBuilder builder = new PredicateBuilder(formatTable.partitionType());
        Predicate predicate =
                PredicateBuilder.and(builder.greaterThan(0, 2022), builder.lessThan(0, 2026));

        FormatTableScan scan =
                ((FormatTableScan) formatTable.newReadBuilder().withFilter(predicate).newScan());
        List<Pair<LinkedHashMap<String, String>, Path>> result = scan.findPartitions();

        // Should find years 2023, 2024, 2025
        assertEquals(36, result.size());
        assertEquals(4, atomicInteger.get());
        List<String> years =
                result.stream()
                        .map(pair -> pair.getKey().get("year"))
                        .sorted()
                        .distinct()
                        .collect(java.util.stream.Collectors.toList());
        assertEquals(Arrays.asList("2023", "2024", "2025"), years);

        atomicInteger.set(0);
        predicate = PredicateBuilder.and(builder.greaterThan(1, 3), builder.lessThan(1, 10));
        predicate = PredicateBuilder.and(predicate, builder.equal(0, 2024));

        scan = ((FormatTableScan) formatTable.newReadBuilder().withFilter(predicate).newScan());
        result = scan.findPartitions();
        assertEquals(6, result.size());
        // equals predicate reduce 1 io
        assertEquals(1, atomicInteger.get());
        List<String> month =
                result.stream()
                        .map(pair -> pair.getKey().get("month"))
                        .sorted()
                        .distinct()
                        .collect(java.util.stream.Collectors.toList());
        assertEquals(Arrays.asList("4", "5", "6", "7", "8", "9"), month);

        atomicInteger.set(0);
        predicate = PredicateBuilder.and(builder.greaterThan(1, 3), builder.lessThan(1, 10));

        scan = ((FormatTableScan) formatTable.newReadBuilder().withFilter(predicate).newScan());
        result = scan.findPartitions();
        assertEquals(30, result.size());
        // equals predicate reduce 1 io
        assertEquals(6, atomicInteger.get());
        month =
                result.stream()
                        .map(pair -> pair.getKey().get("month"))
                        .sorted()
                        .distinct()
                        .collect(java.util.stream.Collectors.toList());
        assertEquals(Arrays.asList("4", "5", "6", "7", "8", "9"), month);
    }

    @TestTemplate
    void testFindPartitionsWithAndPartitionPredicate() throws IOException {
        Path tableLocation = new Path(tmpPath.toUri());
        LocalFileIO setupFileIO = LocalFileIO.create();

        // Create partition directories for years 2022 to 2026, 12 months each (60 partitions)
        for (int year = 2022; year <= 2026; year++) {
            String partPath = enablePartitionValueOnly ? String.valueOf(year) : "year=" + year;
            for (int month = 1; month <= 12; month++) {
                String monthPart =
                        enablePartitionValueOnly
                                ? partPath + "/" + month
                                : partPath + "/month=" + month;
                setupFileIO.mkdirs(new Path(tableLocation, monthPart));
            }
        }

        AtomicInteger listCount = new AtomicInteger(0);
        LocalFileIO localFileIO =
                new LocalFileIO() {
                    @Override
                    public FileStatus[] listStatus(Path path) throws IOException {
                        listCount.getAndIncrement();
                        return super.listStatus(path);
                    }
                };

        RowType rowType =
                RowType.builder()
                        .field("year", DataTypes.INT())
                        .field("month", DataTypes.INT())
                        .field("a", DataTypes.INT())
                        .build();

        FormatTable formatTable =
                FormatTable.builder()
                        .fileIO(localFileIO)
                        .identifier(Identifier.create("test_db", "test_table"))
                        .rowType(rowType)
                        .partitionKeys(Arrays.asList("year", "month"))
                        .location(tableLocation.toString())
                        .format(FormatTable.Format.CSV)
                        .options(
                                Collections.singletonMap(
                                        FORMAT_TABLE_PARTITION_ONLY_VALUE_IN_PATH.key(),
                                        String.valueOf(enablePartitionValueOnly)))
                        .build();

        // Simulate a static pushdown (year = 2024) combined with a runtime/DPP partition filter
        // (month > 3 AND month < 10): each is a DefaultPartitionPredicate and the scan combines
        // them via PartitionPredicate.and, exactly like BaseScan / runtime filtering does. The
        // combined predicate must still prune partition directories; otherwise it falls back to a
        // full table-root listing.
        PredicateBuilder builder = new PredicateBuilder(formatTable.partitionType());
        PartitionPredicate yearFilter =
                PartitionPredicate.fromPredicate(
                        formatTable.partitionType(), builder.equal(0, 2024));
        PartitionPredicate monthFilter =
                PartitionPredicate.fromPredicate(
                        formatTable.partitionType(),
                        PredicateBuilder.and(builder.greaterThan(1, 3), builder.lessThan(1, 10)));
        PartitionPredicate combined =
                PartitionPredicate.and(Arrays.asList(yearFilter, monthFilter));

        FormatTableScan scan = new FormatTableScan(formatTable, combined, null);
        List<Pair<LinkedHashMap<String, String>, Path>> result = scan.findPartitions();

        // Should prune to year=2024 and list its months only once (months 4-9).
        assertEquals(6, result.size());
        assertEquals(1, listCount.get());
        List<String> months =
                result.stream()
                        .map(pair -> pair.getKey().get("month"))
                        .sorted()
                        .distinct()
                        .collect(java.util.stream.Collectors.toList());
        assertEquals(Arrays.asList("4", "5", "6", "7", "8", "9"), months);
    }

    @TestTemplate
    void testFindPartitionsWithOrCrossFieldPredicate() throws IOException {
        Path tableLocation = new Path(tmpPath.toUri());
        createYearMonthPartitionDirs(LocalFileIO.create(), tableLocation);

        AtomicInteger listCount = new AtomicInteger(0);
        FormatTable formatTable =
                createYearMonthFormatTable(countingFileIO(listCount), tableLocation);

        // (year = 2024 AND month < 6) OR (year = 2025 AND month >= 6): a cross-field OR that the
        // old per-field split model could not express, so it used to fall back to listing every
        // year directory.
        PredicateBuilder builder = new PredicateBuilder(formatTable.partitionType());
        Predicate orPredicate =
                PredicateBuilder.or(
                        PredicateBuilder.and(builder.equal(0, 2024), builder.lessThan(1, 6)),
                        PredicateBuilder.and(builder.equal(0, 2025), builder.greaterOrEqual(1, 6)));
        PartitionPredicate partitionFilter =
                PartitionPredicate.fromPredicate(formatTable.partitionType(), orPredicate);

        FormatTableScan scan = new FormatTableScan(formatTable, partitionFilter, null);
        List<Pair<LinkedHashMap<String, String>, Path>> result = scan.findPartitions();

        // Only year=2024 and year=2025 are descended: root + 2 year dirs = 3 list calls.
        assertEquals(3, listCount.get());
        // 2024 months 1-5 (5) + 2025 months 6-12 (7) = 12 partitions.
        assertEquals(12, result.size());
        for (Pair<LinkedHashMap<String, String>, Path> pair : result) {
            int year = Integer.parseInt(pair.getKey().get("year"));
            int month = Integer.parseInt(pair.getKey().get("month"));
            assertTrue(
                    (year == 2024 && month < 6) || (year == 2025 && month >= 6),
                    "unexpected partition " + year + "-" + month);
        }
    }

    @TestTemplate
    void testFindPartitionsWithNestedAndOr() throws IOException {
        Path tableLocation = new Path(tmpPath.toUri());
        createYearMonthPartitionDirs(LocalFileIO.create(), tableLocation);

        AtomicInteger listCount = new AtomicInteger(0);
        FormatTable formatTable =
                createYearMonthFormatTable(countingFileIO(listCount), tableLocation);

        // year > 2022 AND (month = 1 OR month = 12)
        PredicateBuilder builder = new PredicateBuilder(formatTable.partitionType());
        Predicate predicate =
                PredicateBuilder.and(
                        builder.greaterThan(0, 2022),
                        PredicateBuilder.or(builder.equal(1, 1), builder.equal(1, 12)));
        PartitionPredicate partitionFilter =
                PartitionPredicate.fromPredicate(formatTable.partitionType(), predicate);

        FormatTableScan scan = new FormatTableScan(formatTable, partitionFilter, null);
        List<Pair<LinkedHashMap<String, String>, Path>> result = scan.findPartitions();

        // year=2022 pruned; 2023-2026 descended: root + 4 year dirs = 5 list calls.
        assertEquals(5, listCount.get());
        // 4 years x months {1, 12} = 8 partitions.
        assertEquals(8, result.size());
        for (Pair<LinkedHashMap<String, String>, Path> pair : result) {
            int year = Integer.parseInt(pair.getKey().get("year"));
            int month = Integer.parseInt(pair.getKey().get("month"));
            assertTrue(
                    year > 2022 && (month == 1 || month == 12),
                    "unexpected partition " + year + "-" + month);
        }
    }

    @TestTemplate
    void testFindPartitionsWithOrSingleField() throws IOException {
        Path tableLocation = new Path(tmpPath.toUri());
        createYearMonthPartitionDirs(LocalFileIO.create(), tableLocation);

        AtomicInteger listCount = new AtomicInteger(0);
        FormatTable formatTable =
                createYearMonthFormatTable(countingFileIO(listCount), tableLocation);

        // year = 2023 OR year = 2025 (single-field OR now prunes too)
        PredicateBuilder builder = new PredicateBuilder(formatTable.partitionType());
        Predicate predicate = PredicateBuilder.or(builder.equal(0, 2023), builder.equal(0, 2025));
        PartitionPredicate partitionFilter =
                PartitionPredicate.fromPredicate(formatTable.partitionType(), predicate);

        FormatTableScan scan = new FormatTableScan(formatTable, partitionFilter, null);
        List<Pair<LinkedHashMap<String, String>, Path>> result = scan.findPartitions();

        // Only year=2023 and year=2025 descended: root + 2 = 3 list calls.
        assertEquals(3, listCount.get());
        // 2 years x 12 months = 24 partitions.
        assertEquals(24, result.size());
        List<String> years =
                result.stream()
                        .map(pair -> pair.getKey().get("year"))
                        .distinct()
                        .sorted()
                        .collect(java.util.stream.Collectors.toList());
        assertEquals(Arrays.asList("2023", "2025"), years);
    }

    @TestTemplate
    void testFindPartitionsWithPrefixAndOrPredicate() throws IOException {
        Path tableLocation = new Path(tmpPath.toUri());
        createYearMonthPartitionDirs(LocalFileIO.create(), tableLocation);

        AtomicInteger listCount = new AtomicInteger(0);
        FormatTable formatTable =
                createYearMonthFormatTable(countingFileIO(listCount), tableLocation);

        // year = 2024 is pushed into the scan-path prefix. The remaining OR on month must not read
        // the unbound prefix slot and accidentally prune the whole subtree.
        PredicateBuilder builder = new PredicateBuilder(formatTable.partitionType());
        Predicate predicate =
                PredicateBuilder.and(
                        builder.equal(0, 2024),
                        PredicateBuilder.or(builder.equal(1, 1), builder.equal(1, 12)));
        PartitionPredicate partitionFilter =
                PartitionPredicate.fromPredicate(formatTable.partitionType(), predicate);

        FormatTableScan scan = new FormatTableScan(formatTable, partitionFilter, null);
        List<Pair<LinkedHashMap<String, String>, Path>> result = scan.findPartitions();

        // The equality prefix narrows the scan to year=2024, so only its month directory is listed.
        assertEquals(1, listCount.get());
        assertEquals(2, result.size());
        List<String> months =
                result.stream()
                        .map(pair -> pair.getKey().get("month"))
                        .distinct()
                        .sorted()
                        .collect(java.util.stream.Collectors.toList());
        assertEquals(Arrays.asList("1", "12"), months);
        assertTrue(result.stream().allMatch(pair -> "2024".equals(pair.getKey().get("year"))));
    }

    @TestTemplate
    void testFindPartitionsWithNullDefaultPartitionInOrPredicate() throws IOException {
        Path tableLocation = new Path(tmpPath.toUri());

        AtomicInteger listCount = new AtomicInteger(0);
        FormatTable formatTable = createDtHourFormatTable(countingFileIO(listCount), tableLocation);
        createDtHourPartitionDirs(
                LocalFileIO.create(),
                tableLocation,
                formatTable.defaultPartName(),
                Arrays.asList(
                        Pair.of(formatTable.defaultPartName(), "10"),
                        Pair.of("20260625", "10"),
                        Pair.of("20260624", "20"),
                        Pair.of("20260101", "10")));

        // The default partition name should be treated as null while pruning.
        PredicateBuilder builder = new PredicateBuilder(formatTable.partitionType());
        Predicate predicate =
                PredicateBuilder.or(
                        builder.isNull(0),
                        PredicateBuilder.and(
                                builder.equal(0, BinaryString.fromString("20260625")),
                                builder.equal(1, BinaryString.fromString("10"))));
        PartitionPredicate partitionFilter =
                PartitionPredicate.fromPredicate(formatTable.partitionType(), predicate);

        FormatTableScan scan = new FormatTableScan(formatTable, partitionFilter, null);
        List<Pair<LinkedHashMap<String, String>, Path>> result = scan.findPartitions();

        // Only the default dt partition and dt=20260625 are descended: root + 2 = 3 list calls.
        assertEquals(3, listCount.get());
        assertEquals(2, result.size());
        assertContainsDefaultAndConcreteDt(result, formatTable.defaultPartName());
    }

    @TestTemplate
    void testListPartitionEntriesWithNullDefaultPartitionInOnlyValuePath() throws IOException {
        Path tableLocation = new Path(tmpPath.toUri());
        FormatTable formatTable = createDtHourFormatTable(LocalFileIO.create(), tableLocation);
        createDtHourPartitionDirs(
                LocalFileIO.create(),
                tableLocation,
                formatTable.defaultPartName(),
                Arrays.asList(
                        Pair.of(formatTable.defaultPartName(), "10"), Pair.of("20260625", "10")));

        FormatTableScan scan = new FormatTableScan(formatTable, null, null);
        List<org.apache.paimon.manifest.PartitionEntry> entries = scan.listPartitionEntries();

        assertEquals(2, entries.size());
        assertTrue(entries.stream().anyMatch(entry -> entry.partition().isNullAt(0)));
        assertTrue(
                entries.stream()
                        .anyMatch(
                                entry ->
                                        !entry.partition().isNullAt(0)
                                                && BinaryString.fromString("20260625")
                                                        .equals(entry.partition().getString(0))));
    }

    private void createYearMonthPartitionDirs(LocalFileIO fileIO, Path tableLocation)
            throws IOException {
        for (int year = 2022; year <= 2026; year++) {
            String partPath = enablePartitionValueOnly ? String.valueOf(year) : "year=" + year;
            for (int month = 1; month <= 12; month++) {
                String monthPart =
                        enablePartitionValueOnly
                                ? partPath + "/" + month
                                : partPath + "/month=" + month;
                fileIO.mkdirs(new Path(tableLocation, monthPart));
            }
        }
    }

    private void createDtHourPartitionDirs(
            LocalFileIO fileIO,
            Path tableLocation,
            String defaultPartName,
            List<Pair<String, String>> partitions)
            throws IOException {
        for (Pair<String, String> partition : partitions) {
            String dt = partition.getLeft();
            String hour = partition.getRight();
            String partPath;
            if (enablePartitionValueOnly) {
                partPath = dt + "/" + hour;
            } else {
                partPath =
                        String.format(
                                "dt=%s/hour=%s",
                                dt == null ? defaultPartName : dt,
                                hour == null ? defaultPartName : hour);
            }
            fileIO.mkdirs(new Path(tableLocation, partPath));
        }
    }

    private FormatTable createYearMonthFormatTable(LocalFileIO fileIO, Path tableLocation) {
        RowType rowType =
                RowType.builder()
                        .field("year", DataTypes.INT())
                        .field("month", DataTypes.INT())
                        .field("a", DataTypes.INT())
                        .build();
        return FormatTable.builder()
                .fileIO(fileIO)
                .identifier(Identifier.create("test_db", "test_table"))
                .rowType(rowType)
                .partitionKeys(Arrays.asList("year", "month"))
                .location(tableLocation.toString())
                .format(FormatTable.Format.CSV)
                .options(
                        Collections.singletonMap(
                                FORMAT_TABLE_PARTITION_ONLY_VALUE_IN_PATH.key(),
                                String.valueOf(enablePartitionValueOnly)))
                .build();
    }

    private FormatTable createDtHourFormatTable(LocalFileIO fileIO, Path tableLocation) {
        RowType rowType =
                RowType.builder()
                        .field("dt", DataTypes.STRING())
                        .field("hour", DataTypes.STRING())
                        .field("a", DataTypes.INT())
                        .build();
        return FormatTable.builder()
                .fileIO(fileIO)
                .identifier(Identifier.create("test_db", "test_table"))
                .rowType(rowType)
                .partitionKeys(Arrays.asList("dt", "hour"))
                .location(tableLocation.toString())
                .format(FormatTable.Format.CSV)
                .options(
                        Collections.singletonMap(
                                FORMAT_TABLE_PARTITION_ONLY_VALUE_IN_PATH.key(),
                                String.valueOf(enablePartitionValueOnly)))
                .build();
    }

    private void assertContainsDefaultAndConcreteDt(
            List<Pair<LinkedHashMap<String, String>, Path>> result, String defaultPartName) {
        List<String> dts =
                result.stream()
                        .map(pair -> pair.getKey().get("dt"))
                        .distinct()
                        .sorted()
                        .collect(java.util.stream.Collectors.toList());
        assertEquals(Arrays.asList("20260625", defaultPartName), dts);
        assertTrue(
                result.stream()
                        .anyMatch(
                                pair ->
                                        defaultPartName.equals(pair.getKey().get("dt"))
                                                && "10".equals(pair.getKey().get("hour"))));
        assertTrue(
                result.stream()
                        .anyMatch(
                                pair ->
                                        "20260625".equals(pair.getKey().get("dt"))
                                                && "10".equals(pair.getKey().get("hour"))));
    }

    private LocalFileIO countingFileIO(AtomicInteger listCount) {
        return new LocalFileIO() {
            @Override
            public FileStatus[] listStatus(Path path) throws IOException {
                listCount.getAndIncrement();
                return super.listStatus(path);
            }
        };
    }
}
