/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

#include "paimon/format/parquet/page_filtered_row_group_reader.h"

#include <algorithm>
#include <cstdint>
#include <cstring>
#include <functional>
#include <iostream>
#include <limits>
#include <map>
#include <memory>
#include <mutex>
#include <optional>
#include <string>
#include <utility>
#include <vector>

#include "arrow/api.h"
#include "arrow/array/array_nested.h"
#include "arrow/c/abi.h"
#include "arrow/c/bridge.h"
#include "arrow/io/api.h"
#include "arrow/ipc/json_simple.h"
#include "gtest/gtest.h"
#include "paimon/common/utils/arrow/arrow_input_stream_adapter.h"
#include "paimon/common/utils/arrow/mem_utils.h"
#include "paimon/common/utils/stream_utils.h"
#include "paimon/defs.h"
#include "paimon/format/parquet/parquet_file_batch_reader.h"
#include "paimon/format/parquet/parquet_format_defs.h"
#include "paimon/format/parquet/parquet_format_writer.h"
#include "paimon/fs/file_system.h"
#include "paimon/memory/memory_pool.h"
#include "paimon/predicate/literal.h"
#include "paimon/predicate/predicate_builder.h"
#include "paimon/result.h"
#include "paimon/status.h"
#include "paimon/testing/utils/read_result_collector.h"
#include "paimon/testing/utils/testharness.h"
#include "paimon/utils/roaring_bitmap32.h"
#include "parquet/arrow/reader.h"
#include "parquet/column_page.h"
#include "parquet/file_reader.h"
#include "parquet/properties.h"

namespace paimon {
class Predicate;
}  // namespace paimon

namespace paimon::parquet::test {

class ReadAtTrackingInputStream : public InputStream {
 public:
    explicit ReadAtTrackingInputStream(std::shared_ptr<InputStream> input)
        : input_(std::move(input)) {}

    Status Seek(int64_t offset, SeekOrigin origin) override {
        return input_->Seek(offset, origin);
    }

    Result<int64_t> GetPos() const override {
        return input_->GetPos();
    }

    Result<int64_t> Read(char* buffer, int64_t size) override {
        return input_->Read(buffer, size);
    }

    Result<int64_t> Read(char* buffer, int64_t size, int64_t offset) override {
        RecordPositionalRead(offset, size);
        return input_->Read(buffer, size, offset);
    }

    void ReadAsync(char* buffer, int64_t size, int64_t offset,
                   std::function<void(Status)>&& callback) override {
        RecordPositionalRead(offset, size);
        input_->ReadAsync(buffer, size, offset, std::move(callback));
    }

    Result<std::string> GetUri() const override {
        return input_->GetUri();
    }

    Result<int64_t> Length() const override {
        return input_->Length();
    }

    Status Close() override {
        return input_->Close();
    }

    void ClearReadAtRanges() {
        std::lock_guard<std::mutex> lock(mutex_);
        read_at_ranges_.clear();
    }

    std::vector<arrow::io::ReadRange> GetReadAtRanges() const {
        std::lock_guard<std::mutex> lock(mutex_);
        return read_at_ranges_;
    }

 private:
    void RecordPositionalRead(int64_t offset, int64_t size) {
        std::lock_guard<std::mutex> lock(mutex_);
        read_at_ranges_.push_back({offset, size});
    }

    std::shared_ptr<InputStream> input_;
    mutable std::mutex mutex_;
    std::vector<arrow::io::ReadRange> read_at_ranges_;
};

/// Test fixture for page-level filtering.
/// Creates Parquet files with multiple row groups and small page sizes to ensure
/// multiple pages per row group, enabling page-level filtering tests.
class PageFilteredRowGroupReaderTest : public ::testing::Test {
 public:
    void SetUp() override {
        pool_ = GetDefaultPool();
        arrow_pool_ = GetArrowPool(pool_);
        dir_ = paimon::test::UniqueTestDirectory::Create();
        ASSERT_TRUE(dir_);
        fs_ = dir_->GetFileSystem();
    }

    /// Write a Parquet file with controlled page boundaries.
    /// @param file_name Output file name
    /// @param struct_array Data to write
    /// @param write_batch_size Controls page size (number of rows per page)
    /// @param max_row_group_length Controls row group size
    void WriteTestFile(
        const std::string& file_name, const std::shared_ptr<arrow::StructArray>& struct_array,
        int32_t write_batch_size, int64_t max_row_group_length, bool enable_dictionary = false,
        int64_t data_page_size = 1, bool enable_page_index = true,
        ::parquet::ParquetDataPageVersion data_page_version = ::parquet::ParquetDataPageVersion::V1,
        int64_t dictionary_page_size_limit = -1) {
        auto data_type = struct_array->struct_type();
        auto data_schema = arrow::schema(data_type->fields());
        auto data_arrow_array = std::make_unique<ArrowArray>();
        ASSERT_TRUE(arrow::ExportArray(*struct_array, data_arrow_array.get()).ok());
        ASSERT_OK_AND_ASSIGN(std::shared_ptr<OutputStream> out,
                             fs_->Create(file_name, /*overwrite=*/false));
        ::parquet::WriterProperties::Builder builder;
        builder.write_batch_size(write_batch_size);
        builder.max_row_group_length(max_row_group_length);
        if (enable_dictionary) {
            builder.enable_dictionary();
            if (dictionary_page_size_limit >= 0) {
                builder.dictionary_pagesize_limit(dictionary_page_size_limit);
            }
        } else {
            builder.disable_dictionary();  // Ensure page index min/max are meaningful
        }
        if (enable_page_index) {
            builder.enable_write_page_index();
        } else {
            builder.disable_write_page_index();
        }
        builder.data_page_version(data_page_version);
        // Data page size controls when a page is flushed. The default of 1 byte forces a new
        // page after every write_batch_size rows (each batch becomes one page), giving pages
        // aligned across columns. A larger byte-based value combined with write_batch_size=1
        // instead lets columns of different physical widths flush pages at different row
        // counts, producing intentionally misaligned pages across leaves.
        builder.data_pagesize(data_page_size);
        auto writer_properties = builder.build();
        ASSERT_OK_AND_ASSIGN(
            auto format_writer,
            ParquetFormatWriter::Create(out, data_schema, writer_properties,
                                        DEFAULT_PARQUET_WRITER_MAX_MEMORY_USE, arrow_pool_));
        ASSERT_OK(format_writer->AddBatch(data_arrow_array.get()));
        ASSERT_OK(format_writer->Finish());
        ASSERT_OK(out->Close());
    }

    /// Read back a Parquet file with an optional predicate and page index filter enabled.
    /// Returns the collected result as a ChunkedArray.
    void ReadWithPredicateImpl(const std::string& file_name,
                               const std::shared_ptr<arrow::Schema>& read_schema,
                               const std::shared_ptr<Predicate>& predicate,
                               std::shared_ptr<arrow::ChunkedArray>* out,
                               int32_t batch_size = 1024) {
        ASSERT_OK_AND_ASSIGN(std::shared_ptr<InputStream> in, fs_->Open(file_name));
        ASSERT_OK_AND_ASSIGN(int64_t length, in->Length());
        auto in_stream = std::make_shared<ArrowInputStreamAdapter>(in, length, arrow_pool_);

        std::map<std::string, std::string> options;
        options[PARQUET_READ_ENABLE_PAGE_INDEX_FILTER] = "true";
        ASSERT_OK_AND_ASSIGN(auto batch_reader, ParquetFileBatchReader::Create(
                                                    std::move(in_stream), options, batch_size,
                                                    /*file_metadata=*/nullptr,
                                                    /*storage_read_bytes=*/nullptr, arrow_pool_));
        auto c_schema = std::make_unique<ArrowSchema>();
        ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok());
        ASSERT_OK(batch_reader->SetReadSchema(c_schema.get(), predicate,
                                              /*selection_bitmap=*/std::nullopt));
        ASSERT_OK_AND_ASSIGN(*out,
                             paimon::test::ReadResultCollector::CollectResult(batch_reader.get()));
    }

    /// Read back a Parquet file with a predicate, a bitmap, and page index filter enabled.
    void ReadWithPredicateAndBitmapImpl(
        const std::string& file_name, const std::shared_ptr<arrow::Schema>& read_schema,
        const std::shared_ptr<Predicate>& predicate, const RoaringBitmap32& bitmap,
        std::shared_ptr<arrow::ChunkedArray>* out,
        const std::map<std::string, std::string> options = {}, int32_t batch_size = 1024,
        std::vector<arrow::io::ReadRange>* read_at_ranges = nullptr) {
        ASSERT_OK_AND_ASSIGN(std::shared_ptr<InputStream> in, fs_->Open(file_name));
        ASSERT_OK_AND_ASSIGN(int64_t length, in->Length());
        std::shared_ptr<ReadAtTrackingInputStream> tracking_in;
        if (read_at_ranges != nullptr) {
            tracking_in = std::make_shared<ReadAtTrackingInputStream>(in);
            in = tracking_in;
        }
        auto in_stream = std::make_shared<ArrowInputStreamAdapter>(in, length, arrow_pool_);

        ASSERT_OK_AND_ASSIGN(
            auto batch_reader,
            ParquetFileBatchReader::Create(std::move(in_stream), options, batch_size, nullptr,
                                           /*storage_read_bytes=*/nullptr, arrow_pool_));
        auto c_schema = std::make_unique<ArrowSchema>();
        ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok());
        ASSERT_OK(batch_reader->SetReadSchema(c_schema.get(), predicate, bitmap));
        if (tracking_in) {
            // Ignore footer and schema metadata reads. The test only checks I/O issued while
            // consuming data pages.
            tracking_in->ClearReadAtRanges();
        }
        ASSERT_OK_AND_ASSIGN(*out,
                             paimon::test::ReadResultCollector::CollectResult(batch_reader.get()));
        if (tracking_in) {
            *read_at_ranges = tracking_in->GetReadAtRanges();
        }
    }

 protected:
    std::shared_ptr<arrow::MemoryPool> arrow_pool_;
    std::shared_ptr<MemoryPool> pool_;
    std::shared_ptr<FileSystem> fs_;
    std::unique_ptr<paimon::test::UniqueTestDirectory> dir_;
};

// Helper: build a StructArray with N rows of int32 "val" column with sequential values.
// val[i] = i for i in [0, N).
static std::shared_ptr<arrow::StructArray> MakeSequentialIntData(int32_t num_rows) {
    arrow::Int32Builder val_builder;
    EXPECT_TRUE(val_builder.Reserve(num_rows).ok());
    for (int32_t i = 0; i < num_rows; ++i) {
        val_builder.UnsafeAppend(i);
    }
    auto val_array = val_builder.Finish().ValueOrDie();
    auto field = arrow::field("val", arrow::int32());
    return arrow::StructArray::Make({val_array}, {field}).ValueOrDie();
}

// Helper: build a StructArray with two int32 columns: "a" and "b".
// a[i] = i, b[i] = i * 10, for i in [0, N).
static std::shared_ptr<arrow::StructArray> MakeTwoColumnData(int32_t num_rows) {
    arrow::Int32Builder a_builder, b_builder;
    EXPECT_TRUE(a_builder.Reserve(num_rows).ok());
    EXPECT_TRUE(b_builder.Reserve(num_rows).ok());
    for (int32_t i = 0; i < num_rows; ++i) {
        a_builder.UnsafeAppend(i);
        b_builder.UnsafeAppend(i * 10);
    }
    auto a_array = a_builder.Finish().ValueOrDie();
    auto b_array = b_builder.Finish().ValueOrDie();
    auto field_a = arrow::field("a", arrow::int32());
    auto field_b = arrow::field("b", arrow::int32());
    return arrow::StructArray::Make({a_array, b_array}, {field_a, field_b}).ValueOrDie();
}

static void AssertUnselectedPageHeadersNotRead(
    const std::vector<::parquet::PageLocation>& page_locations,
    const std::vector<int32_t>& selected_pages,
    const std::vector<arrow::io::ReadRange>& read_at_ranges) {
    for (int32_t page_idx = 0; page_idx < static_cast<int32_t>(page_locations.size()); ++page_idx) {
        if (std::find(selected_pages.begin(), selected_pages.end(), page_idx) !=
            selected_pages.end()) {
            continue;
        }
        const int64_t page_header_offset = page_locations[page_idx].offset;
        for (const auto& read_range : read_at_ranges) {
            ASSERT_FALSE(read_range.offset <= page_header_offset &&
                         page_header_offset < read_range.offset + read_range.length)
                << "unselected page " << page_idx << " header at " << page_header_offset
                << " was covered by positional ReadAt [" << read_range.offset << ", "
                << read_range.offset + read_range.length << ")";
        }
    }
}

static void AssertReadRangeCovered(const std::vector<arrow::io::ReadRange>& read_ranges,
                                   int64_t expected_offset, int64_t expected_size) {
    ASSERT_GT(expected_size, 0);
    ASSERT_TRUE(std::any_of(
        read_ranges.begin(), read_ranges.end(),
        [expected_offset, expected_size](const arrow::io::ReadRange& read_range) {
            return read_range.offset <= expected_offset &&
                   expected_offset + expected_size <= read_range.offset + read_range.length;
        }))
        << "expected range [" << expected_offset << ", " << expected_offset + expected_size
        << ") was not covered by any positional read";
}

static void AssertSelectedPagesRead(const std::vector<::parquet::PageLocation>& page_locations,
                                    const std::vector<int32_t>& selected_pages,
                                    const std::vector<arrow::io::ReadRange>& read_ranges) {
    for (int32_t page_idx : selected_pages) {
        ASSERT_GE(page_idx, 0);
        ASSERT_LT(page_idx, static_cast<int32_t>(page_locations.size()));
        const auto& page = page_locations[page_idx];
        AssertReadRangeCovered(read_ranges, page.offset, page.compressed_page_size);
    }
}

/// Test: page-level filtering correctly skips non-matching pages.
///
/// Scenario: 100 rows, 10 rows per page, 1 row group.
/// val[i] = i. Predicate: val >= 50. Pages 0-4 (rows 0-49) should be skipped,
/// pages 5-9 (rows 50-99) should be read.
TEST_F(PageFilteredRowGroupReaderTest, SingleRowGroupPartialPageMatch) {
    std::string file_name = dir_->Str() + "/single_rg_partial.parquet";
    auto data = MakeSequentialIntData(100);
    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100);

    auto read_schema = arrow::schema({arrow::field("val", arrow::int32())});
    auto predicate = PredicateBuilder::GreaterOrEqual(
        /*field_index=*/0, /*field_name=*/"val", FieldType::INT, Literal(50));

    std::shared_ptr<arrow::ChunkedArray> result;
    ReadWithPredicateImpl(file_name, read_schema, predicate, &result);

    // Should get rows 50-99 = 50 rows
    ASSERT_TRUE(result);
    ASSERT_EQ(50, result->length());

    // Verify actual values
    auto flat = result->chunk(0);
    auto struct_arr = std::dynamic_pointer_cast<arrow::StructArray>(flat);
    ASSERT_TRUE(struct_arr);
    auto val_arr = std::dynamic_pointer_cast<arrow::Int32Array>(struct_arr->field(0));
    ASSERT_TRUE(val_arr);
    for (int32_t i = 0; i < 50; ++i) {
        ASSERT_EQ(50 + i, val_arr->Value(i)) << "Mismatch at index " << i;
    }
}

/// Test: predicate matches all pages → same as unfiltered read.
TEST_F(PageFilteredRowGroupReaderTest, AllPagesMatch) {
    std::string file_name = dir_->Str() + "/all_match.parquet";
    auto data = MakeSequentialIntData(100);
    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100);

    auto read_schema = arrow::schema({arrow::field("val", arrow::int32())});
    auto predicate = PredicateBuilder::GreaterOrEqual(
        /*field_index=*/0, /*field_name=*/"val", FieldType::INT, Literal(0));

    std::shared_ptr<arrow::ChunkedArray> result;
    ReadWithPredicateImpl(file_name, read_schema, predicate, &result);
    ASSERT_TRUE(result);
    ASSERT_EQ(100, result->length());
}

/// Test: predicate matches no pages → empty result.
TEST_F(PageFilteredRowGroupReaderTest, NoPagesMatch) {
    std::string file_name = dir_->Str() + "/no_match.parquet";
    auto data = MakeSequentialIntData(100);
    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100);

    auto read_schema = arrow::schema({arrow::field("val", arrow::int32())});
    auto predicate = PredicateBuilder::GreaterThan(
        /*field_index=*/0, /*field_name=*/"val", FieldType::INT, Literal(999));

    std::shared_ptr<arrow::ChunkedArray> result;
    ReadWithPredicateImpl(file_name, read_schema, predicate, &result);
    // No matching rows; result should be null (empty)
    ASSERT_FALSE(result);
}

/// Test: multiple row groups, page filtering active on some.
///
/// 200 rows, 10 rows per page, 50 rows per row group → 4 row groups.
/// Predicate: val >= 150. Row groups 0-2 (rows 0-149) should be eliminated entirely.
/// Row group 3 (rows 150-199): all pages match → full read, no page filtering.
TEST_F(PageFilteredRowGroupReaderTest, MultipleRowGroupsFullElimination) {
    std::string file_name = dir_->Str() + "/multi_rg_elim.parquet";
    auto data = MakeSequentialIntData(200);
    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50);

    auto read_schema = arrow::schema({arrow::field("val", arrow::int32())});
    auto predicate = PredicateBuilder::GreaterOrEqual(
        /*field_index=*/0, /*field_name=*/"val", FieldType::INT, Literal(150));

    std::shared_ptr<arrow::ChunkedArray> result;
    ReadWithPredicateImpl(file_name, read_schema, predicate, &result);
    ASSERT_TRUE(result);
    ASSERT_EQ(50, result->length());

    // Verify values are 150-199
    auto flat = result->chunk(0);
    auto struct_arr = std::dynamic_pointer_cast<arrow::StructArray>(flat);
    ASSERT_TRUE(struct_arr);
    auto val_arr = std::dynamic_pointer_cast<arrow::Int32Array>(struct_arr->field(0));
    for (int32_t i = 0; i < 50; ++i) {
        ASSERT_EQ(150 + i, val_arr->Value(i));
    }
}

/// Test: multiple row groups, partial page match within a row group.
///
/// 200 rows, 10 rows per page, 100 rows per row group → 2 row groups.
/// Predicate: val >= 50 AND val < 150.
/// Row group 0 (rows 0-99): pages 0-4 skipped, pages 5-9 read → 50 rows
/// Row group 1 (rows 100-199): pages 0-4 read, pages 5-9 skipped → 50 rows
/// Total: 100 rows
TEST_F(PageFilteredRowGroupReaderTest, MultipleRowGroupsPartialPageMatch) {
    std::string file_name = dir_->Str() + "/multi_rg_partial.parquet";
    auto data = MakeSequentialIntData(200);
    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100);

    auto read_schema = arrow::schema({arrow::field("val", arrow::int32())});
    ASSERT_OK_AND_ASSIGN(
        auto predicate,
        PredicateBuilder::And(
            {PredicateBuilder::GreaterOrEqual(/*field_index=*/0, /*field_name=*/"val",
                                              FieldType::INT, Literal(50)),
             PredicateBuilder::LessThan(/*field_index=*/0, /*field_name=*/"val", FieldType::INT,
                                        Literal(150))}));

    std::shared_ptr<arrow::ChunkedArray> result;
    ReadWithPredicateImpl(file_name, read_schema, predicate, &result);
    ASSERT_TRUE(result);
    ASSERT_EQ(100, result->length());

    // Collect all values and verify they are 50-149
    int64_t offset = 0;
    for (int i = 0; i < result->num_chunks(); ++i) {
        auto struct_arr = std::dynamic_pointer_cast<arrow::StructArray>(result->chunk(i));
        ASSERT_TRUE(struct_arr);
        auto val_arr = std::dynamic_pointer_cast<arrow::Int32Array>(struct_arr->field(0));
        for (int64_t j = 0; j < val_arr->length(); ++j) {
            ASSERT_EQ(50 + offset, val_arr->Value(j)) << "Mismatch at offset " << offset;
            ++offset;
        }
    }
    ASSERT_EQ(100, offset);
}

/// Test: two columns remain aligned after page-level filtering.
///
/// 100 rows, a[i] = i, b[i] = i*10. 10 rows per page.
/// Predicate on "a": a >= 50. After filtering, b should be b[50..99] = {500, 510, ..., 990}.
TEST_F(PageFilteredRowGroupReaderTest, MultiColumnAlignment) {
    std::string file_name = dir_->Str() + "/multi_col.parquet";
    auto data = MakeTwoColumnData(100);
    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100);

    auto read_schema =
        arrow::schema({arrow::field("a", arrow::int32()), arrow::field("b", arrow::int32())});
    auto predicate = PredicateBuilder::GreaterOrEqual(
        /*field_index=*/0, /*field_name=*/"a", FieldType::INT, Literal(50));

    std::shared_ptr<arrow::ChunkedArray> result;
    ReadWithPredicateImpl(file_name, read_schema, predicate, &result);
    ASSERT_TRUE(result);
    ASSERT_EQ(50, result->length());

    auto struct_arr = std::dynamic_pointer_cast<arrow::StructArray>(result->chunk(0));
    ASSERT_TRUE(struct_arr);
    auto a_arr = std::dynamic_pointer_cast<arrow::Int32Array>(struct_arr->field(0));
    auto b_arr = std::dynamic_pointer_cast<arrow::Int32Array>(struct_arr->field(1));
    for (int32_t i = 0; i < 50; ++i) {
        ASSERT_EQ(50 + i, a_arr->Value(i));
        ASSERT_EQ((50 + i) * 10, b_arr->Value(i));
    }
}

/// Test: predicate matches pages in the middle of a row group.
///
/// 100 rows, 10 rows per page. Predicate: val >= 30 AND val < 70.
/// Pages 0-2 (rows 0-29) skipped, pages 3-6 (rows 30-69) read, pages 7-9 (rows 70-99) skipped.
TEST_F(PageFilteredRowGroupReaderTest, MiddlePagesMatch) {
    std::string file_name = dir_->Str() + "/middle_pages.parquet";
    auto data = MakeSequentialIntData(100);
    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100);

    auto read_schema = arrow::schema({arrow::field("val", arrow::int32())});
    ASSERT_OK_AND_ASSIGN(
        auto predicate,
        PredicateBuilder::And(
            {PredicateBuilder::GreaterOrEqual(/*field_index=*/0, /*field_name=*/"val",
                                              FieldType::INT, Literal(30)),
             PredicateBuilder::LessThan(/*field_index=*/0, /*field_name=*/"val", FieldType::INT,
                                        Literal(70))}));

    std::shared_ptr<arrow::ChunkedArray> result;
    ReadWithPredicateImpl(file_name, read_schema, predicate, &result);
    ASSERT_TRUE(result);
    ASSERT_EQ(40, result->length());

    int64_t offset = 0;
    for (int i = 0; i < result->num_chunks(); ++i) {
        auto struct_arr = std::dynamic_pointer_cast<arrow::StructArray>(result->chunk(i));
        auto val_arr = std::dynamic_pointer_cast<arrow::Int32Array>(struct_arr->field(0));
        for (int64_t j = 0; j < val_arr->length(); ++j) {
            ASSERT_EQ(30 + offset, val_arr->Value(j));
            ++offset;
        }
    }
    ASSERT_EQ(40, offset);
}

/// Test: no predicate → all data returned (no filtering).
TEST_F(PageFilteredRowGroupReaderTest, NoPredicate) {
    std::string file_name = dir_->Str() + "/no_predicate.parquet";
    auto data = MakeSequentialIntData(100);
    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100);

    auto read_schema = arrow::schema({arrow::field("val", arrow::int32())});

    std::shared_ptr<arrow::ChunkedArray> result;
    ReadWithPredicateImpl(file_name, read_schema, /*predicate=*/nullptr, &result);
    ASSERT_NE(nullptr, result);
    ASSERT_EQ(100, result->length());
}

/// Test: page filtering with EQUAL predicate that matches a single page.
///
/// 100 rows, 10 rows per page. Predicate: val == 55.
/// Only page 5 (rows 50-59) should match, containing value 55.
TEST_F(PageFilteredRowGroupReaderTest, EqualPredicateSinglePageMatch) {
    std::string file_name = dir_->Str() + "/equal_single_page.parquet";
    auto data = MakeSequentialIntData(100);
    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100);

    auto read_schema = arrow::schema({arrow::field("val", arrow::int32())});
    auto predicate = PredicateBuilder::Equal(
        /*field_index=*/0, /*field_name=*/"val", FieldType::INT, Literal(55));

    std::shared_ptr<arrow::ChunkedArray> result;
    ReadWithPredicateImpl(file_name, read_schema, predicate, &result);
    ASSERT_TRUE(result);
    // Page 5 has rows 50-59, which includes 55. The entire page is returned.
    ASSERT_EQ(10, result->length());

    auto struct_arr = std::dynamic_pointer_cast<arrow::StructArray>(result->chunk(0));
    auto val_arr = std::dynamic_pointer_cast<arrow::Int32Array>(struct_arr->field(0));
    for (int32_t i = 0; i < 10; ++i) {
        ASSERT_EQ(50 + i, val_arr->Value(i));
    }
}

/// Test: page filtering with LessThan predicate.
///
/// 100 rows, 10 rows per page. Predicate: val < 25.
/// Pages 0-2 (rows 0-29) match (page 2 has min=20 < 25).
/// Pages 3-9 don't match.
TEST_F(PageFilteredRowGroupReaderTest, LessThanPredicatePageMatch) {
    std::string file_name = dir_->Str() + "/less_than.parquet";
    auto data = MakeSequentialIntData(100);
    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100);

    auto read_schema = arrow::schema({arrow::field("val", arrow::int32())});
    auto predicate = PredicateBuilder::LessThan(
        /*field_index=*/0, /*field_name=*/"val", FieldType::INT, Literal(25));

    std::shared_ptr<arrow::ChunkedArray> result;
    ReadWithPredicateImpl(file_name, read_schema, predicate, &result);
    ASSERT_TRUE(result);
    // Pages 0 (0-9), 1 (10-19), 2 (20-29) match because their min < 25.
    // Page 2 has min=20, max=29, and 20 < 25, so it matches.
    ASSERT_EQ(30, result->length());

    auto struct_arr = std::dynamic_pointer_cast<arrow::StructArray>(result->chunk(0));
    auto val_arr = std::dynamic_pointer_cast<arrow::Int32Array>(struct_arr->field(0));
    for (int32_t i = 0; i < 30; ++i) {
        ASSERT_EQ(i, val_arr->Value(i));
    }
}

/// Test: large data with multiple row groups and page filtering.
///
/// 1000 rows, 10 rows per page, 200 rows per row group → 5 row groups.
/// Predicate: val >= 500 AND val < 700.
/// Row groups 0,1 (rows 0-399): all pages eliminated
/// Row group 2 (rows 400-599): pages 0-9 (400-499) eliminated, pages 10-19 (500-599) read
/// Row group 3 (rows 600-799): pages 0-9 (600-699) read, pages 10-19 (700-799) eliminated
/// Row group 4 (rows 800-999): all pages eliminated
/// Total: 200 rows (500-699)
TEST_F(PageFilteredRowGroupReaderTest, LargeDataMultiRowGroupPageFilter) {
    std::string file_name = dir_->Str() + "/large_data.parquet";
    auto data = MakeSequentialIntData(1000);
    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/200);

    auto read_schema = arrow::schema({arrow::field("val", arrow::int32())});
    ASSERT_OK_AND_ASSIGN(
        auto predicate,
        PredicateBuilder::And(
            {PredicateBuilder::GreaterOrEqual(/*field_index=*/0, /*field_name=*/"val",
                                              FieldType::INT, Literal(500)),
             PredicateBuilder::LessThan(/*field_index=*/0, /*field_name=*/"val", FieldType::INT,
                                        Literal(700))}));

    std::shared_ptr<arrow::ChunkedArray> result;
    ReadWithPredicateImpl(file_name, read_schema, predicate, &result);
    ASSERT_TRUE(result);
    ASSERT_EQ(200, result->length());

    // Verify values are 500-699
    int64_t offset = 0;
    for (int i = 0; i < result->num_chunks(); ++i) {
        auto struct_arr = std::dynamic_pointer_cast<arrow::StructArray>(result->chunk(i));
        auto val_arr = std::dynamic_pointer_cast<arrow::Int32Array>(struct_arr->field(0));
        for (int64_t j = 0; j < val_arr->length(); ++j) {
            ASSERT_EQ(500 + offset, val_arr->Value(j)) << "Mismatch at offset " << offset;
            ++offset;
        }
    }
    ASSERT_EQ(200, offset);
}

/// Test: string column page filtering.
///
/// Write 40 rows with string values: "aaa_00", "aaa_01", ..., "aaa_09",
/// "bbb_10", ..., "bbb_19", "ccc_20", ..., "ccc_29", "ddd_30", ..., "ddd_39".
/// 10 rows per page → 4 pages. Predicate: val >= "ccc" should match pages 2-3.
TEST_F(PageFilteredRowGroupReaderTest, StringColumnPageFilter) {
    std::string file_name = dir_->Str() + "/string_filter.parquet";

    arrow::StringBuilder str_builder;
    ASSERT_TRUE(str_builder.Reserve(40).ok());
    std::vector<std::string> prefixes = {"aaa", "bbb", "ccc", "ddd"};
    for (int32_t i = 0; i < 40; ++i) {
        std::string val = prefixes[i / 10] + "_" + (i < 10 ? "0" : "") + std::to_string(i);
        ASSERT_TRUE(str_builder.Append(val).ok());
    }
    auto str_array = str_builder.Finish().ValueOrDie();
    auto field = arrow::field("val", arrow::utf8());
    auto struct_arr = arrow::StructArray::Make({str_array}, {field}).ValueOrDie();

    WriteTestFile(file_name, struct_arr, /*write_batch_size=*/10, /*max_row_group_length=*/40);

    auto read_schema = arrow::schema({field});
    auto predicate = PredicateBuilder::GreaterOrEqual(
        /*field_index=*/0, /*field_name=*/"val", FieldType::STRING,
        Literal(FieldType::STRING, "ccc", 3));

    std::shared_ptr<arrow::ChunkedArray> result;
    ReadWithPredicateImpl(file_name, read_schema, predicate, &result);
    ASSERT_TRUE(result);
    // Pages 2 (ccc_20..ccc_29) and 3 (ddd_30..ddd_39) should match.
    ASSERT_EQ(20, result->length());
}

/// Test: ComputePageRanges returns only matching page byte ranges.
///
/// 100 rows, 10 rows per page, 1 row group with page index enabled.
/// RowRanges = [50, 59] (page 5 only). Should return exactly 1 page range per column.
TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesPartialMatch) {
    std::string file_name = dir_->Str() + "/compute_ranges_partial.parquet";
    auto data = MakeSequentialIntData(100);
    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100);

    // Open as raw ParquetFileReader
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<InputStream> in, fs_->Open(file_name));
    ASSERT_OK_AND_ASSIGN(int64_t length, in->Length());
    auto in_stream = std::make_shared<ArrowInputStreamAdapter>(in, length, arrow_pool_);
    auto parquet_reader = ::parquet::ParquetFileReader::Open(in_stream);
    ASSERT_TRUE(parquet_reader);

    // Single page match: rows [50, 59] = page 5
    RowRanges row_ranges;
    row_ranges.Add(RowRanges::Range(50, 59));
    auto page_index_reader = parquet_reader->GetPageIndexReader();
    ASSERT_TRUE(page_index_reader);
    auto rg_page_index_reader = page_index_reader->RowGroup(0);
    auto ranges = PageFilteredRowGroupReader::ComputePageRanges(
        TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/row_ranges),
        /*column_indices=*/{0}, /*row_group_page_index_reader=*/rg_page_index_reader,
        parquet_reader.get());

    // Should have exactly 1 range (page 5 of column 0, no dictionary since disabled)
    ASSERT_EQ(1, ranges.size());
    ASSERT_GT(ranges[0].offset, 0);
    ASSERT_GT(ranges[0].length, 0);
}

/// Test: ComputePageRanges returns all page ranges when RowRanges covers entire row group.
TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesAllMatch) {
    std::string file_name = dir_->Str() + "/compute_ranges_all.parquet";
    auto data = MakeSequentialIntData(100);
    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100);

    ASSERT_OK_AND_ASSIGN(std::shared_ptr<InputStream> in, fs_->Open(file_name));
    ASSERT_OK_AND_ASSIGN(int64_t length, in->Length());
    auto in_stream = std::make_shared<ArrowInputStreamAdapter>(in, length, arrow_pool_);
    auto parquet_reader = ::parquet::ParquetFileReader::Open(in_stream);

    // All rows match
    RowRanges row_ranges;
    row_ranges.Add(RowRanges::Range(0, 99));
    auto page_index_reader = parquet_reader->GetPageIndexReader();
    ASSERT_TRUE(page_index_reader);
    auto rg_page_index_reader = page_index_reader->RowGroup(0);
    auto ranges = PageFilteredRowGroupReader::ComputePageRanges(
        TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/row_ranges),
        /*column_indices=*/{0},
        /*row_group_page_index_reader=*/rg_page_index_reader, parquet_reader.get());

    // 10 pages, all matching
    ASSERT_EQ(10, ranges.size());
    for (const auto& r : ranges) {
        ASSERT_GT(r.offset, 0);
        ASSERT_GT(r.length, 0);
    }
}

/// Test: ComputePageRanges returns no page ranges for empty RowRanges.
TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesNoMatch) {
    std::string file_name = dir_->Str() + "/compute_ranges_none.parquet";
    auto data = MakeSequentialIntData(100);
    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100);

    ASSERT_OK_AND_ASSIGN(std::shared_ptr<InputStream> in, fs_->Open(file_name));
    ASSERT_OK_AND_ASSIGN(int64_t length, in->Length());
    auto in_stream = std::make_shared<ArrowInputStreamAdapter>(in, length, arrow_pool_);
    auto parquet_reader = ::parquet::ParquetFileReader::Open(in_stream);

    RowRanges row_ranges;  // empty
    auto page_index_reader = parquet_reader->GetPageIndexReader();
    ASSERT_TRUE(page_index_reader);
    auto rg_page_index_reader = page_index_reader->RowGroup(0);
    auto ranges = PageFilteredRowGroupReader::ComputePageRanges(
        TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/row_ranges),
        /*column_indices=*/{0},
        /*row_group_page_index_reader=*/rg_page_index_reader, parquet_reader.get());

    ASSERT_EQ(0, ranges.size());
}

/// Test: ComputePageRanges with multiple columns returns ranges for each column.
TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesMultiColumn) {
    std::string file_name = dir_->Str() + "/compute_ranges_multi_col.parquet";
    auto data = MakeTwoColumnData(100);
    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100);

    ASSERT_OK_AND_ASSIGN(std::shared_ptr<InputStream> in, fs_->Open(file_name));
    ASSERT_OK_AND_ASSIGN(int64_t length, in->Length());
    auto in_stream = std::make_shared<ArrowInputStreamAdapter>(in, length, arrow_pool_);
    auto parquet_reader = ::parquet::ParquetFileReader::Open(in_stream);

    // Match page 5 only (rows 50-59)
    RowRanges row_ranges;
    row_ranges.Add(RowRanges::Range(50, 59));

    auto page_index_reader = parquet_reader->GetPageIndexReader();
    ASSERT_TRUE(page_index_reader);
    auto rg_page_index_reader = page_index_reader->RowGroup(0);
    auto ranges = PageFilteredRowGroupReader::ComputePageRanges(
        TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/row_ranges),
        /*column_indices=*/{0, 1}, /*row_group_page_index_reader=*/rg_page_index_reader,
        parquet_reader.get());

    // 1 matching page per column = 2 ranges total
    ASSERT_EQ(2, ranges.size());
    // Ranges should be at different offsets (different columns)
    ASSERT_NE(ranges[0].offset, ranges[1].offset);
}

/// Test: ComputePageRanges with multiple matching pages.
///
/// 100 rows, 10 per page. RowRanges = [20,29] + [70,79] = pages 2 and 7.
TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesMultiplePages) {
    std::string file_name = dir_->Str() + "/compute_ranges_multi_page.parquet";
    auto data = MakeSequentialIntData(100);
    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100);

    ASSERT_OK_AND_ASSIGN(std::shared_ptr<InputStream> in, fs_->Open(file_name));
    ASSERT_OK_AND_ASSIGN(int64_t length, in->Length());
    auto in_stream = std::make_shared<ArrowInputStreamAdapter>(in, length, arrow_pool_);
    auto parquet_reader = ::parquet::ParquetFileReader::Open(in_stream);

    RowRanges row_ranges;
    row_ranges.Add(RowRanges::Range(20, 29));
    row_ranges.Add(RowRanges::Range(70, 79));

    auto page_index_reader = parquet_reader->GetPageIndexReader();
    ASSERT_TRUE(page_index_reader);
    auto rg_page_index_reader = page_index_reader->RowGroup(0);
    auto ranges = PageFilteredRowGroupReader::ComputePageRanges(
        TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true, /*ranges=*/row_ranges),
        /*column_indices=*/{0},
        /*row_group_page_index_reader=*/rg_page_index_reader, parquet_reader.get());

    // 2 matching pages for 1 column
    ASSERT_EQ(2, ranges.size());
    // Pages should be at increasing offsets
    ASSERT_LT(ranges[0].offset, ranges[1].offset);
}

/// Test: variable-length columns are streamed across multiple offset-normalized
/// RecordBatches when batch_size is smaller than the matched row count, instead of
/// being concatenated into a single RecordBatch via CombineChunks.
///
/// This verifies the alignment with Arrow's standard TableBatchReader path:
/// multi-chunk binary/string columns split along chunk + batch_size boundaries. It
/// asserts correctness and the multi-batch shape.
TEST_F(PageFilteredRowGroupReaderTest, StringColumnMultiBatchStreaming) {
    std::string file_name = dir_->Str() + "/string_multi_batch.parquet";

    arrow::StringBuilder str_builder;
    ASSERT_TRUE(str_builder.Reserve(60).ok());
    // 6 pages of 10 rows each: prefix "p0_".."p5_" so each page has a distinct min/max.
    for (int32_t i = 0; i < 60; ++i) {
        std::string val =
            "p" + std::to_string(i / 10) + "_" + (i < 10 ? "0" : "") + std::to_string(i);
        ASSERT_TRUE(str_builder.Append(val).ok());
    }
    auto str_array = str_builder.Finish().ValueOrDie();
    auto field = arrow::field("val", arrow::utf8());
    auto struct_arr = arrow::StructArray::Make({str_array}, {field}).ValueOrDie();

    WriteTestFile(file_name, struct_arr, /*write_batch_size=*/10, /*max_row_group_length=*/60);

    // Predicate matches pages 2..5 (40 rows: "p2_20".."p5_59"). batch_size=7 forces
    // the wrapper to surface multiple batches per page-filtered RG.
    auto read_schema = arrow::schema({field});
    auto predicate = PredicateBuilder::GreaterOrEqual(
        /*field_index=*/0, /*field_name=*/"val", FieldType::STRING,
        Literal(FieldType::STRING, "p2", 2));

    std::shared_ptr<arrow::ChunkedArray> result;
    ReadWithPredicateImpl(file_name, read_schema, predicate, &result, /*batch_size=*/7);
    ASSERT_TRUE(result);
    ASSERT_EQ(40, result->length());

    // Multi-batch shape: with 40 matched rows and batch_size=7 we expect at least
    // ceil(40/7)=6 chunks. Anything > 1 already proves we did not collapse to a single
    // post-CombineChunks RecordBatch.
    ASSERT_GT(result->num_chunks(), 1);

    // Content correctness: rows arrive in the original page order, "p2_20" through "p5_59".
    int64_t seen = 0;
    for (int i = 0; i < result->num_chunks(); ++i) {
        auto struct_chunk = std::dynamic_pointer_cast<arrow::StructArray>(result->chunk(i));
        ASSERT_TRUE(struct_chunk);
        auto str_chunk = std::dynamic_pointer_cast<arrow::StringArray>(struct_chunk->field(0));
        ASSERT_TRUE(str_chunk);
        for (int64_t j = 0; j < str_chunk->length(); ++j) {
            int32_t row = 20 + static_cast<int32_t>(seen);
            std::string expected =
                "p" + std::to_string(row / 10) + "_" + (row < 10 ? "0" : "") + std::to_string(row);
            ASSERT_EQ(expected, str_chunk->GetString(j));
            ++seen;
        }
    }
    ASSERT_EQ(40, seen);
}

TEST_F(PageFilteredRowGroupReaderTest, NormalizesSlicedBatchOffsets) {
    std::string file_name = dir_->Str() + "/normalized_sliced_offsets.parquet";
    std::shared_ptr<arrow::StructArray> data = MakeSequentialIntData(60);
    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/60);

    std::shared_ptr<arrow::Schema> read_schema =
        arrow::schema({arrow::field("val", arrow::int32())});
    std::shared_ptr<Predicate> predicate = PredicateBuilder::GreaterOrEqual(
        /*field_index=*/0, /*field_name=*/"val", FieldType::INT, Literal(20));

    std::shared_ptr<arrow::ChunkedArray> result;
    ReadWithPredicateImpl(file_name, read_schema, predicate, &result, /*batch_size=*/7);
    ASSERT_TRUE(result);
    ASSERT_EQ(40, result->length());
    ASSERT_GT(result->num_chunks(), 1);
}

/// Test: end-to-end page-filtered read produces correct results when using page-level PreBuffer.
///
/// This exercises the full path: ComputePageRanges → PreBufferRanges → CachedInputStream →
/// ReadFilteredRowGroup with page_ranges.
TEST_F(PageFilteredRowGroupReaderTest, EndToEndPageLevelPreBuffer) {
    std::string file_name = dir_->Str() + "/e2e_page_prebuffer.parquet";
    auto data = MakeSequentialIntData(100);
    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100);

    // Read via the standard ParquetFileBatchReader path (page index enabled)
    auto read_schema = arrow::schema({arrow::field("val", arrow::int32())});
    auto predicate = PredicateBuilder::Equal(
        /*field_index=*/0, /*field_name=*/"val", FieldType::INT, Literal(55));

    // Use small batch_size to verify batched consumption of page-filtered results
    std::shared_ptr<arrow::ChunkedArray> result;
    ReadWithPredicateImpl(file_name, read_schema, predicate, &result, /*batch_size=*/3);
    ASSERT_TRUE(result);
    // Page 5 (rows 50-59) matches, should return 10 rows
    ASSERT_EQ(10, result->length());

    // Verify actual values across chunks
    int64_t offset = 0;
    for (int i = 0; i < result->num_chunks(); ++i) {
        auto struct_arr = std::dynamic_pointer_cast<arrow::StructArray>(result->chunk(i));
        ASSERT_TRUE(struct_arr);
        auto val_arr = std::dynamic_pointer_cast<arrow::Int32Array>(struct_arr->field(0));
        for (int64_t j = 0; j < val_arr->length(); ++j) {
            ASSERT_EQ(50 + offset, val_arr->Value(j));
            ++offset;
        }
    }
    ASSERT_EQ(10, offset);
}

/// Test: ComputePageRanges with dictionary encoding produces correct chunk_end.
///
/// When dictionary encoding is enabled, the column chunk layout is:
///   [Dictionary Page] [Data Page 0] [Data Page 1] ... [Data Page N]
/// And total_compressed_size covers the entire chunk starting from dictionary_page_offset.
///
/// The bug: chunk_end = data_page_offset + total_compressed_size is wrong because
/// total_compressed_size already includes the dictionary page size. The correct
/// chunk_end should be dictionary_page_offset + total_compressed_size.
///
/// This test verifies that:
/// 1. No range exceeds the true chunk boundary (overshoot regression).
/// 2. At least one non-dictionary data-page range is present (not truncated).
/// 3. The maximum range_end equals true_chunk_end when requesting all rows.
/// 4. End-to-end reads with page-level filtering return correct query results.
TEST_F(PageFilteredRowGroupReaderTest, ComputePageRangesWithDictionaryEncoding) {
    std::string file_name = dir_->Str() + "/compute_ranges_dict.parquet";

    // Use low-cardinality data to ensure dictionary encoding is actually used.
    // 100 rows with values cycling through 0..9 → dictionary will have 10 entries.
    arrow::Int32Builder val_builder;
    ASSERT_TRUE(val_builder.Reserve(100).ok());
    for (int32_t i = 0; i < 100; ++i) {
        val_builder.UnsafeAppend(i % 10);
    }
    auto val_array = val_builder.Finish().ValueOrDie();
    auto field = arrow::field("val", arrow::int32());
    auto struct_array = arrow::StructArray::Make({val_array}, {field}).ValueOrDie();

    // Write with dictionary encoding enabled and 1 row per page.
    // Each page has min==max==val for that row, enabling precise page-level skipping.
    WriteTestFile(file_name, struct_array, /*write_batch_size=*/1,
                  /*max_row_group_length=*/100, /*enable_dictionary=*/true);

    // Open the file and verify metadata confirms dictionary page presence
    ASSERT_OK_AND_ASSIGN(std::shared_ptr<InputStream> in, fs_->Open(file_name));
    ASSERT_OK_AND_ASSIGN(uint64_t length, in->Length());
    auto in_stream = std::make_shared<ArrowInputStreamAdapter>(in, length, arrow_pool_);
    auto parquet_reader = ::parquet::ParquetFileReader::Open(in_stream);
    ASSERT_TRUE(parquet_reader);

    auto file_metadata = parquet_reader->metadata();
    auto rg_metadata = file_metadata->RowGroup(0);
    auto col_chunk = rg_metadata->ColumnChunk(0);

    // Precondition: dictionary page must exist for this test to be meaningful
    ASSERT_TRUE(col_chunk->has_dictionary_page());

    int64_t dict_offset = col_chunk->dictionary_page_offset();
    int64_t data_page_offset = col_chunk->data_page_offset();
    int64_t total_compressed_size = col_chunk->total_compressed_size();

    // The true chunk end is dict_offset + total_compressed_size
    int64_t true_chunk_end = dict_offset + total_compressed_size;
    // The buggy chunk end would be data_page_offset + total_compressed_size
    int64_t buggy_chunk_end = data_page_offset + total_compressed_size;

    // Sanity: dict page is before data pages, so buggy end > true end
    ASSERT_LT(dict_offset, data_page_offset);
    ASSERT_GT(buggy_chunk_end, true_chunk_end);
    // Now call ComputePageRanges with all rows matching
    RowRanges row_ranges;
    row_ranges.Add(RowRanges::Range(0, 99));

    auto page_index_reader = parquet_reader->GetPageIndexReader();
    ASSERT_TRUE(page_index_reader);
    auto rg_page_index_reader = page_index_reader->RowGroup(0);
    auto ranges = PageFilteredRowGroupReader::ComputePageRanges(
        TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/false, /*ranges=*/row_ranges),
        /*column_indices=*/{0}, /*row_group_page_index_reader=*/rg_page_index_reader,
        parquet_reader.get());

    ASSERT_FALSE(ranges.empty());

    // --- Check 1: No range should extend beyond the true chunk end ---
    // With the bug, the last data page's range would use chunk_end = data_page_offset +
    // total_compressed_size, which overshoots by the dictionary page size.
    for (auto& range : ranges) {
        int64_t range_end = range.offset + range.length;
        ASSERT_LE(range_end, true_chunk_end);
    }

    // --- Check 2: At least one non-dictionary data-page range is present ---
    // Guards against truncation: if only the dictionary range is returned, the test
    // would still pass the overshoot check but miss that data pages are lost.
    int data_page_range_count = 0;
    for (const auto& range : ranges) {
        if (range.offset >= data_page_offset) {
            ++data_page_range_count;
        }
    }
    ASSERT_GE(data_page_range_count, 1);

    // --- Check 3: Maximum range_end equals true_chunk_end when requesting all rows ---
    int64_t max_range_end = 0;
    for (const auto& range : ranges) {
        int64_t range_end = range.offset + range.length;
        max_range_end = std::max(max_range_end, range_end);
    }
    ASSERT_EQ(max_range_end, true_chunk_end);

    // --- Check 4: No range exceeds file size ---
    for (const auto& range : ranges) {
        ASSERT_LE(range.offset + range.length, static_cast<int64_t>(length));
    }

    // --- End-to-end check 1: read all rows (no predicate filtering) ---
    // Verifies that reading a dictionary-encoded file with page index enabled
    // returns all 100 rows with correct values.
    auto read_schema = arrow::schema({field});
    auto predicate_all = PredicateBuilder::GreaterOrEqual(
        /*field_index=*/0, /*field_name=*/"val", FieldType::INT, Literal(0));
    std::shared_ptr<arrow::ChunkedArray> result_all;
    ReadWithPredicateImpl(file_name, read_schema, predicate_all, &result_all);
    ASSERT_TRUE(result_all);
    ASSERT_EQ(100, result_all->length());

    // --- End-to-end check 2: full range query with page level skipping ---
    // Build expected array: val = i % 10 for i in [0, 100), wrapped in a struct.
    // Concatenate all chunks and compare with Equals
    auto actual_struct_arr = arrow::Concatenate(result_all->chunks()).ValueOrDie();
    ASSERT_TRUE(actual_struct_arr->Equals(struct_array));

    // --- End-to-end check 3: partial-row query with page-level skipping ---
    // Predicate val >= 7 skips pages where val < 7, keeping only val in {7,8,9}.
    // Out of 100 rows, 30 rows satisfy val >= 7 (3 per cycle × 10 cycles).
    auto predicate_partial = PredicateBuilder::GreaterOrEqual(
        /*field_index=*/0, /*field_name=*/"val", FieldType::INT, Literal(7));
    std::shared_ptr<arrow::ChunkedArray> result_partial;
    ReadWithPredicateImpl(file_name, read_schema, predicate_partial, &result_partial);
    ASSERT_TRUE(result_partial);

    // Build expected StructArray and compare with Equals
    arrow::Int32Builder expected_builder;
    ASSERT_TRUE(expected_builder.Reserve(30).ok());
    for (int32_t i = 0; i < 100; ++i) {
        if (i % 10 >= 7) {
            expected_builder.UnsafeAppend(i % 10);
        }
    }
    auto expected_array = expected_builder.Finish().ValueOrDie();
    auto expected_struct = arrow::StructArray::Make({expected_array}, {field}).ValueOrDie();
    auto partial_concat = arrow::Concatenate(result_partial->chunks()).ValueOrDie();
    ASSERT_TRUE(partial_concat->Equals(expected_struct));
}
/// Helper: build an Int32Array with sequential values 0..N-1.
static std::shared_ptr<arrow::Array> MakeIdColumn(int32_t num_rows) {
    arrow::Int32Builder id_builder;
    EXPECT_TRUE(id_builder.Reserve(num_rows).ok());
    for (int32_t i = 0; i < num_rows; ++i) {
        id_builder.UnsafeAppend(i);
    }
    return id_builder.Finish().ValueOrDie();
}

/// Helper: build a struct<x: int32, y: int32> array (without id column).
/// x[i] = i * 100, y[i] = i * 100 + 1, for i in [0, N).
static std::shared_ptr<arrow::StructArray> MakeNestedStructData(int32_t num_rows) {
    arrow::Int32Builder x_builder, y_builder;
    EXPECT_TRUE(x_builder.Reserve(num_rows).ok());
    EXPECT_TRUE(y_builder.Reserve(num_rows).ok());
    for (int32_t i = 0; i < num_rows; ++i) {
        x_builder.UnsafeAppend(i * 100);
        y_builder.UnsafeAppend(i * 100 + 1);
    }
    auto x_array = x_builder.Finish().ValueOrDie();
    auto y_array = y_builder.Finish().ValueOrDie();

    auto field_x = arrow::field("x", arrow::int32());
    auto field_y = arrow::field("y", arrow::int32());
    return arrow::StructArray::Make({x_array, y_array}, {field_x, field_y}).ValueOrDie();
}

/// Test: rowgroup-level filtering on a file with nested struct columns.
///
/// Schema: { id: int32, info: struct<x: int32, y: int32> }
/// Parquet leaf columns: [id=0, info.x=1, info.y=2]
/// 100 rows, 10 per page, 2 row groups.
/// Predicate: id >= 70 → page 0-7 skipped, paged 8-9 read → 30 rows expected.
/// The read schema requests both "id" and "info" columns.
TEST_F(PageFilteredRowGroupReaderTest, NestedStructColumnPageFilter) {
    std::string file_name = dir_->Str() + "/nested_struct_filter.parquet";

    auto field_x = arrow::field("x", arrow::int32());
    auto field_y = arrow::field("y", arrow::int32());
    auto field_id = arrow::field("id", arrow::int32());
    auto field_info = arrow::field("info", arrow::struct_({field_x, field_y}));

    auto id_array = MakeIdColumn(100);
    auto info_array = MakeNestedStructData(100);
    auto data =
        arrow::StructArray::Make({id_array, info_array}, {field_id, field_info}).ValueOrDie();
    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50);

    auto read_schema = arrow::schema({field_id, field_info});

    auto predicate = PredicateBuilder::GreaterOrEqual(
        /*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(70));

    std::shared_ptr<arrow::ChunkedArray> result;
    ReadWithPredicateImpl(file_name, read_schema, predicate, &result);

    // Should get rows 70-99 = 30 rows
    ASSERT_TRUE(result);
    ASSERT_EQ(30, result->length());

    // Build expected result: rows 50-99 from the original data
    auto expected = data->Slice(70, 30);
    ASSERT_TRUE(expected->Equals(result->chunk(0)));
}

/// Test: Page-level filtering reading only the predicate column (no nested column in read schema).
///
/// This verifies that when reading only the "id" column (without the nested struct)
///
/// Schema: { id: int32, info: struct<x: int32, y: int32> }
/// Read schema: { id: int32 }
/// Predicate on "id": id >= 70. Page-level filtering active → rows 70-99 (30 rows).
TEST_F(PageFilteredRowGroupReaderTest, NestedStructColumnOnlyReadIdField) {
    std::string file_name = dir_->Str() + "/nested_struct_only_nested.parquet";

    auto field_x = arrow::field("x", arrow::int32());
    auto field_y = arrow::field("y", arrow::int32());
    auto field_id = arrow::field("id", arrow::int32());
    auto field_info = arrow::field("info", arrow::struct_({field_x, field_y}));

    auto id_array = MakeIdColumn(100);
    auto info_array = MakeNestedStructData(100);
    auto data =
        arrow::StructArray::Make({id_array, info_array}, {field_id, field_info}).ValueOrDie();
    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50);

    // Read "id" column only
    auto read_schema = arrow::schema({field_id});

    // Predicate is on "id" (field_index=0 in file schema)
    auto predicate = PredicateBuilder::GreaterOrEqual(
        /*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(70));

    std::shared_ptr<arrow::ChunkedArray> result;
    ReadWithPredicateImpl(file_name, read_schema, predicate, &result);

    // Should get rows 70-99 = 30 rows
    ASSERT_TRUE(result);
    ASSERT_EQ(30, result->length());

    auto result_struct = std::dynamic_pointer_cast<arrow::StructArray>(result->chunk(0));
    ASSERT_TRUE(result_struct);
    ASSERT_TRUE(data->field(0)->Slice(70, 30)->Equals(result_struct->field(0)));
}

/// Helper: build a list<item: int32> array (without id column).
/// tags[i] = [i*10, i*10+1], for i in [0, N).
static std::shared_ptr<arrow::Array> MakeListColumnData(int32_t num_rows) {
    auto value_builder = std::make_shared<arrow::Int32Builder>();
    arrow::ListBuilder list_builder(arrow::default_memory_pool(), value_builder);
    for (int32_t i = 0; i < num_rows; ++i) {
        EXPECT_TRUE(list_builder.Append().ok());
        EXPECT_TRUE(value_builder->Append(i * 10).ok());
        EXPECT_TRUE(value_builder->Append(i * 10 + 1).ok());
    }
    return list_builder.Finish().ValueOrDie();
}

/// Helper: build a map<utf8, int32> array (without id column).
/// props[i] = {"k_i": i * 100}, for i in [0, N).
static std::shared_ptr<arrow::Array> MakeMapColumnData(int32_t num_rows) {
    auto key_builder = std::make_shared<arrow::StringBuilder>();
    auto value_builder = std::make_shared<arrow::Int32Builder>();
    arrow::MapBuilder map_builder(arrow::default_memory_pool(), key_builder, value_builder);
    for (int32_t i = 0; i < num_rows; ++i) {
        EXPECT_TRUE(map_builder.Append().ok());
        std::string key = "k_" + std::to_string(i);
        EXPECT_TRUE(key_builder->Append(key).ok());
        EXPECT_TRUE(value_builder->Append(i * 100).ok());
    }
    return map_builder.Finish().ValueOrDie();
}

/// Test: rowgroup-level filtering on a file with a list column.
///
/// Schema: { id: int32, tags: list<item: int32> }
/// 100 rows, 10 per page, 2 row groups.
/// Predicate: id >= 70 → page 0-7 skipped, page 8-9 read → 30 rows expected.
TEST_F(PageFilteredRowGroupReaderTest, NestedListColumnPageFilter) {
    std::string file_name = dir_->Str() + "/nested_list_filter.parquet";

    auto field_id = arrow::field("id", arrow::int32());
    auto field_tags = arrow::field("tags", arrow::list(arrow::field("item", arrow::int32())));

    auto id_array = MakeIdColumn(100);
    auto tags_array = MakeListColumnData(100);
    auto data =
        arrow::StructArray::Make({id_array, tags_array}, {field_id, field_tags}).ValueOrDie();
    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50);

    auto read_schema = arrow::schema({field_id, field_tags});

    auto predicate = PredicateBuilder::GreaterOrEqual(
        /*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(70));

    std::shared_ptr<arrow::ChunkedArray> result;
    ReadWithPredicateImpl(file_name, read_schema, predicate, &result);

    ASSERT_TRUE(result);
    ASSERT_EQ(30, result->length());

    // Build expected result: rows 70-99 from the original data
    auto expected = data->Slice(70, 30);
    ASSERT_TRUE(expected->Equals(result->chunk(0)));
}

/// Test: rowgroup filtering on a file with a map column.
///
/// Schema: { id: int32, props: map<utf8, int32> }
/// 100 rows, 10 per page, 2 row groups.
/// Predicate: id >= 70 → page 0-7 skipped, page 8-9 read → 30 rows expected.
TEST_F(PageFilteredRowGroupReaderTest, NestedMapColumnPageFilter) {
    std::string file_name = dir_->Str() + "/nested_map_filter.parquet";

    auto field_id = arrow::field("id", arrow::int32());
    auto field_props = arrow::field("props", arrow::map(arrow::utf8(), arrow::int32()));

    auto id_array = MakeIdColumn(100);
    auto props_array = MakeMapColumnData(100);
    auto data =
        arrow::StructArray::Make({id_array, props_array}, {field_id, field_props}).ValueOrDie();
    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50);

    auto read_schema = arrow::schema({field_id, field_props});

    auto predicate = PredicateBuilder::GreaterOrEqual(
        /*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(70));

    std::shared_ptr<arrow::ChunkedArray> result;
    ReadWithPredicateImpl(file_name, read_schema, predicate, &result);

    ASSERT_TRUE(result);
    ASSERT_EQ(30, result->length());

    // Build expected result: rows 70-99 from the original data
    auto expected = data->Slice(70, 30);
    ASSERT_TRUE(expected->Equals(result->chunk(0)));
}

/// Test: rowgroup-level filtering with multiple adjacent nested columns (struct + list).
///
/// Schema: { id: int32, info: struct<x: int32, y: int32>, tags: list<item: int32> }
/// This tests the boundary handling when two nested fields are adjacent in the schema.
/// Predicate: id >= 70 → page 0-7 skipped, page 8-9 read → 30 rows expected.
TEST_F(PageFilteredRowGroupReaderTest, MultipleAdjacentNestedColumns) {
    std::string file_name = dir_->Str() + "/multi_nested.parquet";

    auto field_x = arrow::field("x", arrow::int32());
    auto field_y = arrow::field("y", arrow::int32());
    auto field_id = arrow::field("id", arrow::int32());
    auto field_info = arrow::field("info", arrow::struct_({field_x, field_y}));
    auto field_tags = arrow::field("tags", arrow::list(arrow::field("item", arrow::int32())));

    auto id_array = MakeIdColumn(100);
    auto info_array = MakeNestedStructData(100);
    auto tags_array = MakeListColumnData(100);
    auto data = arrow::StructArray::Make({id_array, info_array, tags_array},
                                         {field_id, field_info, field_tags})
                    .ValueOrDie();

    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50);

    auto read_schema = arrow::schema({field_id, field_info, field_tags});
    auto predicate = PredicateBuilder::GreaterOrEqual(
        /*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(70));

    std::shared_ptr<arrow::ChunkedArray> result;
    ReadWithPredicateImpl(file_name, read_schema, predicate, &result);

    ASSERT_TRUE(result);
    ASSERT_EQ(30, result->length());

    // Build expected result: rows 70-99 from the original data
    auto expected = data->Slice(70, 30);
    ASSERT_TRUE(expected->Equals(result->chunk(0)));
}
/// Test: bitmap hits all pages of a subset of row groups (no predicate).
///
/// 200 rows, 10 rows per page, 100 rows per row group → 2 row groups.
/// RG0: rows 0-99, RG1: rows 100-199.
/// Bitmap: {0..99} hits all pages of RG0, RG1 is excluded entirely.
/// Expected: 100 rows (0-99).
TEST_F(PageFilteredRowGroupReaderTest, BitmapAllPagesSomeRowGroups) {
    std::string file_name = dir_->Str() + "/bitmap_all_pages_rg.parquet";
    auto data = MakeSequentialIntData(200);
    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100);

    RoaringBitmap32 bitmap;
    bitmap.AddRange(0, 100);  // hits all of RG0

    auto read_schema = arrow::schema({arrow::field("val", arrow::int32())});
    std::shared_ptr<arrow::ChunkedArray> result;
    ReadWithPredicateAndBitmapImpl(file_name, read_schema, /*predicate=*/nullptr, bitmap, &result);
    ASSERT_TRUE(result);
    ASSERT_EQ(100, result->length());

    auto flat = arrow::Concatenate(result->chunks()).ValueOrDie();
    auto struct_arr = std::dynamic_pointer_cast<arrow::StructArray>(flat);
    ASSERT_TRUE(struct_arr);
    auto val_arr = std::dynamic_pointer_cast<arrow::Int32Array>(struct_arr->field(0));
    for (int32_t i = 0; i < 100; ++i) {
        ASSERT_EQ(i, val_arr->Value(i));
    }
}

/// Test: OffsetIndex lets the reader jump directly to selected data pages.
///
/// The bitmap selects rows from page 1 and page 8. Synchronous positional reads issued while
/// consuming the row group must not cover any unselected page header. Before direct page jumps,
/// SerializedPageReader sequentially Peeked every page header and this assertion failed.
TEST_F(PageFilteredRowGroupReaderTest, DirectOffsetIndexJumpDoesNotReadUnselectedPageHeaders) {
    std::string file_name = dir_->Str() + "/direct_offset_index_jump.parquet";
    auto data = MakeSequentialIntData(100);
    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100);

    std::vector<::parquet::PageLocation> page_locations;
    {
        ASSERT_OK_AND_ASSIGN(std::shared_ptr<InputStream> in, fs_->Open(file_name));
        ASSERT_OK_AND_ASSIGN(int64_t length, in->Length());
        auto in_stream = std::make_shared<ArrowInputStreamAdapter>(in, length, arrow_pool_);
        auto parquet_reader = ::parquet::ParquetFileReader::Open(in_stream);
        ASSERT_TRUE(parquet_reader);

        auto page_index_reader = parquet_reader->GetPageIndexReader();
        ASSERT_TRUE(page_index_reader);
        auto row_group_page_index = page_index_reader->RowGroup(0);
        ASSERT_TRUE(row_group_page_index);
        auto offset_index = row_group_page_index->GetOffsetIndex(0);
        ASSERT_TRUE(offset_index);
        page_locations = offset_index->page_locations();
    }

    ASSERT_EQ(10, page_locations.size());
    for (int32_t page_idx = 0; page_idx < 10; ++page_idx) {
        ASSERT_EQ(page_idx * 10, page_locations[page_idx].first_row_index);
    }

    RoaringBitmap32 bitmap;
    bitmap.Add(15);  // page 1
    bitmap.Add(85);  // page 8

    std::map<std::string, std::string> options;
    options[PARQUET_READ_ROW_RANGES_COALESCE_HOLE_SIZE_LIMIT] = "0";
    options[PARQUET_READ_CACHE_OPTION_HOLE_SIZE_LIMIT] = "0";

    auto read_schema = arrow::schema({arrow::field("val", arrow::int32())});
    std::shared_ptr<arrow::ChunkedArray> result;
    std::vector<arrow::io::ReadRange> read_at_ranges;
    ReadWithPredicateAndBitmapImpl(file_name, read_schema, /*predicate=*/nullptr, bitmap, &result,
                                   options, /*batch_size=*/1024, &read_at_ranges);

    ASSERT_TRUE(result);
    ASSERT_EQ(2, result->length());
    auto flat = arrow::Concatenate(result->chunks()).ValueOrDie();
    auto struct_arr = std::dynamic_pointer_cast<arrow::StructArray>(flat);
    ASSERT_TRUE(struct_arr);
    auto val_arr = std::dynamic_pointer_cast<arrow::Int32Array>(struct_arr->field(0));
    ASSERT_TRUE(val_arr);
    ASSERT_EQ(15, val_arr->Value(0));
    ASSERT_EQ(85, val_arr->Value(1));

    AssertSelectedPagesRead(page_locations, /*selected_pages=*/{1, 8}, read_at_ranges);
    AssertUnselectedPageHeadersNotRead(page_locations, /*selected_pages=*/{1, 8}, read_at_ranges);
}

/// Dictionary pages are column-local. Each leaf must load its own dictionary before the
/// PageReader jumps directly to a selected late data page.
TEST_F(PageFilteredRowGroupReaderTest, DirectOffsetIndexJumpReadsEachLeafDictionary) {
    std::string file_name = dir_->Str() + "/direct_offset_index_jump_dictionary.parquet";

    arrow::Int32Builder a_builder;
    arrow::Int32Builder b_builder;
    ASSERT_TRUE(a_builder.Reserve(100).ok());
    ASSERT_TRUE(b_builder.Reserve(100).ok());
    for (int32_t i = 0; i < 100; ++i) {
        a_builder.UnsafeAppend(i % 7);
        b_builder.UnsafeAppend(i % 5);
    }
    auto a_array = a_builder.Finish().ValueOrDie();
    auto b_array = b_builder.Finish().ValueOrDie();
    auto field_a = arrow::field("a", arrow::int32());
    auto field_b = arrow::field("b", arrow::int32());
    auto data = arrow::StructArray::Make({a_array, b_array}, {field_a, field_b}).ValueOrDie();
    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100,
                  /*enable_dictionary=*/true);

    std::vector<std::vector<::parquet::PageLocation>> page_locations(2);
    std::vector<arrow::io::ReadRange> dictionary_ranges;
    {
        ASSERT_OK_AND_ASSIGN(std::shared_ptr<InputStream> in, fs_->Open(file_name));
        ASSERT_OK_AND_ASSIGN(int64_t length, in->Length());
        auto in_stream = std::make_shared<ArrowInputStreamAdapter>(in, length, arrow_pool_);
        auto parquet_reader = ::parquet::ParquetFileReader::Open(in_stream);
        ASSERT_TRUE(parquet_reader);

        auto row_group = parquet_reader->metadata()->RowGroup(0);
        auto page_index_reader = parquet_reader->GetPageIndexReader();
        ASSERT_TRUE(page_index_reader);
        auto row_group_page_index = page_index_reader->RowGroup(0);
        ASSERT_TRUE(row_group_page_index);

        RowRanges selected_rows;
        selected_rows.Add(RowRanges::Range(95, 95));
        auto ranges = PageFilteredRowGroupReader::ComputePageRanges(
            TargetRowGroup(/*rg_index=*/0, /*is_partially_matched=*/true,
                           /*ranges=*/selected_rows),
            /*column_indices=*/{0, 1}, /*row_group_page_index_reader=*/row_group_page_index,
            parquet_reader.get());

        for (int32_t col_idx = 0; col_idx < 2; ++col_idx) {
            auto column_chunk = row_group->ColumnChunk(col_idx);
            ASSERT_TRUE(column_chunk->has_dictionary_page());
            const int64_t dictionary_offset = column_chunk->dictionary_page_offset();
            const int64_t data_page_offset = column_chunk->data_page_offset();
            ASSERT_LT(dictionary_offset, data_page_offset);

            auto offset_index = row_group_page_index->GetOffsetIndex(col_idx);
            ASSERT_TRUE(offset_index);
            page_locations[col_idx] = offset_index->page_locations();
            ASSERT_EQ(10, page_locations[col_idx].size());

            const auto& selected_page = page_locations[col_idx][9];
            const auto contains_range = [&ranges](int64_t offset, int64_t size) {
                return std::any_of(ranges.begin(), ranges.end(),
                                   [offset, size](const arrow::io::ReadRange& range) {
                                       return range.offset == offset && range.length == size;
                                   });
            };
            ASSERT_TRUE(contains_range(dictionary_offset, data_page_offset - dictionary_offset));
            ASSERT_TRUE(contains_range(selected_page.offset, selected_page.compressed_page_size));
            dictionary_ranges.push_back({dictionary_offset, data_page_offset - dictionary_offset});

            auto page_reader = parquet_reader->RowGroup(0)->GetColumnPageReader(col_idx);
            ASSERT_TRUE(page_reader);
            std::vector<::parquet::Encoding::type> data_page_encodings;
            while (std::shared_ptr<::parquet::Page> page = page_reader->NextPage()) {
                if (page->type() == ::parquet::PageType::DATA_PAGE ||
                    page->type() == ::parquet::PageType::DATA_PAGE_V2) {
                    data_page_encodings.push_back(
                        std::static_pointer_cast<::parquet::DataPage>(page)->encoding());
                }
            }
            ASSERT_EQ(page_locations[col_idx].size(), data_page_encodings.size());
            ASSERT_TRUE(data_page_encodings[9] == ::parquet::Encoding::PLAIN_DICTIONARY ||
                        data_page_encodings[9] == ::parquet::Encoding::RLE_DICTIONARY);
        }
    }

    RoaringBitmap32 bitmap;
    bitmap.Add(95);  // Last data page; every leaf still needs its dictionary first.

    std::map<std::string, std::string> options;
    options[PARQUET_READ_ROW_RANGES_COALESCE_HOLE_SIZE_LIMIT] = "0";
    options[PARQUET_READ_CACHE_OPTION_HOLE_SIZE_LIMIT] = "0";

    auto read_schema = arrow::schema({field_a, field_b});
    std::shared_ptr<arrow::ChunkedArray> result;
    std::vector<arrow::io::ReadRange> read_at_ranges;
    ReadWithPredicateAndBitmapImpl(file_name, read_schema, /*predicate=*/nullptr, bitmap, &result,
                                   options, /*batch_size=*/1024, &read_at_ranges);

    ASSERT_TRUE(result);
    ASSERT_EQ(1, result->length());
    auto flat = arrow::Concatenate(result->chunks()).ValueOrDie();
    auto struct_arr = std::dynamic_pointer_cast<arrow::StructArray>(flat);
    ASSERT_TRUE(struct_arr);
    auto result_a = std::dynamic_pointer_cast<arrow::Int32Array>(struct_arr->field(0));
    auto result_b = std::dynamic_pointer_cast<arrow::Int32Array>(struct_arr->field(1));
    ASSERT_TRUE(result_a);
    ASSERT_TRUE(result_b);
    ASSERT_EQ(95 % 7, result_a->Value(0));
    ASSERT_EQ(95 % 5, result_b->Value(0));

    for (size_t col_idx = 0; col_idx < page_locations.size(); ++col_idx) {
        AssertReadRangeCovered(read_at_ranges, dictionary_ranges[col_idx].offset,
                               dictionary_ranges[col_idx].length);
        AssertSelectedPagesRead(page_locations[col_idx], /*selected_pages=*/{9}, read_at_ranges);
        const auto& leaf_page_locations = page_locations[col_idx];
        AssertUnselectedPageHeadersNotRead(leaf_page_locations, /*selected_pages=*/{9},
                                           read_at_ranges);
    }
}

/// A column can switch from dictionary encoding to PLAIN after the dictionary reaches its size
/// limit. A single direct read plan must decode both kinds of selected data pages correctly.
TEST_F(PageFilteredRowGroupReaderTest, DirectOffsetIndexJumpSupportsDictionaryFallbackToPlain) {
    std::string file_name = dir_->Str() + "/direct_offset_index_dictionary_fallback.parquet";
    auto data = MakeSequentialIntData(1000);
    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/1000,
                  /*enable_dictionary=*/true, /*data_page_size=*/1,
                  /*enable_page_index=*/true, ::parquet::ParquetDataPageVersion::V1,
                  /*dictionary_page_size_limit=*/256);

    std::vector<::parquet::PageLocation> page_locations;
    std::vector<::parquet::Encoding::type> data_page_encodings;
    int64_t dictionary_offset = 0;
    int64_t first_data_page_offset = 0;
    {
        ASSERT_OK_AND_ASSIGN(std::shared_ptr<InputStream> in, fs_->Open(file_name));
        ASSERT_OK_AND_ASSIGN(int64_t length, in->Length());
        auto in_stream = std::make_shared<ArrowInputStreamAdapter>(in, length, arrow_pool_);
        auto parquet_reader = ::parquet::ParquetFileReader::Open(in_stream);
        ASSERT_TRUE(parquet_reader);

        auto column_chunk = parquet_reader->metadata()->RowGroup(0)->ColumnChunk(0);
        ASSERT_TRUE(column_chunk->has_dictionary_page());
        dictionary_offset = column_chunk->dictionary_page_offset();

        auto page_index_reader = parquet_reader->GetPageIndexReader();
        ASSERT_TRUE(page_index_reader);
        auto row_group_page_index = page_index_reader->RowGroup(0);
        ASSERT_TRUE(row_group_page_index);
        auto offset_index = row_group_page_index->GetOffsetIndex(0);
        ASSERT_TRUE(offset_index);
        page_locations = offset_index->page_locations();
        ASSERT_FALSE(page_locations.empty());
        first_data_page_offset = page_locations.front().offset;
        ASSERT_LT(dictionary_offset, first_data_page_offset);

        auto page_reader = parquet_reader->RowGroup(0)->GetColumnPageReader(0);
        ASSERT_TRUE(page_reader);
        while (std::shared_ptr<::parquet::Page> page = page_reader->NextPage()) {
            if (page->type() == ::parquet::PageType::DATA_PAGE ||
                page->type() == ::parquet::PageType::DATA_PAGE_V2) {
                data_page_encodings.push_back(
                    std::static_pointer_cast<::parquet::DataPage>(page)->encoding());
            }
        }
    }
    ASSERT_EQ(page_locations.size(), data_page_encodings.size());

    int32_t dictionary_page_idx = -1;
    int32_t plain_page_idx = -1;
    for (int32_t page_idx = 0; page_idx < static_cast<int32_t>(data_page_encodings.size());
         ++page_idx) {
        const auto encoding = data_page_encodings[page_idx];
        if (dictionary_page_idx < 0 && (encoding == ::parquet::Encoding::PLAIN_DICTIONARY ||
                                        encoding == ::parquet::Encoding::RLE_DICTIONARY)) {
            dictionary_page_idx = page_idx;
        }
        if (encoding == ::parquet::Encoding::PLAIN) {
            plain_page_idx = page_idx;
        }
    }
    ASSERT_GE(dictionary_page_idx, 0);
    ASSERT_GT(plain_page_idx, dictionary_page_idx);

    const auto dictionary_row =
        static_cast<int32_t>(page_locations[dictionary_page_idx].first_row_index);
    const auto plain_row = static_cast<int32_t>(page_locations[plain_page_idx].first_row_index);
    RoaringBitmap32 bitmap;
    bitmap.Add(dictionary_row);
    bitmap.Add(plain_row);

    std::map<std::string, std::string> options;
    options[PARQUET_READ_ROW_RANGES_COALESCE_HOLE_SIZE_LIMIT] = "0";
    options[PARQUET_READ_CACHE_OPTION_HOLE_SIZE_LIMIT] = "0";

    auto read_schema = arrow::schema({arrow::field("val", arrow::int32())});
    std::shared_ptr<arrow::ChunkedArray> result;
    std::vector<arrow::io::ReadRange> read_at_ranges;
    ReadWithPredicateAndBitmapImpl(file_name, read_schema, /*predicate=*/nullptr, bitmap, &result,
                                   options, /*batch_size=*/1024, &read_at_ranges);

    ASSERT_TRUE(result);
    ASSERT_EQ(2, result->length());
    auto flat = arrow::Concatenate(result->chunks()).ValueOrDie();
    auto struct_arr = std::dynamic_pointer_cast<arrow::StructArray>(flat);
    ASSERT_TRUE(struct_arr);
    auto values = std::dynamic_pointer_cast<arrow::Int32Array>(struct_arr->field(0));
    ASSERT_TRUE(values);
    ASSERT_EQ(dictionary_row, values->Value(0));
    ASSERT_EQ(plain_row, values->Value(1));

    AssertReadRangeCovered(read_at_ranges, dictionary_offset,
                           first_data_page_offset - dictionary_offset);
    AssertSelectedPagesRead(page_locations, {dictionary_page_idx, plain_page_idx}, read_at_ranges);
    AssertUnselectedPageHeadersNotRead(page_locations, {dictionary_page_idx, plain_page_idx},
                                       read_at_ranges);
}

/// Empty row selection may read OffsetIndex metadata, but must not read dictionary/data pages.
TEST_F(PageFilteredRowGroupReaderTest, DictionaryEmptySelectionDoesNotReadPages) {
    std::string file_name = dir_->Str() + "/dictionary_empty_bitmap.parquet";
    auto data = MakeSequentialIntData(100);
    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100,
                  /*enable_dictionary=*/true);

    ASSERT_OK_AND_ASSIGN(std::shared_ptr<InputStream> input, fs_->Open(file_name));
    auto tracking_input = std::make_shared<ReadAtTrackingInputStream>(input);
    ASSERT_OK_AND_ASSIGN(int64_t length, tracking_input->Length());
    auto in_stream = std::make_shared<ArrowInputStreamAdapter>(tracking_input, length, arrow_pool_);

    ::parquet::arrow::FileReaderBuilder builder;
    ASSERT_TRUE(builder.Open(in_stream).ok());
    builder.memory_pool(arrow_pool_.get());
    auto arrow_file_reader_result = builder.Build();
    ASSERT_TRUE(arrow_file_reader_result.ok()) << arrow_file_reader_result.status().ToString();
    std::unique_ptr<::parquet::arrow::FileReader> arrow_file_reader =
        std::move(arrow_file_reader_result).ValueOrDie();

    auto column_chunk =
        arrow_file_reader->parquet_reader()->metadata()->RowGroup(0)->ColumnChunk(0);
    ASSERT_TRUE(column_chunk->has_dictionary_page());
    const int64_t column_chunk_offset = column_chunk->dictionary_page_offset();
    const int64_t column_chunk_end = column_chunk_offset + column_chunk->total_compressed_size();
    ASSERT_GE(column_chunk_offset, 0);
    ASSERT_GT(column_chunk_end, column_chunk_offset);

    auto page_index_reader = arrow_file_reader->parquet_reader()->GetPageIndexReader();
    ASSERT_TRUE(page_index_reader);
    auto row_group_page_index = page_index_reader->RowGroup(0);
    ASSERT_TRUE(row_group_page_index);
    RowRanges empty_ranges;
    TargetRowGroup empty_target(/*rg_index=*/0, /*is_partially_matched=*/true,
                                /*ranges=*/empty_ranges);
    auto page_ranges = PageFilteredRowGroupReader::ComputePageRanges(
        empty_target, /*column_indices=*/{0}, /*row_group_page_index_reader=*/row_group_page_index,
        arrow_file_reader->parquet_reader());
    ASSERT_TRUE(page_ranges.empty());

    tracking_input->ClearReadAtRanges();
    ASSERT_OK_AND_ASSIGN(
        std::unique_ptr<arrow::RecordBatchReader> result_reader,
        PageFilteredRowGroupReader::ReadFilteredRowGroup(
            empty_target, /*column_indices=*/{0}, arrow::io::CacheOptions::Defaults(),
            /*pre_buffered=*/false, /*page_ranges=*/{}, /*max_chunksize=*/1024,
            /*row_group_page_index_reader=*/row_group_page_index, arrow_pool_,
            arrow_file_reader.get()));
    std::shared_ptr<arrow::RecordBatch> batch;
    ASSERT_TRUE(result_reader->ReadNext(&batch).ok());
    ASSERT_FALSE(batch);
    for (const auto& read_range : tracking_input->GetReadAtRanges()) {
        const int64_t read_end = read_range.offset + read_range.length;
        ASSERT_TRUE(read_end <= column_chunk_offset || read_range.offset >= column_chunk_end)
            << "empty selection read column chunk [" << column_chunk_offset << ", "
            << column_chunk_end << ") via positional range [" << read_range.offset << ", "
            << read_end << ")";
    }
}

/// The direct page plan must work for DATA_PAGE_V2 as well as DATA_PAGE_V1. Selecting adjacent
/// pages also verifies that the plan cursor advances exactly once per page.
TEST_F(PageFilteredRowGroupReaderTest, DirectOffsetIndexJumpDataPageV2AdjacentPages) {
    std::string file_name = dir_->Str() + "/direct_offset_index_jump_v2.parquet";
    auto data = MakeSequentialIntData(100);
    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100,
                  /*enable_dictionary=*/false, /*data_page_size=*/1,
                  /*enable_page_index=*/true, ::parquet::ParquetDataPageVersion::V2);

    std::vector<::parquet::PageLocation> page_locations;
    {
        ASSERT_OK_AND_ASSIGN(std::shared_ptr<InputStream> in, fs_->Open(file_name));
        ASSERT_OK_AND_ASSIGN(int64_t length, in->Length());
        auto in_stream = std::make_shared<ArrowInputStreamAdapter>(in, length, arrow_pool_);
        auto parquet_reader = ::parquet::ParquetFileReader::Open(in_stream);
        auto page_index_reader = parquet_reader->GetPageIndexReader();
        ASSERT_TRUE(page_index_reader);
        auto row_group_page_index = page_index_reader->RowGroup(0);
        ASSERT_TRUE(row_group_page_index);
        auto offset_index = row_group_page_index->GetOffsetIndex(0);
        ASSERT_TRUE(offset_index);
        page_locations = offset_index->page_locations();
    }
    ASSERT_EQ(10, page_locations.size());

    RoaringBitmap32 bitmap;
    bitmap.Add(45);  // Page 4.
    bitmap.Add(55);  // Adjacent page 5.

    std::map<std::string, std::string> options;
    options[PARQUET_READ_ROW_RANGES_COALESCE_HOLE_SIZE_LIMIT] = "0";
    options[PARQUET_READ_CACHE_OPTION_HOLE_SIZE_LIMIT] = "0";

    auto read_schema = arrow::schema({arrow::field("val", arrow::int32())});
    std::shared_ptr<arrow::ChunkedArray> result;
    std::vector<arrow::io::ReadRange> read_at_ranges;
    ReadWithPredicateAndBitmapImpl(file_name, read_schema, /*predicate=*/nullptr, bitmap, &result,
                                   options, /*batch_size=*/1024, &read_at_ranges);

    ASSERT_TRUE(result);
    ASSERT_EQ(2, result->length());
    auto flat = arrow::Concatenate(result->chunks()).ValueOrDie();
    auto struct_arr = std::dynamic_pointer_cast<arrow::StructArray>(flat);
    ASSERT_TRUE(struct_arr);
    auto values = std::dynamic_pointer_cast<arrow::Int32Array>(struct_arr->field(0));
    ASSERT_TRUE(values);
    ASSERT_EQ(45, values->Value(0));
    ASSERT_EQ(55, values->Value(1));
    AssertSelectedPagesRead(page_locations, /*selected_pages=*/{4, 5}, read_at_ranges);
    AssertUnselectedPageHeadersNotRead(page_locations, /*selected_pages=*/{4, 5}, read_at_ranges);
}

/// Files without OffsetIndex must keep the original full-column reader path.
TEST_F(PageFilteredRowGroupReaderTest, MissingOffsetIndexFallsBackToSequentialRead) {
    std::string file_name = dir_->Str() + "/missing_offset_index_fallback.parquet";
    auto data = MakeSequentialIntData(100);
    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100,
                  /*enable_dictionary=*/false, /*data_page_size=*/1,
                  /*enable_page_index=*/false);

    {
        ASSERT_OK_AND_ASSIGN(std::shared_ptr<InputStream> in, fs_->Open(file_name));
        ASSERT_OK_AND_ASSIGN(int64_t length, in->Length());
        auto in_stream = std::make_shared<ArrowInputStreamAdapter>(in, length, arrow_pool_);
        auto parquet_reader = ::parquet::ParquetFileReader::Open(in_stream);
        auto page_index_reader = parquet_reader->GetPageIndexReader();
        ASSERT_TRUE(page_index_reader);
        ASSERT_FALSE(page_index_reader->RowGroup(0));
    }

    RoaringBitmap32 bitmap;
    bitmap.Add(15);
    bitmap.Add(85);

    auto read_schema = arrow::schema({arrow::field("val", arrow::int32())});
    std::shared_ptr<arrow::ChunkedArray> result;
    ReadWithPredicateAndBitmapImpl(file_name, read_schema, /*predicate=*/nullptr, bitmap, &result);

    ASSERT_TRUE(result);
    ASSERT_EQ(2, result->length());
    auto flat = arrow::Concatenate(result->chunks()).ValueOrDie();
    auto struct_arr = std::dynamic_pointer_cast<arrow::StructArray>(flat);
    ASSERT_TRUE(struct_arr);
    auto values = std::dynamic_pointer_cast<arrow::Int32Array>(struct_arr->field(0));
    ASSERT_TRUE(values);
    ASSERT_EQ(15, values->Value(0));
    ASSERT_EQ(85, values->Value(1));
}

/// Test: bitmap hits partial pages of a row group (no predicate).
///
/// 200 rows, 10 rows per page, 100 rows per row group → 2 row groups.
/// Bitmap: {90..109} hits page 9 of RG0 (rows 90-99), and page 0 of RG1 (rows 100-109).
/// Expected: 20 rows (90-109).
TEST_F(PageFilteredRowGroupReaderTest, BitmapPartialPagesAcrossRowGroups) {
    std::string file_name = dir_->Str() + "/bitmap_partial_pages_rg.parquet";
    auto data = MakeSequentialIntData(200);
    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100);

    RoaringBitmap32 bitmap;
    // {90..109} hits page 9 of RG0 (rows 90-99), and page 0 of RG1 (rows 100-109).
    bitmap.AddRange(90, 110);

    auto read_schema = arrow::schema({arrow::field("val", arrow::int32())});
    std::shared_ptr<arrow::ChunkedArray> result;
    ReadWithPredicateAndBitmapImpl(file_name, read_schema, /*predicate=*/nullptr, bitmap, &result);
    ASSERT_TRUE(result);
    ASSERT_EQ(20, result->length());

    auto flat = arrow::Concatenate(result->chunks()).ValueOrDie();
    auto struct_arr = std::dynamic_pointer_cast<arrow::StructArray>(flat);
    ASSERT_TRUE(struct_arr);
    auto val_arr = std::dynamic_pointer_cast<arrow::Int32Array>(struct_arr->field(0));
    for (int32_t i = 0; i < 20; ++i) {
        ASSERT_EQ(90 + i, val_arr->Value(i));
    }
}

/// Test: bitmap hits all pages of some row groups and partial pages of others.
///
/// 200 rows, 10 rows per page, 100 rows per row group → 2 row groups.
/// Bitmap: {0..99} hits all of RG0 + {120..149} hits pages 2-4 of RG1.
/// Expected: 100 (RG0) + 30 (RG1 partial) = 130 rows.
TEST_F(PageFilteredRowGroupReaderTest, BitmapAllAndPartialPagesMixed) {
    std::string file_name = dir_->Str() + "/bitmap_all_and_partial.parquet";
    auto data = MakeSequentialIntData(200);
    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100);

    RoaringBitmap32 bitmap;
    bitmap.AddRange(0, 100);    // all of RG0
    bitmap.AddRange(120, 150);  // pages 2-4 of RG1

    auto read_schema = arrow::schema({arrow::field("val", arrow::int32())});
    std::shared_ptr<arrow::ChunkedArray> result;
    ReadWithPredicateAndBitmapImpl(file_name, read_schema, /*predicate=*/nullptr, bitmap, &result);
    ASSERT_TRUE(result);
    ASSERT_EQ(130, result->length());

    // Verify: rows 0-99 + 120-149
    auto flat = arrow::Concatenate(result->chunks()).ValueOrDie();
    auto struct_arr = std::dynamic_pointer_cast<arrow::StructArray>(flat);
    ASSERT_TRUE(struct_arr);
    auto val_arr = std::dynamic_pointer_cast<arrow::Int32Array>(struct_arr->field(0));
    for (int32_t i = 0; i < 100; ++i) {
        ASSERT_EQ(i, val_arr->Value(i));
    }
    for (int32_t i = 0; i < 30; ++i) {
        ASSERT_EQ(120 + i, val_arr->Value(100 + i));
    }
}

/// Test: bitmap hits partial pages of a row group, with page-filtered option disabled.
///
/// 200 rows, 10 rows per page, 100 rows per row group → 2 row groups.
/// Bitmap: {120..149} hits pages 2-4 of RG1.
/// Expected: 100 rows (100-199) because page-filtered option is disabled, so page-level bitmap is
/// ignored.
TEST_F(PageFilteredRowGroupReaderTest, BitmapWithPageFilteredOptionDisabled) {
    std::string file_name = dir_->Str() + "/bitmap_all_and_partial.parquet";
    auto data = MakeSequentialIntData(200);
    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100);

    RoaringBitmap32 bitmap;
    bitmap.AddRange(120, 150);  // pages 2-4 of RG1

    auto read_schema = arrow::schema({arrow::field("val", arrow::int32())});
    std::shared_ptr<arrow::ChunkedArray> result;
    std::map<std::string, std::string> options;
    options[PARQUET_READ_ENABLE_PAGE_INDEX_FILTER] = "false";
    ReadWithPredicateAndBitmapImpl(file_name, read_schema, /*predicate=*/nullptr, bitmap, &result,
                                   options);
    ASSERT_TRUE(result);
    ASSERT_EQ(100, result->length());

    // Verify: 100-199
    auto flat = arrow::Concatenate(result->chunks()).ValueOrDie();
    auto struct_arr = std::dynamic_pointer_cast<arrow::StructArray>(flat);
    ASSERT_TRUE(struct_arr);
    auto val_arr = std::dynamic_pointer_cast<arrow::Int32Array>(struct_arr->field(0));
    for (int32_t i = 0; i < 100; ++i) {
        ASSERT_EQ(100 + i, val_arr->Value(i));
    }
}

/// Test: bitmap + predicate both applied, bitmap hits all pages of some row groups.
///
/// 200 rows, 10 rows per page, 100 rows per row group → 2 row groups.
/// Bitmap: {0..99} hits all of RG0.
/// Predicate: val >= 50. Page-level filtering on RG0: pages 5-9.
/// Expected: 50 rows (50-99).
TEST_F(PageFilteredRowGroupReaderTest, BitmapAllPagesWithPredicate) {
    std::string file_name = dir_->Str() + "/bitmap_all_predicate.parquet";
    auto data = MakeSequentialIntData(200);
    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100);

    RoaringBitmap32 bitmap;
    bitmap.AddRange(0, 100);  // hits all of RG0

    auto predicate = PredicateBuilder::GreaterOrEqual(
        /*field_index=*/0, /*field_name=*/"val", FieldType::INT, Literal(50));

    auto read_schema = arrow::schema({arrow::field("val", arrow::int32())});
    std::shared_ptr<arrow::ChunkedArray> result;
    ReadWithPredicateAndBitmapImpl(file_name, read_schema, predicate, bitmap, &result);
    ASSERT_TRUE(result);
    ASSERT_EQ(50, result->length());

    auto flat = arrow::Concatenate(result->chunks()).ValueOrDie();
    auto struct_arr = std::dynamic_pointer_cast<arrow::StructArray>(flat);
    ASSERT_TRUE(struct_arr);
    auto val_arr = std::dynamic_pointer_cast<arrow::Int32Array>(struct_arr->field(0));
    for (int32_t i = 0; i < 50; ++i) {
        ASSERT_EQ(50 + i, val_arr->Value(i));
    }
}

/// Test: bitmap + predicate both applied, bitmap hits partial pages of a row group.
///
/// 200 rows, 10 rows per page, 100 rows per row group → 2 row groups.
/// Bitmap: {30..59} hits pages 3-5 of RG0 (rows 30-59).
/// Predicate: val >= 40. Page-level filtering further narrows to pages 4-5 (rows 40-59).
/// Expected: 20 rows (40-59).
TEST_F(PageFilteredRowGroupReaderTest, BitmapPartialPagesWithPredicate) {
    std::string file_name = dir_->Str() + "/bitmap_partial_predicate.parquet";
    auto data = MakeSequentialIntData(200);
    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100);

    RoaringBitmap32 bitmap;
    bitmap.AddRange(30, 60);  // hits pages 3-5 of RG0

    auto predicate = PredicateBuilder::GreaterOrEqual(
        /*field_index=*/0, /*field_name=*/"val", FieldType::INT, Literal(40));

    auto read_schema = arrow::schema({arrow::field("val", arrow::int32())});
    std::shared_ptr<arrow::ChunkedArray> result;
    ReadWithPredicateAndBitmapImpl(file_name, read_schema, predicate, bitmap, &result);
    ASSERT_TRUE(result);
    ASSERT_EQ(20, result->length());

    auto flat = arrow::Concatenate(result->chunks()).ValueOrDie();
    auto struct_arr = std::dynamic_pointer_cast<arrow::StructArray>(flat);
    auto val_arr = std::dynamic_pointer_cast<arrow::Int32Array>(struct_arr->field(0));
    for (int32_t i = 0; i < 20; ++i) {
        ASSERT_EQ(40 + i, val_arr->Value(i));
    }
}

/// Test: bitmap + predicate both applied, bitmap hits all pages of some RG and
/// partial pages of another.
///
/// 200 rows, 10 rows per page, 100 rows per row group → 2 row groups.
/// Bitmap: {0..99} (all of RG0) + {120..149} (pages 2-4 of RG1).
/// Predicate: val >= 50 AND val < 160.
///   RG0: all pages → page-filtered to val>=50 → rows 50-99 (50 rows)
///   RG1: pages 2-4 (120-149) → page-filtered to val>=50 AND val<160 → all match (30 rows)
/// Expected: 80 rows (50-99 + 120-149).
TEST_F(PageFilteredRowGroupReaderTest, BitmapMixedWithPredicate) {
    std::string file_name = dir_->Str() + "/bitmap_mixed_predicate.parquet";
    auto data = MakeSequentialIntData(200);
    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/100);

    RoaringBitmap32 bitmap;
    bitmap.AddRange(0, 100);    // all of RG0
    bitmap.AddRange(120, 150);  // pages 2-4 of RG1

    ASSERT_OK_AND_ASSIGN(
        auto predicate,
        PredicateBuilder::And(
            {PredicateBuilder::GreaterOrEqual(/*field_index=*/0, /*field_name=*/"val",
                                              FieldType::INT, Literal(50)),
             PredicateBuilder::LessThan(/*field_index=*/0, /*field_name=*/"val", FieldType::INT,
                                        Literal(160))}));

    auto read_schema = arrow::schema({arrow::field("val", arrow::int32())});
    std::shared_ptr<arrow::ChunkedArray> result;
    ReadWithPredicateAndBitmapImpl(file_name, read_schema, predicate, bitmap, &result);
    ASSERT_TRUE(result);
    ASSERT_EQ(80, result->length());

    // Verify: rows 50-99 + 120-149
    auto flat = arrow::Concatenate(result->chunks()).ValueOrDie();
    auto struct_arr = std::dynamic_pointer_cast<arrow::StructArray>(flat);
    ASSERT_TRUE(struct_arr);
    auto val_arr = std::dynamic_pointer_cast<arrow::Int32Array>(struct_arr->field(0));
    for (int32_t i = 0; i < 50; ++i) {
        ASSERT_EQ(50 + i, val_arr->Value(i));
    }
    for (int32_t i = 0; i < 30; ++i) {
        ASSERT_EQ(120 + i, val_arr->Value(50 + i));
    }
}

/// Test: coalesce strategy with default hole_size_limit (32).
///
/// 200 rows, 50 rows per page, 100 rows per row group → 2 row groups.
/// Bitmap: [0,10), [45, 50), [60, 70).
/// - [0,10) and [45,50) are both in RG0 page 0 ([0,49]); gap = 35 > 32, NOT merged.
/// - [45,50) and [60,70) straddle RG0 page 0 ([0,49]) and page 1 ([50,99]); gap = 10 <= 32,
///   merged across the page boundary, so rows 50-59 are read even though not in the bitmap.
/// Expected: 35 rows ([0,10) + [45, 70)).
TEST_F(PageFilteredRowGroupReaderTest, BitmapCoalesceTest) {
    std::string file_name = dir_->Str() + "/scattered_bitmap.parquet";
    auto data = MakeSequentialIntData(200);
    WriteTestFile(file_name, data, /*write_batch_size=*/50, /*max_row_group_length=*/100);

    RoaringBitmap32 bitmap;
    bitmap.AddRange(0, 10);
    bitmap.AddRange(45, 50);
    bitmap.AddRange(60, 70);

    auto read_schema = arrow::schema({arrow::field("val", arrow::int32())});
    std::shared_ptr<arrow::ChunkedArray> result;
    ReadWithPredicateAndBitmapImpl(file_name, read_schema, /*predicate=*/nullptr, bitmap, &result);
    ASSERT_TRUE(result);
    ASSERT_EQ(35, result->length());

    auto flat = arrow::Concatenate(result->chunks()).ValueOrDie();
    auto struct_arr = std::dynamic_pointer_cast<arrow::StructArray>(flat);
    ASSERT_TRUE(struct_arr);
    auto val_arr = std::dynamic_pointer_cast<arrow::Int32Array>(struct_arr->field(0));
    // [0, 9] — not merged (gap to next range = 35 > 32)
    for (int32_t i = 0; i < 10; ++i) {
        ASSERT_EQ(0 + i, val_arr->Value(i));
    }
    // [45, 69] — merged across page boundary (gap = 10 <= 32)
    for (int32_t i = 0; i < 25; ++i) {
        ASSERT_EQ(45 + i, val_arr->Value(10 + i));
    }
}

/// Test: coalesce strategy with hole_size_limit=5 (instead of default 32).
///
/// Same bitmap as BitmapCoalesceTest: [0,10), [45, 50), [60, 70).
/// Both gaps (35 and 10) exceed 5, so no ranges are merged.
/// Expected: 25 rows (exact bitmap selection, no holes filled).
/// Compare with BitmapCoalesceTest which gets 35 rows (gap of 10 is filled with default limit=32).
TEST_F(PageFilteredRowGroupReaderTest, BitmapCoalesceSmallHoleSizeTest) {
    std::string file_name = dir_->Str() + "/coalesce_small_hole.parquet";
    auto data = MakeSequentialIntData(200);
    WriteTestFile(file_name, data, /*write_batch_size=*/50, /*max_row_group_length=*/100);

    RoaringBitmap32 bitmap;
    bitmap.AddRange(0, 10);
    bitmap.AddRange(45, 50);
    bitmap.AddRange(60, 70);

    std::map<std::string, std::string> options;
    options[PARQUET_READ_ROW_RANGES_COALESCE_HOLE_SIZE_LIMIT] = "5";

    auto read_schema = arrow::schema({arrow::field("val", arrow::int32())});
    std::shared_ptr<arrow::ChunkedArray> result;
    ReadWithPredicateAndBitmapImpl(file_name, read_schema, /*predicate=*/nullptr, bitmap, &result,
                                   options);
    ASSERT_TRUE(result);
    ASSERT_EQ(25, result->length());

    auto flat = arrow::Concatenate(result->chunks()).ValueOrDie();
    auto struct_arr = std::dynamic_pointer_cast<arrow::StructArray>(flat);
    ASSERT_TRUE(struct_arr);
    auto val_arr = std::dynamic_pointer_cast<arrow::Int32Array>(struct_arr->field(0));
    ASSERT_TRUE(val_arr);
    // [0, 9] — not merged (gap = 35 > 5)
    for (int32_t i = 0; i < 10; ++i) {
        ASSERT_EQ(0 + i, val_arr->Value(i));
    }
    // [45, 49] — not merged (gap = 10 > 5)
    for (int32_t i = 0; i < 5; ++i) {
        ASSERT_EQ(45 + i, val_arr->Value(10 + i));
    }
    // [60, 69] — not merged
    for (int32_t i = 0; i < 10; ++i) {
        ASSERT_EQ(60 + i, val_arr->Value(15 + i));
    }
}

/// Test: trim strategy for bitmap row filtering.
///
/// Same bitmap as BitmapCoalesceTest: [0,10), [45, 50), [60, 70).
/// Trim produces one range per page (trimmed to first/last selected row in that page):
/// - RG0 page 0 ([0,49]): first selected = 0, last selected = 49 → [0, 49] (50 rows, includes
///   the 35-row gap 10-44 that coalesce keeps as a hole because gap = 35 > 32).
/// - RG0 page 1 ([50,99]): first selected = 60, last selected = 69 → [60, 69] (10 rows; rows
///   50-59 are NOT read, unlike coalesce which merges across the page boundary).
/// Expected: 60 rows ([0, 50) + [60, 70)).
/// Compare with BitmapCoalesceTest which gets 35 rows.
TEST_F(PageFilteredRowGroupReaderTest, BitmapTrimStrategyTest) {
    std::string file_name = dir_->Str() + "/trim_strategy.parquet";
    auto data = MakeSequentialIntData(200);
    WriteTestFile(file_name, data, /*write_batch_size=*/50, /*max_row_group_length=*/100);

    RoaringBitmap32 bitmap;
    bitmap.AddRange(0, 10);
    bitmap.AddRange(45, 50);
    bitmap.AddRange(60, 70);

    std::map<std::string, std::string> options;
    options[PARQUET_READ_BITMAP_ROW_RANGE_REFINING_STRATEGY] = "trim";

    auto read_schema = arrow::schema({arrow::field("val", arrow::int32())});
    std::shared_ptr<arrow::ChunkedArray> result;
    ReadWithPredicateAndBitmapImpl(file_name, read_schema, /*predicate=*/nullptr, bitmap, &result,
                                   options);
    ASSERT_TRUE(result);
    ASSERT_EQ(60, result->length());

    auto flat = arrow::Concatenate(result->chunks()).ValueOrDie();
    auto struct_arr = std::dynamic_pointer_cast<arrow::StructArray>(flat);
    ASSERT_TRUE(struct_arr);
    auto val_arr = std::dynamic_pointer_cast<arrow::Int32Array>(struct_arr->field(0));
    ASSERT_TRUE(val_arr);
    // RG0 page 0 trimmed to [0, 49] — includes gap 10-44 that coalesce skips
    for (int32_t i = 0; i < 50; ++i) {
        ASSERT_EQ(0 + i, val_arr->Value(i));
    }
    // RG0 page 1 trimmed to [60, 69] — rows 50-59 not read (unlike coalesce)
    for (int32_t i = 0; i < 10; ++i) {
        ASSERT_EQ(60 + i, val_arr->Value(50 + i));
    }
}

/// Test: invalid strategy value returns Status::Invalid.
///
/// 200 rows, 50 rows per page, 100 rows per row group → 2 row groups.
/// Bitmap: [0,10), [45, 50), [60, 70) — same as BitmapCoalesceTest.
/// Strategy: "invalid" (not one of "coalesce", "trim", "none").
/// Expected: SetReadSchema returns Status::Invalid.
TEST_F(PageFilteredRowGroupReaderTest, BitmapInvalidStrategyTest) {
    std::string file_name = dir_->Str() + "/invalid_strategy.parquet";
    auto data = MakeSequentialIntData(200);
    WriteTestFile(file_name, data, /*write_batch_size=*/50, /*max_row_group_length=*/100);

    RoaringBitmap32 bitmap;
    bitmap.AddRange(0, 10);
    bitmap.AddRange(45, 50);
    bitmap.AddRange(60, 70);

    std::map<std::string, std::string> options;
    options[PARQUET_READ_BITMAP_ROW_RANGE_REFINING_STRATEGY] = "invalid";

    ASSERT_OK_AND_ASSIGN(std::shared_ptr<InputStream> in, fs_->Open(file_name));
    ASSERT_OK_AND_ASSIGN(int64_t length, in->Length());
    auto in_stream = std::make_shared<ArrowInputStreamAdapter>(in, length, arrow_pool_);

    ASSERT_OK_AND_ASSIGN(auto batch_reader, ParquetFileBatchReader::Create(
                                                std::move(in_stream), options, 1024, nullptr,
                                                /*storage_read_bytes=*/nullptr, arrow_pool_));

    auto read_schema = arrow::schema({arrow::field("val", arrow::int32())});
    auto c_schema = std::make_unique<ArrowSchema>();
    ASSERT_TRUE(arrow::ExportSchema(*read_schema, c_schema.get()).ok());

    auto status = batch_reader->SetReadSchema(c_schema.get(), /*predicate=*/nullptr, bitmap);
    ASSERT_FALSE(status.ok());
    ASSERT_TRUE(status.IsInvalid());
}

/// Test: trim strategy with multiple columns — intersection of per-column trimmed ranges.
///
/// 200 rows, 50 rows per page, 100 rows per row group → 2 row groups.
/// Two columns: a[i] = i, b[i] = i * 10.
/// Bitmap: [0,10), [45, 50), [60, 70) — same as BitmapTrimStrategyTest.
/// Both columns share the same page boundaries, so their trimmed ranges are identical.
/// The intersection is the same as either column alone:
/// - RG0 page 0 ([0,49]): trimmed to [0, 49] (50 rows)
/// - RG0 page 1 ([50,99]): trimmed to [60, 69] (10 rows)
/// Expected: 60 rows. Both columns must remain aligned after trimming.
TEST_F(PageFilteredRowGroupReaderTest, BitmapTrimMultiColumnTest) {
    std::string file_name = dir_->Str() + "/trim_multi_col.parquet";
    auto data = MakeTwoColumnData(200);
    WriteTestFile(file_name, data, /*write_batch_size=*/50, /*max_row_group_length=*/100);

    RoaringBitmap32 bitmap;
    bitmap.AddRange(0, 10);
    bitmap.AddRange(45, 50);
    bitmap.AddRange(60, 70);

    std::map<std::string, std::string> options;
    options[PARQUET_READ_BITMAP_ROW_RANGE_REFINING_STRATEGY] = "trim";

    auto read_schema =
        arrow::schema({arrow::field("a", arrow::int32()), arrow::field("b", arrow::int32())});
    std::shared_ptr<arrow::ChunkedArray> result;
    ReadWithPredicateAndBitmapImpl(file_name, read_schema, /*predicate=*/nullptr, bitmap, &result,
                                   options);
    ASSERT_TRUE(result);
    ASSERT_EQ(60, result->length());

    auto flat = arrow::Concatenate(result->chunks()).ValueOrDie();
    auto struct_arr = std::dynamic_pointer_cast<arrow::StructArray>(flat);
    ASSERT_TRUE(struct_arr);
    auto a_arr = std::dynamic_pointer_cast<arrow::Int32Array>(struct_arr->field(0));
    auto b_arr = std::dynamic_pointer_cast<arrow::Int32Array>(struct_arr->field(1));
    ASSERT_TRUE(a_arr);
    ASSERT_TRUE(b_arr);
    // RG0 page 0 trimmed to [0, 49]
    for (int32_t i = 0; i < 50; ++i) {
        ASSERT_EQ(i, a_arr->Value(i));
        ASSERT_EQ(i * 10, b_arr->Value(i));
    }
    // RG0 page 1 trimmed to [60, 69]
    for (int32_t i = 0; i < 10; ++i) {
        ASSERT_EQ(60 + i, a_arr->Value(50 + i));
        ASSERT_EQ((60 + i) * 10, b_arr->Value(50 + i));
    }
}

/// Test: predicate pushdown with all nested column types (struct, list, map).
///
/// Schema: { id: int32, info: struct<x: int32, y: int32>,
///           tags: list<item: int32>, props: map<utf8, int32> }
/// 100 rows, 10 rows per page, 50 rows per row group → 2 row groups.
/// Predicate: id in [15, 29] or id in [80, 99] (Between is inclusive).
/// Read schema: full schema (all columns).
/// Page-level filtering (10 rows/page):
///   Between(15, 29) → pages 1-2 (rows 10-29)
///   Between(80, 99) → pages 8-9 (rows 80-99)
///   Total: 40 rows.
TEST_F(PageFilteredRowGroupReaderTest, MultipleNestedColumns) {
    std::string file_name = dir_->Str() + "/multi_nested_columns.parquet";

    auto field_x = arrow::field("x", arrow::int32());
    auto field_y = arrow::field("y", arrow::int32());
    auto field_id = arrow::field("id", arrow::int32());
    auto field_info = arrow::field("info", arrow::struct_({field_x, field_y}));
    auto field_tags = arrow::field("tags", arrow::list(arrow::field("item", arrow::int32())));
    auto field_props = arrow::field("props", arrow::map(arrow::utf8(), arrow::int32()));

    // Build data with all nested column types using shared helpers
    auto id_array = MakeIdColumn(100);
    auto info_array = MakeNestedStructData(100);
    auto tags_array = MakeListColumnData(100);
    auto props_array = MakeMapColumnData(100);
    auto data = arrow::StructArray::Make({id_array, info_array, tags_array, props_array},
                                         {field_id, field_info, field_tags, field_props})
                    .ValueOrDie();

    // Write: 10 rows per page, 50 rows per row group → 2 row groups
    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50);

    // Read full schema
    auto read_schema = arrow::schema({field_id, field_info, field_tags, field_props});

    // predicate: id in [15, 29] or id in [80, 99]
    ASSERT_OK_AND_ASSIGN(
        auto predicate, PredicateBuilder::Or(
                            {PredicateBuilder::Between(/*field_index=*/0, /*field_name=*/"id",
                                                       FieldType::INT, Literal(15), Literal(29)),
                             PredicateBuilder::Between(/*field_index=*/0, /*field_name=*/"id",
                                                       FieldType::INT, Literal(80), Literal(99))}));

    std::shared_ptr<arrow::ChunkedArray> result;
    ReadWithPredicateImpl(file_name, read_schema, predicate, &result,
                          /*batch_size=*/1024);

    // Page-level filtering (10 rows/page):
    //   Between(15, 29) → pages 1-2 (rows 10-29)
    //   Between(80, 99) → pages 8-9 (rows 80-99)
    //   Total: 40 rows
    ASSERT_TRUE(result);
    ASSERT_EQ(40, result->length());

    auto expected =
        arrow::ChunkedArray::Make({data->Slice(10, 20), data->Slice(80, 20)}).ValueOrDie();
    ASSERT_TRUE(result->Equals(expected));
}

/// Test: sub-column projection of a struct type with page-level filtering.
///
/// Schema: { id: int32, info: struct<x: int32, y: int32> }
/// Read schema: { info: struct<x: int32> } — project only x, not y.
/// Predicate: id >= 70 → 30 rows expected.
/// Verifies that reading a sub-column of a nested struct works correctly
/// with page-level filtering and the ColumnReader tree (GetColumn + filter_leaves).
TEST_F(PageFilteredRowGroupReaderTest, NestedStructSubColumnProjection) {
    std::string file_name = dir_->Str() + "/nested_struct_subcol.parquet";

    auto field_x = arrow::field("x", arrow::int32());
    auto field_y = arrow::field("y", arrow::int32());
    auto field_id = arrow::field("id", arrow::int32());
    auto field_info = arrow::field("info", arrow::struct_({field_x, field_y}));

    auto id_array = MakeIdColumn(100);
    auto info_array = MakeNestedStructData(100);
    auto data =
        arrow::StructArray::Make({id_array, info_array}, {field_id, field_info}).ValueOrDie();
    WriteTestFile(file_name, data, /*write_batch_size=*/10, /*max_row_group_length=*/50);

    // Read only info.x (sub-column projection: only x, not y)
    auto read_schema = arrow::schema({arrow::field("info", arrow::struct_({field_x}))});

    auto predicate = PredicateBuilder::GreaterOrEqual(
        /*field_index=*/0, /*field_name=*/"id", FieldType::INT, Literal(70));

    std::shared_ptr<arrow::ChunkedArray> result;
    ReadWithPredicateImpl(file_name, read_schema, predicate, &result);

    ASSERT_TRUE(result);
    ASSERT_EQ(30, result->length());

    // Result is struct<info: struct<x: int32>>
    auto result_struct = std::dynamic_pointer_cast<arrow::StructArray>(result->chunk(0));
    ASSERT_TRUE(result_struct);
    ASSERT_EQ(1, result_struct->num_fields());

    auto info_result = std::dynamic_pointer_cast<arrow::StructArray>(result_struct->field(0));
    ASSERT_TRUE(info_result);
    ASSERT_EQ(1, info_result->num_fields());

    auto x_arr = std::dynamic_pointer_cast<arrow::Int32Array>(info_result->field(0));
    ASSERT_TRUE(x_arr);
    for (int32_t i = 0; i < 30; ++i) {
        ASSERT_EQ((70 + i) * 100, x_arr->Value(i)) << "Mismatch at index " << i;
    }
}

/// Helper: build a struct with a flat key plus several nested columns of different
/// physical widths / repetition, so that with a byte-based data page size their
/// leaves flush pages at different row counts (misaligned pages):
///   key:   int64            (fixed 8B  -> ~5 rows/page)
///   s:     struct<x:int32, y:int64>   (x ~10 rows/page, y ~5 rows/page)
///   tags:  list<int32>       (2 values/row -> ~5 rows/page)
///   props: map<utf8,int32>   (variable-width utf8 key -> irregular rows/page)
/// key/s.x/s.y encode the row index (= i); tags/props reuse the shared list/map
/// helpers. Correctness is verified per row by deep-comparing against this array.
static std::shared_ptr<arrow::StructArray> MakeMisalignedNestedData(int32_t num_rows) {
    arrow::Int64Builder key_builder;
    arrow::Int32Builder x_builder;
    arrow::Int64Builder y_builder;
    EXPECT_TRUE(key_builder.Reserve(num_rows).ok());
    EXPECT_TRUE(x_builder.Reserve(num_rows).ok());
    EXPECT_TRUE(y_builder.Reserve(num_rows).ok());
    for (int32_t i = 0; i < num_rows; ++i) {
        key_builder.UnsafeAppend(i);
        x_builder.UnsafeAppend(i);
        y_builder.UnsafeAppend(i);
    }
    auto key_array = key_builder.Finish().ValueOrDie();
    auto x_array = x_builder.Finish().ValueOrDie();
    auto y_array = y_builder.Finish().ValueOrDie();

    auto field_x = arrow::field("x", arrow::int32());
    auto field_y = arrow::field("y", arrow::int64());
    auto s_array = arrow::StructArray::Make({x_array, y_array}, {field_x, field_y}).ValueOrDie();

    // Repeated nested leaves (list<int32> and map<utf8,int32>) paginate on value
    // bytes, so they misalign with the struct's fixed-width leaves too.
    auto tags_array = MakeListColumnData(num_rows);
    auto props_array = MakeMapColumnData(num_rows);

    auto field_key = arrow::field("key", arrow::int64());
    auto field_s = arrow::field("s", arrow::struct_({field_x, field_y}));
    auto field_tags = arrow::field("tags", arrow::list(arrow::field("item", arrow::int32())));
    auto field_props = arrow::field("props", arrow::map(arrow::utf8(), arrow::int32()));
    return arrow::StructArray::Make({key_array, s_array, tags_array, props_array},
                                    {field_key, field_s, field_tags, field_props})
        .ValueOrDie();
}

/// Test: page-level filtering across multiple nested columns whose leaf pages are
/// MISALIGNED, within a SINGLE row group. The file mixes a flat key, a
/// struct<int32,int64>, a list<int32> and a map<utf8,int32>; with write_batch_size=1
/// and a byte-based data page size every leaf flushes pages at a different (and, for
/// the utf8 map key, irregular) row count.
/// Bitmap: [0,15), [77, 87) (to avoid bitmap hole filling)
/// Expected: 25 rows
TEST_F(PageFilteredRowGroupReaderTest, NestedColumnsMisalignedPagesSingleRowGroup) {
    std::string file_name = dir_->Str() + "/nested_misaligned_single_rg.parquet";
    constexpr int32_t kNumRows = 100;
    auto data = MakeMisalignedNestedData(kNumRows);

    // write_batch_size=1 + a byte-based data page size makes the int32 and int64 leaves
    // flush pages at different row counts, i.e. deliberately misaligned pages.
    // max_row_group_length=kNumRows keeps all rows in a single row group.
    WriteTestFile(file_name, data, /*write_batch_size=*/1, /*max_row_group_length=*/kNumRows,
                  /*enable_dictionary=*/false, /*data_page_size=*/40);

    auto read_schema =
        arrow::schema({arrow::field("key", arrow::int64()),
                       arrow::field("s", arrow::struct_({arrow::field("x", arrow::int32()),
                                                         arrow::field("y", arrow::int64())})),
                       arrow::field("tags", arrow::list(arrow::field("item", arrow::int32()))),
                       arrow::field("props", arrow::map(arrow::utf8(), arrow::int32()))});

    // bitmap: [0,15), [77, 87)
    RoaringBitmap32 bitmap;
    bitmap.AddRange(0, 15);
    bitmap.AddRange(77, 87);

    std::shared_ptr<arrow::ChunkedArray> result;
    ReadWithPredicateAndBitmapImpl(file_name, read_schema, nullptr, bitmap, &result);
    ASSERT_TRUE(result);

    int64_t total = 0;
    auto top = std::dynamic_pointer_cast<arrow::StructArray>(result->chunk(0));
    ASSERT_TRUE(top);
    auto key_arr = std::dynamic_pointer_cast<arrow::Int64Array>(top->field(0));
    for (int64_t i = 0; i < key_arr->length(); ++i) {
        int64_t k = key_arr->Value(i);
        // Full-row deep compare across ALL columns (struct + list + map): the returned
        // row must equal the original row identified by key k. This is what catches a
        // desync in the repeated list/map leaves, whose reassembly also relies on the
        // per-leaf skip/read staying row-consistent.
        ASSERT_TRUE(data->Slice(k, 1)->Equals(*top->Slice(i, 1)))
            << "row content mismatch at key " << k;
        ++total;
    }

    for (int64_t i = 0; i < 15; ++i) {
        ASSERT_EQ(i, key_arr->Value(i));
    }
    for (int64_t i = 0; i < 10; ++i) {
        ASSERT_EQ(77 + i, key_arr->Value(15 + i));
    }

    ASSERT_EQ(total, 25);
}

/// Test: same misaligned nested layout as the single-row-group case above, but split
/// across MULTIPLE row groups (max_row_group_length=40 -> row groups of 40/40/20).
/// The selection bitmap spans row-group boundaries so the reader must keep the
/// per-leaf skip/read row-consistent both across misaligned pages and across row
/// groups.
/// Bitmap: [0,15), [77, 87) (to avoid bitmap hole filling)
/// Expected: 25 rows -> keys 0..14 and 77..86
TEST_F(PageFilteredRowGroupReaderTest, NestedColumnsMisalignedPagesMultiRowGroup) {
    std::string file_name = dir_->Str() + "/nested_misaligned_multi_rg.parquet";
    constexpr int32_t kNumRows = 100;
    auto data = MakeMisalignedNestedData(kNumRows);

    // write_batch_size=1 + byte-based data page size -> misaligned leaf pages.
    // max_row_group_length=40 -> 3 row groups (40, 40, 20).
    WriteTestFile(file_name, data, /*write_batch_size=*/1, /*max_row_group_length=*/40,
                  /*enable_dictionary=*/false, /*data_page_size=*/40);

    auto read_schema =
        arrow::schema({arrow::field("key", arrow::int64()),
                       arrow::field("s", arrow::struct_({arrow::field("x", arrow::int32()),
                                                         arrow::field("y", arrow::int64())})),
                       arrow::field("tags", arrow::list(arrow::field("item", arrow::int32()))),
                       arrow::field("props", arrow::map(arrow::utf8(), arrow::int32()))});

    // bitmap: [0,15), [77, 87) -> spans all three row groups.
    RoaringBitmap32 bitmap;
    bitmap.AddRange(0, 15);
    bitmap.AddRange(77, 87);

    std::shared_ptr<arrow::ChunkedArray> result;
    ReadWithPredicateAndBitmapImpl(file_name, read_schema, nullptr, bitmap, &result);
    ASSERT_TRUE(result);

    std::vector<int64_t> keys;

    auto concated = arrow::Concatenate(result->chunks()).ValueOrDie();
    auto top = std::dynamic_pointer_cast<arrow::StructArray>(concated);
    ASSERT_TRUE(top);
    auto key_arr = std::dynamic_pointer_cast<arrow::Int64Array>(top->field(0));
    ASSERT_TRUE(key_arr);
    for (int64_t i = 0; i < key_arr->length(); ++i) {
        int64_t k = key_arr->Value(i);
        ASSERT_TRUE(data->Slice(k, 1)->Equals(*top->Slice(i, 1)))
            << "row content mismatch at key " << k;
        keys.push_back(k);
    }

    std::vector<int64_t> expected;
    for (int64_t i = 0; i < 15; ++i) {
        expected.push_back(i);
    }
    for (int64_t i = 77; i < 87; ++i) {
        expected.push_back(i);
    }
    ASSERT_EQ(keys, expected);
}

/// Helper: like MakeMisalignedNestedData but sprinkles nulls into the nested columns
/// so that definition levels vary per row (which also perturbs page boundaries):
///   key:   int64, always non-null (= i, used to identify the row)
///   s:     struct<x:int32, y:int64>; whole struct null when i%11==0, otherwise
///          x null when i%5==0 and y null when i%7==0
///   tags:  list<int32>; null when i%6==0, otherwise [i*10, i*10+1]
///   props: map<utf8,int32>; null when i%8==0, otherwise {"k_i": i*100}
/// Correctness is verified per row by deep-comparing against this array.
static std::shared_ptr<arrow::StructArray> MakeMisalignedNestedDataWithNulls(int32_t num_rows) {
    arrow::Int64Builder key_builder;
    EXPECT_TRUE(key_builder.Reserve(num_rows).ok());
    for (int32_t i = 0; i < num_rows; ++i) {
        key_builder.UnsafeAppend(i);
    }
    auto key_array = key_builder.Finish().ValueOrDie();

    // s: struct<x:int32, y:int64> with nulls at both leaf and struct level.
    auto field_x = arrow::field("x", arrow::int32());
    auto field_y = arrow::field("y", arrow::int64());
    auto x_builder = std::make_shared<arrow::Int32Builder>();
    auto y_builder = std::make_shared<arrow::Int64Builder>();
    arrow::StructBuilder s_builder(arrow::struct_({field_x, field_y}), arrow::default_memory_pool(),
                                   {x_builder, y_builder});
    for (int32_t i = 0; i < num_rows; ++i) {
        if (i % 11 == 0) {
            // AppendNull() also appends nulls to the child builders, keeping lengths in sync.
            EXPECT_TRUE(s_builder.AppendNull().ok());
            continue;
        }
        EXPECT_TRUE(s_builder.Append().ok());
        if (i % 5 == 0) {
            EXPECT_TRUE(x_builder->AppendNull().ok());
        } else {
            EXPECT_TRUE(x_builder->Append(i).ok());
        }
        if (i % 7 == 0) {
            EXPECT_TRUE(y_builder->AppendNull().ok());
        } else {
            EXPECT_TRUE(y_builder->Append(i).ok());
        }
    }
    auto s_array = s_builder.Finish().ValueOrDie();

    // tags: list<int32> with null lists.
    auto item_builder = std::make_shared<arrow::Int32Builder>();
    arrow::ListBuilder tags_builder(arrow::default_memory_pool(), item_builder);
    for (int32_t i = 0; i < num_rows; ++i) {
        if (i % 6 == 0) {
            EXPECT_TRUE(tags_builder.AppendNull().ok());
        } else {
            EXPECT_TRUE(tags_builder.Append().ok());
            EXPECT_TRUE(item_builder->Append(i * 10).ok());
            EXPECT_TRUE(item_builder->Append(i * 10 + 1).ok());
        }
    }
    auto tags_array = tags_builder.Finish().ValueOrDie();

    // props: map<utf8,int32> with null maps.
    auto map_key_builder = std::make_shared<arrow::StringBuilder>();
    auto map_val_builder = std::make_shared<arrow::Int32Builder>();
    arrow::MapBuilder props_builder(arrow::default_memory_pool(), map_key_builder, map_val_builder);
    for (int32_t i = 0; i < num_rows; ++i) {
        if (i % 8 == 0) {
            EXPECT_TRUE(props_builder.AppendNull().ok());
        } else {
            EXPECT_TRUE(props_builder.Append().ok());
            EXPECT_TRUE(map_key_builder->Append("k_" + std::to_string(i)).ok());
            EXPECT_TRUE(map_val_builder->Append(i * 100).ok());
        }
    }
    auto props_array = props_builder.Finish().ValueOrDie();

    auto field_key = arrow::field("key", arrow::int64());
    auto field_s = arrow::field("s", arrow::struct_({field_x, field_y}));
    auto field_tags = arrow::field("tags", arrow::list(arrow::field("item", arrow::int32())));
    auto field_props = arrow::field("props", arrow::map(arrow::utf8(), arrow::int32()));
    return arrow::StructArray::Make({key_array, s_array, tags_array, props_array},
                                    {field_key, field_s, field_tags, field_props})
        .ValueOrDie();
}

/// Test: nested columns containing NULLs, with MISALIGNED leaf pages, split across
/// MULTIPLE row groups. This combines the three stress dimensions: null-driven
/// definition levels, byte-based misaligned pages, and row-group boundaries that the
/// selection bitmap crosses. Every returned row is deep-compared (nulls included)
/// against the original data identified by its non-null key.
/// Bitmap: [0,15), [77, 87) (to avoid bitmap hole filling)
/// Expected: 25 rows -> keys 0..14 and 77..86
TEST_F(PageFilteredRowGroupReaderTest, NestedColumnsWithNullsMisalignedPagesMultiRowGroup) {
    std::string file_name = dir_->Str() + "/nested_nulls_misaligned_multi_rg.parquet";
    constexpr int32_t kNumRows = 100;
    auto data = MakeMisalignedNestedDataWithNulls(kNumRows);

    // write_batch_size=1 + byte-based data page size -> misaligned leaf pages.
    // max_row_group_length=40 -> 3 row groups (40, 40, 20).
    WriteTestFile(file_name, data, /*write_batch_size=*/1, /*max_row_group_length=*/40,
                  /*enable_dictionary=*/false, /*data_page_size=*/40);

    auto read_schema =
        arrow::schema({arrow::field("key", arrow::int64()),
                       arrow::field("s", arrow::struct_({arrow::field("x", arrow::int32()),
                                                         arrow::field("y", arrow::int64())})),
                       arrow::field("tags", arrow::list(arrow::field("item", arrow::int32()))),
                       arrow::field("props", arrow::map(arrow::utf8(), arrow::int32()))});

    // bitmap: [0,15), [77, 87) -> spans all three row groups.
    RoaringBitmap32 bitmap;
    bitmap.AddRange(0, 15);
    bitmap.AddRange(77, 87);

    std::shared_ptr<arrow::ChunkedArray> result;
    ReadWithPredicateAndBitmapImpl(file_name, read_schema, nullptr, bitmap, &result);
    ASSERT_TRUE(result);

    std::vector<int64_t> keys;

    auto concated = arrow::Concatenate(result->chunks()).ValueOrDie();
    auto top = std::dynamic_pointer_cast<arrow::StructArray>(concated);
    ASSERT_TRUE(top);
    auto key_arr = std::dynamic_pointer_cast<arrow::Int64Array>(top->field(0));
    ASSERT_TRUE(key_arr);
    for (int64_t i = 0; i < key_arr->length(); ++i) {
        ASSERT_FALSE(key_arr->IsNull(i)) << "key column must stay non-null";
        int64_t k = key_arr->Value(i);
        // Deep compare including nulls across struct/list/map leaves.
        ASSERT_TRUE(data->Slice(k, 1)->Equals(*top->Slice(i, 1)))
            << "row content mismatch at key " << k;
        keys.push_back(k);
    }

    std::vector<int64_t> expected;
    for (int64_t i = 0; i < 15; ++i) {
        expected.push_back(i);
    }
    for (int64_t i = 77; i < 87; ++i) {
        expected.push_back(i);
    }
    ASSERT_EQ(keys, expected);
}

}  // namespace paimon::parquet::test
