/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "paimon/core/operation/data_evolution_split_read.h"

#include <algorithm>
#include <limits>
#include <map>
#include <string>
#include <string_view>
#include <thread>
#include <unordered_set>
#include <utility>

#include "arrow/array/array_base.h"
#include "arrow/array/array_binary.h"
#include "arrow/array/array_nested.h"
#include "arrow/c/bridge.h"
#include "paimon/common/catalog/catalog_context.h"
#include "paimon/common/data/blob_defs.h"
#include "paimon/common/data/blob_utils.h"
#include "paimon/common/data/blob_view_struct.h"
#include "paimon/common/file_index/bitmap/apply_bitmap_index_batch_reader.h"
#include "paimon/common/global_index/complete_index_score_batch_reader.h"
#include "paimon/common/reader/blob_fallback_batch_reader.h"
#include "paimon/common/reader/blob_view_resolving_batch_reader.h"
#include "paimon/common/reader/complete_row_kind_batch_reader.h"
#include "paimon/common/reader/concat_batch_reader.h"
#include "paimon/common/table/special_fields.h"
#include "paimon/common/types/data_field.h"
#include "paimon/common/utils/arrow/status_utils.h"
#include "paimon/common/utils/object_utils.h"
#include "paimon/common/utils/path_util.h"
#include "paimon/common/utils/range_helper.h"
#include "paimon/core/core_options.h"
#include "paimon/core/global_index/indexed_split_impl.h"
#include "paimon/core/utils/blob_view_lookup.h"
namespace paimon {
int64_t DataEvolutionSplitRead::BlobBunch::RowCount() const {
    if (files_.empty()) {
        return 0;
    }
    if (!has_row_ids_selection_) {
        // Add enforces the union range to be contiguous
        return union_end_row_id_ - union_first_row_id_;
    }
    // with a row-ids selection the scan may have pruned files, leaving holes in the union
    int64_t row_count = 0;
    for (const auto& range : Range::SortAndMergeOverlap(ranges_, /*adjacent=*/true)) {
        row_count += range.Count();
    }
    return row_count;
}

Status DataEvolutionSplitRead::BlobBunch::Add(const std::shared_ptr<DataFileMeta>& file) {
    if (!BlobUtils::IsBlobFile(file->file_name)) {
        return Status::Invalid("Only blob file can be added to a blob bunch.");
    }
    PAIMON_ASSIGN_OR_RAISE(int64_t first_row_id, file->NonNullFirstRowId());
    if (!files_.empty()) {
        // Blob files for the same field may span schema ids.
        if (file->write_cols != files_[0]->write_cols) {
            return Status::Invalid("All files in a blob bunch should have the same write columns.");
        }
    }
    // files sharing a max sequence number form one layer, whose row id ranges must be disjoint;
    // overlaps across layers are the expected shape of partial updates and are kept for the
    // row-level placeholder fallback
    auto [layer_iter, layer_inserted] =
        sequence_group_end_.try_emplace(file->max_sequence_number, 0);
    if (!layer_inserted && first_row_id < layer_iter->second) {
        return Status::Invalid(fmt::format(
            "Blob files with the same max sequence number should not have overlapping row id "
            "ranges: file {} (max sequence number {}) starts at row id {} before the previous "
            "file's end {}",
            file->file_name, file->max_sequence_number, first_row_id, layer_iter->second));
    }
    if (!has_row_ids_selection_ && !files_.empty() && first_row_id > union_end_row_id_) {
        // a hole no layer covers cannot be aligned with the data files
        return Status::Invalid(
            fmt::format("Blob file first row id should be continuous, expect {} but got {}",
                        union_end_row_id_, first_row_id));
    }
    int64_t end_row_id = first_row_id + file->row_count;
    layer_iter->second = end_row_id;
    union_first_row_id_ = std::min(union_first_row_id_, first_row_id);
    union_end_row_id_ = std::max(union_end_row_id_, end_row_id);
    ranges_.emplace_back(first_row_id, end_row_id - 1);
    files_.push_back(file);
    if (!has_row_ids_selection_ && expected_row_count_ >= 0 && RowCount() > expected_row_count_) {
        return Status::Invalid(
            fmt::format("Blob files row count exceed the expect {}", expected_row_count_));
    }
    return Status::OK();
}
DataEvolutionSplitRead::DataEvolutionSplitRead(
    const std::shared_ptr<FileStorePathFactory>& path_factory,
    const std::shared_ptr<InternalReadContext>& context,
    const std::shared_ptr<MemoryPool>& memory_pool, const std::shared_ptr<Executor>& executor)
    : AbstractSplitRead(path_factory, context,
                        std::make_unique<SchemaManager>(context->GetCoreOptions().GetFileSystem(),
                                                        context->GetPath(),
                                                        context->GetCoreOptions().GetBranch()),
                        memory_pool, executor) {}

bool DataEvolutionSplitRead::HasIndexScoreField(const std::shared_ptr<arrow::Schema>& read_schema) {
    return read_schema->GetFieldIndex(SpecialFields::IndexScore().Name()) != -1;
}

std::vector<std::string> DataEvolutionSplitRead::HasBlobViewField(
    const CoreOptions& options, const std::shared_ptr<arrow::Schema>& read_schema) {
    std::vector<std::string> read_blob_view_fields;
    std::vector<std::string> blob_view_fields = options.GetBlobViewFields();
    for (const auto& blob : blob_view_fields) {
        if (read_schema->GetFieldByName(blob)) {
            read_blob_view_fields.push_back(blob);
        }
    }
    return read_blob_view_fields;
}

Result<std::unique_ptr<BatchReader>> DataEvolutionSplitRead::CreateReader(
    const std::shared_ptr<Split>& split) {
    if (auto indexed_split = std::dynamic_pointer_cast<IndexedSplitImpl>(split)) {
        PAIMON_RETURN_NOT_OK(indexed_split->Validate());
        const auto& data_split = indexed_split->GetDataSplit();
        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> batch_reader,
                               InnerCreateReader(data_split, indexed_split->RowRanges()));
        if (HasIndexScoreField(raw_read_schema_)) {
            batch_reader = std::make_unique<CompleteIndexScoreBatchReader>(
                std::move(batch_reader), indexed_split->Scores(), pool_);
        }
        return WrapWithBlobViewResolverIfNeeded(data_split, std::move(batch_reader));
    } else if (auto data_split = std::dynamic_pointer_cast<DataSplit>(split)) {
        if (HasIndexScoreField(raw_read_schema_)) {
            return Status::Invalid(
                "Invalid read schema, read _INDEX_SCORE while split cannot cast to IndexedSplit");
        }
        PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> inner_reader,
                               InnerCreateReader(data_split, /*row_ranges=*/std::nullopt));
        return WrapWithBlobViewResolverIfNeeded(data_split, std::move(inner_reader));
    }
    return Status::Invalid("Invalid Split, cannot cast to IndexedSplit or DataSplit");
}

Result<std::unique_ptr<BatchReader>> DataEvolutionSplitRead::WrapWithBlobViewResolverIfNeeded(
    const std::shared_ptr<DataSplit>& data_split,
    std::unique_ptr<BatchReader>&& inner_reader) const {
    if (!options_.BlobViewResolveEnabled()) {
        // preserve serialized BlobViewStruct bytes, e.g. for forwarding blob view values to
        // another blob-view table
        return std::move(inner_reader);
    }
    std::vector<std::string> read_blob_view_fields = HasBlobViewField(options_, raw_read_schema_);
    if (read_blob_view_fields.empty()) {
        return std::move(inner_reader);
    }
    std::optional<std::string> warehouse_path = options_.GetBlobViewUpstreamWarehouse();
    if (!warehouse_path) {
        return Status::Invalid(
            "invalid config for blob view, supposed to set BLOB_VIEW_UPSTREAM_WAREHOUSE");
    }
    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BatchReader> pre_reader,
                           CreateBlobViewReader(data_split, read_blob_view_fields));
    PAIMON_ASSIGN_OR_RAISE(std::unordered_set<BlobViewStruct> blob_view_structs,
                           ExtractBlobViewStructs(pre_reader.get()));
    auto catalog_context = std::make_shared<CatalogContext>(
        warehouse_path.value(), options_.ToMap(), options_.GetFileSystem());
    // use global thread number
    uint32_t cpu_count = std::thread::hardware_concurrency();
    uint32_t thread_num = cpu_count > 0 ? cpu_count : 1;
    PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<Executor> executor, CreateDefaultExecutor(thread_num));
    PAIMON_ASSIGN_OR_RAISE(
        BlobViewResolver resolver,
        BlobViewLookup::CreateResolver(blob_view_structs, catalog_context, pool_, executor));
    return std::make_unique<BlobViewResolvingBatchReader>(
        std::move(inner_reader), std::move(read_blob_view_fields), std::move(resolver), pool_);
}

Result<std::unique_ptr<BatchReader>> DataEvolutionSplitRead::CreateBlobViewReader(
    const std::shared_ptr<DataSplit>& data_split,
    const std::vector<std::string>& read_blob_view_fields) const {
    auto split_impl = dynamic_cast<DataSplitImpl*>(data_split.get());
    if (split_impl == nullptr) {
        return Status::Invalid("unexpected error, split cast to impl failed");
    }
    assert(raw_read_schema_->num_fields() > 0);

    arrow::FieldVector blob_view_arrow_fields;
    blob_view_arrow_fields.reserve(read_blob_view_fields.size());
    for (const auto& field_name : read_blob_view_fields) {
        auto field = raw_read_schema_->GetFieldByName(field_name);
        if (field == nullptr) {
            return Status::Invalid(
                fmt::format("Blob view field {} not found in read schema.", field_name));
        }
        blob_view_arrow_fields.push_back(std::move(field));
    }
    auto blob_view_schema = arrow::schema(std::move(blob_view_arrow_fields));

    PAIMON_ASSIGN_OR_RAISE(
        std::shared_ptr<DataFilePathFactory> data_file_path_factory,
        path_factory_->CreateDataFilePathFactory(split_impl->Partition(), split_impl->Bucket()));

    // skip blob files: they only contain blob payloads, not the blob-view columns.
    std::vector<std::shared_ptr<DataFileMeta>> data_files;
    data_files.reserve(split_impl->DataFiles().size());
    for (const auto& file : split_impl->DataFiles()) {
        if (!BlobUtils::IsBlobFile(file->file_name)) {
            data_files.push_back(file);
        }
    }
    PAIMON_ASSIGN_OR_RAISE(
        std::vector<std::unique_ptr<FileBatchReader>> raw_file_readers,
        CreateRawFileReaders(split_impl->Partition(), data_files, blob_view_schema,
                             /*predicate=*/nullptr, /*dv_factory=*/nullptr,
                             /*row_ranges=*/std::nullopt, data_file_path_factory,
                             /*extra_format_options=*/{}));

    auto batch_readers =
        ObjectUtils::MoveVector<std::unique_ptr<BatchReader>>(std::move(raw_file_readers));
    return std::make_unique<ConcatBatchReader>(std::move(batch_readers), pool_);
}

Result<std::unordered_set<BlobViewStruct>> DataEvolutionSplitRead::ExtractBlobViewStructs(
    BatchReader* reader) {
    if (reader == nullptr) {
        return Status::Invalid("invalid reader in ExtractBlobViewStructs, reader is nullptr");
    }
    std::unordered_set<BlobViewStruct> blob_view_structs;
    while (true) {
        PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader->NextBatch());
        if (BatchReader::IsEofBatch(batch)) {
            break;
        }
        auto& [c_array, c_schema] = batch;
        PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Array> arrow_array,
                                          arrow::ImportArray(c_array.get(), c_schema.get()));
        auto struct_array = std::dynamic_pointer_cast<arrow::StructArray>(arrow_array);
        if (struct_array == nullptr) {
            return Status::Invalid(
                "invalid array in ExtractBlobViewStructs, batch array is not a StructArray.");
        }

        for (int32_t field_idx = 0; field_idx < struct_array->num_fields(); ++field_idx) {
            auto binary_array =
                std::dynamic_pointer_cast<arrow::LargeBinaryArray>(struct_array->field(field_idx));
            if (binary_array == nullptr) {
                return Status::Invalid(
                    "invalid array in ExtractBlobViewStructs, blob view column is not a "
                    "LargeBinaryArray.");
            }
            for (int64_t row = 0; row < binary_array->length(); ++row) {
                if (binary_array->IsNull(row)) {
                    continue;
                }
                std::string_view bytes = binary_array->GetView(row);
                PAIMON_ASSIGN_OR_RAISE(
                    bool is_view, BlobViewStruct::IsBlobViewStruct(bytes.data(), bytes.size()));
                if (!is_view) {
                    return Status::Invalid(
                        "blob-view-field requires blob field value to be a serialized "
                        "BlobViewStruct.");
                }
                PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BlobViewStruct> view_struct,
                                       BlobViewStruct::Deserialize(bytes.data(), bytes.size()));
                blob_view_structs.insert(*view_struct);
            }
        }
    }
    return blob_view_structs;
}

Result<std::unique_ptr<BatchReader>> DataEvolutionSplitRead::InnerCreateReader(
    const std::shared_ptr<DataSplit>& data_split,
    const std::optional<std::vector<Range>>& row_ranges) const {
    auto split_impl = dynamic_cast<DataSplitImpl*>(data_split.get());
    if (split_impl == nullptr) {
        return Status::Invalid("unexpected error, split cast to impl failed");
    }
    assert(raw_read_schema_->num_fields() > 0);
    PAIMON_ASSIGN_OR_RAISE(
        std::shared_ptr<DataFilePathFactory> data_file_path_factory,
        path_factory_->CreateDataFilePathFactory(split_impl->Partition(), split_impl->Bucket()));
    auto metas = split_impl->DataFiles();
    PAIMON_ASSIGN_OR_RAISE(std::vector<std::vector<std::shared_ptr<DataFileMeta>>> split_by_row_id,
                           MergeRangesAndSort(std::move(metas)));

    std::vector<std::unique_ptr<BatchReader>> sub_readers;
    for (const std::vector<std::shared_ptr<DataFileMeta>>& need_merge_files : split_by_row_id) {
        if (need_merge_files.size() == 1) {
            // No need to merge fields, just create a single file reader
            PAIMON_ASSIGN_OR_RAISE(
                std::vector<std::unique_ptr<FileBatchReader>> raw_file_readers,
                CreateRawFileReaders(split_impl->Partition(), need_merge_files, raw_read_schema_,
                                     /*predicate=*/nullptr,
                                     /*dv_factory=*/nullptr, row_ranges, data_file_path_factory,
                                     /*extra_format_options=*/{}));
            assert(raw_file_readers.size() == 1);
            sub_readers.push_back(std::move(raw_file_readers[0]));
        } else {
            PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<DataEvolutionFileReader> evolution_reader,
                                   CreateUnionReader(split_impl->Partition(), need_merge_files,
                                                     row_ranges, data_file_path_factory));
            sub_readers.push_back(std::move(evolution_reader));
        }
    }
    auto concat_batch_reader = std::make_unique<ConcatBatchReader>(std::move(sub_readers), pool_);
    PAIMON_ASSIGN_OR_RAISE(
        std::unique_ptr<BatchReader> batch_reader,
        ApplyPredicateFilterIfNeeded(std::move(concat_batch_reader), context_->GetPredicate()));
    return std::make_unique<CompleteRowKindBatchReader>(std::move(batch_reader), pool_);
}

Result<std::unique_ptr<FileBatchReader>> DataEvolutionSplitRead::ApplyIndexAndDvReaderIfNeeded(
    std::unique_ptr<FileBatchReader>&& file_reader, const std::shared_ptr<DataFileMeta>& file,
    const std::shared_ptr<arrow::Schema>& data_schema,
    const std::shared_ptr<arrow::Schema>& read_schema, const std::shared_ptr<Predicate>& predicate,
    DeletionVector::Factory dv_factory, const std::optional<std::vector<Range>>& row_ranges,
    const std::shared_ptr<DataFilePathFactory>& data_file_path_factory) const {
    if (dv_factory) {
        return Status::Invalid("DataEvolutionSplitRead do not support deletion vector");
    }
    if (predicate) {
        assert(false);
        // as DataEvolutionSplitRead will skip predicate
        return Status::Invalid("DataEvolutionSplitRead do not support predicate");
    }
    PAIMON_ASSIGN_OR_RAISE(std::optional<RoaringBitmap32> selection_row_ids,
                           file->ToFileSelection(row_ranges));
    ::ArrowSchema c_read_schema;
    PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*read_schema, &c_read_schema));
    PAIMON_RETURN_NOT_OK(
        file_reader->SetReadSchema(&c_read_schema, /*predicate=*/nullptr, selection_row_ids));

    std::unique_ptr<FileBatchReader> reader;
    if (!file_reader->SupportPreciseBitmapSelection() && selection_row_ids) {
        // Some formats (for example blob) return an accurate batch result, where
        // ApplyBitmapIndexBatchReader is not necessary
        reader = std::make_unique<ApplyBitmapIndexBatchReader>(
            std::move(file_reader), std::move(selection_row_ids).value());
    } else {
        reader = std::move(file_reader);
    }

    return std::move(reader);
}

Result<std::vector<std::shared_ptr<DataEvolutionSplitRead::FieldBunch>>>
DataEvolutionSplitRead::SplitFieldBunches(
    const std::vector<std::shared_ptr<DataFileMeta>>& need_merge_files,
    const std::function<Result<int32_t>(const std::shared_ptr<DataFileMeta>&)>&
        blob_field_to_field_id,
    bool has_row_ranges_selection) {
    std::vector<std::shared_ptr<DataEvolutionSplitRead::FieldBunch>> fields_files;
    std::map<int32_t, std::shared_ptr<DataEvolutionSplitRead::BlobBunch>> blob_bunch_map;
    int64_t row_count = -1;
    for (const auto& file : need_merge_files) {
        if (BlobUtils::IsBlobFile(file->file_name)) {
            PAIMON_ASSIGN_OR_RAISE(int32_t field_id, blob_field_to_field_id(file));
            auto iter = blob_bunch_map.find(field_id);
            if (iter == blob_bunch_map.end()) {
                auto blob_bunch = std::make_shared<DataEvolutionSplitRead::BlobBunch>(
                    row_count, has_row_ranges_selection);
                PAIMON_RETURN_NOT_OK(blob_bunch->Add(file));
                blob_bunch_map.insert(std::make_pair(field_id, blob_bunch));
            } else {
                assert(iter->second);
                PAIMON_RETURN_NOT_OK(iter->second->Add(file));
            }
        } else {
            // Normal file, just add it to the current merge split
            fields_files.push_back(std::make_shared<DataEvolutionSplitRead::DataBunch>(file));
            row_count = file->row_count;
        }
    }
    for (const auto& [field_id, blob_bunch] : blob_bunch_map) {
        fields_files.push_back(blob_bunch);
    }
    return fields_files;
}

Result<std::unique_ptr<DataEvolutionFileReader>> DataEvolutionSplitRead::CreateUnionReader(
    const BinaryRow& partition, const std::vector<std::shared_ptr<DataFileMeta>>& need_merge_files,
    const std::optional<std::vector<Range>>& row_ranges,
    const std::shared_ptr<DataFilePathFactory>& data_file_path_factory) const {
    auto blob_field_to_field_id =
        [&](const std::shared_ptr<DataFileMeta>& file_meta) -> Result<int32_t> {
        if (!BlobUtils::IsBlobFile(file_meta->file_name)) {
            return Status::Invalid("Only blob file need to call this method.");
        }
        auto data_schema = context_->GetTableSchema();
        if (data_schema->Id() != file_meta->schema_id) {
            PAIMON_ASSIGN_OR_RAISE(data_schema, schema_manager_->ReadSchema(file_meta->schema_id));
        }
        const auto& write_cols = file_meta->write_cols;
        if (write_cols == std::nullopt || write_cols.value().size() != 1) {
            return Status::Invalid("Blob file only has one blob field");
        }
        PAIMON_ASSIGN_OR_RAISE(DataField data_field, data_schema->GetField(write_cols.value()[0]));
        return data_field.Id();
    };

    PAIMON_ASSIGN_OR_RAISE(
        std::vector<std::shared_ptr<DataEvolutionSplitRead::FieldBunch>> fields_files,
        SplitFieldBunches(need_merge_files, blob_field_to_field_id,
                          /*has_row_ranges_selection=*/row_ranges.has_value()));

    assert(!fields_files.empty());
    int64_t row_count = fields_files[0]->RowCount();
    PAIMON_ASSIGN_OR_RAISE(int64_t first_row_id, fields_files[0]->Files()[0]->NonNullFirstRowId());
    if (!row_ranges) {
        for (const auto& bunch : fields_files) {
            if (bunch->RowCount() != row_count) {
                return Status::Invalid(
                    "All files in a field merge split should have the same row count.");
            }
            PAIMON_ASSIGN_OR_RAISE(int64_t current_bunch_first_row_id,
                                   bunch->Files()[0]->NonNullFirstRowId());
            if (current_bunch_first_row_id != first_row_id) {
                return Status::Invalid(
                    "All files in a field merge split should have the same first row id and could "
                    "not be null.");
            }
        }
    }

    // Init all we need to create a compound reader
    PAIMON_ASSIGN_OR_RAISE(std::vector<DataField> all_read_fields,
                           DataField::ConvertArrowSchemaToDataFields(raw_read_schema_));
    std::vector<std::unique_ptr<BatchReader>> file_batch_readers(fields_files.size());
    std::vector<int32_t> read_field_ids = DataField::GetAllFieldIds(all_read_fields);
    // which row the read field index belongs to
    std::vector<int32_t> reader_offsets(all_read_fields.size(), -1);
    // which field index in the reading row
    std::vector<int32_t> field_offsets(all_read_fields.size(), -1);

    for (int32_t file_idx = 0; file_idx < static_cast<int32_t>(fields_files.size()); file_idx++) {
        const auto& bunch = fields_files[file_idx];
        const auto& first_file = bunch->Files()[0];
        int64_t schema_id = first_file->schema_id;
        std::shared_ptr<TableSchema> data_schema = context_->GetTableSchema();
        if (schema_id != data_schema->Id()) {
            PAIMON_ASSIGN_OR_RAISE(data_schema, schema_manager_->ReadSchema(schema_id));
        }
        PAIMON_ASSIGN_OR_RAISE(
            std::vector<DataField> data_fields,
            ProjectFieldsForRowTrackingAndDataEvolution(data_schema, first_file->write_cols));
        std::vector<int32_t> data_field_ids = DataField::GetAllFieldIds(data_fields);
        std::vector<DataField> read_fields_in_file;
        for (int32_t read_field_idx = 0;
             read_field_idx < static_cast<int32_t>(read_field_ids.size()); read_field_idx++) {
            // -1 indicates that the read fields are not matched
            if (reader_offsets[read_field_idx] != -1) {
                continue;
            }
            for (int32_t data_field_id : data_field_ids) {
                // Check if the read field index matches the file field
                // index
                if (data_field_id == read_field_ids[read_field_idx]) {
                    // "file_idx" is the reader index, and "read_fields_in_file.size()"
                    // is the offset the that row
                    reader_offsets[read_field_idx] = file_idx;
                    field_offsets[read_field_idx] = read_fields_in_file.size();
                    read_fields_in_file.push_back(all_read_fields[read_field_idx]);
                    break;
                }
            }
        }

        if (!read_fields_in_file.empty()) {
            // create new FieldMappingReader for read partial fields
            auto file_read_schema = DataField::ConvertDataFieldsToArrowSchema(read_fields_in_file);
            auto blob_bunch = std::dynamic_pointer_cast<BlobBunch>(bunch);
            if (blob_bunch && !blob_bunch->SequentialReadOptimize()) {
                // blob files span multiple max sequence number layers: placeholder entries of
                // newer layers must fall back row by row to older layers
                PAIMON_ASSIGN_OR_RAISE(
                    file_batch_readers[file_idx],
                    CreateBlobFallbackReader(partition, blob_bunch->Files(), file_read_schema,
                                             row_ranges, data_file_path_factory));
            } else {
                PAIMON_ASSIGN_OR_RAISE(
                    std::vector<std::unique_ptr<FileBatchReader>> file_readers,
                    CreateRawFileReaders(partition, bunch->Files(), file_read_schema,
                                         /*predicate=*/nullptr, /*dv_factory=*/{}, row_ranges,
                                         data_file_path_factory,
                                         /*extra_format_options=*/{}));
                if (file_readers.size() == 1) {
                    file_batch_readers[file_idx] = std::move(file_readers[0]);
                } else {
                    auto raw_readers = ObjectUtils::MoveVector<std::unique_ptr<BatchReader>>(
                        std::move(file_readers));
                    // Concat multiple blob files that map to the same data file.
                    file_batch_readers[file_idx] =
                        std::make_unique<ConcatBatchReader>(std::move(raw_readers), pool_);
                }
            }
        }
    }
    // TODO(xinyu.lxy): check nullable when reader_offsets[read_field_idx] = -1
    return DataEvolutionFileReader::Create(std::move(file_batch_readers), raw_read_schema_,
                                           options_.GetReadBatchSize(), reader_offsets,
                                           field_offsets, pool_);
}

namespace {
/// Selected row ids in [from, to] as sorted disjoint ranges: the whole range without a
/// selection, otherwise its intersection with the (possibly overlapping) selected ranges.
std::vector<Range> SelectedRangesInRange(int64_t from, int64_t to,
                                         const std::optional<std::vector<Range>>& row_ranges) {
    if (!row_ranges) {
        return {Range(from, to)};
    }
    std::vector<Range> selected;
    Range gap_range(from, to);
    for (const auto& range : Range::SortAndMergeOverlap(row_ranges.value(), /*adjacent=*/true)) {
        std::optional<Range> intersection = Range::Intersection(gap_range, range);
        if (intersection) {
            selected.push_back(*intersection);
        }
    }
    return selected;
}
}  // namespace

Result<std::unique_ptr<BatchReader>> DataEvolutionSplitRead::CreateBlobFallbackReader(
    const BinaryRow& partition, const std::vector<std::shared_ptr<DataFileMeta>>& files,
    const std::shared_ptr<arrow::Schema>& file_read_schema,
    const std::optional<std::vector<Range>>& row_ranges,
    const std::shared_ptr<DataFilePathFactory>& data_file_path_factory) const {
    int64_t union_first_row_id = std::numeric_limits<int64_t>::max();
    int64_t union_last_row_id = std::numeric_limits<int64_t>::min();
    using FileWithFirstRowId = std::pair<int64_t, std::shared_ptr<DataFileMeta>>;
    std::map<int64_t, std::vector<FileWithFirstRowId>, std::greater<>> sequence_groups;
    for (const auto& file : files) {
        PAIMON_ASSIGN_OR_RAISE(int64_t first_row_id, file->NonNullFirstRowId());
        union_first_row_id = std::min(union_first_row_id, first_row_id);
        union_last_row_id = std::max(union_last_row_id, first_row_id + file->row_count - 1);
        sequence_groups[file->max_sequence_number].emplace_back(first_row_id, file);
    }
    // the blob format reader must emit placeholder sentinels instead of failing on them
    const std::map<std::string, std::string> blob_format_options = {
        {BlobDefs::kEmitPlaceholderSentinelKey, "true"}};
    std::vector<std::vector<BlobFallbackBatchReader::Segment>> groups;
    groups.reserve(sequence_groups.size());
    for (auto& [max_sequence_number, group_files] : sequence_groups) {
        std::stable_sort(group_files.begin(), group_files.end(),
                         [](const FileWithFirstRowId& f1, const FileWithFirstRowId& f2) {
                             return f1.first < f2.first;
                         });
        std::vector<BlobFallbackBatchReader::Segment> segments;
        // pad row ids this layer does not cover with placeholder gaps, so that every layer spans
        // the same union range and the groups can be stepped in lockstep
        int64_t next_row_id = union_first_row_id;
        for (const auto& [first_row_id, file] : group_files) {
            if (first_row_id < next_row_id) {
                return Status::Invalid(fmt::format(
                    "Blob files with the same max sequence number should not have overlapping "
                    "row id ranges: file {} (max sequence number {}) starts at row id {} before "
                    "the previous file's end {}",
                    file->file_name, max_sequence_number, first_row_id, next_row_id));
            }
            if (first_row_id > next_row_id) {
                std::vector<Range> gap_selected_ranges =
                    SelectedRangesInRange(next_row_id, first_row_id - 1, row_ranges);
                if (!gap_selected_ranges.empty()) {
                    segments.push_back(
                        BlobFallbackBatchReader::Segment{nullptr, std::move(gap_selected_ranges)});
                }
            }
            PAIMON_ASSIGN_OR_RAISE(
                std::vector<std::unique_ptr<FileBatchReader>> file_readers,
                CreateRawFileReaders(partition, {file}, file_read_schema,
                                     /*predicate=*/nullptr, /*dv_factory=*/{}, row_ranges,
                                     data_file_path_factory, blob_format_options));
            if (file_readers.size() != 1) {
                return Status::Invalid("Unexpected: blob fallback file reader was skipped.");
            }
            segments.push_back(BlobFallbackBatchReader::Segment{std::move(file_readers[0]), {}});
            next_row_id = first_row_id + file->row_count;
        }
        if (next_row_id <= union_last_row_id) {
            std::vector<Range> gap_selected_ranges =
                SelectedRangesInRange(next_row_id, union_last_row_id, row_ranges);
            if (!gap_selected_ranges.empty()) {
                segments.push_back(
                    BlobFallbackBatchReader::Segment{nullptr, std::move(gap_selected_ranges)});
            }
        }
        groups.push_back(std::move(segments));
    }
    PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<BlobFallbackBatchReader> fallback_reader,
                           BlobFallbackBatchReader::Create(std::move(groups), file_read_schema,
                                                           options_.GetReadBatchSize(), pool_));
    return std::move(fallback_reader);
}

Result<bool> DataEvolutionSplitRead::Match(const std::shared_ptr<Split>& split,
                                           bool force_keep_delete) const {
    return true;
}

Result<std::vector<std::vector<std::shared_ptr<DataFileMeta>>>>
DataEvolutionSplitRead::MergeRangesAndSort(std::vector<std::shared_ptr<DataFileMeta>>&& files) {
    // group by row id range
    RangeHelper<std::shared_ptr<DataFileMeta>> range_helper(
        [](const std::shared_ptr<DataFileMeta>& meta) -> Result<int64_t> {
            return meta->NonNullFirstRowId();
        },
        [](const std::shared_ptr<DataFileMeta>& meta) -> Result<int64_t> {
            PAIMON_ASSIGN_OR_RAISE(int64_t first_row_id, meta->NonNullFirstRowId());
            return first_row_id + meta->row_count - 1;
        });

    PAIMON_ASSIGN_OR_RAISE(std::vector<std::vector<std::shared_ptr<DataFileMeta>>> result,
                           range_helper.MergeOverlappingRanges(std::move(files)));
    // in group, sort by blob file and max_seq
    for (auto& group : result) {
        // split to data files and blob files
        std::vector<std::shared_ptr<DataFileMeta>> data_files;
        data_files.reserve(group.size());
        std::vector<std::shared_ptr<DataFileMeta>> blob_files;
        blob_files.reserve(group.size());
        for (auto& meta : group) {
            if (BlobUtils::IsBlobFile(meta->file_name)) {
                blob_files.push_back(std::move(meta));
            } else {
                data_files.push_back(std::move(meta));
            }
        }

        // data files sort by reversed max sequence number
        std::stable_sort(
            data_files.begin(), data_files.end(),
            [](const std::shared_ptr<DataFileMeta>& m1, const std::shared_ptr<DataFileMeta>& m2) {
                return m1->max_sequence_number > m2->max_sequence_number;
            });

        PAIMON_ASSIGN_OR_RAISE(bool all_range_same, range_helper.AreAllRangesSame(data_files));
        if (!all_range_same) {
            return Status::Invalid("Data files should be all row id ranges same.");
        }

        // blob files sort by first row id then by reversed max sequence number
        std::stable_sort(
            blob_files.begin(), blob_files.end(),
            [](const std::shared_ptr<DataFileMeta>& m1, const std::shared_ptr<DataFileMeta>& m2) {
                if (m1->first_row_id.value() != m2->first_row_id.value()) {
                    return m1->first_row_id.value() < m2->first_row_id.value();
                }
                return m1->max_sequence_number > m2->max_sequence_number;
            });
        // concat data files and blob files
        group.clear();
        group.insert(group.end(), std::make_move_iterator(data_files.begin()),
                     std::make_move_iterator(data_files.end()));
        group.insert(group.end(), std::make_move_iterator(blob_files.begin()),
                     std::make_move_iterator(blob_files.end()));
    }
    return result;
}
}  // namespace paimon
