# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

import io
import os
import shutil
import struct
import tempfile
import unittest

import pyarrow as pa

from pypaimon import CatalogFactory, Schema
from pypaimon.schema.schema_change import SchemaChange
from pypaimon.table.file_store_table import FileStoreTable
from pypaimon.table.row.blob import Blob
from pypaimon.write.commit_message import CommitMessage


class _StreamingOnlyBlob(Blob):
    def __init__(self, data: bytes):
        self.data = data
        self.opened = False

    def to_data(self) -> bytes:
        raise AssertionError("streaming blob should not be materialized")

    def to_descriptor(self):
        raise RuntimeError("streaming blob has no descriptor")

    def new_input_stream(self):
        self.opened = True
        return io.BytesIO(self.data)


class _RecordBatchWithoutSelect:
    def __init__(self, batch):
        self.schema = batch.schema
        self._batch = batch

    def column(self, index):
        return self._batch.column(index)


class DedicatedFormatWriterTest(unittest.TestCase):
    """Tests for DedicatedFormatWriter functionality with paimon table operations."""

    @classmethod
    def setUpClass(cls):
        """Set up test environment."""
        cls.temp_dir = tempfile.mkdtemp()
        cls.warehouse = os.path.join(cls.temp_dir, 'warehouse')
        # Create catalog for table operations
        cls.catalog = CatalogFactory.create({
            'warehouse': cls.warehouse
        })
        cls.catalog.create_database('test_db', False)

    @classmethod
    def tearDownClass(cls):
        """Clean up test environment."""
        try:
            shutil.rmtree(cls.temp_dir)
        except OSError:
            pass

    def test_dedicated_format_writer_basic_functionality(self):
        """Test basic DedicatedFormatWriter functionality with paimon table."""
        from pypaimon import Schema

        # Create schema with normal and blob columns
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('name', pa.string()),
            ('blob_data', pa.large_binary()),  # This will be detected as blob
        ])

        # Create Paimon schema
        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true'
            }
        )

        # Create table
        self.catalog.create_table('test_db.blob_writer_test', schema, False)
        table = self.catalog.get_table('test_db.blob_writer_test')

        # Test data
        test_data = pa.Table.from_pydict({
            'id': [1, 2, 3],
            'name': ['Alice', 'Bob', 'Charlie'],
            'blob_data': [b'blob_data_1', b'blob_data_2', b'blob_data_3']
        }, schema=pa_schema)

        # Test DedicatedFormatWriter initialization using proper table API
        # Use proper table API to create writer
        write_builder = table.new_batch_write_builder()
        blob_writer = write_builder.new_write()

        # Write test data using BatchTableWrite API
        blob_writer.write_arrow(test_data)

        # Test prepare commit
        commit_messages = blob_writer.prepare_commit()
        self.assertIsInstance(commit_messages, list)
        self.assertGreater(len(commit_messages), 0)

        # Verify commit message structure
        for commit_msg in commit_messages:
            self.assertIsInstance(commit_msg.new_files, list)
            self.assertGreater(len(commit_msg.new_files), 0)

            # Verify file metadata structure
            for file_meta in commit_msg.new_files:
                self.assertIsNotNone(file_meta.file_name)
                self.assertGreater(file_meta.file_size, 0)
                self.assertGreater(file_meta.row_count, 0)

        blob_writer.close()

    def test_split_data_with_pyarrow_6_record_batch_api(self):
        from pypaimon.write.writer.dedicated_format_writer import DedicatedFormatWriter

        payload_type = pa.list_(
            pa.field('item', pa.large_binary(), nullable=False)
        )
        schema = pa.schema(
            [
                pa.field('id', pa.int32(), nullable=False),
                pa.field('payloads', payload_type),
                pa.field('embedding', pa.list_(pa.float32())),
            ],
            metadata={b'table': b'array-blob'},
        )
        batch = pa.RecordBatch.from_arrays(
            [
                pa.array([1, 2], type=pa.int32()),
                pa.array([[b'a'], []], type=payload_type),
                pa.array([[1.0], [2.0]], type=pa.list_(pa.float32())),
            ],
            schema=schema,
        )

        writer = object.__new__(DedicatedFormatWriter)
        writer.normal_column_names = ['id']
        writer.blob_file_column_names = ['payloads']
        writer.vector_write_columns = ['embedding']

        normal_data, blob_data_map, vector_data = writer._split_data(
            _RecordBatchWithoutSelect(batch)
        )

        self.assertEqual(normal_data.column(0).to_pylist(), [1, 2])
        self.assertFalse(normal_data.schema.field('id').nullable)
        self.assertEqual(
            blob_data_map['payloads'].column(0).to_pylist(),
            [[b'a'], []],
        )
        self.assertFalse(
            blob_data_map['payloads'].schema.field('payloads').type.value_field.nullable
        )
        self.assertEqual(vector_data.column(0).to_pylist(), [[1.0], [2.0]])
        self.assertEqual(normal_data.schema.metadata, {b'table': b'array-blob'})

    def test_dedicated_format_writer_schema_detection(self):
        """Test that DedicatedFormatWriter correctly detects blob columns from schema."""
        from pypaimon import Schema

        # Test schema with blob column
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('blob_field', pa.large_binary()),
        ])

        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true'
            }
        )
        self.catalog.create_table('test_db.blob_detection_test', schema, False)
        table = self.catalog.get_table('test_db.blob_detection_test')

        # Use proper table API to create writer
        write_builder = table.new_batch_write_builder()
        blob_writer = write_builder.new_write()

        # Test that DedicatedFormatWriter was created internally
        # We can verify this by checking the internal data writers
        test_data = pa.Table.from_pydict({
            'id': [1, 2, 3],
            'blob_field': [b'blob1', b'blob2', b'blob3']
        }, schema=pa_schema)

        # Write data to trigger writer creation
        blob_writer.write_arrow(test_data)

        # Verify that a DedicatedFormatWriter was created internally
        data_writers = blob_writer.file_store_write.data_writers
        self.assertGreater(len(data_writers), 0)

        # Check that the writer is a DedicatedFormatWriter
        for writer in data_writers.values():
            from pypaimon.write.writer.dedicated_format_writer import DedicatedFormatWriter
            self.assertIsInstance(writer, DedicatedFormatWriter)

        blob_writer.close()

    def test_dedicated_format_writer_no_blob_column(self):
        """Test that DedicatedFormatWriter raises error when no blob column is found."""
        from pypaimon import Schema

        # Test schema without blob column
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('name', pa.string()),
        ])

        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true'
            }
        )
        self.catalog.create_table('test_db.no_blob_test', schema, False)
        table = self.catalog.get_table('test_db.no_blob_test')

        # Use proper table API to create writer
        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()

        # Test that a regular writer (not DedicatedFormatWriter) was created
        test_data = pa.Table.from_pydict({
            'id': [1, 2, 3],
            'name': ['Alice', 'Bob', 'Charlie']
        }, schema=pa_schema)

        # Write data to trigger writer creation
        writer.write_arrow(test_data)

        # Verify that a regular writer was created (not DedicatedFormatWriter)
        data_writers = writer.file_store_write.data_writers
        self.assertGreater(len(data_writers), 0)

        # Check that the writer is NOT a DedicatedFormatWriter
        for writer_instance in data_writers.values():
            from pypaimon.write.writer.dedicated_format_writer import DedicatedFormatWriter
            self.assertNotIsInstance(writer_instance, DedicatedFormatWriter)

        writer.close()

    def test_dedicated_format_writer_multiple_blob_columns(self):
        """Test that DedicatedFormatWriter supports multiple blob columns."""
        from pypaimon import Schema

        # Test schema with multiple blob columns
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('blob1', pa.large_binary()),
            ('blob2', pa.large_binary()),
        ])

        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true'
            }
        )
        self.catalog.create_table('test_db.multiple_blob_test', schema, False)
        table = self.catalog.get_table('test_db.multiple_blob_test')

        # Use proper table API to create writer
        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()

        # Test data with multiple blob columns
        test_data = pa.Table.from_pydict({
            'id': [1, 2, 3],
            'blob1': [b'blob1_1', b'blob1_2', b'blob1_3'],
            'blob2': [b'blob2_1', b'blob2_2', b'blob2_3']
        }, schema=pa_schema)

        writer.write_arrow(test_data)
        commit_messages = writer.prepare_commit()
        write_builder.new_commit().commit(commit_messages)
        writer.close()

        all_files = [f for msg in commit_messages for f in msg.new_files]
        blob_files = [f for f in all_files if f.file_name.endswith('.blob')]
        self.assertGreaterEqual(len(blob_files), 2, "Each blob column should produce blob files")

        # Verify row counts can be read back correctly.
        result = table.new_read_builder().new_read().to_arrow(table.new_read_builder().new_scan().plan().splits())
        self.assertEqual(result.num_rows, 3)

    def test_dedicated_format_writer_partial_write_with_write_type(self):
        """Partial write (normal + blob subset) via with_write_type: split must match batch columns."""
        from pypaimon import Schema

        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('name', pa.string()),
            ('blob_data', pa.large_binary()),
        ])
        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true'
            }
        )
        self.catalog.create_table('test_db.blob_partial_write_type', schema, False)
        table = self.catalog.get_table('test_db.blob_partial_write_type')

        partial_schema = pa.schema([('id', pa.int32()), ('blob_data', pa.large_binary())])
        test_data = pa.Table.from_pydict({
            'id': [1, 2],
            'blob_data': [b'a', b'b'],
        }, schema=partial_schema)

        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write().with_write_type(['id', 'blob_data'])
        writer.write_arrow(test_data)
        commit_messages = writer.prepare_commit()
        self.assertGreater(len(commit_messages), 0)
        all_files = [f for msg in commit_messages for f in msg.new_files]
        parquet_files = [f for f in all_files if f.file_name.endswith('.parquet')]
        blob_files = [f for f in all_files if f.file_name.endswith('.blob')]
        self.assertEqual(len(parquet_files), 1)
        self.assertGreaterEqual(len(blob_files), 1)
        self.assertEqual(parquet_files[0].write_cols, ['id'])
        self.assertEqual(parquet_files[0].row_count, 2)
        for bf in blob_files:
            self.assertEqual(bf.write_cols, ['blob_data'])
            self.assertEqual(bf.row_count, 2)
        write_builder.new_commit().commit(commit_messages)
        writer.close()

        read_builder = table.new_read_builder()
        out = read_builder.new_read().to_arrow(read_builder.new_scan().plan().splits())
        self.assertEqual(out.num_rows, 2)
        self.assertEqual(out.column('id').to_pylist(), [1, 2])
        self.assertEqual(out.column('blob_data').to_pylist(), [b'a', b'b'])
        self.assertEqual(out.column('name').to_pylist(), [None, None])

    def test_dedicated_format_writer_partial_write_normal_only_with_write_type(self):
        """Partial write without blob columns in write_cols must not touch blob split paths."""
        from pypaimon import Schema

        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('name', pa.string()),
            ('blob_data', pa.large_binary()),
        ])
        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true'
            }
        )
        self.catalog.create_table('test_db.blob_partial_normal_only', schema, False)
        table = self.catalog.get_table('test_db.blob_partial_normal_only')

        partial_schema = pa.schema([('id', pa.int32()), ('name', pa.string())])
        test_data = pa.Table.from_pydict({'id': [7], 'name': ['n']}, schema=partial_schema)

        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write().with_write_type(['id', 'name'])
        writer.write_arrow(test_data)
        commit_messages = writer.prepare_commit()
        all_files = [f for msg in commit_messages for f in msg.new_files]
        self.assertFalse(any(f.file_name.endswith('.blob') for f in all_files))
        parquet_files = [f for f in all_files if f.file_name.endswith('.parquet')]
        self.assertEqual(len(parquet_files), 1)
        self.assertEqual(parquet_files[0].write_cols, ['id', 'name'])
        self.assertEqual(parquet_files[0].row_count, 1)
        write_builder.new_commit().commit(commit_messages)
        writer.close()

        read_builder = table.new_read_builder()
        out = read_builder.new_read().to_arrow(read_builder.new_scan().plan().splits())
        self.assertEqual(out.column('id').to_pylist(), [7])
        self.assertEqual(out.column('name').to_pylist(), ['n'])
        self.assertEqual(out.column('blob_data').to_pylist(), [None])

    def test_dedicated_format_writer_partial_write_single_blob_of_two_with_write_type(self):
        """with_write_type lists only one blob column: only that column gets .blob files."""
        from pypaimon import Schema

        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('blob1', pa.large_binary()),
            ('blob2', pa.large_binary()),
        ])
        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true'
            }
        )
        self.catalog.create_table('test_db.blob_partial_one_of_two', schema, False)
        table = self.catalog.get_table('test_db.blob_partial_one_of_two')

        partial_schema = pa.schema([('id', pa.int32()), ('blob1', pa.large_binary())])
        test_data = pa.Table.from_pydict({
            'id': [1],
            'blob1': [b'only_b1'],
        }, schema=partial_schema)

        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write().with_write_type(['id', 'blob1'])
        writer.write_arrow(test_data)
        commit_messages = writer.prepare_commit()
        all_files = [f for msg in commit_messages for f in msg.new_files]
        blob_files = [f for f in all_files if f.file_name.endswith('.blob')]
        self.assertEqual(len(blob_files), 1)
        self.assertEqual(blob_files[0].write_cols, ['blob1'])
        write_builder.new_commit().commit(commit_messages)
        writer.close()

    def test_dedicated_format_writer_write_operations(self):
        """Test DedicatedFormatWriter write operations with real data."""
        from pypaimon import Schema

        # Create schema with blob column
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('name', pa.string()),
            ('document', pa.large_binary()),
        ])

        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true'
            }
        )
        self.catalog.create_table('test_db.write_test', schema, False)
        table = self.catalog.get_table('test_db.write_test')

        # Use proper table API to create writer
        write_builder = table.new_batch_write_builder()
        blob_writer = write_builder.new_write()

        # Test data
        test_data = pa.Table.from_pydict({
            'id': [1, 2],
            'name': ['Alice', 'Bob'],
            'document': [b'document_content_1', b'document_content_2']
        }, schema=pa_schema)

        # Test writing data
        for batch in test_data.to_batches():
            blob_writer.write_arrow_batch(batch)

        # Test prepare commit
        commit_messages = blob_writer.prepare_commit()
        self.assertIsInstance(commit_messages, list)

        blob_writer.close()

    def test_dedicated_format_writer_write_large_blob(self):
        """Test DedicatedFormatWriter with large blob data across multiple batches."""
        from pypaimon import Schema

        # Create schema with blob column
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('description', pa.string()),
            ('large_blob', pa.large_binary()),
        ])

        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true'
            }
        )
        self.catalog.create_table('test_db.large_blob_test', schema, False)
        table = self.catalog.get_table('test_db.large_blob_test')

        # Use proper table API to create writer
        write_builder = table.new_batch_write_builder()
        blob_writer = write_builder.new_write()

        # Keep the payload small enough for CI while still exercising large BLOB writes.
        target_size = 512 * 1024  # 512KB in bytes
        blob_pattern = b'LARGE_BLOB_DATA_PATTERN_' + b'X' * 1024  # ~1KB pattern
        pattern_size = len(blob_pattern)
        repetitions = target_size // pattern_size
        large_blob_data = blob_pattern * repetitions

        # Verify the blob size is approximately 512KB
        blob_size_mb = len(large_blob_data) / (1024 * 1024)
        self.assertGreater(blob_size_mb, 0.4)  # Should be at least 400KB
        self.assertLess(blob_size_mb, 0.6)  # Should be less than 600KB

        total_rows = 0

        # Write 5 batches, each with 4 rows (20 rows total).
        for batch_num in range(5):
            batch_data = pa.Table.from_pydict({
                'id': [batch_num * 4 + i for i in range(4)],
                'description': [f'Large blob batch {batch_num}, row {i}' for i in range(4)],
                'large_blob': [large_blob_data] * 4
            }, schema=pa_schema)

            # Write each batch
            for batch in batch_data.to_batches():
                blob_writer.write_arrow_batch(batch)
                total_rows += batch.num_rows

            # Log progress for large data processing
            print(f"Completed batch {batch_num + 1}/5 with {batch.num_rows} rows")

        # Record count is tracked internally by DedicatedFormatWriter

        # Test prepare commit
        commit_messages: CommitMessage = blob_writer.prepare_commit()
        self.assertIsInstance(commit_messages, list)
        # Verify we have commit messages
        self.assertEqual(len(commit_messages), 1)
        commit_message = commit_messages[0]
        normal_file_meta = commit_message.new_files[0]
        blob_file_metas = commit_message.new_files[1:]
        # Validate row count consistency
        parquet_row_count = normal_file_meta.row_count
        blob_row_count_sum = sum(meta.row_count for meta in blob_file_metas)
        self.assertEqual(parquet_row_count, blob_row_count_sum,
                         f"Parquet row count ({parquet_row_count}) should equal "
                         f"sum of blob row counts ({blob_row_count_sum})")

        # Verify commit message structure and file metadata
        total_file_size = 0
        total_row_count = parquet_row_count
        for commit_msg in commit_messages:
            self.assertIsInstance(commit_msg.new_files, list)
            self.assertGreater(len(commit_msg.new_files), 0)

            # Verify file metadata structure
            for file_meta in commit_msg.new_files:
                self.assertIsNotNone(file_meta.file_name)
                self.assertGreater(file_meta.file_size, 0)
                self.assertGreater(file_meta.row_count, 0)
                total_file_size += file_meta.file_size

        self.assertEqual(total_row_count, 20)

        total_size_mb = total_file_size / (1024 * 1024)
        self.assertGreater(total_file_size, len(large_blob_data) * total_row_count)

        total_files = sum(len(commit_msg.new_files) for commit_msg in commit_messages)
        print(f"Total data written: {total_size_mb:.2f}MB across {total_files} files")
        print(f"Total rows processed: {total_row_count}")

        blob_writer.close()

    def test_dedicated_format_writer_abort_functionality(self):
        """Test DedicatedFormatWriter abort functionality."""
        from pypaimon import Schema

        # Create schema with blob column
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('blob_data', pa.large_binary()),
        ])

        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true'
            }
        )
        self.catalog.create_table('test_db.abort_test', schema, False)
        table = self.catalog.get_table('test_db.abort_test')

        # Use proper table API to create writer
        write_builder = table.new_batch_write_builder()
        blob_writer = write_builder.new_write()

        # Test data
        test_data = pa.Table.from_pydict({
            'id': [1, 2],
            'blob_data': [b'blob_1', b'blob_2']
        }, schema=pa_schema)

        # Write some data
        for batch in test_data.to_batches():
            blob_writer.write_arrow_batch(batch)

        # Test abort - BatchTableWrite doesn't have abort method
        # The abort functionality is handled internally by DedicatedFormatWriter

        blob_writer.close()

    def test_dedicated_format_writer_multiple_batches(self):
        """Test DedicatedFormatWriter with multiple batches and verify results."""
        from pypaimon import Schema

        # Create schema with blob column
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('name', pa.string()),
            ('document', pa.large_binary()),
        ])

        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true'
            }
        )
        self.catalog.create_table('test_db.multiple_batches_test', schema, False)
        table = self.catalog.get_table('test_db.multiple_batches_test')

        # Use proper table API to create writer
        write_builder = table.new_batch_write_builder()
        blob_writer = write_builder.new_write()

        # Test data - multiple batches
        batch1_data = pa.Table.from_pydict({
            'id': [1, 2],
            'name': ['Alice', 'Bob'],
            'document': [b'document_1_content', b'document_2_content']
        }, schema=pa_schema)

        batch2_data = pa.Table.from_pydict({
            'id': [3, 4, 5],
            'name': ['Charlie', 'David', 'Eve'],
            'document': [b'document_3_content', b'document_4_content', b'document_5_content']
        }, schema=pa_schema)

        batch3_data = pa.Table.from_pydict({
            'id': [6],
            'name': ['Frank'],
            'document': [b'document_6_content']
        }, schema=pa_schema)

        # Write multiple batches
        total_rows = 0
        for batch in batch1_data.to_batches():
            blob_writer.write_arrow_batch(batch)
            total_rows += batch.num_rows

        for batch in batch2_data.to_batches():
            blob_writer.write_arrow_batch(batch)
            total_rows += batch.num_rows

        for batch in batch3_data.to_batches():
            blob_writer.write_arrow_batch(batch)
            total_rows += batch.num_rows

        # Record count is tracked internally by DedicatedFormatWriter

        # Test prepare commit
        commit_messages = blob_writer.prepare_commit()
        self.assertIsInstance(commit_messages, list)

        # Verify we have committed files
        self.assertGreater(len(commit_messages), 0)

        blob_writer.close()

    def test_dedicated_format_writer_large_batches(self):
        """Test DedicatedFormatWriter with large batches to test rolling behavior."""
        from pypaimon import Schema

        # Create schema with blob column
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('description', pa.string()),
            ('large_blob', pa.large_binary()),
        ])

        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true'
            }
        )
        self.catalog.create_table('test_db.large_batches_test', schema, False)
        table = self.catalog.get_table('test_db.large_batches_test')

        # Use proper table API to create writer
        write_builder = table.new_batch_write_builder()
        blob_writer = write_builder.new_write()

        # Create large batches with substantial blob data
        large_blob_data = b'L' * 10000  # 10KB blob data

        # Batch 1: 100 rows
        batch1_data = pa.Table.from_pydict({
            'id': list(range(1, 101)),
            'description': [f'Description for row {i}' for i in range(1, 101)],
            'large_blob': [large_blob_data] * 100
        }, schema=pa_schema)

        # Batch 2: 50 rows
        batch2_data = pa.Table.from_pydict({
            'id': list(range(101, 151)),
            'description': [f'Description for row {i}' for i in range(101, 151)],
            'large_blob': [large_blob_data] * 50
        }, schema=pa_schema)

        # Write large batches
        total_rows = 0
        for batch in batch1_data.to_batches():
            blob_writer.write_arrow_batch(batch)
            total_rows += batch.num_rows

        for batch in batch2_data.to_batches():
            blob_writer.write_arrow_batch(batch)
            total_rows += batch.num_rows

        # Record count is tracked internally by DedicatedFormatWriter

        # Test prepare commit
        commit_messages = blob_writer.prepare_commit()
        self.assertIsInstance(commit_messages, list)

        # Verify we have committed files
        self.assertGreater(len(commit_messages), 0)

        blob_writer.close()

    def test_dedicated_format_writer_mixed_data_types(self):
        """Test DedicatedFormatWriter with mixed data types in blob column."""
        from pypaimon import Schema

        # Create schema with blob column
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('type', pa.string()),
            ('data', pa.large_binary()),
        ])

        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true'
            }
        )
        self.catalog.create_table('test_db.mixed_data_test', schema, False)
        table = self.catalog.get_table('test_db.mixed_data_test')

        # Use proper table API to create writer
        write_builder = table.new_batch_write_builder()
        blob_writer = write_builder.new_write()

        # Test data with different types of blob content
        test_data = pa.Table.from_pydict({
            'id': [1, 2, 3, 4, 5],
            'type': ['text', 'json', 'binary', 'image', 'pdf'],
            'data': [
                b'This is text content',
                b'{"key": "value", "number": 42}',
                b'\x00\x01\x02\x03\xff\xfe\xfd',
                b'PNG_IMAGE_DATA_PLACEHOLDER',
                b'%PDF-1.4\nPDF_CONTENT_PLACEHOLDER'
            ]
        }, schema=pa_schema)

        # Write mixed data
        total_rows = 0
        for batch in test_data.to_batches():
            blob_writer.write_arrow_batch(batch)
            total_rows += batch.num_rows

        # Record count is tracked internally by DedicatedFormatWriter

        # Test prepare commit
        commit_messages = blob_writer.prepare_commit()
        self.assertIsInstance(commit_messages, list)

        # Verify we have committed files
        self.assertGreater(len(commit_messages), 0)

        # Verify commit message structure
        for commit_msg in commit_messages:
            self.assertIsInstance(commit_msg.new_files, list)
            self.assertGreater(len(commit_msg.new_files), 0)

            # Verify file metadata structure
            for file_meta in commit_msg.new_files:
                self.assertIsNotNone(file_meta.file_name)
                self.assertGreater(file_meta.file_size, 0)
                self.assertGreater(file_meta.row_count, 0)

        # Should have both normal and blob files
        file_names = [f.file_name for f in commit_msg.new_files]
        parquet_files = [f for f in file_names if f.endswith('.parquet')]
        blob_files = [f for f in file_names if f.endswith('.blob')]

        self.assertGreater(len(parquet_files), 0, "Should have at least one parquet file")
        self.assertGreater(len(blob_files), 0, "Should have at least one blob file")

        # Create commit and commit the data
        commit = write_builder.new_commit()
        commit.commit(commit_messages)
        blob_writer.close()

        # Read data back using table API
        read_builder = table.new_read_builder()
        table_scan = read_builder.new_scan()
        table_read = read_builder.new_read()
        result = table_read.to_arrow(table_scan.plan().splits())

        # Verify the data was read back correctly
        self.assertEqual(result.num_rows, 5, "Should have 5 rows")
        self.assertEqual(result.num_columns, 3, "Should have 3 columns")

        # Convert result to pandas for easier comparison
        result_df = result.to_pandas()

        # Verify each row matches the original data
        for i in range(5):
            original_id = test_data.column('id')[i].as_py()
            original_type = test_data.column('type')[i].as_py()
            original_data = test_data.column('data')[i].as_py()

            result_id = result_df.iloc[i]['id']
            result_type = result_df.iloc[i]['type']
            result_data = result_df.iloc[i]['data']

            self.assertEqual(result_id, original_id, f"Row {i + 1}: ID should match")
            self.assertEqual(result_type, original_type, f"Row {i + 1}: Type should match")
            self.assertEqual(result_data, original_data, f"Row {i + 1}: Blob data should match")

    def test_dedicated_format_writer_empty_batches(self):
        """Test DedicatedFormatWriter with empty batches."""
        from pypaimon import Schema

        # Create schema with blob column
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('data', pa.large_binary()),
        ])

        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true'
            }
        )
        self.catalog.create_table('test_db.empty_batches_test', schema, False)
        table = self.catalog.get_table('test_db.empty_batches_test')

        # Use proper table API to create writer
        write_builder = table.new_batch_write_builder()
        blob_writer = write_builder.new_write()

        # Test data with some empty batches
        batch1_data = pa.Table.from_pydict({
            'id': [1, 2],
            'data': [b'data1', b'data2']
        }, schema=pa_schema)

        # Empty batch
        empty_batch = pa.Table.from_pydict({
            'id': [],
            'data': []
        }, schema=pa_schema)

        batch2_data = pa.Table.from_pydict({
            'id': [3],
            'data': [b'data3']
        }, schema=pa_schema)

        # Write batches including empty ones
        total_rows = 0
        for batch in batch1_data.to_batches():
            blob_writer.write_arrow_batch(batch)
            total_rows += batch.num_rows

        for batch in empty_batch.to_batches():
            blob_writer.write_arrow_batch(batch)
            total_rows += batch.num_rows

        for batch in batch2_data.to_batches():
            blob_writer.write_arrow_batch(batch)
            total_rows += batch.num_rows

        # Verify record count (empty batch should not affect count)
        # Record count is tracked internally by DedicatedFormatWriter
        # Record count is tracked internally by DedicatedFormatWriter

        # Test prepare commit
        commit_messages = blob_writer.prepare_commit()
        self.assertIsInstance(commit_messages, list)

        blob_writer.close()

    def test_dedicated_format_writer_rolling_behavior(self):
        """Test DedicatedFormatWriter rolling behavior with multiple commits."""
        from pypaimon import Schema

        # Create schema with blob column
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('content', pa.string()),
            ('blob_data', pa.large_binary()),
        ])

        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true'
            }
        )
        self.catalog.create_table('test_db.rolling_test', schema, False)
        table = self.catalog.get_table('test_db.rolling_test')

        # Use proper table API to create writer
        write_builder = table.new_batch_write_builder()
        blob_writer = write_builder.new_write()

        # Create data that should trigger rolling
        large_content = 'X' * 1000  # Large string content
        large_blob = b'B' * 5000  # Large blob data

        # Write multiple batches to test rolling
        for i in range(10):  # 10 batches
            batch_data = pa.Table.from_pydict({
                'id': [i * 10 + j for j in range(10)],
                'content': [f'{large_content}_{i}_{j}' for j in range(10)],
                'blob_data': [large_blob] * 10
            }, schema=pa_schema)

            for batch in batch_data.to_batches():
                blob_writer.write_arrow_batch(batch)

        # Verify total record count
        # Record count is tracked internally by DedicatedFormatWriter

        # Test prepare commit
        commit_messages = blob_writer.prepare_commit()
        self.assertIsInstance(commit_messages, list)

        # Verify we have committed files
        self.assertGreater(len(commit_messages), 0)

        # Verify file metadata structure
        for commit_msg in commit_messages:
            for file_meta in commit_msg.new_files:
                self.assertIsNotNone(file_meta.file_name)
                self.assertGreater(file_meta.file_size, 0)
                self.assertGreater(file_meta.row_count, 0)

        blob_writer.close()

    def test_blob_write_read_end_to_end(self):
        """Test complete end-to-end blob functionality: write blob data and read it back to verify correctness."""
        from pypaimon import Schema

        # Create schema with blob column
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('name', pa.string()),
            ('description', pa.string()),
            ('blob_data', pa.large_binary()),
        ])

        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true'
            }
        )
        self.catalog.create_table('test_db.blob_write_read_e2e', schema, False)
        table = self.catalog.get_table('test_db.blob_write_read_e2e')

        # Test data with various blob sizes and types
        test_data = pa.Table.from_pydict({
            'id': [1, 2, 3, 4, 5],
            'name': ['Alice', 'Bob', 'Charlie', 'David', 'Eve'],
            'description': ['User 1', 'User 2', 'User 3', 'User 4', 'User 5'],
            'blob_data': [
                b'small_blob_1',
                b'medium_blob_data_2_with_more_content',
                b'large_blob_data_3_with_even_more_content_and_details',
                b'very_large_blob_data_4_with_extensive_content_and_multiple_details_here',  # noqa: E501
                b'extremely_large_blob_data_5_with_comprehensive_content_and_'
                b'extensive_details_covering_multiple_aspects'  # noqa: E501
            ]
        }, schema=pa_schema)

        # Write data using table API
        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(test_data)

        # Commit the data
        commit_messages = writer.prepare_commit()
        self.assertGreater(len(commit_messages), 0)

        # Verify commit message structure
        for commit_msg in commit_messages:
            self.assertIsInstance(commit_msg.new_files, list)
        self.assertGreater(len(commit_msg.new_files), 0)

        # Should have both normal and blob files
        file_names = [f.file_name for f in commit_msg.new_files]
        parquet_files = [f for f in file_names if f.endswith('.parquet')]
        blob_files = [f for f in file_names if f.endswith('.blob')]

        self.assertGreater(len(parquet_files), 0, "Should have at least one parquet file")
        self.assertGreater(len(blob_files), 0, "Should have at least one blob file")

        # Create commit and commit the data
        commit = write_builder.new_commit()
        commit.commit(commit_messages)
        writer.close()

        # Read data back using table API
        read_builder = table.new_read_builder()
        table_scan = read_builder.new_scan()
        table_read = read_builder.new_read()
        result = table_read.to_arrow(table_scan.plan().splits())

        # Verify the data was read back correctly
        self.assertEqual(result.num_rows, 5, "Should have 5 rows")
        self.assertEqual(result.num_columns, 4, "Should have 4 columns")

        # Verify normal columns
        self.assertEqual(result.column('id').to_pylist(), [1, 2, 3, 4, 5], "ID column should match")
        self.assertEqual(result.column('name').to_pylist(), ['Alice', 'Bob', 'Charlie', 'David', 'Eve'],
                         "Name column should match")  # noqa: E501
        self.assertEqual(result.column('description').to_pylist(), ['User 1', 'User 2', 'User 3', 'User 4', 'User 5'],
                         "Description column should match")  # noqa: E501

        # Verify blob data correctness
        blob_data = result.column('blob_data').to_pylist()
        expected_blobs = [
            b'small_blob_1',
            b'medium_blob_data_2_with_more_content',
            b'large_blob_data_3_with_even_more_content_and_details',
            b'very_large_blob_data_4_with_extensive_content_and_multiple_details_here',  # noqa: E501
            b'extremely_large_blob_data_5_with_comprehensive_content_and_extensive_details_covering_multiple_aspects'
            # noqa: E501
        ]

        self.assertEqual(len(blob_data), 5, "Should have 5 blob records")
        self.assertEqual(blob_data, expected_blobs, "Blob data should match exactly")

        # Verify individual blob sizes
        for i, (actual_blob, expected_blob) in enumerate(zip(blob_data, expected_blobs)):
            self.assertEqual(len(actual_blob), len(expected_blob), f"Blob {i + 1} size should match")
            self.assertEqual(actual_blob, expected_blob, f"Blob {i + 1} content should match exactly")

        print(
            f"✅ End-to-end blob write/read test passed: wrote and read back {len(blob_data)} blob records correctly")  # noqa: E501

    def test_null_blob(self):
        from pypaimon import Schema

        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('name', pa.string()),
            ('blob_data', pa.large_binary()),
        ])

        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true'
            }
        )
        self.catalog.create_table('test_db.blob_write_read_e2e_null', schema, False)
        table = self.catalog.get_table('test_db.blob_write_read_e2e_null')

        test_data = pa.Table.from_pydict({
            'id':   [1, 2, 3, 4, 5],
            'name': ['a', 'b', 'c', 'd', 'e'],
            'blob_data': [
                b'first_blob',
                None,
                b'third_blob',
                None,
                b'fifth_blob',
            ],
        }, schema=pa_schema)

        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(test_data)

        commit_messages = writer.prepare_commit()
        write_builder.new_commit().commit(commit_messages)
        writer.close()

        read_builder = table.new_read_builder()
        result = read_builder.new_read().to_arrow(read_builder.new_scan().plan().splits())

        self.assertEqual(result.column('id').to_pylist(), [1, 2, 3, 4, 5])
        self.assertEqual(result.column('name').to_pylist(), ['a', 'b', 'c', 'd', 'e'])
        self.assertEqual(
            result.column('blob_data').to_pylist(),
            [b'first_blob', None, b'third_blob', None, b'fifth_blob'],
        )

    def test_write_row_accepts_streaming_blob(self):
        from pypaimon import Schema
        from pypaimon.table.row.generic_row import GenericRow

        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('blob_data', pa.large_binary()),
        ])
        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true'
            }
        )
        self.catalog.create_table('test_db.blob_write_row_streaming', schema, False)
        table = self.catalog.get_table('test_db.blob_write_row_streaming')

        blob = _StreamingOnlyBlob(b'row-blob')
        row = GenericRow([1, blob], table.fields)

        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_row(row)
        write_builder.new_commit().commit(writer.prepare_commit())
        writer.close()

        result = table.new_read_builder().new_read().to_arrow(
            table.new_read_builder().new_scan().plan().splits())
        self.assertTrue(blob.opened)
        self.assertEqual(result.column('id').to_pylist(), [1])
        self.assertEqual(result.column('blob_data').to_pylist(), [b'row-blob'])

    def test_upsert_by_key_accepts_streaming_blob_row(self):
        from pypaimon import Schema
        from pypaimon.table.row.generic_row import GenericRow

        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('blob_data', pa.large_binary()),
        ])
        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true'
            }
        )
        self.catalog.create_table('test_db.blob_upsert_row_streaming', schema, False)
        table = self.catalog.get_table('test_db.blob_upsert_row_streaming')

        initial = pa.Table.from_pydict({
            'id': [1],
            'blob_data': [b'old-blob'],
        }, schema=pa_schema)
        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(initial)
        write_builder.new_commit().commit(writer.prepare_commit())
        writer.close()

        updated_blob = _StreamingOnlyBlob(b'updated-blob')
        update_row = GenericRow([1, updated_blob], table.fields)
        shadow_blob = _StreamingOnlyBlob(b'shadow-blob')
        shadow_row = GenericRow([2, shadow_blob], table.fields)
        new_blob = _StreamingOnlyBlob(b'new-blob')
        new_row = GenericRow([2, new_blob], table.fields)
        upsert_builder = table.new_batch_write_builder()
        table_update = upsert_builder.new_update().with_update_type(['blob_data'])
        upsert_builder.new_commit().commit(
            table_update.upsert_by_key(
                [update_row, shadow_row, new_row], ['id']))

        result = table.new_read_builder().new_read().to_arrow(
            table.new_read_builder().new_scan().plan().splits())
        rows = sorted(
            result.select(['id', 'blob_data']).to_pylist(),
            key=lambda item: item['id'],
        )
        self.assertTrue(updated_blob.opened)
        self.assertFalse(shadow_blob.opened)
        self.assertTrue(new_blob.opened)
        self.assertEqual(
            rows,
            [
                {'id': 1, 'blob_data': b'updated-blob'},
                {'id': 2, 'blob_data': b'new-blob'},
            ],
        )

    def test_upsert_by_key_rejects_heterogeneous_append_row_fields(self):
        from pypaimon import Schema
        from pypaimon.table.row.generic_row import GenericRow

        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('name', pa.string()),
            ('blob_data', pa.large_binary()),
        ])
        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true'
            }
        )
        self.catalog.create_table(
            'test_db.blob_upsert_row_heterogeneous_fields', schema, False)
        table = self.catalog.get_table(
            'test_db.blob_upsert_row_heterogeneous_fields')

        partial_fields = [
            table.field_dict['id'],
            table.field_dict['blob_data'],
        ]
        partial_row = GenericRow([1, b'a'], partial_fields)
        full_row = GenericRow([2, 'bob', b'b'], table.fields)

        with self.assertRaisesRegex(ValueError, 'same field set'):
            table.new_batch_write_builder().new_update().upsert_by_key(
                [partial_row, full_row], ['id'])

    def test_update_blob_column(self):
        from pypaimon import Schema
        from pypaimon.read.reader.format_blob_reader import FormatBlobReader
        from pypaimon.write.blob_format_writer import BlobFormatWriter

        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('name', pa.string()),
            ('blob_data', pa.large_binary()),
        ])

        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true'
            }
        )
        self.catalog.create_table('test_db.blob_update_column', schema, False)
        table = self.catalog.get_table('test_db.blob_update_column')

        initial = pa.Table.from_pydict({
            'id': [1, 2, 3],
            'name': ['a', 'b', 'c'],
            'blob_data': [b'blob-1', b'blob-2', b'blob-3'],
        }, schema=pa_schema)

        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(initial)
        write_builder.new_commit().commit(writer.prepare_commit())
        writer.close()

        update_builder = table.new_batch_write_builder()
        table_update = update_builder.new_update().with_update_type(['blob_data'])
        update_data = pa.Table.from_pydict({
            '_ROW_ID': pa.array([1], type=pa.int64()),
            'blob_data': pa.array([b'updated-blob-2'], type=pa.large_binary()),
        })
        update_messages = table_update.update_by_arrow_with_row_id(update_data)
        update_builder.new_commit().commit(update_messages)

        update_files = [f for msg in update_messages for f in msg.new_files]
        update_blob_files = [f for f in update_files if f.file_name.endswith('.blob')]
        self.assertEqual(len(update_blob_files), 1)
        self.assertEqual(update_blob_files[0].row_count, 2)
        self.assertEqual(update_blob_files[0].first_row_id, 0)
        self.assertTrue(all(f.write_cols == ['blob_data'] for f in update_files))
        blob_fields = [field for field in table.fields if field.name == 'blob_data']
        blob_reader = FormatBlobReader(
            file_io=table.file_io,
            file_path=update_blob_files[0].file_path,
            read_fields=['blob_data'],
            full_fields=blob_fields,
            push_down_predicate=None,
            blob_as_descriptor=False,
        )
        update_blob_lengths = list(blob_reader.blob_lengths)
        blob_reader.close()
        self.assertEqual(
            update_blob_lengths.count(BlobFormatWriter.PLACE_HOLDER_LENGTH),
            1,
        )

        read_builder = table.new_read_builder()
        result = read_builder.new_read().to_arrow(read_builder.new_scan().plan().splits())
        by_id = {
            row['id']: row['blob_data']
            for row in result.select(['id', 'blob_data']).to_pylist()
        }
        self.assertEqual(by_id, {
            1: b'blob-1',
            2: b'updated-blob-2',
            3: b'blob-3',
        })

    def test_array_blob_column_write_read_and_update(self):
        from pypaimon import Schema
        from pypaimon.read.reader.format_blob_reader import FormatBlobReader
        from pypaimon.write.blob_format_writer import BlobFormatWriter

        array_blob_type = pa.list_(pa.large_binary())
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('payloads', array_blob_type),
        ])

        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true',
            },
        )
        self.catalog.create_table('test_db.array_blob_update_column', schema, False)
        table = self.catalog.get_table('test_db.array_blob_update_column')

        initial = pa.Table.from_pydict({
            'id': [1, 2, 3],
            'payloads': pa.array(
                [[b'blob-1a', None, b'blob-1b'], None, [b'blob-3']],
                type=array_blob_type,
            ),
        }, schema=pa_schema)

        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(initial)
        write_builder.new_commit().commit(writer.prepare_commit())
        writer.close()

        read_builder = table.new_read_builder().with_projection(['id', 'payloads'])
        result = read_builder.new_read().to_arrow(read_builder.new_scan().plan().splits())
        self.assertEqual(
            {
                row['id']: row['payloads']
                for row in result.select(['id', 'payloads']).to_pylist()
            },
            {
                1: [b'blob-1a', None, b'blob-1b'],
                2: None,
                3: [b'blob-3'],
            },
        )

        row_id_builder = table.new_read_builder().with_projection(['id', '_ROW_ID'])
        row_id_result = row_id_builder.new_read().to_arrow(
            row_id_builder.new_scan().plan().splits())
        row_ids_by_id = {
            row['id']: row['_ROW_ID']
            for row in row_id_result.select(['id', '_ROW_ID']).to_pylist()
        }

        update_builder = table.new_batch_write_builder()
        table_update = update_builder.new_update().with_update_type(['payloads'])
        update_data = pa.Table.from_pydict({
            '_ROW_ID': pa.array([row_ids_by_id[2]], type=pa.int64()),
            'payloads': pa.array([[b'updated-2']], type=array_blob_type),
        })
        update_messages = table_update.update_by_arrow_with_row_id(update_data)

        update_files = [f for msg in update_messages for f in msg.new_files]
        update_blob_files = [f for f in update_files if f.file_name.endswith('.blob')]
        self.assertEqual(len(update_blob_files), 1)
        self.assertEqual(update_blob_files[0].row_count, 2)
        blob_fields = [field for field in table.fields if field.name == 'payloads']
        blob_reader = FormatBlobReader(
            file_io=table.file_io,
            file_path=update_blob_files[0].file_path,
            read_fields=['payloads'],
            full_fields=blob_fields,
            push_down_predicate=None,
            blob_as_descriptor=False,
        )
        update_blob_lengths = list(blob_reader.blob_lengths)
        blob_reader.close()
        self.assertEqual(
            update_blob_lengths.count(BlobFormatWriter.PLACE_HOLDER_LENGTH),
            1,
        )

        update_builder.new_commit().commit(update_messages)

        read_builder = table.new_read_builder().with_projection(['id', 'payloads'])
        result = read_builder.new_read().to_arrow(read_builder.new_scan().plan().splits())
        self.assertEqual(
            {
                row['id']: row['payloads']
                for row in result.select(['id', 'payloads']).to_pylist()
            },
            {
                1: [b'blob-1a', None, b'blob-1b'],
                2: [b'updated-2'],
                3: [b'blob-3'],
            },
        )

    def test_map_blob_column_write_read_and_update(self):
        from pypaimon.read.reader.format_blob_reader import FormatBlobReader
        from pypaimon.table.row.blob import BlobDescriptor
        from pypaimon.write.blob_format_writer import BlobFormatWriter

        map_blob_type = pa.map_(pa.string(), pa.large_binary())
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('payloads', map_blob_type),
        ])
        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true',
            },
        )
        table_name = 'test_db.map_blob_update_column'
        self.catalog.create_table(table_name, schema, False)
        table = self.catalog.get_table(table_name)

        descriptor_body = b'descriptor-map-value'
        descriptor_prefix = b'prefix-'
        descriptor_path = os.path.join(self.temp_dir, 'map_blob_descriptor_source')
        with open(descriptor_path, 'wb') as source:
            source.write(descriptor_prefix + descriptor_body + b'-suffix')
        source_descriptor = BlobDescriptor(
            descriptor_path,
            len(descriptor_prefix),
            len(descriptor_body),
        )

        initial = pa.Table.from_pydict({
            'id': [1, 2, 3, 4, 5, 6],
            'payloads': pa.array(
                [
                    [('first', b'blob-1'), ('null', None), ('empty', b'')],
                    None,
                    [],
                    [('last', b'blob-4')],
                    [('descriptor', source_descriptor.serialize())],
                    [('duplicate', b'first'), ('duplicate', b'last')],
                ],
                type=map_blob_type,
            ),
        }, schema=pa_schema)

        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(initial)
        write_builder.new_commit().commit(writer.prepare_commit())
        writer.close()

        read_builder = table.new_read_builder().with_projection(['id', 'payloads'])
        result = read_builder.new_read().to_arrow(
            read_builder.new_scan().plan().splits())
        self.assertEqual(
            {
                row['id']: None if row['payloads'] is None else dict(row['payloads'])
                for row in result.select(['id', 'payloads']).to_pylist()
            },
            {
                1: {'first': b'blob-1', 'null': None, 'empty': b''},
                2: None,
                3: {},
                4: {'last': b'blob-4'},
                5: {'descriptor': descriptor_body},
                6: {'duplicate': b'last'},
            },
        )

        descriptor_table = table.copy({'blob-as-descriptor': 'true'})
        descriptor_builder = descriptor_table.new_read_builder()
        descriptor_result = descriptor_builder.new_read().to_arrow(
            descriptor_builder.new_scan().plan().splits())
        descriptor_rows = {
            row['id']: None if row['payloads'] is None else dict(row['payloads'])
            for row in descriptor_result.select(['id', 'payloads']).to_pylist()
        }
        output_descriptor = BlobDescriptor.deserialize(
            descriptor_rows[5]['descriptor']
        )
        self.assertNotEqual(output_descriptor.uri, descriptor_path)
        self.assertGreater(output_descriptor.offset, 0)
        self.assertEqual(output_descriptor.length, len(descriptor_body))
        output_blob = Blob.from_descriptor(
            table.file_io.uri_reader_factory.create(output_descriptor.uri),
            output_descriptor,
        )
        self.assertEqual(output_blob.to_data(), descriptor_body)

        row_id_builder = table.new_read_builder().with_projection(['id', '_ROW_ID'])
        row_id_result = row_id_builder.new_read().to_arrow(
            row_id_builder.new_scan().plan().splits())
        row_ids_by_id = {
            row['id']: row['_ROW_ID']
            for row in row_id_result.select(['id', '_ROW_ID']).to_pylist()
        }

        update_builder = table.new_batch_write_builder()
        table_update = update_builder.new_update().with_update_type(['payloads'])
        update_data = pa.Table.from_pydict({
            '_ROW_ID': pa.array([row_ids_by_id[2]], type=pa.int64()),
            'payloads': pa.array(
                [[('updated', b'updated-2'), ('null', None)]],
                type=map_blob_type,
            ),
        })
        update_messages = table_update.update_by_arrow_with_row_id(update_data)

        update_files = [f for msg in update_messages for f in msg.new_files]
        update_blob_files = [f for f in update_files if f.file_name.endswith('.blob')]
        self.assertEqual(len(update_blob_files), 1)
        self.assertEqual(update_blob_files[0].row_count, 2)
        blob_fields = [field for field in table.fields if field.name == 'payloads']
        blob_reader = FormatBlobReader(
            file_io=table.file_io,
            file_path=update_blob_files[0].file_path,
            read_fields=['payloads'],
            full_fields=blob_fields,
            push_down_predicate=None,
            blob_as_descriptor=False,
        )
        update_blob_lengths = list(blob_reader.blob_lengths)
        blob_reader.close()
        self.assertEqual(
            update_blob_lengths.count(BlobFormatWriter.PLACE_HOLDER_LENGTH),
            1,
        )

        update_builder.new_commit().commit(update_messages)

        read_builder = table.new_read_builder().with_projection(['id', 'payloads'])
        result = read_builder.new_read().to_arrow(
            read_builder.new_scan().plan().splits())
        self.assertEqual(
            {
                row['id']: None if row['payloads'] is None else dict(row['payloads'])
                for row in result.select(['id', 'payloads']).to_pylist()
            },
            {
                1: {'first': b'blob-1', 'null': None, 'empty': b''},
                2: {'updated': b'updated-2', 'null': None},
                3: {},
                4: {'last': b'blob-4'},
                5: {'descriptor': descriptor_body},
                6: {'duplicate': b'last'},
            },
        )

    def test_map_blob_value_not_null(self):
        map_blob_type = pa.map_(
            pa.field('key', pa.string(), nullable=False),
            pa.field('value', pa.large_binary(), nullable=False),
        )
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('payloads', map_blob_type),
        ])
        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true',
            },
        )
        table_name = 'test_db.map_blob_value_not_null'
        self.catalog.create_table(table_name, schema, False)
        table = self.catalog.get_table(table_name)

        valid_data = pa.Table.from_pydict({
            'id': [1, 2, 3],
            'payloads': pa.array(
                [[('first', b'alpha'), ('empty', b'')], [], None],
                type=map_blob_type,
            ),
        }, schema=pa_schema)
        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(valid_data)
        write_builder.new_commit().commit(writer.prepare_commit())
        writer.close()

        read_builder = table.new_read_builder()
        result = read_builder.new_read().to_arrow(
            read_builder.new_scan().plan().splits())
        self.assertFalse(result.schema.field('payloads').type.item_field.nullable)
        self.assertEqual(
            {
                row['id']: None if row['payloads'] is None else dict(row['payloads'])
                for row in result.select(['id', 'payloads']).to_pylist()
            },
            {
                1: {'first': b'alpha', 'empty': b''},
                2: {},
                3: None,
            },
        )

        nullable_map_type = pa.map_(pa.string(), pa.large_binary())
        incompatible_schema = pa.schema([
            ('id', pa.int32()),
            ('payloads', nullable_map_type),
        ])
        incompatible_data = pa.Table.from_pydict({
            'id': [4],
            'payloads': pa.array([[('bad', None)]], type=nullable_map_type),
        }, schema=incompatible_schema)
        incompatible_writer = table.new_batch_write_builder().new_write()
        with self.assertRaisesRegex(ValueError, "Input schema isn't consistent"):
            incompatible_writer.write_arrow(incompatible_data)
        incompatible_writer.abort()

        invalid_data = pa.Table.from_pydict({
            'id': [5],
            'payloads': pa.array([[('bad', None)]], type=map_blob_type),
        }, schema=pa_schema)
        invalid_writer = table.new_batch_write_builder().new_write()
        with self.assertRaisesRegex(ValueError, "does not allow null values"):
            invalid_writer.write_arrow(invalid_data)
        invalid_writer.abort()

    def test_array_blob_element_not_null(self):
        array_blob_type = pa.list_(
            pa.field('item', pa.large_binary(), nullable=False)
        )
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('payloads', array_blob_type),
        ])
        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true',
            },
        )
        table_name = 'test_db.array_blob_element_not_null'
        self.catalog.create_table(table_name, schema, False)
        table = self.catalog.get_table(table_name)

        valid_data = pa.Table.from_pydict({
            'id': [1, 2, 3],
            'payloads': pa.array(
                [[b'alpha', b''], [], None],
                type=array_blob_type,
            ),
        }, schema=pa_schema)
        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(valid_data)
        write_builder.new_commit().commit(writer.prepare_commit())
        writer.close()

        read_builder = table.new_read_builder()
        result = read_builder.new_read().to_arrow(
            read_builder.new_scan().plan().splits())
        self.assertFalse(
            result.schema.field('payloads').type.value_field.nullable
        )
        self.assertEqual(
            {
                row['id']: row['payloads']
                for row in result.select(['id', 'payloads']).to_pylist()
            },
            {
                1: [b'alpha', b''],
                2: [],
                3: None,
            },
        )

        nullable_array_type = pa.list_(pa.large_binary())
        incompatible_schema = pa.schema([
            ('id', pa.int32()),
            ('payloads', nullable_array_type),
        ])
        incompatible_data = pa.Table.from_pydict({
            'id': [4],
            'payloads': pa.array([[b'bad', None]], type=nullable_array_type),
        }, schema=incompatible_schema)
        incompatible_writer = table.new_batch_write_builder().new_write()
        with self.assertRaisesRegex(ValueError, "Input schema isn't consistent"):
            incompatible_writer.write_arrow(incompatible_data)
        incompatible_writer.abort()

        invalid_data = pa.Table.from_pydict({
            'id': [5],
            'payloads': pa.array([[b'bad', None]], type=array_blob_type),
        }, schema=pa_schema)
        invalid_writer = table.new_batch_write_builder().new_write()
        with self.assertRaisesRegex(ValueError, "does not allow null elements"):
            invalid_writer.write_arrow(invalid_data)
        invalid_writer.abort()

    def test_blob_update_single_row_at_first_position(self):
        from pypaimon import Schema

        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('blob_data', pa.large_binary()),
        ])
        schema = Schema.from_pyarrow_schema(pa_schema, options={
            'row-tracking.enabled': 'true',
            'data-evolution.enabled': 'true',
        })
        self.catalog.create_table(
            'test_db.blob_update_first_row', schema, False)
        table = self.catalog.get_table('test_db.blob_update_first_row')

        initial = pa.Table.from_pydict({
            'id': list(range(10)),
            'blob_data': [f'blob-{i}'.encode() for i in range(10)],
        }, schema=pa_schema)
        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(initial)
        write_builder.new_commit().commit(writer.prepare_commit())
        writer.close()

        table = self.catalog.get_table('test_db.blob_update_first_row')
        update_builder = table.new_batch_write_builder()
        table_update = update_builder.new_update().with_update_type(
            ['blob_data'])
        update_data = pa.Table.from_pydict({
            '_ROW_ID': pa.array([0], type=pa.int64()),
            'blob_data': pa.array([b'updated-0'], type=pa.large_binary()),
        })
        update_messages = table_update.update_by_arrow_with_row_id(update_data)
        update_builder.new_commit().commit(update_messages)

        update_blob_files = [
            f for msg in update_messages for f in msg.new_files
            if f.file_name.endswith('.blob')
        ]
        self.assertEqual(len(update_blob_files), 1)
        self.assertEqual(update_blob_files[0].row_count, 1)
        self.assertEqual(update_blob_files[0].first_row_id, 0)

        read_builder = table.new_read_builder()
        result = read_builder.new_read().to_arrow(
            read_builder.new_scan().plan().splits())
        by_id = {
            row['id']: row['blob_data']
            for row in result.select(['id', 'blob_data']).to_pylist()
        }
        self.assertEqual(by_id[0], b'updated-0')
        self.assertEqual(by_id[9], b'blob-9')

    def test_blob_update_all_rows_full_span(self):
        from pypaimon import Schema

        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('blob_data', pa.large_binary()),
        ])
        schema = Schema.from_pyarrow_schema(pa_schema, options={
            'row-tracking.enabled': 'true',
            'data-evolution.enabled': 'true',
        })
        self.catalog.create_table(
            'test_db.blob_update_full', schema, False)
        table = self.catalog.get_table('test_db.blob_update_full')

        num_rows = 5
        initial = pa.Table.from_pydict({
            'id': list(range(num_rows)),
            'blob_data': [f'blob-{i}'.encode() for i in range(num_rows)],
        }, schema=pa_schema)
        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(initial)
        write_builder.new_commit().commit(writer.prepare_commit())
        writer.close()

        table = self.catalog.get_table('test_db.blob_update_full')
        update_builder = table.new_batch_write_builder()
        table_update = update_builder.new_update().with_update_type(
            ['blob_data'])
        update_data = pa.Table.from_pydict({
            '_ROW_ID': pa.array(list(range(num_rows)), type=pa.int64()),
            'blob_data': pa.array(
                [f'updated-{i}'.encode() for i in range(num_rows)],
                type=pa.large_binary()),
        })
        update_messages = table_update.update_by_arrow_with_row_id(update_data)
        update_builder.new_commit().commit(update_messages)

        update_blob_files = [
            f for msg in update_messages for f in msg.new_files
            if f.file_name.endswith('.blob')
        ]
        self.assertEqual(len(update_blob_files), 1)
        self.assertEqual(update_blob_files[0].row_count, num_rows)
        self.assertEqual(update_blob_files[0].first_row_id, 0)

        read_builder = table.new_read_builder()
        result = read_builder.new_read().to_arrow(
            read_builder.new_scan().plan().splits())
        by_id = {
            row['id']: row['blob_data']
            for row in result.select(['id', 'blob_data']).to_pylist()
        }
        for i in range(num_rows):
            self.assertEqual(by_id[i], f'updated-{i}'.encode())

    def test_blob_update_multiple_blob_columns(self):
        from pypaimon import Schema

        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('blob_a', pa.large_binary()),
            ('blob_b', pa.large_binary()),
        ])
        schema = Schema.from_pyarrow_schema(pa_schema, options={
            'row-tracking.enabled': 'true',
            'data-evolution.enabled': 'true',
        })
        self.catalog.create_table(
            'test_db.blob_update_two_cols', schema, False)
        table = self.catalog.get_table('test_db.blob_update_two_cols')

        initial = pa.Table.from_pydict({
            'id': list(range(5)),
            'blob_a': [f'a-{i}'.encode() for i in range(5)],
            'blob_b': [f'b-{i}'.encode() for i in range(5)],
        }, schema=pa_schema)
        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(initial)
        write_builder.new_commit().commit(writer.prepare_commit())
        writer.close()

        table = self.catalog.get_table('test_db.blob_update_two_cols')
        update_builder = table.new_batch_write_builder()
        table_update = update_builder.new_update().with_update_type(
            ['blob_a', 'blob_b'])
        update_data = pa.Table.from_pydict({
            '_ROW_ID': pa.array([2], type=pa.int64()),
            'blob_a': pa.array([b'a-updated'], type=pa.large_binary()),
            'blob_b': pa.array([b'b-updated'], type=pa.large_binary()),
        })
        update_messages = table_update.update_by_arrow_with_row_id(update_data)
        update_builder.new_commit().commit(update_messages)

        update_blob_files = [
            f for msg in update_messages for f in msg.new_files
            if f.file_name.endswith('.blob')
        ]
        self.assertEqual(len(update_blob_files), 2)
        for f in update_blob_files:
            self.assertEqual(f.row_count, 3)
            self.assertEqual(f.first_row_id, 0)
        cols_written = sorted(f.write_cols[0] for f in update_blob_files)
        self.assertEqual(cols_written, ['blob_a', 'blob_b'])

        read_builder = table.new_read_builder()
        result = read_builder.new_read().to_arrow(
            read_builder.new_scan().plan().splits())
        rows = result.select(['id', 'blob_a', 'blob_b']).to_pylist()
        by_id = {r['id']: r for r in rows}
        self.assertEqual(by_id[2]['blob_a'], b'a-updated')
        self.assertEqual(by_id[2]['blob_b'], b'b-updated')
        self.assertEqual(by_id[0]['blob_a'], b'a-0')
        self.assertEqual(by_id[4]['blob_b'], b'b-4')

    def test_blob_update_across_multiple_file_groups(self):
        from pypaimon import Schema

        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('blob_data', pa.large_binary()),
        ])
        schema = Schema.from_pyarrow_schema(pa_schema, options={
            'row-tracking.enabled': 'true',
            'data-evolution.enabled': 'true',
        })
        self.catalog.create_table(
            'test_db.blob_update_multi_group', schema, False)
        table = self.catalog.get_table('test_db.blob_update_multi_group')

        # Two separate commits -> two data files with distinct first_row_id.
        first = pa.Table.from_pydict({
            'id': [0, 1, 2, 3, 4],
            'blob_data': [f'a-{i}'.encode() for i in range(5)],
        }, schema=pa_schema)
        wb = table.new_batch_write_builder()
        w = wb.new_write()
        w.write_arrow(first)
        wb.new_commit().commit(w.prepare_commit())
        w.close()

        table = self.catalog.get_table('test_db.blob_update_multi_group')
        second = pa.Table.from_pydict({
            'id': [5, 6, 7, 8, 9],
            'blob_data': [f'b-{i}'.encode() for i in range(5)],
        }, schema=pa_schema)
        wb = table.new_batch_write_builder()
        w = wb.new_write()
        w.write_arrow(second)
        wb.new_commit().commit(w.prepare_commit())
        w.close()

        # Touch one row in each group: _ROW_ID 0 (group A, pos 0)
        # and _ROW_ID 7 (group B, pos 2).
        table = self.catalog.get_table('test_db.blob_update_multi_group')
        update_builder = table.new_batch_write_builder()
        table_update = update_builder.new_update().with_update_type(
            ['blob_data'])
        update_data = pa.Table.from_pydict({
            '_ROW_ID': pa.array([0, 7], type=pa.int64()),
            'blob_data': pa.array(
                [b'updated-0', b'updated-7'],
                type=pa.large_binary()),
        })
        update_messages = table_update.update_by_arrow_with_row_id(update_data)
        update_builder.new_commit().commit(update_messages)

        update_blob_files = [
            f for msg in update_messages for f in msg.new_files
            if f.file_name.endswith('.blob')
        ]
        self.assertEqual(len(update_blob_files), 2)
        by_first = {f.first_row_id: f for f in update_blob_files}
        self.assertEqual(set(by_first), {0, 5})
        self.assertEqual(by_first[0].row_count, 1)
        self.assertEqual(by_first[5].row_count, 3)

        read_builder = table.new_read_builder()
        result = read_builder.new_read().to_arrow(
            read_builder.new_scan().plan().splits())
        by_id = {
            row['id']: row['blob_data']
            for row in result.select(['id', 'blob_data']).to_pylist()
        }
        self.assertEqual(by_id[0], b'updated-0')
        self.assertEqual(by_id[7], b'updated-7')
        self.assertEqual(by_id[3], b'a-3')
        self.assertEqual(by_id[5], b'b-0')

    def test_blob_update_scattered_endpoints_spans_full_range(self):
        from pypaimon import Schema

        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('blob_data', pa.large_binary()),
        ])
        schema = Schema.from_pyarrow_schema(pa_schema, options={
            'row-tracking.enabled': 'true',
            'data-evolution.enabled': 'true',
        })
        self.catalog.create_table(
            'test_db.blob_update_scattered', schema, False)
        table = self.catalog.get_table('test_db.blob_update_scattered')

        num_rows = 100
        initial = pa.Table.from_pydict({
            'id': list(range(num_rows)),
            'blob_data': [f'blob-{i}'.encode() for i in range(num_rows)],
        }, schema=pa_schema)
        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(initial)
        write_builder.new_commit().commit(writer.prepare_commit())
        writer.close()

        table = self.catalog.get_table('test_db.blob_update_scattered')
        update_builder = table.new_batch_write_builder()
        table_update = update_builder.new_update().with_update_type(
            ['blob_data'])
        update_data = pa.Table.from_pydict({
            '_ROW_ID': pa.array([0, 99], type=pa.int64()),
            'blob_data': pa.array(
                [b'updated-blob-0', b'updated-blob-99'],
                type=pa.large_binary()),
        })
        update_messages = table_update.update_by_arrow_with_row_id(update_data)
        update_builder.new_commit().commit(update_messages)

        update_blob_files = [
            f for msg in update_messages for f in msg.new_files
            if f.file_name.endswith('.blob')
        ]
        self.assertEqual(len(update_blob_files), 1)
        self.assertEqual(update_blob_files[0].row_count, num_rows)
        self.assertEqual(update_blob_files[0].first_row_id, 0)

        read_builder = table.new_read_builder()
        result = read_builder.new_read().to_arrow(
            read_builder.new_scan().plan().splits())
        by_id = {
            row['id']: row['blob_data']
            for row in result.select(['id', 'blob_data']).to_pylist()
        }
        self.assertEqual(by_id[0], b'updated-blob-0')
        self.assertEqual(by_id[99], b'updated-blob-99')
        self.assertEqual(by_id[50], b'blob-50')

    def test_blob_update_shrinks_delta_to_max_updated_row(self):
        from pypaimon import Schema

        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('blob_data', pa.large_binary()),
        ])
        schema = Schema.from_pyarrow_schema(pa_schema, options={
            'row-tracking.enabled': 'true',
            'data-evolution.enabled': 'true',
        })
        self.catalog.create_table(
            'test_db.blob_update_max_span', schema, False)
        table = self.catalog.get_table('test_db.blob_update_max_span')

        num_rows = 100
        initial = pa.Table.from_pydict({
            'id': list(range(num_rows)),
            'blob_data': [f'blob-{i}'.encode() for i in range(num_rows)],
        }, schema=pa_schema)
        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(initial)
        write_builder.new_commit().commit(writer.prepare_commit())
        writer.close()

        table = self.catalog.get_table('test_db.blob_update_max_span')
        update_builder = table.new_batch_write_builder()
        table_update = update_builder.new_update().with_update_type(
            ['blob_data'])
        update_data = pa.Table.from_pydict({
            '_ROW_ID': pa.array([50], type=pa.int64()),
            'blob_data': pa.array(
                [b'updated-blob-50'], type=pa.large_binary()),
        })
        update_messages = table_update.update_by_arrow_with_row_id(update_data)
        update_builder.new_commit().commit(update_messages)

        update_blob_files = [
            f for msg in update_messages for f in msg.new_files
            if f.file_name.endswith('.blob')
        ]
        self.assertEqual(len(update_blob_files), 1)
        self.assertEqual(update_blob_files[0].row_count, 51)
        self.assertEqual(update_blob_files[0].first_row_id, 0)

        read_builder = table.new_read_builder()
        result = read_builder.new_read().to_arrow(
            read_builder.new_scan().plan().splits())
        by_id = {
            row['id']: row['blob_data']
            for row in result.select(['id', 'blob_data']).to_pylist()
        }
        self.assertEqual(by_id[50], b'updated-blob-50')
        self.assertEqual(by_id[0], b'blob-0')
        self.assertEqual(by_id[99], b'blob-99')

    def test_update_blob_column_with_rolling_files(self):
        from pypaimon import Schema

        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('blob_data', pa.large_binary()),
        ])

        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true',
                'blob.target-file-size': '1KB',
            }
        )
        self.catalog.create_table('test_db.blob_update_column_rolling', schema, False)
        table = self.catalog.get_table('test_db.blob_update_column_rolling')

        initial = pa.Table.from_pydict({
            'id': [1, 2, 3, 4, 5],
            'blob_data': [b'blob-1', b'blob-2', b'blob-3', b'blob-4', b'blob-5'],
        }, schema=pa_schema)

        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(initial)
        write_builder.new_commit().commit(writer.prepare_commit())
        writer.close()

        row_id_builder = table.new_read_builder().with_projection(['id', '_ROW_ID'])
        row_id_result = row_id_builder.new_read().to_arrow(
            row_id_builder.new_scan().plan().splits())
        row_ids_by_id = {
            row['id']: row['_ROW_ID']
            for row in row_id_result.select(['id', '_ROW_ID']).to_pylist()
        }

        update_builder = table.new_batch_write_builder()
        table_update = update_builder.new_update().with_update_type(['blob_data'])
        payload = b'x' * 2048
        update_ids = [1, 2, 3, 4, 5]
        update_data = pa.Table.from_pydict({
            '_ROW_ID': pa.array([row_ids_by_id[id_] for id_ in update_ids], type=pa.int64()),
            'blob_data': pa.array(
                [payload + str(id_ - 1).encode() for id_ in update_ids],
                type=pa.large_binary(),
            ),
        })
        update_messages = table_update.update_by_arrow_with_row_id(update_data)

        update_blob_files = [
            file
            for msg in update_messages
            for file in msg.new_files
            if file.file_name.endswith('.blob')
        ]
        self.assertGreater(len(update_blob_files), 1)

        update_builder.new_commit().commit(update_messages)

        read_builder = table.new_read_builder()
        result = read_builder.new_read().to_arrow(read_builder.new_scan().plan().splits())
        by_id = {
            row['id']: row['blob_data']
            for row in result.select(['id', 'blob_data']).to_pylist()
        }
        self.assertEqual(by_id, {
            1: payload + b'0',
            2: payload + b'1',
            3: payload + b'2',
            4: payload + b'3',
            5: payload + b'4',
        })

    def test_update_partial_blob_column_with_rolling_files(self):
        from pypaimon import Schema
        from pypaimon.read.reader.format_blob_reader import FormatBlobReader
        from pypaimon.write.blob_format_writer import BlobFormatWriter

        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('blob_data', pa.large_binary()),
        ])

        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true',
                'blob.target-file-size': '1KB',
            }
        )
        self.catalog.create_table('test_db.blob_update_partial_column_rolling', schema, False)
        table = self.catalog.get_table('test_db.blob_update_partial_column_rolling')

        initial = pa.Table.from_pydict({
            'id': [1, 2, 3, 4, 5],
            'blob_data': [b'blob-1', b'blob-2', b'blob-3', b'blob-4', b'blob-5'],
        }, schema=pa_schema)

        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(initial)
        write_builder.new_commit().commit(writer.prepare_commit())
        writer.close()

        row_id_builder = table.new_read_builder().with_projection(['id', '_ROW_ID'])
        row_id_result = row_id_builder.new_read().to_arrow(
            row_id_builder.new_scan().plan().splits())
        row_ids_by_id = {
            row['id']: row['_ROW_ID']
            for row in row_id_result.select(['id', '_ROW_ID']).to_pylist()
        }

        update_builder = table.new_batch_write_builder()
        table_update = update_builder.new_update().with_update_type(['blob_data'])
        payload = b'x' * 2048
        update_ids = [2, 4]
        update_data = pa.Table.from_pydict({
            '_ROW_ID': pa.array([row_ids_by_id[id_] for id_ in update_ids], type=pa.int64()),
            'blob_data': pa.array(
                [payload + str(id_).encode() for id_ in update_ids],
                type=pa.large_binary(),
            ),
        })
        update_messages = table_update.update_by_arrow_with_row_id(update_data)

        update_blob_files = [
            file
            for msg in update_messages
            for file in msg.new_files
            if file.file_name.endswith('.blob')
        ]
        self.assertGreater(len(update_blob_files), 1)

        update_blob_lengths = []
        blob_fields = [field for field in table.fields if field.name == 'blob_data']
        for blob_file in update_blob_files:
            blob_reader = FormatBlobReader(
                file_io=table.file_io,
                file_path=blob_file.file_path,
                read_fields=['blob_data'],
                full_fields=blob_fields,
                push_down_predicate=None,
                blob_as_descriptor=False,
            )
            update_blob_lengths.extend(blob_reader.blob_lengths)
            blob_reader.close()
        self.assertEqual(
            update_blob_lengths.count(BlobFormatWriter.PLACE_HOLDER_LENGTH),
            2,
        )

        update_builder.new_commit().commit(update_messages)

        read_builder = table.new_read_builder()
        result = read_builder.new_read().to_arrow(read_builder.new_scan().plan().splits())
        by_id = {
            row['id']: row['blob_data']
            for row in result.select(['id', 'blob_data']).to_pylist()
        }
        self.assertEqual(by_id, {
            1: b'blob-1',
            2: payload + b'2',
            3: b'blob-3',
            4: payload + b'4',
            5: b'blob-5',
        })

    def test_blob_write_read_partition(self):
        """Test complete end-to-end blob functionality: write blob data and read it back to verify correctness."""
        from pypaimon import Schema

        # Create schema with blob column
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('name', pa.string()),
            ('description', pa.string()),
            ('blob_data', pa.large_binary()),
        ])

        schema = Schema.from_pyarrow_schema(
            pa_schema, partition_keys=['name'],
            # pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true'
            }
        )
        self.catalog.create_table('test_db.blob_write_read_partition', schema, False)
        table = self.catalog.get_table('test_db.blob_write_read_partition')

        # Test data with various blob sizes and types
        test_data = pa.Table.from_pydict({
            'id': [1, 2, 3, 4, 5],
            'name': ['Alice', 'Alice', 'David', 'David', 'David'],
            'description': ['User 1', 'User 2', 'User 3', 'User 4', 'User 5'],
            'blob_data': [
                b'small_blob_1',
                b'medium_blob_data_2_with_more_content',
                b'large_blob_data_3_with_even_more_content_and_details',
                b'very_large_blob_data_4_with_extensive_content_and_multiple_details_here',  # noqa: E501
                b'extremely_large_blob_data_5_with_comprehensive_content_and_'
                b'extensive_details_covering_multiple_aspects'
                # noqa: E501
            ]
        }, schema=pa_schema)

        # Write data using table API
        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(test_data)

        # Commit the data
        commit_messages = writer.prepare_commit()

        # Create commit and commit the data
        commit = write_builder.new_commit()
        commit.commit(commit_messages)
        writer.close()

        # Read data back using table API
        read_builder = table.new_read_builder()
        table_scan = read_builder.new_scan()
        table_read = read_builder.new_read()
        splits = table_scan.plan().splits()
        result = table_read.to_arrow(splits)

        # Verify the data was read back correctly
        self.assertEqual(result.num_rows, 5, "Should have 5 rows")
        self.assertEqual(result.num_columns, 4, "Should have 4 columns")

        # Verify normal columns
        self.assertEqual(result.column('id').to_pylist(), [1, 2, 3, 4, 5], "ID column should match")
        self.assertEqual(result.column('name').to_pylist(), ['Alice', 'Alice', 'David', 'David', 'David'],
                         "Name column should match")  # noqa: E501
        self.assertEqual(result.column('description').to_pylist(), ['User 1', 'User 2', 'User 3', 'User 4', 'User 5'],
                         "Description column should match")  # noqa: E501

        # Verify blob data correctness
        blob_data = result.column('blob_data').to_pylist()
        expected_blobs = [
            b'small_blob_1',
            b'medium_blob_data_2_with_more_content',
            b'large_blob_data_3_with_even_more_content_and_details',
            b'very_large_blob_data_4_with_extensive_content_and_multiple_details_here',  # noqa: E501
            b'extremely_large_blob_data_5_with_comprehensive_content_and_extensive_details_covering_multiple_aspects'
            # noqa: E501
        ]

        self.assertEqual(len(blob_data), 5, "Should have 5 blob records")
        self.assertEqual(blob_data, expected_blobs, "Blob data should match exactly")

        # Verify individual blob sizes
        for i, (actual_blob, expected_blob) in enumerate(zip(blob_data, expected_blobs)):
            self.assertEqual(len(actual_blob), len(expected_blob), f"Blob {i + 1} size should match")
            self.assertEqual(actual_blob, expected_blob, f"Blob {i + 1} content should match exactly")

        print(
            f"✅ End-to-end blob write/read test passed: wrote and read back {len(blob_data)} blob records correctly")  # noqa: E501

    def test_blob_write_read_end_to_end_with_descriptor(self):
        """Test end-to-end blob functionality using blob descriptors."""
        import random
        from pypaimon import Schema
        from pypaimon.table.row.blob import BlobDescriptor, Blob
        from pypaimon.common.uri_reader import UriReaderFactory
        from pypaimon.common.options.config import CatalogOptions

        # Create schema with blob column
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('name', pa.string()),
            ('picture', pa.large_binary()),
        ])

        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true',
                'blob-as-descriptor': 'true'
            }
        )

        # Create table
        self.catalog.create_table('test_db.blob_descriptor_test', schema, False)
        table: FileStoreTable = self.catalog.get_table('test_db.blob_descriptor_test')

        # Create test blob data (1MB)
        blob_data = bytearray(1024 * 1024)
        random.seed(42)  # For reproducible tests
        for i in range(len(blob_data)):
            blob_data[i] = random.randint(0, 255)
        blob_data = bytes(blob_data)

        # Create external blob file
        external_blob_path = os.path.join(self.temp_dir, 'external_blob')
        with open(external_blob_path, 'wb') as f:
            f.write(blob_data)

        # Create blob descriptor pointing to external file
        blob_descriptor = BlobDescriptor(external_blob_path, 0, len(blob_data))

        # Create test data with blob descriptor
        test_data = pa.Table.from_pydict({
            'id': [1],
            'name': ['paimon'],
            'picture': [blob_descriptor.serialize()]
        }, schema=pa_schema)

        # Write data using table API
        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(test_data)

        # Commit the data
        commit_messages = writer.prepare_commit()
        commit = write_builder.new_commit()
        commit.commit(commit_messages)

        # Read data back
        read_builder = table.new_read_builder()
        table_scan = read_builder.new_scan()
        table_read = read_builder.new_read()
        result = table_read.to_arrow(table_scan.plan().splits())

        # Verify the data was written and read correctly
        self.assertEqual(result.num_rows, 1, "Should have 1 row")
        self.assertEqual(result.column('id').to_pylist(), [1], "ID should match")
        self.assertEqual(result.column('name').to_pylist(), ['paimon'], "Name should match")

        # Get the blob descriptor bytes from the result
        picture_bytes = result.column('picture').to_pylist()[0]
        self.assertIsInstance(picture_bytes, bytes, "Picture should be bytes")

        # Deserialize the blob descriptor
        new_blob_descriptor = BlobDescriptor.deserialize(picture_bytes)

        # The URI might be different if the blob was stored in the table's data directory
        # Let's verify the descriptor properties and try to read the data
        # Note: offset might be non-zero due to blob file format overhead
        self.assertGreaterEqual(new_blob_descriptor.offset, 0, "Offset should be non-negative")
        self.assertEqual(new_blob_descriptor.length, len(blob_data), "Length should match")

        # Create URI reader factory and read the blob data
        catalog_options = {CatalogOptions.WAREHOUSE.key(): self.warehouse}
        uri_reader_factory = UriReaderFactory(catalog_options)
        uri_reader = uri_reader_factory.create(new_blob_descriptor.uri)
        blob = Blob.from_descriptor(uri_reader, new_blob_descriptor)

        blob_descriptor_from_blob = blob.to_descriptor()
        self.assertEqual(
            blob_descriptor_from_blob.uri,
            new_blob_descriptor.uri,
            f"URI should be preserved. Expected: {new_blob_descriptor.uri}, Got: {blob_descriptor_from_blob.uri}"
        )

        # Verify the blob data matches the original
        self.assertEqual(blob.to_data(), blob_data, "Blob data should match original")

        print("✅ Blob descriptor end-to-end test passed:")
        print("   - Created external blob file and descriptor")
        print("   - Wrote and read blob descriptor successfully")
        print("   - Verified blob data can be read from descriptor")
        print("   - Tested blob-as-descriptor=true read output mode")

    def test_blob_descriptor_fields_mixed_mode(self):
        import random
        from pypaimon import Schema
        from pypaimon.table.row.blob import BlobDescriptor

        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('pic1', pa.large_binary()),
            ('pic2', pa.large_binary()),
        ])

        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true',
                'blob-descriptor-field': 'pic2'
            }
        )
        self.catalog.create_table('test_db.blob_mixed_mode_test', schema, False)
        table = self.catalog.get_table('test_db.blob_mixed_mode_test')

        random.seed(7)
        pic1_data = bytes(bytearray([random.randint(0, 255) for _ in range(1024)]))

        external_blob_path = os.path.join(self.temp_dir, 'mixed_external_blob')
        pic2_data = b'pic2_descriptor_payload'
        with open(external_blob_path, 'wb') as f:
            f.write(pic2_data)
        descriptor = BlobDescriptor(external_blob_path, 0, len(pic2_data))

        test_data = pa.Table.from_pydict({
            'id': [1],
            'pic1': [pic1_data],
            'pic2': [descriptor.serialize()]
        }, schema=pa_schema)

        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(test_data)
        commit_messages = writer.prepare_commit()
        write_builder.new_commit().commit(commit_messages)
        writer.close()

        all_files = [f for msg in commit_messages for f in msg.new_files]
        blob_files = [f for f in all_files if f.file_name.endswith('.blob')]
        self.assertGreaterEqual(len(blob_files), 1)
        self.assertTrue(all(f.write_cols == ['pic1'] for f in blob_files))

        result = table.new_read_builder().new_read().to_arrow(table.new_read_builder().new_scan().plan().splits())
        self.assertEqual(result.num_rows, 1)
        self.assertEqual(result.column('pic1').to_pylist()[0], pic1_data)
        self.assertEqual(result.column('pic2').to_pylist()[0], pic2_data)

    def test_blob_view_fields_resolve_upstream_blob(self):
        from pypaimon import Schema
        from pypaimon.common.options.core_options import CoreOptions
        from pypaimon.table.row.blob import BlobViewStruct

        source_schema = pa.schema([
            ('id', pa.int32()),
            ('picture', pa.large_binary()),
        ])
        source = Schema.from_pyarrow_schema(
            source_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true',
            }
        )
        self.catalog.create_table('test_db.blob_view_source', source, False)
        source_table = self.catalog.get_table('test_db.blob_view_source')
        payloads = [b'view-source-0', b'view-source-1']

        write_builder = source_table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(pa.Table.from_pydict({
            'id': [1, 2],
            'picture': payloads,
        }, schema=source_schema))
        commit_messages = writer.prepare_commit()
        write_builder.new_commit().commit(commit_messages)
        writer.close()

        picture_field_id = next(
            field.id for field in source_table.table_schema.fields if field.name == 'picture'
        )
        view_values = [
            BlobViewStruct('test_db.blob_view_source', picture_field_id, 0).serialize(),
            BlobViewStruct('test_db.blob_view_source', picture_field_id, 1).serialize(),
        ]

        target_schema = pa.schema([
            ('id', pa.int32()),
            ('picture', pa.large_binary()),
        ])
        target = Schema.from_pyarrow_schema(
            target_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true',
                'blob-view-field': 'picture',
            }
        )
        self.catalog.create_table('test_db.blob_view_target', target, False)
        target_table = self.catalog.get_table('test_db.blob_view_target')

        target_write_builder = target_table.new_batch_write_builder()
        target_writer = target_write_builder.new_write()
        target_writer.write_arrow(pa.Table.from_pydict({
            'id': [10, 11],
            'picture': view_values,
        }, schema=target_schema))
        target_commit_messages = target_writer.prepare_commit()
        target_write_builder.new_commit().commit(target_commit_messages)
        target_writer.close()

        all_target_files = [f for msg in target_commit_messages for f in msg.new_files]
        self.assertFalse(
            any(f.file_name.endswith('.blob') for f in all_target_files),
            "Blob view fields should be stored inline without writing new blob files",
        )

        result = target_table.new_read_builder().new_read().to_arrow(
            target_table.new_read_builder().new_scan().plan().splits()
        ).sort_by('id')
        self.assertEqual(result.column('picture').to_pylist(), payloads)

        descriptor_table = target_table.copy({CoreOptions.BLOB_AS_DESCRIPTOR.key(): 'true'})
        descriptor_result = descriptor_table.new_read_builder().new_read().to_arrow(
            descriptor_table.new_read_builder().new_scan().plan().splits()
        ).sort_by('id')
        # With blob-as-descriptor=true, view fields return BlobDescriptor bytes
        from pypaimon.table.row.blob import BlobDescriptor
        for value in descriptor_result.column('picture').to_pylist():
            self.assertTrue(
                BlobDescriptor.is_blob_descriptor(value),
                "Expected BlobDescriptor bytes when blob-as-descriptor=true"
            )

    def test_blob_view_default_read_uses_upstream_file_io(self):
        from pypaimon import Schema
        from pypaimon.table.row.blob import BlobViewStruct

        class GuardedFileIO:
            def __init__(self, wrapped, forbidden_uris):
                self._wrapped = wrapped
                self._forbidden_uris = forbidden_uris
                self.forbidden_reads = []

            def new_input_stream(self, path):
                path = str(path)
                if path in self._forbidden_uris:
                    self.forbidden_reads.append(path)
                    raise AssertionError(
                        "Downstream file_io must not read upstream blob {}.".format(path)
                    )
                return self._wrapped.new_input_stream(path)

            def __getattr__(self, name):
                return getattr(self._wrapped, name)

        source_schema = pa.schema([
            ('id', pa.int32()),
            ('picture', pa.large_binary()),
        ])
        source = Schema.from_pyarrow_schema(
            source_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true',
            }
        )
        self.catalog.create_table('test_db.blob_view_guard_source', source, False)
        source_table = self.catalog.get_table('test_db.blob_view_guard_source')
        payloads = [b'guard-source-0', b'guard-source-1']

        write_builder = source_table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(pa.Table.from_pydict({
            'id': [1, 2],
            'picture': payloads,
        }, schema=source_schema))
        source_commit_messages = writer.prepare_commit()
        write_builder.new_commit().commit(source_commit_messages)
        writer.close()

        source_blob_paths = {
            str(f.file_path)
            for msg in source_commit_messages
            for f in msg.new_files
            if f.file_name.endswith('.blob')
        }
        self.assertGreater(len(source_blob_paths), 0)

        picture_field_id = next(
            field.id for field in source_table.table_schema.fields if field.name == 'picture'
        )
        view_values = [
            BlobViewStruct('test_db.blob_view_guard_source', picture_field_id, 0).serialize(),
            BlobViewStruct('test_db.blob_view_guard_source', picture_field_id, 1).serialize(),
        ]

        target_schema = pa.schema([
            ('id', pa.int32()),
            ('picture', pa.large_binary()),
        ])
        target = Schema.from_pyarrow_schema(
            target_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true',
                'blob-view-field': 'picture',
            }
        )
        self.catalog.create_table('test_db.blob_view_guard_target', target, False)
        target_table = self.catalog.get_table('test_db.blob_view_guard_target')

        target_write_builder = target_table.new_batch_write_builder()
        target_writer = target_write_builder.new_write()
        target_writer.write_arrow(pa.Table.from_pydict({
            'id': [10, 11],
            'picture': view_values,
        }, schema=target_schema))
        target_commit_messages = target_writer.prepare_commit()
        target_write_builder.new_commit().commit(target_commit_messages)
        target_writer.close()

        all_target_files = [f for msg in target_commit_messages for f in msg.new_files]
        self.assertFalse(
            any(f.file_name.endswith('.blob') for f in all_target_files),
            "Blob view fields should be stored inline without writing new blob files",
        )

        original_file_io = target_table.file_io
        guarded_file_io = GuardedFileIO(original_file_io, source_blob_paths)
        target_table.file_io = guarded_file_io
        try:
            read_builder = target_table.new_read_builder()
            result = read_builder.new_read().to_arrow(
                read_builder.new_scan().plan().splits()
            ).sort_by('id')
        finally:
            target_table.file_io = original_file_io

        self.assertEqual(result.column('picture').to_pylist(), payloads)
        self.assertEqual(guarded_file_io.forbidden_reads, [])

    def test_blob_view_resolve_disabled_preserves_references(self):
        from pypaimon import Schema
        from pypaimon.common.options.core_options import CoreOptions
        from pypaimon.table.row.blob import BlobViewStruct

        source_schema = pa.schema([
            ('id', pa.int32()),
            ('picture', pa.large_binary()),
        ])
        source = Schema.from_pyarrow_schema(
            source_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true',
            }
        )
        self.catalog.create_table('test_db.blob_view_resolve_source', source, False)
        source_table = self.catalog.get_table('test_db.blob_view_resolve_source')
        payloads = [b'resolve-source-0', b'resolve-source-1']

        write_builder = source_table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(pa.Table.from_pydict({
            'id': [1, 2],
            'picture': payloads,
        }, schema=source_schema))
        commit_messages = writer.prepare_commit()
        write_builder.new_commit().commit(commit_messages)
        writer.close()

        picture_field_id = next(
            field.id for field in source_table.table_schema.fields if field.name == 'picture'
        )
        view_values = [
            BlobViewStruct('test_db.blob_view_resolve_source', picture_field_id, 0).serialize(),
            BlobViewStruct('test_db.blob_view_resolve_source', picture_field_id, 1).serialize(),
        ]

        target_schema = pa.schema([
            ('id', pa.int32()),
            ('picture', pa.large_binary()),
        ])
        target = Schema.from_pyarrow_schema(
            target_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true',
                'blob-view-field': 'picture',
            }
        )
        self.catalog.create_table('test_db.blob_view_resolve_target', target, False)
        target_table = self.catalog.get_table('test_db.blob_view_resolve_target')

        target_write_builder = target_table.new_batch_write_builder()
        target_writer = target_write_builder.new_write()
        target_writer.write_arrow(pa.Table.from_pydict({
            'id': [10, 11],
            'picture': view_values,
        }, schema=target_schema))
        target_commit_messages = target_writer.prepare_commit()
        target_write_builder.new_commit().commit(target_commit_messages)
        target_writer.close()

        # Default (resolve enabled): view fields are resolved to real blob data.
        resolved_result = target_table.new_read_builder().new_read().to_arrow(
            target_table.new_read_builder().new_scan().plan().splits()
        ).sort_by('id')
        self.assertEqual(resolved_result.column('picture').to_pylist(), payloads)

        # resolve disabled: view fields keep the original BlobViewStruct bytes.
        preserve_table = target_table.copy(
            {CoreOptions.BLOB_VIEW_RESOLVE_ENABLED.key(): 'false'}
        )
        preserve_result = preserve_table.new_read_builder().new_read().to_arrow(
            preserve_table.new_read_builder().new_scan().plan().splits()
        ).sort_by('id')
        preserved_values = preserve_result.column('picture').to_pylist()
        self.assertEqual(preserved_values, view_values)
        for value in preserved_values:
            self.assertTrue(
                BlobViewStruct.is_blob_view_struct(value),
                "Expected original BlobViewStruct bytes when resolve disabled"
            )

    def test_blob_view_resolves_null_upstream_value(self):
        from pypaimon import Schema
        from pypaimon.table.row.blob import BlobViewStruct

        source_schema = pa.schema([
            ('id', pa.int32()),
            ('picture', pa.large_binary()),
        ])
        source = Schema.from_pyarrow_schema(
            source_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true',
            }
        )
        self.catalog.create_table('test_db.blob_view_null_source', source, False)
        source_table = self.catalog.get_table('test_db.blob_view_null_source')
        # Row 0 has a real blob value, row 1 has a null blob value.
        payloads = [b'null-source-0', None]

        write_builder = source_table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(pa.Table.from_pydict({
            'id': [1, 2],
            'picture': payloads,
        }, schema=source_schema))
        commit_messages = writer.prepare_commit()
        write_builder.new_commit().commit(commit_messages)
        writer.close()

        picture_field_id = next(
            field.id for field in source_table.table_schema.fields if field.name == 'picture'
        )
        view_values = [
            BlobViewStruct('test_db.blob_view_null_source', picture_field_id, 0).serialize(),
            BlobViewStruct('test_db.blob_view_null_source', picture_field_id, 1).serialize(),
        ]

        target_schema = pa.schema([
            ('id', pa.int32()),
            ('picture', pa.large_binary()),
        ])
        target = Schema.from_pyarrow_schema(
            target_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true',
                'blob-view-field': 'picture',
            }
        )
        self.catalog.create_table('test_db.blob_view_null_target', target, False)
        target_table = self.catalog.get_table('test_db.blob_view_null_target')

        target_write_builder = target_table.new_batch_write_builder()
        target_writer = target_write_builder.new_write()
        target_writer.write_arrow(pa.Table.from_pydict({
            'id': [10, 11],
            'picture': view_values,
        }, schema=target_schema))
        target_commit_messages = target_writer.prepare_commit()
        target_write_builder.new_commit().commit(target_commit_messages)
        target_writer.close()

        # View referencing a real upstream value resolves to data; view
        # referencing a null upstream value resolves to None (not an error).
        result = target_table.new_read_builder().new_read().to_arrow(
            target_table.new_read_builder().new_scan().plan().splits()
        ).sort_by('id')
        self.assertEqual(result.column('picture').to_pylist(), [b'null-source-0', None])

    def test_blob_view_fields_rejects_non_view_input(self):
        from pypaimon import Schema

        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('picture', pa.large_binary()),
        ])
        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true',
                'blob-view-field': 'picture',
            }
        )
        self.catalog.create_table('test_db.blob_view_reject_test', schema, False)
        table = self.catalog.get_table('test_db.blob_view_reject_test')

        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()
        bad_data = pa.Table.from_pydict({
            'id': [1],
            'picture': [b'not-a-view-struct'],
        }, schema=pa_schema)

        with self.assertRaises(ValueError) as context:
            writer.write_arrow(bad_data)
        self.assertIn("blob-view-field", str(context.exception))

    def test_blob_inline_fields_reject_overlap_and_unknown_fields(self):
        from pypaimon import Schema

        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('picture', pa.large_binary()),
        ])
        base_options = {
            'row-tracking.enabled': 'true',
            'data-evolution.enabled': 'true',
        }

        overlap_options = dict(base_options)
        overlap_options.update({
            'blob-descriptor-field': 'picture',
            'blob-view-field': 'picture',
        })
        overlap_schema = Schema.from_pyarrow_schema(pa_schema, options=overlap_options)
        with self.assertRaises(ValueError) as overlap_context:
            self.catalog.create_table(
                'test_db.blob_overlap_reject', overlap_schema, False)
        self.assertIn("must not overlap", str(overlap_context.exception))

        unknown_options = dict(base_options)
        unknown_options.update({'blob-view-field': 'missing_picture'})
        unknown_schema = Schema.from_pyarrow_schema(pa_schema, options=unknown_options)
        with self.assertRaises(ValueError) as unknown_context:
            self.catalog.create_table(
                'test_db.blob_unknown_reject', unknown_schema, False)
        self.assertIn("must be blob fields", str(unknown_context.exception))

    def test_blob_view_prescan_with_limit(self):
        """Test that limit is correctly pushed down to prescan reader.
        
        Regression test for: prescan should only scan up to limit rows,
        not the entire split.
        """
        from pypaimon import Schema
        from pypaimon.table.row.blob import BlobViewStruct

        # Create source table with multiple rows
        source_schema = pa.schema([
            ('id', pa.int32()),
            ('picture', pa.large_binary()),
        ])
        source = Schema.from_pyarrow_schema(
            source_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true',
            }
        )
        self.catalog.create_table('test_db.blob_view_limit_source', source, False)
        source_table = self.catalog.get_table('test_db.blob_view_limit_source')

        # Write 10 rows
        num_rows = 10
        payloads = [f'payload-{i}'.encode() for i in range(num_rows)]
        write_builder = source_table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(pa.Table.from_pydict({
            'id': list(range(num_rows)),
            'picture': payloads,
        }, schema=source_schema))
        commit_messages = writer.prepare_commit()
        write_builder.new_commit().commit(commit_messages)
        writer.close()

        picture_field_id = next(
            field.id for field in source_table.table_schema.fields if field.name == 'picture'
        )
        view_values = [
            BlobViewStruct('test_db.blob_view_limit_source', picture_field_id, i).serialize()
            for i in range(num_rows)
        ]

        # Create target table with blob-view-field
        target_schema = pa.schema([
            ('id', pa.int32()),
            ('picture', pa.large_binary()),
        ])
        target = Schema.from_pyarrow_schema(
            target_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true',
                'blob-view-field': 'picture',
            }
        )
        self.catalog.create_table('test_db.blob_view_limit_target', target, False)
        target_table = self.catalog.get_table('test_db.blob_view_limit_target')

        target_write_builder = target_table.new_batch_write_builder()
        target_writer = target_write_builder.new_write()
        target_writer.write_arrow(pa.Table.from_pydict({
            'id': list(range(num_rows)),
            'picture': view_values,
        }, schema=target_schema))
        target_commit_messages = target_writer.prepare_commit()
        target_write_builder.new_commit().commit(target_commit_messages)
        target_writer.close()

        # Test with limit: should only return first 3 rows
        read_builder = target_table.new_read_builder()
        read_builder.with_limit(3)
        result = read_builder.new_read().to_arrow(
            read_builder.new_scan().plan().splits()
        )
        self.assertEqual(result.num_rows, 3, "LIMIT should be respected in blob view prescan")
        self.assertEqual(result.column('id').to_pylist(), [0, 1, 2])

    def test_blob_view_prescan_only_collects_limited_view_structs(self):
        """Verify that the prescan stage only collects as many BlobViewStructs as
        the limit allows, instead of scanning the entire split.

        Unlike test_blob_view_prescan_with_limit (which only checks the final
        output), this test patches BlobViewLookup.preload to capture the exact
        list of view structs collected during prescan and asserts its length
        equals the limit.
        """
        from unittest import mock

        from pypaimon import Schema
        from pypaimon.table.row.blob import BlobViewStruct
        from pypaimon.utils.blob_view_lookup import BlobViewLookup

        source_schema = pa.schema([
            ('id', pa.int32()),
            ('picture', pa.large_binary()),
        ])
        source = Schema.from_pyarrow_schema(
            source_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true',
            }
        )
        self.catalog.create_table('test_db.blob_view_prescan_count_source', source, False)
        source_table = self.catalog.get_table('test_db.blob_view_prescan_count_source')

        num_rows = 10
        payloads = [f'payload-{i}'.encode() for i in range(num_rows)]
        write_builder = source_table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(pa.Table.from_pydict({
            'id': list(range(num_rows)),
            'picture': payloads,
        }, schema=source_schema))
        commit_messages = writer.prepare_commit()
        write_builder.new_commit().commit(commit_messages)
        writer.close()

        picture_field_id = next(
            field.id for field in source_table.table_schema.fields if field.name == 'picture'
        )
        view_values = [
            BlobViewStruct('test_db.blob_view_prescan_count_source', picture_field_id, i).serialize()
            for i in range(num_rows)
        ]

        target_schema = pa.schema([
            ('id', pa.int32()),
            ('picture', pa.large_binary()),
        ])
        target = Schema.from_pyarrow_schema(
            target_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true',
                'blob-view-field': 'picture',
            }
        )
        self.catalog.create_table('test_db.blob_view_prescan_count_target', target, False)
        target_table = self.catalog.get_table('test_db.blob_view_prescan_count_target')

        target_write_builder = target_table.new_batch_write_builder()
        target_writer = target_write_builder.new_write()
        target_writer.write_arrow(pa.Table.from_pydict({
            'id': list(range(num_rows)),
            'picture': view_values,
        }, schema=target_schema))
        target_commit_messages = target_writer.prepare_commit()
        target_write_builder.new_commit().commit(target_commit_messages)
        target_writer.close()

        captured_view_structs = []
        original_preload = BlobViewLookup.preload

        def capturing_preload(lookup_self, view_structs):
            captured_view_structs.append(list(view_structs))
            return original_preload(lookup_self, view_structs)

        limit = 3
        read_builder = target_table.new_read_builder()
        read_builder.with_limit(limit)
        with mock.patch.object(BlobViewLookup, 'preload', autospec=True,
                               side_effect=capturing_preload):
            result = read_builder.new_read().to_arrow(
                read_builder.new_scan().plan().splits()
            )

        self.assertEqual(result.num_rows, limit)
        self.assertEqual(len(captured_view_structs), 1,
                         "preload should be invoked exactly once during prescan")
        self.assertEqual(
            len(captured_view_structs[0]), limit,
            "prescan should only collect as many view structs as the limit allows")

    def test_to_arrow_batch_reader(self):
        import random
        from pypaimon import Schema
        from pypaimon.table.row.blob import BlobDescriptor

        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('pic1', pa.large_binary()),
            ('pic2', pa.large_binary()),
        ])

        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true',
                'blob-descriptor-field': 'pic2'
            }
        )
        self.catalog.create_table('test_db.test_to_arrow_batch_reader', schema, False)
        table = self.catalog.get_table('test_db.test_to_arrow_batch_reader')

        random.seed(8)
        pic1_data = bytes(bytearray([random.randint(0, 255) for _ in range(1024)]))

        external_blob_path = os.path.join(self.temp_dir, 'mixed_external_blob_batch_reader')
        pic2_data = b'pic2_batch_reader_payload'
        with open(external_blob_path, 'wb') as f:
            f.write(pic2_data)
        descriptor = BlobDescriptor(external_blob_path, 0, len(pic2_data))

        test_data = pa.Table.from_pydict({
            'id': [1],
            'pic1': [pic1_data],
            'pic2': [descriptor.serialize()]
        }, schema=pa_schema)

        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(test_data)
        commit_messages = writer.prepare_commit()
        write_builder.new_commit().commit(commit_messages)
        writer.close()

        read_builder = table.new_read_builder()
        splits = read_builder.new_scan().plan().splits()
        batch_reader = read_builder.new_read().to_arrow_batch_reader(splits)

        batches = []
        while True:
            try:
                batch = batch_reader.read_next_batch()
            except StopIteration:
                break
            if batch.num_rows > 0:
                batches.append(batch)

        self.assertGreater(len(batches), 0, "Should have at least one batch")
        result = pa.Table.from_batches(batches)
        self.assertEqual(result.num_rows, 1)
        self.assertEqual(result.column('pic1').to_pylist()[0], pic1_data)
        self.assertEqual(result.column('pic2').to_pylist()[0], pic2_data)

    def test_blob_descriptor_fields_rejects_non_descriptor_input(self):
        from pypaimon import Schema

        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('pic1', pa.large_binary()),
            ('pic2', pa.large_binary()),
        ])

        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true',
                'blob-descriptor-field': 'pic2'
            }
        )
        self.catalog.create_table('test_db.blob_mixed_mode_reject_test', schema, False)
        table = self.catalog.get_table('test_db.blob_mixed_mode_reject_test')

        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()

        bad_data = pa.Table.from_pydict({
            'id': [1],
            'pic1': [b'good'],
            'pic2': [b'not-a-descriptor']
        }, schema=pa_schema)

        with self.assertRaises(ValueError) as context:
            writer.write_arrow(bad_data)
        self.assertIn("blob-descriptor-field", str(context.exception))

    def test_blob_write_read_large_data_end_to_end(self):
        """Test end-to-end blob functionality with large blob data (1MB per blob)."""
        from pypaimon import Schema

        # Create schema with blob column
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('metadata', pa.string()),
            ('large_blob', pa.large_binary()),
        ])

        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true'
            }
        )
        self.catalog.create_table('test_db.blob_large_write_read_e2e', schema, False)
        table = self.catalog.get_table('test_db.blob_large_write_read_e2e')

        # Create large blob data (1MB per blob)
        large_blob_size = 1024 * 1024  # 1MB
        blob_pattern = b'LARGE_BLOB_PATTERN_' + b'X' * 1024  # ~1KB pattern
        pattern_size = len(blob_pattern)
        repetitions = large_blob_size // pattern_size
        large_blob_data = blob_pattern * repetitions

        # Test data with large blobs
        test_data = pa.Table.from_pydict({
            'id': [1, 2, 3],
            'metadata': ['Large blob 1', 'Large blob 2', 'Large blob 3'],
            'large_blob': [large_blob_data, large_blob_data, large_blob_data]
        }, schema=pa_schema)

        # Write data
        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(test_data)

        # Commit the data
        commit_messages = writer.prepare_commit()
        self.assertGreater(len(commit_messages), 0)

        # Verify commit message structure
        for commit_msg in commit_messages:
            self.assertIsInstance(commit_msg.new_files, list)
        self.assertGreater(len(commit_msg.new_files), 0)

        # Should have both normal and blob files
        file_names = [f.file_name for f in commit_msg.new_files]
        parquet_files = [f for f in file_names if f.endswith('.parquet')]
        blob_files = [f for f in file_names if f.endswith('.blob')]

        self.assertGreater(len(parquet_files), 0, "Should have at least one parquet file")
        self.assertGreater(len(blob_files), 0, "Should have at least one blob file")

        commit = write_builder.new_commit()
        commit.commit(commit_messages)
        writer.close()

        # Read data back
        read_builder = table.new_read_builder()
        table_scan = read_builder.new_scan()
        table_read = read_builder.new_read()
        result = table_read.to_arrow(table_scan.plan().splits())

        # Verify the data
        self.assertEqual(result.num_rows, 3, "Should have 3 rows")
        self.assertEqual(result.num_columns, 3, "Should have 3 columns")

        # Verify normal columns
        self.assertEqual(result.column('id').to_pylist(), [1, 2, 3], "ID column should match")
        self.assertEqual(result.column('metadata').to_pylist(), ['Large blob 1', 'Large blob 2', 'Large blob 3'],
                         "Metadata column should match")  # noqa: E501

        # Verify blob data integrity
        blob_data = result.column('large_blob').to_pylist()
        self.assertEqual(len(blob_data), 3, "Should have 3 blob records")

        for i, blob in enumerate(blob_data):
            self.assertEqual(len(blob), len(large_blob_data), f"Blob {i + 1} should be {large_blob_size} bytes")
            self.assertEqual(blob, large_blob_data, f"Blob {i + 1} content should match exactly")
            print(f"✅ Verified large blob {i + 1}: {len(blob)} bytes")

        print(
            f"✅ Large blob end-to-end test passed: wrote and read back {len(blob_data)} large blob records correctly")  # noqa: E501

    def test_blob_write_read_large_data_volume(self):
        from pypaimon import Schema

        # Create schema with blob column
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('batch_id', pa.int32()),
            ('metadata', pa.string()),
            ('large_blob', pa.large_binary()),
        ])

        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true'
            }
        )
        self.catalog.create_table('test_db.blob_large_data_volume', schema, False)
        table = self.catalog.get_table('test_db.blob_large_data_volume')

        large_blob_size = 5 * 1024
        blob_pattern = b'LARGE_BLOB_PATTERN_' + b'X' * 1024  # ~1KB pattern
        pattern_size = len(blob_pattern)
        repetitions = large_blob_size // pattern_size
        large_blob_data = blob_pattern * repetitions

        num_row = 20000
        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()
        expected = pa.Table.from_pydict({
            'id': [1] * num_row,
            'batch_id': [11] * num_row,
            'metadata': [f'Large blob batch {11}'] * num_row,
            'large_blob': [i.to_bytes(2, byteorder='little') + large_blob_data for i in range(num_row)]
        }, schema=pa_schema)
        writer.write_arrow(expected)

        # Commit all data at once
        commit_messages = writer.prepare_commit()
        commit = write_builder.new_commit()
        commit.commit(commit_messages)
        writer.close()

        # Read data back
        read_builder = table.new_read_builder()
        table_scan = read_builder.new_scan()
        table_read = read_builder.new_read()
        result = table_read.to_arrow(table_scan.plan().splits())

        # Verify the data
        self.assertEqual(result.num_rows, num_row)
        self.assertEqual(result.num_columns, 4)

        self.assertEqual(expected, result)

    def test_blob_write_read_large_data_volume_rolling(self):
        from pypaimon import Schema

        # Create schema with blob column
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('batch_id', pa.int32()),
            ('metadata', pa.string()),
            ('large_blob', pa.large_binary()),
        ])

        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true',
                'blob.target-file-size': '21MB'
            }
        )
        self.catalog.create_table('test_db.large_data_volume_rolling', schema, False)
        table = self.catalog.get_table('test_db.large_data_volume_rolling')

        # Create large blob data
        large_blob_size = 5 * 1024  #
        blob_pattern = b'LARGE_BLOB_PATTERN_' + b'X' * 1024  # ~1KB pattern
        pattern_size = len(blob_pattern)
        repetitions = large_blob_size // pattern_size
        large_blob_data = blob_pattern * repetitions

        actual_size = len(large_blob_data)
        print(f"Created blob data: {actual_size:,} bytes ({actual_size / (1024 * 1024):.2f} MB)")
        num_row = 20000
        expected = pa.Table.from_pydict({
            'id': [1] * num_row,
            'batch_id': [11] * num_row,
            'metadata': [f'Large blob batch {11}'] * num_row,
            'large_blob': [i.to_bytes(2, byteorder='little') + large_blob_data for i in range(num_row)]
        }, schema=pa_schema)
        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(expected)

        commit_messages = writer.prepare_commit()
        commit = write_builder.new_commit()
        commit.commit(commit_messages)
        writer.close()

        # Read data back
        read_builder = table.new_read_builder()
        table_scan = read_builder.new_scan()
        table_read = read_builder.new_read()
        result = table_read.to_arrow(table_scan.plan().splits())
        self.assertEqual(expected, result)

    def test_blob_write_read_mixed_sizes_end_to_end(self):
        """Test end-to-end blob functionality with mixed blob sizes."""
        from pypaimon import Schema

        # Create schema with blob column
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('size_category', pa.string()),
            ('blob_data', pa.large_binary()),
        ])

        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true'
            }
        )
        self.catalog.create_table('test_db.blob_mixed_sizes_write_read_e2e', schema, False)
        table = self.catalog.get_table('test_db.blob_mixed_sizes_write_read_e2e')

        # Create blobs of different sizes
        tiny_blob = b'tiny'
        small_blob = b'small_blob_data' * 10  # ~140 bytes
        medium_blob = b'medium_blob_data' * 100  # ~1.4KB
        large_blob = b'large_blob_data' * 1000  # ~14KB
        huge_blob = b'huge_blob_data' * 10000  # ~140KB

        test_data = pa.Table.from_pydict({
            'id': [1, 2, 3, 4, 5],
            'size_category': ['tiny', 'small', 'medium', 'large', 'huge'],
            'blob_data': [tiny_blob, small_blob, medium_blob, large_blob, huge_blob]
        }, schema=pa_schema)

        # Write data
        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(test_data)

        # Commit
        commit_messages = writer.prepare_commit()
        self.assertGreater(len(commit_messages), 0)

        # Verify commit message structure
        for commit_msg in commit_messages:
            self.assertIsInstance(commit_msg.new_files, list)
        self.assertGreater(len(commit_msg.new_files), 0)

        # Should have both normal and blob files
        file_names = [f.file_name for f in commit_msg.new_files]
        parquet_files = [f for f in file_names if f.endswith('.parquet')]
        blob_files = [f for f in file_names if f.endswith('.blob')]

        self.assertGreater(len(parquet_files), 0, "Should have at least one parquet file")
        self.assertGreater(len(blob_files), 0, "Should have at least one blob file")

        commit = write_builder.new_commit()
        commit.commit(commit_messages)
        writer.close()

        # Read back
        read_builder = table.new_read_builder()
        table_scan = read_builder.new_scan()
        table_read = read_builder.new_read()
        result = table_read.to_arrow(table_scan.plan().splits())

        # Verify
        self.assertEqual(result.num_rows, 5, "Should have 5 rows")
        self.assertEqual(result.num_columns, 3, "Should have 3 columns")

        # Verify normal columns
        self.assertEqual(result.column('id').to_pylist(), [1, 2, 3, 4, 5], "ID column should match")
        self.assertEqual(result.column('size_category').to_pylist(), ['tiny', 'small', 'medium', 'large', 'huge'],
                         "Size category column should match")  # noqa: E501

        # Verify blob data
        blob_data = result.column('blob_data').to_pylist()
        expected_blobs = [tiny_blob, small_blob, medium_blob, large_blob, huge_blob]

        self.assertEqual(len(blob_data), 5, "Should have 5 blob records")
        self.assertEqual(blob_data, expected_blobs, "Blob data should match exactly")

        # Verify sizes
        sizes = [len(blob) for blob in blob_data]
        expected_sizes = [len(blob) for blob in expected_blobs]
        self.assertEqual(sizes, expected_sizes, "Blob sizes should match")

        # Verify individual blob content
        for i, (actual_blob, expected_blob) in enumerate(zip(blob_data, expected_blobs)):
            self.assertEqual(actual_blob, expected_blob, f"Blob {i + 1} content should match exactly")

        print(
            f"✅ Mixed sizes end-to-end test passed: wrote and read back blobs ranging from {min(sizes)} to {max(sizes)} bytes")  # noqa: E501

    def test_blob_write_read_large_data_end_to_end_with_rolling(self):
        """Test end-to-end blob functionality with large blob data and rolling behavior."""
        from pypaimon import Schema

        # Create schema with blob column
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('batch_id', pa.int32()),
            ('metadata', pa.string()),
            ('large_blob', pa.large_binary()),
        ])

        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true',
                'blob.target-file-size': '5MB'
            }
        )
        self.catalog.create_table('test_db.blob_large_rolling_e2e', schema, False)
        table = self.catalog.get_table('test_db.blob_large_rolling_e2e')

        num_blobs = 10
        large_blob_size = 2 * 1024 * 1024  # 2MB
        blob_pattern = b'LARGE_BLOB_PATTERN_' + b'X' * 1024  # ~1KB pattern
        pattern_size = len(blob_pattern)
        repetitions = large_blob_size // pattern_size
        large_blob_data = blob_pattern * repetitions

        # Verify the blob size is approximately 2MB.
        actual_size = len(large_blob_data)
        print(f"Created blob data: {actual_size:,} bytes ({actual_size / (1024 * 1024):.2f} MB)")

        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()

        for batch_id in range(num_blobs):
            # Create test data for this batch
            test_data = pa.Table.from_pydict({
                'id': [batch_id + 1],
                'batch_id': [batch_id],
                'metadata': [f'Large blob batch {batch_id + 1}'],
                'large_blob': [large_blob_data]
            }, schema=pa_schema)

            # Write data
            writer.write_arrow(test_data)

            if (batch_id + 1) % 5 == 0:
                print(f"✅ Written batch {batch_id + 1}/{num_blobs}: {len(large_blob_data):,} bytes")

        print(f"✅ Successfully wrote all {num_blobs} batches of large blobs")

        # Commit all data at once
        commit_messages = writer.prepare_commit()
        self.assertGreater(len(commit_messages), 0)

        # Verify commit message structure
        for commit_msg in commit_messages:
            self.assertIsInstance(commit_msg.new_files, list)
        self.assertGreater(len(commit_msg.new_files), 0)

        # Should have both normal and blob files
        file_names = [f.file_name for f in commit_msg.new_files]
        parquet_files = [f for f in file_names if f.endswith('.parquet')]
        blob_files = [f for f in file_names if f.endswith('.blob')]

        self.assertGreater(len(parquet_files), 0, "Should have at least one parquet file")
        self.assertGreater(len(blob_files), 0, "Should have at least one blob file")

        # Commit the data
        commit = write_builder.new_commit()
        commit.commit(commit_messages)
        writer.close()

        print(
            f"✅ Successfully committed {len(commit_messages)} commit messages "
            f"with {num_blobs} batches of large blobs"
        )

        # Read data back
        read_builder = table.new_read_builder()
        table_scan = read_builder.new_scan()
        table_read = read_builder.new_read()
        splits = table_scan.plan().splits()
        result = table_read.to_arrow(splits)

        self.assertEqual(sum([s.row_count for s in splits]), num_blobs * 2)

        # Verify the data
        self.assertEqual(result.num_rows, num_blobs, f"Should have {num_blobs} rows")
        self.assertEqual(result.num_columns, 4, "Should have 4 columns")

        # Verify normal columns
        expected_ids = list(range(1, num_blobs + 1))
        expected_batch_ids = list(range(num_blobs))
        expected_metadata = [f'Large blob batch {i}' for i in range(1, num_blobs + 1)]

        self.assertEqual(result.column('id').to_pylist(), expected_ids, "ID column should match")
        self.assertEqual(result.column('batch_id').to_pylist(), expected_batch_ids,
                         "Batch ID column should match")  # noqa: E501
        self.assertEqual(result.column('metadata').to_pylist(), expected_metadata,
                         "Metadata column should match")  # noqa: E501

        # Verify blob data integrity
        blob_data = result.column('large_blob').to_pylist()
        self.assertEqual(len(blob_data), num_blobs, f"Should have {num_blobs} blob records")

        # Verify each blob
        for i, blob in enumerate(blob_data):
            self.assertEqual(len(blob), len(large_blob_data), f"Blob {i + 1} should be {large_blob_size:,} bytes")
            self.assertEqual(blob, large_blob_data, f"Blob {i + 1} content should match exactly")

            if (i + 1) % 5 == 0:
                print(f"✅ Verified blob {i + 1}/{num_blobs}: {len(blob):,} bytes")

        # Verify total data size
        total_blob_size = sum(len(blob) for blob in blob_data)
        expected_total_size = num_blobs * len(large_blob_data)
        self.assertEqual(total_blob_size, expected_total_size,
                         f"Total blob size should be {expected_total_size:,} bytes")

        print("✅ Large blob rolling end-to-end test passed:")
        print(f"   - Wrote and read back {num_blobs} large blobs")
        print(
            f"   - Total data size: {total_blob_size:,} bytes ({total_blob_size / (1024 * 1024 * 1024):.2f} GB)")  # noqa: E501
        print("   - All blob content verified as correct")

    def test_blob_read_row_by_row_iterator(self):
        """Test reading blob data row by row using to_iterator()."""
        from pypaimon import Schema
        from pypaimon.table.row.blob import Blob
        from pypaimon.table.row.internal_row import RowKind

        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('name', pa.string()),
            ('blob_data', pa.large_binary()),
        ])

        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true'
            }
        )
        self.catalog.create_table('test_db.blob_iterator_test', schema, False)
        table = self.catalog.get_table('test_db.blob_iterator_test')

        expected_data = {
            1: {'name': 'Alice', 'blob': b'blob_1'},
            2: {'name': 'Bob', 'blob': b'blob_2_data'},
            3: {'name': 'Charlie', 'blob': b'blob_3_content'},
            4: {'name': 'David', 'blob': b'blob_4_large_content'},
            5: {'name': 'Eve', 'blob': b'blob_5_very_large_content_data'}
        }

        test_data = pa.Table.from_pydict({
            'id': list(expected_data.keys()),
            'name': [expected_data[i]['name'] for i in expected_data.keys()],
            'blob_data': [expected_data[i]['blob'] for i in expected_data.keys()]
        }, schema=pa_schema)

        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(test_data)
        commit_messages = writer.prepare_commit()
        commit = write_builder.new_commit()
        commit.commit(commit_messages)
        writer.close()

        # Verify blob files were created
        file_names = [f.file_name for f in commit_messages[0].new_files]
        self.assertGreater(
            len([f for f in file_names if f.endswith('.blob')]), 0,
            "Should have at least one blob file")

        # Read using to_iterator
        iterator = table.new_read_builder().new_read().to_iterator(
            table.new_read_builder().new_scan().plan().splits())

        rows = []
        value = next(iterator, None)
        while value is not None:
            rows.append(value)
            value = next(iterator, None)

        self.assertEqual(len(rows), 5, "Should have 5 rows")

        for row in rows:
            row_id = row.get_field(0)
            self.assertIn(row_id, expected_data, f"ID {row_id} should be in expected data")

            expected = expected_data[row_id]
            self.assertEqual(row.get_field(1), expected['name'], f"Row {row_id}: name should match")

            row_blob = row.get_field(2)
            blob_bytes = row_blob.to_data() if isinstance(row_blob, Blob) else row_blob
            self.assertIsInstance(blob_bytes, bytes, f"Row {row_id}: blob should be bytes")
            self.assertEqual(blob_bytes, expected['blob'], f"Row {row_id}: blob data should match")
            self.assertEqual(len(blob_bytes), len(expected['blob']), f"Row {row_id}: blob size should match")

            self.assertIn(
                row.get_row_kind(),
                [RowKind.INSERT, RowKind.UPDATE_BEFORE, RowKind.UPDATE_AFTER, RowKind.DELETE],
                f"Row {row_id}: RowKind should be valid")

    def test_blob_as_descriptor_target_file_size_rolling(self):
        import random
        import os
        from pypaimon import Schema
        from pypaimon.table.row.blob import BlobDescriptor

        pa_schema = pa.schema([('id', pa.int32()), ('name', pa.string()), ('blob_data', pa.large_binary())])
        schema = Schema.from_pyarrow_schema(pa_schema, options={
            'row-tracking.enabled': 'true', 'data-evolution.enabled': 'true',
            'blob-as-descriptor': 'true', 'target-file-size': '1MB'
        })
        self.catalog.create_table('test_db.blob_target_size_test', schema, False)
        table = self.catalog.get_table('test_db.blob_target_size_test')

        # Create 5 external blob files (2MB each, > target-file-size)
        num_blobs, blob_size = 5, 2 * 1024 * 1024
        random.seed(42)
        descriptors = []
        for i in range(num_blobs):
            path = os.path.join(self.temp_dir, f'external_blob_{i}')
            data = bytes(bytearray([random.randint(0, 255) for _ in range(blob_size)]))
            with open(path, 'wb') as f:
                f.write(data)
            descriptors.append(BlobDescriptor(path, 0, len(data)))

        # Write data
        test_data = pa.Table.from_pydict({
            'id': list(range(1, num_blobs + 1)),
            'name': [f'item_{i}' for i in range(1, num_blobs + 1)],
            'blob_data': [d.serialize() for d in descriptors]
        }, schema=pa_schema)
        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(test_data)
        commit_messages = writer.prepare_commit()
        write_builder.new_commit().commit(commit_messages)
        writer.close()

        # Check blob files
        all_files = [f for msg in commit_messages for f in msg.new_files]
        blob_files = [f for f in all_files if f.file_name.endswith('.blob')]
        total_size = sum(f.file_size for f in blob_files)

        # Verify that rolling works correctly: should have multiple files when total size exceeds target
        # Each blob is 2MB, target-file-size is 1MB, so should have multiple files
        self.assertGreater(
            len(blob_files), 1,
            f"Should have multiple blob files when total size ({total_size / 1024 / 1024:.2f}MB) exceeds target (1MB), "
            f"but got {len(blob_files)} file(s)"
        )

        # Verify data integrity
        result = table.new_read_builder().new_read().to_arrow(table.new_read_builder().new_scan().plan().splits())
        self.assertEqual(result.num_rows, num_blobs)
        self.assertEqual(result.column('id').to_pylist(), list(range(1, num_blobs + 1)))

    def test_blob_file_name_format_with_shared_uuid(self):
        import random
        import re
        from pypaimon import Schema
        from pypaimon.table.row.blob import BlobDescriptor

        # Create schema with blob column
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('name', pa.string()),
            ('blob_data', pa.large_binary())
        ])
        schema = Schema.from_pyarrow_schema(pa_schema, options={
            'row-tracking.enabled': 'true',
            'data-evolution.enabled': 'true',
            'blob-as-descriptor': 'true',
            'blob.target-file-size': '1MB'  # Small target size to trigger multiple rollings
        })

        self.catalog.create_table('test_db.blob_file_name_test', schema, False)
        table = self.catalog.get_table('test_db.blob_file_name_test')

        # Create multiple external blob files (2MB each, > target-file-size)
        # This will trigger multiple blob file rollings
        num_blobs, blob_size = 5, 2 * 1024 * 1024
        random.seed(789)
        descriptors = []
        for i in range(num_blobs):
            path = os.path.join(self.temp_dir, f'external_blob_{i}')
            data = bytes(bytearray([random.randint(0, 255) for _ in range(blob_size)]))
            with open(path, 'wb') as f:
                f.write(data)
            descriptors.append(BlobDescriptor(path, 0, len(data)))

        # Write data
        test_data = pa.Table.from_pydict({
            'id': list(range(1, num_blobs + 1)),
            'name': [f'item_{i}' for i in range(1, num_blobs + 1)],
            'blob_data': [d.serialize() for d in descriptors]
        }, schema=pa_schema)

        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(test_data)
        commit_messages = writer.prepare_commit()
        write_builder.new_commit().commit(commit_messages)
        writer.close()

        # Extract blob files from commit messages
        all_files = [f for msg in commit_messages for f in msg.new_files]
        blob_files = [f for f in all_files if f.file_name.endswith('.blob')]

        # Should have multiple blob files due to rolling
        self.assertGreater(len(blob_files), 1, "Should have multiple blob files due to rolling")

        # Verify file name format: data-{uuid}-{count}.blob
        file_name_pattern = re.compile(
            r'^data-[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-'
            r'[a-f0-9]{12}-(\d+)\.blob$'
        )

        first_file_name = blob_files[0].file_name
        self.assertTrue(
            file_name_pattern.match(first_file_name),
            f"File name should match expected format: data-{{uuid}}-{{count}}.blob, got: {first_file_name}"
        )

        first_match = file_name_pattern.match(first_file_name)
        first_counter = int(first_match.group(1))

        # Extract UUID (everything between "data-" and last "-")
        uuid_start = len("data-")
        uuid_end = first_file_name.rfind('-', uuid_start)
        shared_uuid = first_file_name[uuid_start:uuid_end]

        # Verify all blob files use the same UUID and have sequential counters
        for i, blob_file in enumerate(blob_files):
            file_name = blob_file.file_name
            match = file_name_pattern.match(file_name)

            self.assertIsNotNone(
                match,
                f"File name should match expected format: data-{{uuid}}-{{count}}.blob, got: {file_name}"
            )

            counter = int(match.group(1))

            # Extract UUID from this file
            file_uuid = file_name[uuid_start:file_name.rfind('-', uuid_start)]

            self.assertEqual(
                file_uuid,
                shared_uuid,
                f"All blob files should use the same UUID. Expected: {shared_uuid}, got: {file_uuid} in {file_name}"
            )

            self.assertEqual(
                counter,
                first_counter + i,
                f"File counter should be sequential. Expected: {first_counter + i}, got: {counter} in {file_name}"
            )

        # Verify data integrity
        result = table.new_read_builder().new_read().to_arrow(table.new_read_builder().new_scan().plan().splits())
        self.assertEqual(result.num_rows, num_blobs)
        self.assertEqual(result.column('id').to_pylist(), list(range(1, num_blobs + 1)))

    def test_blob_as_descriptor_sequence_number_increment(self):
        import os
        from pypaimon import Schema
        from pypaimon.table.row.blob import BlobDescriptor

        # Create schema with blob column (descriptor input is auto-detected in write path)
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('name', pa.string()),
            ('blob_data', pa.large_binary())
        ])
        schema = Schema.from_pyarrow_schema(pa_schema, options={
            'row-tracking.enabled': 'true',
            'data-evolution.enabled': 'true',
            'blob-as-descriptor': 'true'
        })

        self.catalog.create_table('test_db.blob_sequence_test', schema, False)
        table = self.catalog.get_table('test_db.blob_sequence_test')

        # Create multiple external blob files
        num_blobs = 10
        descriptors = []
        for i in range(num_blobs):
            path = os.path.join(self.temp_dir, f'external_blob_seq_{i}')
            data = f"blob data {i}".encode('utf-8')
            with open(path, 'wb') as f:
                f.write(data)
            descriptors.append(BlobDescriptor(path, 0, len(data)))

        # Write data row by row and verify blob sequence-number continuity
        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()

        # Write each row separately to ensure one-by-one writing
        for i in range(num_blobs):
            test_data = pa.Table.from_pydict({
                'id': [i + 1],
                'name': [f'item_{i}'],
                'blob_data': [descriptors[i].serialize()]
            }, schema=pa_schema)
            writer.write_arrow(test_data)

        commit_messages = writer.prepare_commit()
        write_builder.new_commit().commit(commit_messages)
        writer.close()

        # Extract blob files from commit messages
        all_files = [f for msg in commit_messages for f in msg.new_files]
        blob_files = [f for f in all_files if f.file_name.endswith('.blob')]

        # Verify that we have at least one blob file
        self.assertGreater(len(blob_files), 0, "Should have at least one blob file")

        # Verify sequence numbers for each blob file
        for blob_file in blob_files:
            min_seq = blob_file.min_sequence_number
            max_seq = blob_file.max_sequence_number
            row_count = blob_file.row_count

            # Critical assertion: min_seq should NOT equal max_seq when there are multiple rows
            if row_count > 1:
                self.assertNotEqual(
                    min_seq, max_seq,
                    f"Sequence numbers should be different for files with multiple rows. "
                    f"File: {blob_file.file_name}, row_count: {row_count}, "
                    f"min_seq: {min_seq}, max_seq: {max_seq}. "
                    f"This indicates sequence generator was not incremented for each row."
                )
                self.assertEqual(
                    max_seq - min_seq + 1, row_count,
                    f"Sequence number range should match row count. "
                    f"File: {blob_file.file_name}, row_count: {row_count}, "
                    f"min_seq: {min_seq}, max_seq: {max_seq}, "
                    f"expected range: {row_count}, actual range: {max_seq - min_seq + 1}"
                )
            else:
                # For single row files, min_seq == max_seq is acceptable
                self.assertEqual(
                    min_seq, max_seq,
                    f"Single row file should have min_seq == max_seq. "
                    f"File: {blob_file.file_name}, min_seq: {min_seq}, max_seq: {max_seq}"
                )

        # Verify data integrity
        result = table.new_read_builder().new_read().to_arrow(table.new_read_builder().new_scan().plan().splits())
        self.assertEqual(result.num_rows, num_blobs)
        self.assertEqual(result.column('id').to_pylist(), list(range(1, num_blobs + 1)))

    def test_blob_non_descriptor_sequence_number_increment(self):
        from pypaimon import Schema

        # Create schema with blob column (default read output is blob bytes)
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('name', pa.string()),
            ('blob_data', pa.large_binary())
        ])
        schema = Schema.from_pyarrow_schema(pa_schema, options={
            'row-tracking.enabled': 'true',
            'data-evolution.enabled': 'true',
            'blob-as-descriptor': 'false'  # Read output as blob bytes
        })

        self.catalog.create_table('test_db.blob_sequence_non_desc_test', schema, False)
        table = self.catalog.get_table('test_db.blob_sequence_non_desc_test')

        # Create test data with multiple rows in a batch
        num_rows = 10
        test_data = pa.Table.from_pydict({
            'id': list(range(1, num_rows + 1)),
            'name': [f'item_{i}' for i in range(num_rows)],
            'blob_data': [f'blob data {i}'.encode('utf-8') for i in range(num_rows)]
        }, schema=pa_schema)

        # Write data as a batch and verify blob sequence-number continuity
        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(test_data)

        commit_messages = writer.prepare_commit()
        write_builder.new_commit().commit(commit_messages)
        writer.close()

        # Extract blob files from commit messages
        all_files = [f for msg in commit_messages for f in msg.new_files]
        blob_files = [f for f in all_files if f.file_name.endswith('.blob')]

        # Verify that we have at least one blob file
        self.assertGreater(len(blob_files), 0, "Should have at least one blob file")

        # Verify sequence numbers for each blob file
        for blob_file in blob_files:
            min_seq = blob_file.min_sequence_number
            max_seq = blob_file.max_sequence_number
            row_count = blob_file.row_count

            # Critical assertion: min_seq should NOT equal max_seq when there are multiple rows
            if row_count > 1:
                self.assertNotEqual(
                    min_seq, max_seq,
                    f"Sequence numbers should be different for files with multiple rows. "
                    f"File: {blob_file.file_name}, row_count: {row_count}, "
                    f"min_seq: {min_seq}, max_seq: {max_seq}. "
                    f"This indicates sequence generator was not incremented for each row in batch."
                )
                self.assertEqual(
                    max_seq - min_seq + 1, row_count,
                    f"Sequence number range should match row count. "
                    f"File: {blob_file.file_name}, row_count: {row_count}, "
                    f"min_seq: {min_seq}, max_seq: {max_seq}, "
                    f"expected range: {row_count}, actual range: {max_seq - min_seq + 1}"
                )
            else:
                # For single row files, min_seq == max_seq is acceptable
                self.assertEqual(
                    min_seq, max_seq,
                    f"Single row file should have min_seq == max_seq. "
                    f"File: {blob_file.file_name}, min_seq: {min_seq}, max_seq: {max_seq}"
                )

        print("✅ Blob sequence number increment test passed (batch write)")

    def test_blob_stats_schema_with_custom_column_name(self):
        from pypaimon import Schema

        # Create schema with blob column using a custom name (not 'blob_data')
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('name', pa.string()),
            ('my_custom_blob', pa.large_binary())  # Custom blob column name
        ])
        schema = Schema.from_pyarrow_schema(pa_schema, options={
            'row-tracking.enabled': 'true',
            'data-evolution.enabled': 'true'
        })

        self.catalog.create_table('test_db.blob_custom_name_test', schema, False)
        table = self.catalog.get_table('test_db.blob_custom_name_test')

        # Write data
        test_data = pa.Table.from_pydict({
            'id': [1, 2, 3],
            'name': ['Alice', 'Bob', 'Charlie'],
            'my_custom_blob': [b'blob1', b'blob2', b'blob3']
        }, schema=pa_schema)

        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(test_data)
        commit_messages = writer.prepare_commit()
        write_builder.new_commit().commit(commit_messages)
        writer.close()

        # Extract blob files from commit messages
        all_files = [f for msg in commit_messages for f in msg.new_files]
        blob_files = [f for f in all_files if f.file_name.endswith('.blob')]

        # Verify we have at least one blob file
        self.assertGreater(len(blob_files), 0, "Should have at least one blob file")

        # Verify that the stats schema uses the actual blob column name, not hardcoded 'blob_data'
        for blob_file in blob_files:
            value_stats = blob_file.value_stats
            if value_stats is not None and value_stats.min_values is not None:
                # Get the field names from the stats
                # The stats should use 'my_custom_blob', not 'blob_data'
                min_values = value_stats.min_values
                if hasattr(min_values, 'fields') and len(min_values.fields) > 0:
                    # Check if the field name matches the actual blob column name
                    field_name = min_values.fields[0].name
                    self.assertEqual(
                        field_name, 'my_custom_blob',
                        f"Blob stats field name should be 'my_custom_blob' (actual column name), "
                        f"but got '{field_name}'. This indicates the field name was hardcoded "
                        f"instead of using the blob_column parameter."
                    )

        # Verify data integrity
        result = table.new_read_builder().new_read().to_arrow(table.new_read_builder().new_scan().plan().splits())
        self.assertEqual(result.num_rows, 3)
        self.assertEqual(result.column('id').to_pylist(), [1, 2, 3])
        self.assertEqual(result.column('name').to_pylist(), ['Alice', 'Bob', 'Charlie'])

    def test_blob_file_name_format_with_shared_uuid_non_descriptor_mode(self):
        import random
        import re
        from pypaimon import Schema

        # Create schema with blob column (blob bytes read output)
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('name', pa.string()),
            ('blob_data', pa.large_binary())
        ])
        schema = Schema.from_pyarrow_schema(pa_schema, options={
            'row-tracking.enabled': 'true',
            'data-evolution.enabled': 'true',
            'blob-as-descriptor': 'false',  # Read output as blob bytes
            'target-file-size': '1MB'  # Small target size to trigger multiple rollings
        })

        self.catalog.create_table('test_db.blob_file_name_test_non_desc', schema, False)
        table = self.catalog.get_table('test_db.blob_file_name_test_non_desc')

        num_blobs, blob_size = 5, 2 * 1024 * 1024
        random.seed(123)
        blob_data_list = []
        for i in range(num_blobs):
            blob_data = bytes(bytearray([random.randint(0, 255) for _ in range(blob_size)]))
            blob_data_list.append(blob_data)

        # Write data in batches to trigger multiple file rollings
        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()

        # Write data that will trigger rolling
        test_data = pa.Table.from_pydict({
            'id': list(range(1, num_blobs + 1)),
            'name': [f'item_{i}' for i in range(1, num_blobs + 1)],
            'blob_data': blob_data_list
        }, schema=pa_schema)

        writer.write_arrow(test_data)
        commit_messages = writer.prepare_commit()
        write_builder.new_commit().commit(commit_messages)
        writer.close()

        # Extract blob files from commit messages
        all_files = [f for msg in commit_messages for f in msg.new_files]
        blob_files = [f for f in all_files if f.file_name.endswith('.blob')]

        # Should have at least one blob file (may have multiple if rolling occurred)
        self.assertGreaterEqual(len(blob_files), 1, "Should have at least one blob file")

        # Verify file name format: data-{uuid}-{count}.blob
        file_name_pattern = re.compile(
            r'^data-[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-'
            r'[a-f0-9]{12}-(\d+)\.blob$'
        )

        first_file_name = blob_files[0].file_name
        self.assertTrue(
            file_name_pattern.match(first_file_name),
            f"File name should match expected format: data-{{uuid}}-{{count}}.blob, got: {first_file_name}"
        )

        # Extract UUID and counter from first file name
        first_match = file_name_pattern.match(first_file_name)
        first_counter = int(first_match.group(1))

        # Extract UUID (everything between "data-" and last "-")
        uuid_start = len("data-")
        uuid_end = first_file_name.rfind('-', uuid_start)
        shared_uuid = first_file_name[uuid_start:uuid_end]

        # Verify all blob files use the same UUID and have sequential counters
        for i, blob_file in enumerate(blob_files):
            file_name = blob_file.file_name
            match = file_name_pattern.match(file_name)

            self.assertIsNotNone(
                match,
                f"File name should match expected format: data-{{uuid}}-{{count}}.blob, got: {file_name}"
            )

            counter = int(match.group(1))

            # Extract UUID from this file
            file_uuid = file_name[uuid_start:file_name.rfind('-', uuid_start)]

            self.assertEqual(
                file_uuid,
                shared_uuid,
                f"All blob files should use the same UUID. Expected: {shared_uuid}, got: {file_uuid} in {file_name}"
            )

            self.assertEqual(
                counter,
                first_counter + i,
                f"File counter should be sequential. Expected: {first_counter + i}, got: {counter} in {file_name}"
            )

        # Verify data integrity
        result = table.new_read_builder().new_read().to_arrow(table.new_read_builder().new_scan().plan().splits())
        self.assertEqual(result.num_rows, num_blobs)
        self.assertEqual(result.column('id').to_pylist(), list(range(1, num_blobs + 1)))

    def test_blob_non_descriptor_target_file_size_rolling(self):
        """Test that blob.target-file-size is respected in blob write path."""
        from pypaimon import Schema

        # Create schema with blob column (blob write path)
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('blob_data', pa.large_binary()),
        ])

        # Test with blob.target-file-size set to a small value
        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true',
                'blob.target-file-size': '1MB'  # Set blob-specific target file size
            }
        )

        self.catalog.create_table('test_db.blob_non_descriptor_rolling', schema, False)
        table = self.catalog.get_table('test_db.blob_non_descriptor_rolling')

        # Write multiple blobs that together exceed the target size
        # Each blob is 0.6MB, so 3 blobs = 1.8MB > 1MB target
        num_blobs = 3
        blob_size = int(0.6 * 1024 * 1024)  # 0.6MB per blob

        test_data = pa.Table.from_pydict({
            'id': list(range(1, num_blobs + 1)),
            'blob_data': [b'x' * blob_size for _ in range(num_blobs)]
        }, schema=pa_schema)

        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(test_data)
        commit_messages = writer.prepare_commit()
        commit = write_builder.new_commit()
        commit.commit(commit_messages)
        writer.close()

        # Extract blob files from commit messages
        all_files = [f for msg in commit_messages for f in msg.new_files]
        blob_files = [f for f in all_files if f.file_name.endswith('.blob')]

        # The key test: verify that blob.target-file-size is used instead of target-file-size
        # If target-file-size (default 256MB for append-only) was used, we'd have 1 file
        # If blob.target-file-size (1MB) is used, we should have multiple files
        total_data_size = num_blobs * blob_size

        # Verify that the rolling logic used blob_target_file_size (1MB) not target_file_size (256MB)
        # If target_file_size was used, all data would fit in one file
        # If blob_target_file_size was used, data should be split
        if total_data_size > 1024 * 1024:  # Total > 1MB
            self.assertGreater(
                len(blob_files), 1,
                f"Should have multiple blob files when using blob.target-file-size (1MB). "
                f"Total data size: {total_data_size / 1024 / 1024:.2f}MB, "
                f"got {len(blob_files)} file(s). "
                f"This indicates blob.target-file-size was ignored and target-file-size was used instead."
            )

        # Verify data integrity
        result = table.new_read_builder().new_read().to_arrow(
            table.new_read_builder().new_scan().plan().splits()
        )
        self.assertEqual(result.num_rows, num_blobs)
        self.assertEqual(result.column('id').to_pylist(), list(range(1, num_blobs + 1)))

    def test_blob_write_read_large_data_with_rolling_with_shard(self):
        """
        Test writing and reading large blob data with file rolling and sharding.

        Test workflow:
        - Creates a table with blob column and small target file size
        - Writes 4 batches of 40 records each (160 total records)
        - Each record contains a 64KB blob
        - Reads data using 3-way sharding (shard 0 - 3)
        - Verifies blob data integrity and size
        - Compares concatenated sharded results with full table scan
        """

        # Create schema with blob column
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('record_id_of_batch', pa.int32()),
            ('metadata', pa.string()),
            ('large_blob', pa.large_binary()),
        ])

        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true',
                'blob.target-file-size': '512KB'
            }
        )
        self.catalog.create_table('test_db.blob_rolling_with_shard', schema, False)
        table = self.catalog.get_table('test_db.blob_rolling_with_shard')

        # Create large blob data
        large_blob_size = 64 * 1024
        blob_pattern = b'LARGE_BLOB_PATTERN_' + b'X' * 1024  # ~1KB pattern
        pattern_size = len(blob_pattern)
        repetitions = large_blob_size // pattern_size
        large_blob_data = blob_pattern * repetitions

        actual_size = len(large_blob_data)
        print(f"Created blob data: {actual_size:,} bytes ({actual_size / (1024 * 1024):.2f} MB)")
        # Write 4 batches of 40 records
        for i in range(4):
            write_builder = table.new_batch_write_builder()
            writer = write_builder.new_write()
            # Write 40 records
            for record_id in range(40):
                test_data = pa.Table.from_pydict({
                    'id': [i * 40 + record_id + 1],  # Unique ID for each row
                    'record_id_of_batch': [record_id],
                    'metadata': [f'Large blob batch {record_id + 1}'],
                    'large_blob': [large_blob_data]
                }, schema=pa_schema)
                writer.write_arrow(test_data)

            commit_messages = writer.prepare_commit()
            commit = write_builder.new_commit()
            commit.commit(commit_messages)
            writer.close()

        # Read data back
        read_builder = table.new_read_builder()
        table_scan = read_builder.new_scan().with_shard(0, 3)
        table_read = read_builder.new_read()
        result = table_read.to_arrow(table_scan.plan().splits())

        # Verify the data
        self.assertEqual(result.num_rows, 54, "Should have 54 rows")
        self.assertEqual(result.num_columns, 4, "Should have 4 columns")

        # Verify blob data integrity
        blob_data = result.column('large_blob').to_pylist()
        self.assertEqual(len(blob_data), 54, "Should have 54 blob records")
        # Verify each blob
        for i, blob in enumerate(blob_data):
            self.assertEqual(len(blob), len(large_blob_data), f"Blob {i + 1} should be {large_blob_size:,} bytes")
            self.assertEqual(blob, large_blob_data, f"Blob {i + 1} content should match exactly")
        splits = read_builder.new_scan().plan().splits()
        expected = table_read.to_arrow(splits)
        splits1 = read_builder.new_scan().with_shard(0, 3).plan().splits()
        actual1 = table_read.to_arrow(splits1)
        splits2 = read_builder.new_scan().with_shard(1, 3).plan().splits()
        actual2 = table_read.to_arrow(splits2)
        splits3 = read_builder.new_scan().with_shard(2, 3).plan().splits()
        actual3 = table_read.to_arrow(splits3)
        actual = pa.concat_tables([actual1, actual2, actual3]).sort_by('id')

        # Verify the data
        self.assertEqual(actual.num_rows, 160, "Should have 160 rows")
        self.assertEqual(actual.num_columns, 4, "Should have 4 columns")
        self.assertEqual(actual.column('id').to_pylist(), list(range(1, 161)), "ID column should match")
        self.assertEqual(actual, expected)

    def test_blob_rolling_with_shard(self):
        """
        - Writes 30 records
        - Each record contains a 3MB blob
        """

        # Create schema with blob column
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('metadata', pa.string()),
            ('large_blob', pa.large_binary()),
        ])

        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true',
                'blob.target-file-size': '10MB'
            }
        )
        self.catalog.create_table('test_db.blob_rolling_with_shard1', schema, False)
        table = self.catalog.get_table('test_db.blob_rolling_with_shard1')

        # Create large blob data
        large_blob_size = 3 * 1024 * 1024  #
        blob_pattern = b'LARGE_BLOB_PATTERN_' + b'X' * 1024  # ~1KB pattern
        pattern_size = len(blob_pattern)
        repetitions = large_blob_size // pattern_size
        large_blob_data = blob_pattern * repetitions

        actual_size = len(large_blob_data)
        print(f"Created blob data: {actual_size:,} bytes ({actual_size / (1024 * 1024):.2f} MB)")

        # Write 30 records
        for i in range(3):
            write_builder = table.new_batch_write_builder()
            writer = write_builder.new_write()
            for record_id in range(10):
                test_data = pa.Table.from_pydict({
                    'id': [record_id * i],  # Unique ID for each row
                    'metadata': [f'Large blob batch {record_id + 1}'],
                    'large_blob': [struct.pack('<I', record_id) + large_blob_data]
                }, schema=pa_schema)
                writer.write_arrow(test_data)

            commit_messages = writer.prepare_commit()
            commit = write_builder.new_commit()
            commit.commit(commit_messages)
            writer.close()

        # Read data back
        read_builder = table.new_read_builder()
        table_scan = read_builder.new_scan().with_shard(1, 3)
        table_read = read_builder.new_read()
        result = table_read.to_arrow(table_scan.plan().splits())

        # Verify the data
        self.assertEqual(result.num_rows, 10, "Should have 10 rows")
        self.assertEqual(result.num_columns, 3, "Should have 3 columns")

        # Verify blob data integrity
        blob_data = result.column('large_blob').to_pylist()
        self.assertEqual(len(blob_data), 10, "Should have 94 blob records")
        # Verify each blob
        for i, blob in enumerate(blob_data):
            self.assertEqual(len(blob), len(large_blob_data) + 4, f"Blob {i + 1} should be {large_blob_size:,} bytes")
        splits = read_builder.new_scan().plan().splits()
        expected = table_read.to_arrow(splits)
        splits1 = read_builder.new_scan().with_shard(0, 3).plan().splits()
        actual1 = table_read.to_arrow(splits1)
        splits2 = read_builder.new_scan().with_shard(1, 3).plan().splits()
        actual2 = table_read.to_arrow(splits2)
        splits3 = read_builder.new_scan().with_shard(2, 3).plan().splits()
        actual3 = table_read.to_arrow(splits3)
        actual = pa.concat_tables([actual1, actual2, actual3])

        # Verify the data
        self.assertEqual(actual.num_rows, 30, "Should have 30 rows")
        self.assertEqual(actual.num_columns, 3, "Should have 3 columns")
        self.assertEqual(actual, expected)

    def test_blob_large_data_volume_with_shard(self):

        # Create schema with blob column
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('batch_id', pa.int32()),
            ('metadata', pa.string()),
            ('large_blob', pa.large_binary()),
        ])

        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true'
            }
        )
        self.catalog.create_table('test_db.blob_large_data_volume_with_shard', schema, False)
        table = self.catalog.get_table('test_db.blob_large_data_volume_with_shard')

        large_blob_size = 5 * 1024
        blob_pattern = b'LARGE_BLOB_PATTERN_' + b'X' * 1024  # ~1KB pattern
        pattern_size = len(blob_pattern)
        repetitions = large_blob_size // pattern_size
        large_blob_data = blob_pattern * repetitions

        for i in range(3):
            num_row = 6666
            if i == 0:
                num_row += 1
            write_builder = table.new_batch_write_builder()
            writer = write_builder.new_write()
            expected = pa.Table.from_pydict({
                'id': [1] * num_row,
                'batch_id': [11] * num_row,
                'metadata': [f'Large blob batch {11}'] * num_row,
                'large_blob': [i.to_bytes(2, byteorder='little') + large_blob_data for i in range(num_row)]
            }, schema=pa_schema)
            writer.write_arrow(expected)

            # Commit all data at once
            commit_messages = writer.prepare_commit()
            commit = write_builder.new_commit()
            commit.commit(commit_messages)
            writer.close()

        # Read data back
        read_builder = table.new_read_builder()
        table_scan = read_builder.new_scan().with_shard(2, 3)
        table_read = read_builder.new_read()
        result = table_read.to_arrow(table_scan.plan().splits())

        # Verify the data
        self.assertEqual(6666, result.num_rows)
        self.assertEqual(4, result.num_columns)

        self.assertEqual(expected, result)
        splits = read_builder.new_scan().plan().splits()
        expected = table_read.to_arrow(splits)
        splits1 = read_builder.new_scan().with_shard(0, 3).plan().splits()
        actual1 = table_read.to_arrow(splits1)
        splits2 = read_builder.new_scan().with_shard(1, 3).plan().splits()
        actual2 = table_read.to_arrow(splits2)
        splits3 = read_builder.new_scan().with_shard(2, 3).plan().splits()
        actual3 = table_read.to_arrow(splits3)
        actual = pa.concat_tables([actual1, actual2, actual3]).sort_by('id')
        self.assertEqual(actual, expected)

    def test_dedicated_format_writer_with_slice(self):
        """Test DedicatedFormatWriter with mixed data types in blob column."""

        # Create schema with blob column
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('type', pa.string()),
            ('data', pa.large_binary()),
        ])

        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true'
            }
        )
        self.catalog.create_table('test_db.with_slice_test', schema, False)
        table = self.catalog.get_table('test_db.with_slice_test')

        # Use proper table API to create writer
        write_builder = table.new_batch_write_builder()
        blob_writer = write_builder.new_write()

        # Test data with different types of blob content
        test_data = pa.Table.from_pydict({
            'id': [0, 1, 2, 3, 4],
            'type': ['text', 'json', 'binary', 'image', 'pdf'],
            'data': [
                b'This is text content',
                b'{"key": "value", "number": 42}',
                b'\x00\x01\x02\x03\xff\xfe\xfd',
                b'PNG_IMAGE_DATA_PLACEHOLDER',
                b'%PDF-1.4\nPDF_CONTENT_PLACEHOLDER'
            ]
        }, schema=pa_schema)

        # Write mixed data
        total_rows = 0
        for batch in test_data.to_batches():
            blob_writer.write_arrow_batch(batch)
            total_rows += batch.num_rows

        # Test prepare commit
        commit_messages = blob_writer.prepare_commit()
        # Create commit and commit the data
        commit = write_builder.new_commit()
        commit.commit(commit_messages)
        blob_writer.close()

        # Read data back using table API
        read_builder = table.new_read_builder()
        table_scan = read_builder.new_scan().with_slice(2, 4)
        table_read = read_builder.new_read()
        splits = table_scan.plan().splits()
        result = table_read.to_arrow(splits)

        # Verify the data was read back correctly
        self.assertEqual(result.num_rows, 2, "Should have 2 rows")
        self.assertEqual(result.num_columns, 3, "Should have 3 columns")
        self.assertEqual(result["id"].unique().to_pylist(), [2, 3], "Get incorrect column ID")

    def test_dedicated_format_writer_with_shard(self):
        """Test DedicatedFormatWriter with mixed data types in blob column."""

        # Create schema with blob column
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('type', pa.string()),
            ('data', pa.large_binary()),
        ])

        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true'
            }
        )
        self.catalog.create_table('test_db.with_shard_test', schema, False)
        table = self.catalog.get_table('test_db.with_shard_test')

        # Use proper table API to create writer
        write_builder = table.new_batch_write_builder()
        blob_writer = write_builder.new_write()

        # Test data with different types of blob content
        test_data = pa.Table.from_pydict({
            'id': [1, 2, 3, 4, 5],
            'type': ['text', 'json', 'binary', 'image', 'pdf'],
            'data': [
                b'This is text content',
                b'{"key": "value", "number": 42}',
                b'\x00\x01\x02\x03\xff\xfe\xfd',
                b'PNG_IMAGE_DATA_PLACEHOLDER',
                b'%PDF-1.4\nPDF_CONTENT_PLACEHOLDER'
            ]
        }, schema=pa_schema)

        # Write mixed data
        total_rows = 0
        for batch in test_data.to_batches():
            blob_writer.write_arrow_batch(batch)
            total_rows += batch.num_rows

        # Test prepare commit
        commit_messages = blob_writer.prepare_commit()
        # Create commit and commit the data
        commit = write_builder.new_commit()
        commit.commit(commit_messages)
        blob_writer.close()

        # Read data back using table API
        read_builder = table.new_read_builder()
        table_scan = read_builder.new_scan().with_shard(0, 2)
        table_read = read_builder.new_read()
        splits = table_scan.plan().splits()
        result = table_read.to_arrow(splits)

        # Verify the data was read back correctly
        self.assertEqual(result.num_rows, 3, "Should have 3 rows")
        self.assertEqual(result.num_columns, 3, "Should have 3 columns")

        # Read data back using table API
        read_builder = table.new_read_builder()
        table_scan = read_builder.new_scan().with_shard(1, 2)
        table_read = read_builder.new_read()
        splits = table_scan.plan().splits()
        result = table_read.to_arrow(splits)

        # Verify the data was read back correctly
        self.assertEqual(result.num_rows, 2, "Should have 2 rows")
        self.assertEqual(result.num_columns, 3, "Should have 3 columns")

    def test_blob_write_read_large_data_volume_rolling_with_shard(self):
        """
        - Writes 10000 records
        - Each record contains a 5KB blob
        - 'blob.target-file-size': '10MB'
        - each blob file contains 2000 records
        """
        # Create schema with blob column
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('batch_id', pa.int32()),
            ('metadata', pa.string()),
            ('large_blob', pa.large_binary()),
        ])

        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true',
                'blob.target-file-size': '10MB'
            }
        )
        self.catalog.create_table('test_db.test_blob_write_read_large_data_volume_rolling_with_shard', schema, False)
        table = self.catalog.get_table('test_db.test_blob_write_read_large_data_volume_rolling_with_shard')

        # Create large blob data
        large_blob_size = 5 * 1024  #
        blob_pattern = b'LARGE_BLOB_PATTERN_' + b'X' * 1024  # ~1KB pattern
        pattern_size = len(blob_pattern)
        repetitions = large_blob_size // pattern_size
        large_blob_data = blob_pattern * repetitions

        actual_size = len(large_blob_data)
        print(f"Created blob data: {actual_size:,} bytes ({actual_size / (1024 * 1024):.2f} MB)")
        # Write 10000 records of data
        num_row = 10000
        expected = pa.Table.from_pydict({
            'id': [1] * num_row,
            'batch_id': [11] * num_row,
            'metadata': [f'Large blob batch {11}'] * num_row,
            'large_blob': [i.to_bytes(2, byteorder='little') + large_blob_data for i in range(num_row)]
        }, schema=pa_schema)
        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(expected)

        commit_messages = writer.prepare_commit()
        commit = write_builder.new_commit()
        commit.commit(commit_messages)
        writer.close()

        # Read data back
        read_builder = table.new_read_builder()
        table_scan = read_builder.new_scan()
        table_read = read_builder.new_read()
        result = table_read.to_arrow(table_scan.plan().splits())
        self.assertEqual(expected, result)

        splits = read_builder.new_scan().plan().splits()
        expected = table_read.to_arrow(splits)
        splits1 = read_builder.new_scan().with_shard(0, 3).plan().splits()
        actual1 = table_read.to_arrow(splits1)
        splits2 = read_builder.new_scan().with_shard(1, 3).plan().splits()
        actual2 = table_read.to_arrow(splits2)
        splits3 = read_builder.new_scan().with_shard(2, 3).plan().splits()
        actual3 = table_read.to_arrow(splits3)
        actual = pa.concat_tables([actual1, actual2, actual3]).sort_by('id')

        self.assertEqual(actual, expected)

    def test_concurrent_blob_writes_with_retry(self):
        """Test concurrent blob writes to verify retry mechanism works correctly."""
        import threading
        from pypaimon import Schema

        # Run the test 10 times to verify stability
        iter_num = 2
        for test_iteration in range(iter_num):
            # Create a unique table for each iteration
            table_name = f'test_db.blob_concurrent_writes_{test_iteration}'

            # Create schema with blob column
            pa_schema = pa.schema([
                ('id', pa.int32()),
                ('thread_id', pa.int32()),
                ('metadata', pa.string()),
                ('blob_data', pa.large_binary()),
            ])

            schema = Schema.from_pyarrow_schema(
                pa_schema,
                options={
                    'row-tracking.enabled': 'true',
                    'data-evolution.enabled': 'true',
                    # Concurrent commits are expected in this test; enlarge retry budget
                    # to reduce flaky failures from transient commit conflicts.
                    'commit.max-retries': '50',
                    'commit.max-retry-wait': '30s'
                }
            )
            self.catalog.create_table(table_name, schema, False)
            table = self.catalog.get_table(table_name)

            write_results = []
            write_errors = []

            # Create blob pattern for testing
            blob_size = 5 * 1024  # 5KB
            blob_pattern = b'BLOB_PATTERN_' + b'X' * 1024
            pattern_size = len(blob_pattern)
            repetitions = blob_size // pattern_size
            base_blob_data = blob_pattern * repetitions

            def write_blob_data(thread_id, start_id):
                """Write blob data in a separate thread."""
                try:
                    threading.current_thread().name = f"Iter{test_iteration}-Thread-{thread_id}"
                    write_builder = table.new_batch_write_builder()
                    table_write = write_builder.new_write()
                    table_commit = write_builder.new_commit()

                    # Create unique blob data for this thread
                    data = {
                        'id': list(range(start_id, start_id + 5)),
                        'thread_id': [thread_id] * 5,
                        'metadata': [f'thread{thread_id}_blob_{i}' for i in range(5)],
                        'blob_data': [i.to_bytes(2, byteorder='little') + base_blob_data for i in range(5)]
                    }
                    pa_table = pa.Table.from_pydict(data, schema=pa_schema)

                    table_write.write_arrow(pa_table)
                    commit_messages = table_write.prepare_commit()

                    table_commit.commit(commit_messages)
                    table_write.close()
                    table_commit.close()

                    write_results.append({
                        'thread_id': thread_id,
                        'start_id': start_id,
                        'success': True
                    })
                except Exception as e:
                    write_errors.append({
                        'thread_id': thread_id,
                        'error': str(e)
                    })

            # Create and start multiple threads. Keep this modest (3 vs. the
            # original 10) because GHA runners under load can't drain 10
            # simultaneously-conflicting commits even with
            # ``commit.max-retries=50`` (50 attempts * 30s back-off ~25 min,
            # still timing out in CI). At 5 threads we still saw a different
            # flake — read end occasionally observed only 4 of the 5 commits'
            # rows (race between commit visibility and the immediate read).
            # Three threads exercises the retry path while keeping the
            # contention density low enough that GHA can drain reliably.
            threads = []
            num_threads = 3
            for i in range(num_threads):
                thread = threading.Thread(
                    target=write_blob_data,
                    args=(i, i * 10)
                )
                threads.append(thread)
                thread.start()

            # Wait for all threads to complete
            for thread in threads:
                thread.join()

            # Verify all writes succeeded (retry mechanism should handle conflicts)
            self.assertEqual(num_threads, len(write_results),
                             f"Iteration {test_iteration}: Expected {num_threads} successful writes, "
                             f"got {len(write_results)}. Errors: {write_errors}")
            self.assertEqual(0, len(write_errors),
                             f"Iteration {test_iteration}: Expected no errors, but got: {write_errors}")

            read_builder = table.new_read_builder()
            table_scan = read_builder.new_scan()
            table_read = read_builder.new_read()
            actual = table_read.to_arrow(table_scan.plan().splits()).sort_by('id')

            # Verify data rows
            self.assertEqual(num_threads * 5, actual.num_rows,
                             f"Iteration {test_iteration}: Expected {num_threads * 5} rows")

            # Verify id column
            ids = actual.column('id').to_pylist()
            expected_ids = []
            for i in range(num_threads):
                expected_ids.extend(range(i * 10, i * 10 + 5))
            expected_ids.sort()

            self.assertEqual(ids, expected_ids,
                             f"Iteration {test_iteration}: IDs mismatch")

            # Verify blob data integrity (spot check)
            blob_data_list = actual.column('blob_data').to_pylist()
            for i in range(0, len(blob_data_list), 100):  # Check every 100th blob
                blob = blob_data_list[i]
                self.assertGreater(len(blob), 2, f"Blob {i} should have data")
                # Verify blob contains the pattern
                self.assertIn(b'BLOB_PATTERN_', blob, f"Blob {i} should contain pattern")

            # Verify snapshot count (should have num_threads snapshots)
            snapshot_manager = table.snapshot_manager()
            latest_snapshot = snapshot_manager.get_latest_snapshot()
            self.assertIsNotNone(latest_snapshot,
                                 f"Iteration {test_iteration}: Latest snapshot should not be None")
            self.assertEqual(latest_snapshot.id, num_threads,
                             f"Iteration {test_iteration}: Expected snapshot ID {num_threads}, "
                             f"got {latest_snapshot.id}")

            print(f"✓ Blob Table Iteration {test_iteration + 1}/{iter_num} completed successfully")

    def test_blob_data_with_ray(self):
        try:
            import ray
            if not ray.is_initialized():
                ray.init(ignore_reinit_error=True, num_cpus=2)
        except ImportError:
            self.skipTest("Ray is not available")

        from pypaimon import Schema
        from pypaimon.table.row.blob import BlobDescriptor

        pa_schema = pa.schema([
            ('text', pa.string()),
            ('video_path', pa.string()),
            ('video_bytes', pa.large_binary())  # Blob column
        ])

        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true',
                'blob-field': 'video_bytes',
                'blob-as-descriptor': 'true'
            }
        )

        self.catalog.create_table('test_db.test_blob_data_with_ray', schema, False)
        table = self.catalog.get_table('test_db.test_blob_data_with_ray')

        num_rows = 10
        blob_descriptors = []
        for i in range(num_rows):
            blob_size = 1024 * (i + 1)  # Varying sizes: 1KB, 2KB, ..., 10KB
            blob_data = b'X' * blob_size
            blob_file_path = os.path.join(self.temp_dir, f'blob_{i}.mp4')
            with open(blob_file_path, 'wb') as f:
                f.write(blob_data)
            blob_descriptor = BlobDescriptor(blob_file_path, 0, blob_size)
            blob_descriptors.append(blob_descriptor.serialize())

        test_data = pa.Table.from_pydict({
            'text': [f'text_{i}' for i in range(num_rows)],
            'video_path': [f'video_{i}.mp4' for i in range(num_rows)],
            'video_bytes': blob_descriptors
        }, schema=pa_schema)

        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(test_data)
        commit_messages = writer.prepare_commit()
        commit = write_builder.new_commit()
        commit.commit(commit_messages)
        writer.close()

        read_builder = table.new_read_builder()
        table_scan = read_builder.new_scan()
        table_read = read_builder.new_read()
        splits = table_scan.plan().splits()

        total_split_row_count = sum([s.row_count for s in splits])
        self.assertEqual(total_split_row_count, num_rows * 2,
                         f"Total split row count should be {num_rows}, got {total_split_row_count}")

        total_merged_count = 0
        for split in splits:
            merged_count = split.merged_row_count()
            if merged_count is not None:
                total_merged_count += merged_count
                self.assertLessEqual(
                    merged_count, split.row_count,
                    f"merged_row_count ({merged_count}) should be <= row_count ({split.row_count})")

        if total_merged_count > 0:
            self.assertEqual(
                total_merged_count, num_rows,
                f"Total merged_row_count should be {num_rows}, got {total_merged_count}")

        ray_dataset = table_read.to_ray(splits, override_num_blocks=2)

        ray_count = ray_dataset.count()
        self.assertEqual(
            ray_count,
            num_rows,
            f"Ray dataset count() should be {num_rows}, got {ray_count}. "
        )

        df = ray_dataset.to_pandas()
        self.assertEqual(len(df), num_rows,
                         f"Actual data rows should be {num_rows}, got {len(df)}")

        self.assertEqual(list(df['text']), [f'text_{i}' for i in range(num_rows)])
        self.assertEqual(list(df['video_path']), [f'video_{i}.mp4' for i in range(num_rows)])

    def test_blob_with_row_id_equal(self):
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('name', pa.string()),
            ('data', pa.large_binary()),
        ])

        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true',
                'blob.target-file-size': '500 KB',
            }
        )
        self.catalog.create_table('test_db.blob_rowid_equal', schema, False)
        table = self.catalog.get_table('test_db.blob_rowid_equal')

        blob_values = [bytes([i]) * (248 * 1024) for i in range(20)]
        write_builder = table.new_batch_write_builder()
        tw = write_builder.new_write()
        tc = write_builder.new_commit()
        batch = pa.Table.from_pydict({
            'id': list(range(20)),
            'name': [f'item_{i}' for i in range(20)],
            'data': blob_values,
        }, schema=pa_schema)
        tw.write_arrow(batch)
        tc.commit(tw.prepare_commit())
        tw.close()
        tc.close()

        from pypaimon.common.predicate_builder import PredicateBuilder
        from pypaimon.table.special_fields import SpecialFields

        rb = table.new_read_builder()
        rb.with_projection(['id', 'name', 'data', SpecialFields.ROW_ID.name])
        fields = list(table.fields)
        fields.append(SpecialFields.ROW_ID)
        pb = PredicateBuilder(fields)
        pred = pb.equal(SpecialFields.ROW_ID.name, 5)
        rb.with_filter(pred)

        scan = rb.new_scan()
        splits = scan.plan().splits()
        read = rb.new_read()
        result = read.to_arrow(splits)
        self.assertEqual(result.num_rows, 1)
        self.assertEqual(result.column('id').to_pylist(), [5])
        self.assertEqual(result.column('name').to_pylist(), ['item_5'])
        self.assertEqual(result.column('data').to_pylist(), [blob_values[5]])
        self.assertEqual(result.column(SpecialFields.ROW_ID.name).to_pylist(), [5])

    def test_rename_blob_column_should_fail(self):
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('name', pa.string()),
            ('blob_col', pa.large_binary()),
        ])

        schema = Schema.from_pyarrow_schema(
            pa_schema,
            options={
                'row-tracking.enabled': 'true',
                'data-evolution.enabled': 'true',
            }
        )
        self.catalog.create_table('test_db.blob_rename_test', schema, False)

        with self.assertRaises(RuntimeError) as ctx:
            self.catalog.alter_table(
                'test_db.blob_rename_test',
                [SchemaChange.rename_column('blob_col', 'blob_col_renamed')],
                False
            )
        self.assertIn(
            'Cannot rename BLOB, ARRAY<BLOB> or MAP<X, BLOB> column',
            str(ctx.exception),
        )

    def test_nested_field_named_blob_not_treated_as_blob(self):
        """Regression: a ROW field with a nested column whose name contains
        'blob' must NOT be treated as a top-level BLOB column.  Previously
        the substring match would falsely classify such fields, causing
        create_table to require row-tracking and data-evolution options."""
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('payload', pa.struct([
                ('blob_name', pa.string()),
                ('value', pa.int64()),
            ])),
        ])
        schema = Schema.from_pyarrow_schema(pa_schema)
        self.catalog.create_table(
            'test_db.nested_blob_name_no_error', schema, False)
        table = self.catalog.get_table('test_db.nested_blob_name_no_error')
        self.assertIsNotNone(table)

    def test_blob_table_partial_update_non_blob_column(self):
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('name', pa.string()),
            ('picture', pa.large_binary()),
        ])
        opts = {
            'row-tracking.enabled': 'true',
            'data-evolution.enabled': 'true',
        }
        s = Schema.from_pyarrow_schema(pa_schema, options=opts)
        table_name = 'test_db.blob_de_seq'
        self.catalog.create_table(table_name, s, False)

        table = self.catalog.get_table(table_name)
        wb = table.new_batch_write_builder()
        w = wb.new_write()
        w.write_arrow(pa.Table.from_pydict(
            {'id': [1, 2], 'name': ['a', 'b'], 'picture': [None, None]},
            schema=pa_schema,
        ))
        wb.new_commit().commit(w.prepare_commit())
        w.close()

        from pypaimon.snapshot.snapshot import BATCH_COMMIT_IDENTIFIER
        from pypaimon.write.table_update_by_row_id import TableUpdateByRowId

        table = self.catalog.get_table(table_name)
        rb = table.new_read_builder()
        rb = rb.with_projection(['name', '_ROW_ID'])
        splits = rb.new_scan().plan().splits()
        source = rb.new_read().to_arrow(splits)

        update_data = pa.table({
            '_ROW_ID': source.column('_ROW_ID'),
            'name': pa.array(['updated', 'updated'], type=pa.string()),
        })
        updater = TableUpdateByRowId(
            table, '_test_', BATCH_COMMIT_IDENTIFIER,
        )
        msgs = updater.update_columns(update_data, ['name'])
        table.new_batch_write_builder().new_commit().commit(msgs)

        table = self.catalog.get_table(table_name)
        rb = table.new_read_builder()
        splits = rb.new_scan().plan().splits()
        result = rb.new_read().to_arrow(splits).sort_by('id').to_pydict()
        self.assertEqual(result['name'], ['updated', 'updated'])

    def test_blob_table_partial_update_non_blob_column_with_rolling_files(self):
        from pypaimon.manifest.schema.data_file_meta import DataFileMeta
        from pypaimon.snapshot.snapshot import BATCH_COMMIT_IDENTIFIER
        from pypaimon.write.table_update_by_row_id import TableUpdateByRowId

        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('name', pa.string()),
            ('picture', pa.large_binary()),
        ])
        opts = {
            'row-tracking.enabled': 'true',
            'data-evolution.enabled': 'true',
            'target-file-size': '1KB',
        }
        s = Schema.from_pyarrow_schema(pa_schema, options=opts)
        table_name = 'test_db.blob_de_seq_rolling'
        self.catalog.create_table(table_name, s, False)

        write_schema = pa.schema([
            ('id', pa.int32()),
            ('name', pa.string()),
        ])
        table = self.catalog.get_table(table_name)
        wb = table.new_batch_write_builder()
        w = wb.new_write().with_write_type(['id', 'name'])
        for start in (0, 1000):
            ids = list(range(start, start + 1000))
            w.write_arrow(pa.Table.from_pydict(
                {
                    'id': ids,
                    'name': [f'name_{i}_' + 'x' * 2048 for i in ids],
                },
                schema=write_schema,
            ))
        commit_messages = w.prepare_commit()
        normal_files = [
            f for msg in commit_messages for f in msg.new_files
            if not DataFileMeta.is_blob_file(f.file_name)
            and not DataFileMeta.is_vector_file(f.file_name)
        ]
        self.assertGreaterEqual(len(normal_files), 2)
        for file in normal_files:
            self.assertEqual(file.min_sequence_number, 0)
            self.assertEqual(file.max_sequence_number, file.row_count - 1)
        wb.new_commit().commit(commit_messages)
        w.close()

        table = self.catalog.get_table(table_name)
        rb = table.new_read_builder().with_projection(['id', 'name', '_ROW_ID'])
        splits = rb.new_scan().plan().splits()
        source = rb.new_read().to_arrow(splits).sort_by('id')

        update_data = pa.table({
            '_ROW_ID': source.column('_ROW_ID'),
            'name': pa.array(['updated'] * source.num_rows, type=pa.string()),
        })
        updater = TableUpdateByRowId(
            table, '_test_', BATCH_COMMIT_IDENTIFIER,
        )
        msgs = updater.update_columns(update_data, ['name'])
        update_normal_files = [
            f for msg in msgs for f in msg.new_files
            if not DataFileMeta.is_blob_file(f.file_name)
            and not DataFileMeta.is_vector_file(f.file_name)
        ]
        self.assertGreaterEqual(len(update_normal_files), 2)
        for file in update_normal_files:
            self.assertEqual(file.min_sequence_number, 0)
            self.assertEqual(file.max_sequence_number, file.row_count - 1)
        table.new_batch_write_builder().new_commit().commit(msgs)

        table = self.catalog.get_table(table_name)
        rb = table.new_read_builder().with_projection(['id', 'name'])
        splits = rb.new_scan().plan().splits()
        result = rb.new_read().to_arrow(splits).sort_by('id').to_pydict()
        self.assertEqual(result['id'], list(range(2000)))
        self.assertEqual(result['name'], ['updated'] * 2000)


class GetBlobTest(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        cls.temp_dir = tempfile.mkdtemp()
        cls.warehouse = os.path.join(cls.temp_dir, 'warehouse')
        cls.catalog = CatalogFactory.create({'warehouse': cls.warehouse})
        cls.catalog.create_database('test_db', False)

        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('name', pa.string()),
            ('picture', pa.large_binary()),
        ])
        schema = Schema.from_pyarrow_schema(pa_schema, options={
            'row-tracking.enabled': 'true',
            'data-evolution.enabled': 'true',
        })
        cls.catalog.create_table('test_db.get_blob_test', schema, False)
        cls.table = cls.catalog.get_table('test_db.get_blob_test')

        data = pa.Table.from_pydict({
            'id': [1, 2, 3],
            'name': ['a', 'b', 'c'],
            'picture': [b'img_data_1', b'img_data_2', b'img_data_3'],
        }, schema=pa_schema)

        write_builder = cls.table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(data)
        commit_messages = writer.prepare_commit()
        commit = write_builder.new_commit()
        commit.commit(commit_messages)
        writer.close()

    @classmethod
    def tearDownClass(cls):
        shutil.rmtree(cls.temp_dir, ignore_errors=True)

    def test_get_blob_access(self):
        read_builder = self.table.new_read_builder()
        splits = read_builder.new_scan().plan().splits()
        read = read_builder.new_read()

        results = []
        for row in read.to_iterator(splits):
            blob = row.get_blob(2)
            self.assertIsNotNone(blob)
            results.append((row.get_field(0), blob.to_data()))

        self.assertEqual(len(results), 3)
        results.sort(key=lambda x: x[0])
        self.assertEqual(results[0], (1, b'img_data_1'))
        self.assertEqual(results[1], (2, b'img_data_2'))
        self.assertEqual(results[2], (3, b'img_data_3'))

    def test_get_blob_access_with_limit(self):
        read_builder = self.table.new_read_builder().with_limit(2)
        splits = read_builder.new_scan().plan().splits()
        read = read_builder.new_read()

        results = []
        for row in read.to_iterator(splits):
            blob = row.get_blob(2)
            self.assertIsNotNone(blob)
            results.append((row.get_field(0), blob.to_data()))

        self.assertEqual(len(results), 2)
        for row_id, data in results:
            self.assertIn(row_id, (1, 2, 3))
            self.assertIn(data, (b'img_data_1', b'img_data_2', b'img_data_3'))

    def test_get_blob_streaming(self):
        read_builder = self.table.new_read_builder()
        splits = read_builder.new_scan().plan().splits()
        read = read_builder.new_read()

        results = []
        for row in read.to_iterator(splits):
            with row.get_blob(2).new_input_stream() as stream:
                results.append((row.get_field(0), stream.read()))
        self.assertEqual(len(results), 3)
        results.sort(key=lambda x: x[0])
        self.assertEqual(results[0], (1, b'img_data_1'))
        self.assertEqual(results[1], (2, b'img_data_2'))
        self.assertEqual(results[2], (3, b'img_data_3'))

    def test_get_blob_non_blob_field_raises(self):
        read_builder = self.table.new_read_builder()
        splits = read_builder.new_scan().plan().splits()
        read = read_builder.new_read()

        for row in read.to_iterator(splits):
            with self.assertRaises(TypeError):
                row.get_blob(0)
            break

    def test_to_iterator_yields_all_rows(self):
        read_builder = self.table.new_read_builder()
        splits = read_builder.new_scan().plan().splits()
        read = read_builder.new_read()

        count = 0
        for row in read.to_iterator(splits):
            self.assertIsNotNone(row.get_field(0))
            self.assertIsNotNone(row.get_field(1))
            count += 1
        self.assertEqual(count, 3)


class GetBlobMultiColumnTest(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        cls.temp_dir = tempfile.mkdtemp()
        cls.catalog = CatalogFactory.create({'warehouse': os.path.join(cls.temp_dir, 'warehouse')})
        cls.catalog.create_database('test_db', False)

        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('cover', pa.large_binary()),
            ('thumb', pa.large_binary()),
        ])
        schema = Schema.from_pyarrow_schema(pa_schema, options={
            'row-tracking.enabled': 'true',
            'data-evolution.enabled': 'true',
        })
        cls.catalog.create_table('test_db.get_blob_multi', schema, False)
        cls.table = cls.catalog.get_table('test_db.get_blob_multi')

        write_builder = cls.table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(pa.Table.from_pydict({
            'id': [1, 2],
            'cover': [b'cover_1', b'cover_2'],
            'thumb': [b'thumb_1', b'thumb_2'],
        }, schema=pa_schema))
        write_builder.new_commit().commit(writer.prepare_commit())
        writer.close()

    @classmethod
    def tearDownClass(cls):
        shutil.rmtree(cls.temp_dir, ignore_errors=True)

    def test_get_blob_each_column_independent(self):
        read_builder = self.table.new_read_builder()
        splits = read_builder.new_scan().plan().splits()
        read = read_builder.new_read()

        results = []
        for row in read.to_iterator(splits):
            results.append((row.get_field(0),
                            row.get_blob(1).to_data(),
                            row.get_blob(2).to_data()))
        results.sort(key=lambda x: x[0])
        self.assertEqual(results, [(1, b'cover_1', b'thumb_1'),
                                   (2, b'cover_2', b'thumb_2')])


class GetBlobThroughDescriptorConvertReaderTest(unittest.TestCase):
    """Pin BlobDescriptorConvertReader's file_io / blob_field_indices
    propagation. Configuring blob-descriptor-field puts the wrapper on the
    read chain; reading via to_iterator() + row.get_blob() is the only path
    that consumes those propagated attributes. Regressions on either line
    would silently pass the to_arrow()-based blob_descriptor tests."""

    @classmethod
    def setUpClass(cls):
        cls.temp_dir = tempfile.mkdtemp()
        cls.catalog = CatalogFactory.create({'warehouse': os.path.join(cls.temp_dir, 'warehouse')})
        cls.catalog.create_database('test_db', False)

        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('pic1', pa.large_binary()),
            ('pic2', pa.large_binary()),
        ])
        schema = Schema.from_pyarrow_schema(pa_schema, options={
            'row-tracking.enabled': 'true',
            'data-evolution.enabled': 'true',
            'blob-descriptor-field': 'pic2',
        })
        cls.catalog.create_table('test_db.get_blob_via_descriptor_convert', schema, False)
        cls.table = cls.catalog.get_table('test_db.get_blob_via_descriptor_convert')

        from pypaimon.table.row.blob import BlobDescriptor
        cls.pic1_data = b'inline_pic1_payload'
        cls.pic2_data = b'external_pic2_payload'
        cls.pic2_path = os.path.join(cls.temp_dir, 'pic2_external_blob')
        with open(cls.pic2_path, 'wb') as f:
            f.write(cls.pic2_data)

        write_builder = cls.table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(pa.Table.from_pydict({
            'id': [1],
            'pic1': [cls.pic1_data],
            'pic2': [BlobDescriptor(cls.pic2_path, 0, len(cls.pic2_data)).serialize()],
        }, schema=pa_schema))
        write_builder.new_commit().commit(writer.prepare_commit())
        writer.close()

    @classmethod
    def tearDownClass(cls):
        shutil.rmtree(cls.temp_dir, ignore_errors=True)

    def test_get_blob_resolves_through_descriptor_convert_reader(self):
        read_builder = self.table.new_read_builder()
        splits = read_builder.new_scan().plan().splits()
        read = read_builder.new_read()

        for row in read.to_iterator(splits):
            self.assertEqual(row.get_blob(1).to_data(), self.pic1_data)
            self.assertEqual(row.get_blob(2).to_data(), self.pic2_data)
            break


class GetBlobNonBlobColumnSecurityTest(unittest.TestCase):
    """SSRF defence: get_blob on a non-BLOB column must NOT resolve a
    descriptor URI even when the cell starts with the BLOBDESC magic header."""

    @classmethod
    def setUpClass(cls):
        cls.temp_dir = tempfile.mkdtemp()
        cls.catalog = CatalogFactory.create({'warehouse': os.path.join(cls.temp_dir, 'warehouse')})
        cls.catalog.create_database('test_db', False)

        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('payload', pa.binary()),
        ])
        schema = Schema.from_pyarrow_schema(pa_schema)
        cls.catalog.create_table('test_db.no_blob_col', schema, False)
        cls.table = cls.catalog.get_table('test_db.no_blob_col')

        from pypaimon.table.row.blob import BlobDescriptor
        attacker_bytes = BlobDescriptor(
            "/etc/should-never-be-read-by-paimon", 0, 32
        ).serialize()

        write_builder = cls.table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.write_arrow(pa.Table.from_pydict({
            'id': [1],
            'payload': [attacker_bytes],
        }, schema=pa_schema))
        write_builder.new_commit().commit(writer.prepare_commit())
        writer.close()

    @classmethod
    def tearDownClass(cls):
        shutil.rmtree(cls.temp_dir, ignore_errors=True)

    def test_get_blob_on_non_blob_column_with_magic_bytes_raises(self):
        from unittest.mock import patch

        read_builder = self.table.new_read_builder()
        splits = read_builder.new_scan().plan().splits()
        read = read_builder.new_read()

        with patch.object(
            self.table.file_io.uri_reader_factory, 'create',
            side_effect=AssertionError("URI resolution must not happen on non-BLOB column")
        ) as mock_create:
            for row in read.to_iterator(splits):
                with self.assertRaises(TypeError):
                    row.get_blob(1)
                break
            mock_create.assert_not_called()


class BlobConsumerTest(unittest.TestCase):
    """Tests for BlobConsumer callback functionality."""

    @classmethod
    def setUpClass(cls):
        cls.temp_dir = tempfile.mkdtemp()
        cls.warehouse = os.path.join(cls.temp_dir, 'warehouse')
        cls.catalog = CatalogFactory.create({'warehouse': cls.warehouse})
        cls.catalog.create_database('test_db', False)

    @classmethod
    def tearDownClass(cls):
        shutil.rmtree(cls.temp_dir, ignore_errors=True)

    def test_blob_consumer_basic(self):
        """Consumer receives one BlobDescriptor per blob written, None for nulls."""
        from pypaimon.table.row.blob import Blob, BlobDescriptor
        from pypaimon.common.uri_reader import FileUriReader

        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('name', pa.string()),
            ('blob_data', pa.large_binary()),
        ])
        schema = Schema.from_pyarrow_schema(pa_schema, options={
            'row-tracking.enabled': 'true',
            'data-evolution.enabled': 'true',
        })
        self.catalog.create_table('test_db.blob_consumer_basic', schema, False)
        table = self.catalog.get_table('test_db.blob_consumer_basic')

        blob_bytes = b'hello_blob_consumer'
        received = []

        def my_consumer(field_name, descriptor):
            received.append((field_name, descriptor))
            return True

        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.with_blob_consumer(my_consumer)

        test_data = pa.Table.from_pydict({
            'id': [1, 2, 3],
            'name': ['a', 'b', 'c'],
            'blob_data': [blob_bytes, blob_bytes, None],
        }, schema=pa_schema)
        writer.write_arrow(test_data)
        commit_messages = writer.prepare_commit()
        write_builder.new_commit().commit(commit_messages)
        writer.close()

        self.assertEqual(len(received), 3)

        for field_name, desc in received[:2]:
            self.assertEqual(field_name, 'blob_data')
            self.assertIsInstance(desc, BlobDescriptor)
            uri_reader = FileUriReader(table.file_io)
            blob = Blob.from_descriptor(uri_reader, desc)
            self.assertEqual(blob.to_data(), blob_bytes)

        self.assertEqual(received[2][0], 'blob_data')
        self.assertIsNone(received[2][1])

    def test_blob_consumer_flush_behavior(self):
        """Consumer return value controls flush; verify flush count."""
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('name', pa.string()),
            ('blob_data', pa.large_binary()),
        ])
        schema = Schema.from_pyarrow_schema(pa_schema, options={
            'row-tracking.enabled': 'true',
            'data-evolution.enabled': 'true',
        })
        self.catalog.create_table('test_db.blob_consumer_flush', schema, False)
        table = self.catalog.get_table('test_db.blob_consumer_flush')

        blob_bytes = b'flush_test_blob'
        descriptors = []
        flush_count = [0]

        def my_consumer(field_name, descriptor):
            descriptors.append(descriptor)
            should_flush = len(descriptors) % 2 == 0
            if should_flush:
                flush_count[0] += 1
            return should_flush

        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.with_blob_consumer(my_consumer)

        test_data = pa.Table.from_pydict({
            'id': list(range(5)),
            'name': [f'row{i}' for i in range(5)],
            'blob_data': [blob_bytes] * 5,
        }, schema=pa_schema)
        writer.write_arrow(test_data)
        commit_messages = writer.prepare_commit()
        write_builder.new_commit().commit(commit_messages)
        writer.close()

        self.assertEqual(len(descriptors), 5)
        self.assertEqual(flush_count[0], 2)

        from pypaimon.table.row.blob import Blob
        from pypaimon.common.uri_reader import FileUriReader
        uri_reader = FileUriReader(table.file_io)
        for desc in descriptors:
            self.assertIsNotNone(desc)
            blob = Blob.from_descriptor(uri_reader, desc)
            self.assertEqual(blob.to_data(), blob_bytes)

    def test_blob_consumer_no_consumer_set(self):
        """Without consumer, writing still works normally."""
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('blob_data', pa.large_binary()),
        ])
        schema = Schema.from_pyarrow_schema(pa_schema, options={
            'row-tracking.enabled': 'true',
            'data-evolution.enabled': 'true',
        })
        self.catalog.create_table('test_db.blob_no_consumer', schema, False)
        table = self.catalog.get_table('test_db.blob_no_consumer')

        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()

        test_data = pa.Table.from_pydict({
            'id': [1, 2],
            'blob_data': [b'data1', b'data2'],
        }, schema=pa_schema)
        writer.write_arrow(test_data)
        commit_messages = writer.prepare_commit()
        write_builder.new_commit().commit(commit_messages)
        writer.close()

        result = table.new_read_builder().new_read().to_arrow(
            table.new_read_builder().new_scan().plan().splits())
        self.assertEqual(result.column('blob_data').to_pylist(), [b'data1', b'data2'])

    def test_blob_consumer_chain_call(self):
        """with_blob_consumer returns self for chaining."""
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('blob_data', pa.large_binary()),
        ])
        schema = Schema.from_pyarrow_schema(pa_schema, options={
            'row-tracking.enabled': 'true',
            'data-evolution.enabled': 'true',
        })
        self.catalog.create_table('test_db.blob_consumer_chain', schema, False)
        table = self.catalog.get_table('test_db.blob_consumer_chain')

        write_builder = table.new_batch_write_builder()
        result = write_builder.new_write().with_blob_consumer(lambda f, d: False)
        self.assertIsNotNone(result)
        result.close()

    def test_blob_consumer_abort_preserves_files(self):
        """Abort with consumer must not delete blob files that descriptors point to."""
        from pypaimon.table.row.blob import Blob
        from pypaimon.common.uri_reader import FileUriReader

        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('blob_data', pa.large_binary()),
        ])
        schema = Schema.from_pyarrow_schema(pa_schema, options={
            'row-tracking.enabled': 'true',
            'data-evolution.enabled': 'true',
            'blob.target-file-size': '1KB',
        })
        self.catalog.create_table('test_db.blob_consumer_abort', schema, False)
        table = self.catalog.get_table('test_db.blob_consumer_abort')

        blob_bytes = b'X' * 2048
        received = []

        def my_consumer(field_name, descriptor):
            received.append(descriptor)
            return False

        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()
        writer.with_blob_consumer(my_consumer)

        test_data = pa.Table.from_pydict({
            'id': list(range(5)),
            'blob_data': [blob_bytes] * 5,
        }, schema=pa_schema)
        writer.write_arrow(test_data)

        self.assertGreater(len(received), 0)

        # Capture data writers before close() clears them, then abort each one.
        data_writers = list(writer.file_store_write.data_writers.values())
        self.assertGreater(len(data_writers), 0)
        for dw in data_writers:
            dw.abort()

        # Every descriptor returned to the consumer must still be readable.
        uri_reader = FileUriReader(table.file_io)
        for desc in received:
            self.assertIsNotNone(desc)
            data = Blob.from_descriptor(uri_reader, desc).to_data()
            self.assertEqual(data, blob_bytes)

    def test_blob_consumer_after_write_raises(self):
        """Setting consumer after data has been written must raise."""
        pa_schema = pa.schema([
            ('id', pa.int32()),
            ('blob_data', pa.large_binary()),
        ])
        schema = Schema.from_pyarrow_schema(pa_schema, options={
            'row-tracking.enabled': 'true',
            'data-evolution.enabled': 'true',
        })
        self.catalog.create_table('test_db.blob_consumer_late', schema, False)
        table = self.catalog.get_table('test_db.blob_consumer_late')

        write_builder = table.new_batch_write_builder()
        writer = write_builder.new_write()

        writer.write_arrow(pa.Table.from_pydict({
            'id': [1],
            'blob_data': [b'data'],
        }, schema=pa_schema))

        with self.assertRaises(RuntimeError):
            writer.with_blob_consumer(lambda f, d: False)
        writer.close()


if __name__ == '__main__':
    unittest.main()
