/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
 
package org.apache.xtable.kernel;

import static io.delta.kernel.utils.CloseableIterable.inMemoryIterable;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;

import lombok.Getter;
import lombok.extern.log4j.Log4j2;

import org.apache.hadoop.conf.Configuration;

import scala.collection.JavaConverters;
import scala.collection.Seq;

import com.google.common.annotations.VisibleForTesting;

import io.delta.kernel.Operation;
import io.delta.kernel.Snapshot;
import io.delta.kernel.Table;
import io.delta.kernel.Transaction;
import io.delta.kernel.TransactionBuilder;
import io.delta.kernel.TransactionCommitResult;
import io.delta.kernel.data.Row;
import io.delta.kernel.defaults.engine.DefaultEngine;
import io.delta.kernel.engine.Engine;
import io.delta.kernel.exceptions.TableNotFoundException;
import io.delta.kernel.hook.PostCommitHook;
import io.delta.kernel.internal.SnapshotImpl;
import io.delta.kernel.internal.actions.AddFile;
import io.delta.kernel.internal.actions.Metadata;
import io.delta.kernel.internal.actions.RemoveFile;
import io.delta.kernel.internal.actions.RowBackedAction;
import io.delta.kernel.internal.actions.SingleAction;
import io.delta.kernel.internal.util.Utils;
import io.delta.kernel.types.StructField;
import io.delta.kernel.types.StructType;
import io.delta.kernel.utils.CloseableIterable;

import org.apache.xtable.conversion.TargetTable;
import org.apache.xtable.exception.NotSupportedException;
import org.apache.xtable.exception.UpdateException;
import org.apache.xtable.model.InternalTable;
import org.apache.xtable.model.metadata.TableSyncMetadata;
import org.apache.xtable.model.schema.InternalPartitionField;
import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.storage.InternalFilesDiff;
import org.apache.xtable.model.storage.PartitionFileGroup;
import org.apache.xtable.model.storage.TableFormat;
import org.apache.xtable.spi.sync.ConversionTarget;

/**
 * Implementation of {@link ConversionTarget} for Delta Lake using the Delta Kernel API.
 *
 * <p>This implementation uses Delta Kernel (io.delta.kernel) instead of Delta Standalone for write
 * operations, providing better compatibility with cloud storage (S3, GCS, Azure Blob Storage, HDFS)
 * and improved support for Delta Lake 3.x features.
 *
 * <p><strong>Initialization:</strong> This class supports two initialization patterns:
 *
 * <ul>
 *   <li><strong>Factory/ServiceLoader Pattern:</strong> Use the no-arg constructor followed by
 *       {@link #init(TargetTable, Configuration)}. This is used by {@link
 *       org.apache.xtable.conversion.ConversionTargetFactory}.
 *   <li><strong>Direct Testing Pattern:</strong> Use the constructor with {@link TargetTable} and
 *       {@link Engine} parameters for direct instantiation with custom dependencies in tests.
 * </ul>
 *
 * <p><strong>Important:</strong> Do not mix initialization patterns. If you use the parameterized
 * constructor, do not call {@link #init(TargetTable, Configuration)} afterward, as it will
 * overwrite the custom Engine.
 *
 * <p><strong>Exception Handling:</strong> This implementation only catches {@link
 * io.delta.kernel.exceptions.TableNotFoundException} when checking for table existence, allowing
 * other exceptions (network errors, permission issues, corrupted metadata) to propagate rather than
 * being silently masked. This ensures real errors are visible and fail fast.
 *
 * <p><strong>Known Limitations:</strong>
 *
 * <ul>
 *   <li><strong>Commit Tags:</strong> Delta Kernel 4.0.0 does not support commit tags in commitInfo
 *       (e.g., XTABLE_METADATA tags). This affects source-to-target commit identifier mapping.
 *       Tracked in: https://github.com/apache/incubator-xtable/issues/819
 *   <li><strong>Schema Evolution:</strong> Schema changes are handled through Delta Kernel's
 *       transaction API, which may have different semantics compared to Delta Standalone.
 *   <li><strong>Internal API Usage:</strong> This implementation casts to internal classes
 *       (SnapshotImpl, TableImpl) to access metadata and commit history, as Delta Kernel 4.0.0
 *       lacks public APIs for these operations. These casts are brittle and may break on version
 *       upgrades. Public API alternatives should be used when available.
 * </ul>
 *
 * <p><strong>Implementation Choice:</strong> Delta Kernel API was chosen over Delta Standalone to:
 *
 * <ul>
 *   <li>Support newer Delta Lake features and protocol versions
 *   <li>Align with the Delta Lake community's direction (Kernel is the recommended API)
 *   <li>Reduce dependency on Spark-specific implementations
 * </ul>
 *
 * @see ConversionTarget
 * @see io.delta.kernel.Table
 * @see io.delta.kernel.Transaction
 */
@Log4j2
public class DeltaKernelConversionTarget implements ConversionTarget {
  private static final String DELTA_LOG_RETENTION_DURATION = "delta.logRetentionDuration";

  private DeltaKernelSchemaExtractor schemaExtractor;
  private DeltaKernelPartitionExtractor partitionExtractor;
  private DeltaKernelDataFileUpdatesExtractor dataKernelFileUpdatesExtractor;

  private String basePath;
  private long logRetentionInHours;
  private DeltaKernelConversionTarget.TransactionState transactionState;
  private Engine engine;

  /**
   * No-arg constructor for ServiceLoader instantiation. Must call {@link #init(TargetTable,
   * Configuration)} before use.
   */
  public DeltaKernelConversionTarget() {}

  /**
   * Creates a fully initialized DeltaKernelConversionTarget with custom Engine. Typically used in
   * tests. Do not call {@link #init(TargetTable, Configuration)} after this.
   *
   * @param targetTable the target table configuration
   * @param engine custom Delta Kernel engine instance
   */
  public DeltaKernelConversionTarget(TargetTable targetTable, Engine engine) {
    this(
        targetTable.getBasePath(),
        targetTable.getMetadataRetention().toHours(),
        engine,
        DeltaKernelSchemaExtractor.getInstance(),
        DeltaKernelPartitionExtractor.getInstance(),
        DeltaKernelDataFileUpdatesExtractor.builder()
            .engine(engine)
            .basePath(targetTable.getBasePath())
            // Column statistics are not needed for conversion operations
            .includeColumnStats(false)
            .build());
  }

  @VisibleForTesting
  DeltaKernelConversionTarget(
      String tableDataPath,
      long logRetentionInHours,
      Engine engine,
      DeltaKernelSchemaExtractor schemaExtractor,
      DeltaKernelPartitionExtractor partitionExtractor,
      DeltaKernelDataFileUpdatesExtractor dataKernelFileUpdatesExtractor) {
    initInternal(
        tableDataPath,
        logRetentionInHours,
        engine,
        schemaExtractor,
        partitionExtractor,
        dataKernelFileUpdatesExtractor);
  }

  /**
   * Private initialization helper to avoid code duplication between constructor and init() paths.
   */
  private void initInternal(
      String tableDataPath,
      long logRetentionInHours,
      Engine engine,
      DeltaKernelSchemaExtractor schemaExtractor,
      DeltaKernelPartitionExtractor partitionExtractor,
      DeltaKernelDataFileUpdatesExtractor dataKernelFileUpdatesExtractor) {
    this.basePath = tableDataPath;
    this.schemaExtractor = schemaExtractor;
    this.partitionExtractor = partitionExtractor;
    this.dataKernelFileUpdatesExtractor = dataKernelFileUpdatesExtractor;
    this.engine = engine;
    this.logRetentionInHours = logRetentionInHours;
  }

  @Override
  public void init(TargetTable targetTable, Configuration configuration) {
    if (this.engine != null) {
      throw new IllegalStateException(
          "init() called on an already initialized instance. "
              + "Do not call init() after using the parameterized constructor.");
    }

    Engine engine = DefaultEngine.create(configuration);

    initInternal(
        targetTable.getBasePath(),
        targetTable.getMetadataRetention().toHours(),
        engine,
        DeltaKernelSchemaExtractor.getInstance(),
        DeltaKernelPartitionExtractor.getInstance(),
        DeltaKernelDataFileUpdatesExtractor.builder()
            .engine(engine)
            .basePath(targetTable.getBasePath())
            .includeColumnStats(true)
            .build());
  }

  @Override
  public void beginSync(InternalTable table) {
    this.transactionState =
        new DeltaKernelConversionTarget.TransactionState(engine, logRetentionInHours);
  }

  @Override
  public void syncSchema(InternalSchema schema) {
    transactionState.setLatestSchema(schema);
  }

  @Override
  public void syncPartitionSpec(List<InternalPartitionField> partitionSpec) {
    if (partitionSpec != null) {
      Map<String, StructField> spec =
          partitionExtractor.convertToDeltaPartitionFormat(partitionSpec);
      for (Map.Entry<String, StructField> partitionEntry : spec.entrySet()) {
        String partitionColumnName = partitionEntry.getKey();
        StructField partitionField = partitionEntry.getValue();

        transactionState.addPartitionColumn(partitionColumnName);
        if (partitionField != null
            && transactionState.getLatestSchema().fields().stream()
                .noneMatch(field -> field.getName().equals(partitionField.getName()))) {
          // add generated columns to schema.
          transactionState.addColumn(partitionField);
        }
      }
    }
  }

  @Override
  public void syncMetadata(TableSyncMetadata metadata) {
    transactionState.setMetadata(metadata);
  }

  @Override
  public void syncFilesForSnapshot(List<PartitionFileGroup> partitionedDataFiles) {
    Table table = Table.forPath(engine, basePath);
    // Pass cached snapshot to avoid reloading it
    transactionState.setActions(
        dataKernelFileUpdatesExtractor.applySnapshot(
            table,
            partitionedDataFiles,
            transactionState.getLatestSchemaInternal(),
            transactionState.getCachedSnapshot()));
  }

  @Override
  public void syncFilesForDiff(InternalFilesDiff internalFilesDiff) {
    // Use cached schema directly (already extracted during TransactionState initialization)
    // For new tables, latestSchema is null and applyDiff will handle it appropriately
    transactionState.setActions(
        dataKernelFileUpdatesExtractor.applyDiff(
            internalFilesDiff,
            transactionState.getLatestSchemaInternal(),
            basePath,
            transactionState.getLatestSchema()));
  }

  @Override
  public void completeSync() {
    transactionState.commitTransaction();
    transactionState = null;
  }

  @Override
  public Optional<TableSyncMetadata> getTableMetadata() {
    Table table = Table.forPath(engine, basePath);
    Snapshot snapshot;
    try {
      snapshot = table.getLatestSnapshot(engine);
    } catch (TableNotFoundException e) {
      // Table does not exist yet (first sync) - there is no prior sync metadata to read.
      return Optional.empty();
    }

    // WORKAROUND: Cast to SnapshotImpl (internal class) to access metadata configuration.
    // Delta Kernel 4.0.0 does not provide a public API to access table metadata/configuration.
    // This cast is brittle and may break on Kernel version upgrades.
    Metadata metadata = ((SnapshotImpl) snapshot).getMetadata();

    // Get configuration from metadata
    Map<String, String> configuration = metadata.getConfiguration();
    String metadataJson = configuration.get(TableSyncMetadata.XTABLE_METADATA);

    return TableSyncMetadata.fromJson(metadataJson);
  }

  @Override
  public String getTableFormat() {
    return TableFormat.DELTA;
  }

  @Override
  public Optional<String> getTargetCommitIdentifier(String sourceIdentifier) {
    // Delta Kernel 4.0.0 does not support commit tags in commitInfo, which are required for
    // source-to-target commit identifier mapping. This limitation is documented in:
    // https://github.com/delta-io/delta/issues/6167
    // XTable tracking issue: https://github.com/apache/incubator-xtable/issues/819
    //
    // Unlike DeltaConversionTarget (which uses Delta Standalone with commit tag support),
    // DeltaKernelConversionTarget cannot retrieve commit tags from Delta Kernel's API.
    // Rather than silently scanning all commits (O(n) performance cost) and always returning
    // empty, we explicitly throw an exception to indicate this feature is unsupported.
    //
    // When Delta Kernel adds commit tag support, this method can be reimplemented to:
    // 1. Scan commit history using tableImpl.getChanges(engine, 0, currentVersion, actionSet)
    // 2. Extract tags from CommitInfo.tags MapValue
    // 3. Parse XTABLE_METADATA from tags and match sourceIdentifier
    throw new NotSupportedException(
        "Source-to-target commit identifier mapping is not supported in DeltaKernelConversionTarget. "
            + "Delta Kernel 4.0.0 does not support commit tags in commitInfo. "
            + "See: https://github.com/delta-io/delta/issues/6167");
  }

  private class TransactionState {
    private final Engine engine;
    private final long retentionInHours;
    private final List<String> partitionColumns;
    @Getter private StructType latestSchema;
    @Getter private InternalSchema latestSchemaInternal;
    private TableSyncMetadata metadata;
    private List<RowBackedAction> actions;

    // Cache the snapshot to avoid multiple loads during a single sync cycle
    // This snapshot is used for: schema extraction, table existence check, and file diff operations
    private Snapshot cachedSnapshot;
    private boolean tableExists;

    private TransactionState(Engine engine, long retentionInHours) {
      this.engine = engine;
      this.partitionColumns = new ArrayList<>();
      this.retentionInHours = retentionInHours;

      try {
        Table table = Table.forPath(engine, basePath);
        this.cachedSnapshot = table.getLatestSnapshot(engine);
        this.latestSchema = cachedSnapshot.getSchema();
        this.tableExists = true;
      } catch (TableNotFoundException e) {
        // Expected: table doesn't exist yet on first sync
        this.latestSchema = null;
        this.cachedSnapshot = null;
        this.tableExists = false;
      }
      // Let other exceptions propagate (network issues, permissions, corrupted metadata, etc.)
    }

    /**
     * Adds a partition column name to the list. Package-private to allow access from outer class.
     */
    void addPartitionColumn(String columnName) {
      partitionColumns.add(columnName);
    }

    /**
     * Gets the cached snapshot. Returns null if no snapshot was cached (new table). Package-private
     * to allow access from outer class.
     */
    Snapshot getCachedSnapshot() {
      return cachedSnapshot;
    }

    void setMetadata(TableSyncMetadata metadata) {
      this.metadata = metadata;
    }

    /**
     * Sets the actions to be committed. Converts from Scala Seq to Java List for internal storage.
     */
    void setActions(Seq<RowBackedAction> scalaActions) {
      this.actions = JavaConverters.seqAsJavaList(scalaActions);
    }

    private void addColumn(StructField field) {
      latestSchema = latestSchema.add(field);
      latestSchemaInternal = schemaExtractor.toInternalSchema(latestSchema);
    }

    private void setLatestSchema(InternalSchema schema) {
      this.latestSchemaInternal = schema;
      this.latestSchema = schemaExtractor.fromInternalSchema(schema);
    }

    private void commitTransaction() {
      // Use cached table existence check instead of loading snapshot again
      Operation operation = tableExists ? Operation.WRITE : Operation.CREATE_TABLE;

      Table table = Table.forPath(engine, basePath);
      TransactionBuilder txnBuilder =
          table.createTransactionBuilder(engine, "XTable Delta Sync", operation);

      // LIMITATION: Schema evolution for existing tables is NOT supported in Delta Kernel 4.0.0.
      // The withSchema() method only works during CREATE_TABLE operations. For existing tables:
      // - AddFile/RemoveFile actions are created using the old schema from existing snapshot
      // - If source schema has evolved (columns added/removed/type changed), the Delta table
      //   will have mismatched metadata and data, causing query failures or incorrect results
      // This is a known Delta Kernel limitation: https://github.com/delta-io/delta/issues/4305
      if (!tableExists) {
        txnBuilder = txnBuilder.withSchema(engine, latestSchema);

        if (!partitionColumns.isEmpty()) {
          txnBuilder = txnBuilder.withPartitionColumns(engine, partitionColumns);
        }
      }

      Map<String, String> tableProperties = getConfigurationsForDeltaSync();
      txnBuilder = txnBuilder.withTableProperties(engine, tableProperties);

      Transaction txn = txnBuilder.build(engine);
      List<Row> allActionRows = new ArrayList<>();

      // Iterate through actions (Java List) and convert to Row format
      for (RowBackedAction action : actions) {

        if (action instanceof AddFile) {
          AddFile addFile = (AddFile) action;
          Row wrappedRow = SingleAction.createAddFileSingleAction(addFile.toRow());
          allActionRows.add(wrappedRow);
        } else if (action instanceof RemoveFile) {
          RemoveFile removeFile = (RemoveFile) action;
          Row wrappedRow = SingleAction.createRemoveFileSingleAction(removeFile.toRow());
          allActionRows.add(wrappedRow);
        }
      }

      CloseableIterable<Row> dataActions =
          inMemoryIterable(Utils.toCloseableIterator(allActionRows.iterator()));

      try {
        TransactionCommitResult result = txn.commit(engine, dataActions);

        // Execute PostCommitHooks to create checkpoints and _last_checkpoint metadata file
        List<PostCommitHook> hooks = result.getPostCommitHooks();
        if (hooks != null && !hooks.isEmpty()) {
          for (PostCommitHook hook : hooks) {
            try {
              hook.threadSafeInvoke(engine);
            } catch (Exception hookEx) {
              // Post-commit hooks are optimizations; log but don't fail the transaction
              log.warn(
                  "Post-commit hook failed for table {} but transaction succeeded",
                  basePath,
                  hookEx);
            }
          }
        }
      } catch (Exception e) {
        throw new UpdateException("Failed to commit Delta Kernel transaction", e);
      }

      // NOTE: Delta Kernel API limitations compared to Delta Standalone:
      // - Commit tags (like XTABLE_METADATA in commitInfo.tags) are not yet supported
      // - Operation type metadata (like DeltaOperations.Update) is simplified to
      // Operation.WRITE/CREATE_TABLE
      // - The commit timestamp is managed by Delta Kernel automatically
    }

    private Map<String, String> getConfigurationsForDeltaSync() {
      Map<String, String> configMap = new HashMap<>();
      // Delta Kernel will automatically upgrade protocol versions if/when features that require
      // higher versions are used. Explicitly setting them via table properties is not supported
      // in Delta Kernel 4.0.0 and causes UnknownConfiguration errors.

      configMap.put(TableSyncMetadata.XTABLE_METADATA, metadata.toJson());
      configMap.put(
          DELTA_LOG_RETENTION_DURATION, String.format("interval %d hours", retentionInHours));

      return configMap;
    }
  }
}
