/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.paimon.iceberg;

import org.apache.paimon.annotation.VisibleForTesting;
import org.apache.paimon.catalog.Identifier;
import org.apache.paimon.client.ClientPool;
import org.apache.paimon.fs.Path;
import org.apache.paimon.hive.HiveCatalog;
import org.apache.paimon.hive.HiveTypeUtils;
import org.apache.paimon.hive.pool.CachedClientPool;
import org.apache.paimon.iceberg.metadata.IcebergMetadata;
import org.apache.paimon.options.Options;
import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.types.DataField;
import org.apache.paimon.utils.Preconditions;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.thrift.TException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.annotation.Nullable;

import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.stream.Collectors;

import static org.apache.paimon.iceberg.IcebergCommitCallback.catalogDatabasePath;

/**
 * {@link IcebergMetadataCommitter} to commit Iceberg metadata to Hive metastore, so the table can
 * be visited by Iceberg's Hive catalog.
 */
public class IcebergHiveMetadataCommitter implements IcebergMetadataCommitter {

    private static final Logger LOG = LoggerFactory.getLogger(IcebergHiveMetadataCommitter.class);
    private static final int HIVE_COLUMN_COMMENT_MAX_LENGTH = 255;
    private static final String TRUNCATION_MARKER = "...";

    private final FileStoreTable table;
    private final ClientPool<IMetaStoreClient, TException> clients;
    // Supports syncing to multiple Iceberg databases via semicolon-delimited config
    private final List<String> icebergDatabases;
    private final String icebergTableName;

    public IcebergHiveMetadataCommitter(FileStoreTable table) {
        this.table = table;
        Identifier identifier =
                Preconditions.checkNotNull(
                        table.catalogEnvironment().identifier(),
                        "If you want to sync Paimon Iceberg compatible metadata to Hive, "
                                + "you must use a Paimon table created from a Paimon catalog, "
                                + "instead of a temporary table.");
        Preconditions.checkArgument(
                identifier.getBranchName() == null,
                "Paimon Iceberg compatibility currently does not support branches.");

        Options options = new Options(table.options());
        String uri = options.get(IcebergOptions.URI);
        String hiveConfDir = options.get(IcebergOptions.HIVE_CONF_DIR);
        String hadoopConfDir = options.get(IcebergOptions.HADOOP_CONF_DIR);
        Configuration hadoopConf = new Configuration();
        hadoopConf.setClassLoader(IcebergHiveMetadataCommitter.class.getClassLoader());
        HiveConf hiveConf = HiveCatalog.createHiveConf(hiveConfDir, hadoopConfDir, hadoopConf);

        table.options().forEach(hiveConf::set);
        if (uri != null) {
            hiveConf.set(HiveConf.ConfVars.METASTOREURIS.varname, uri);
        }

        if (hiveConf.get(HiveConf.ConfVars.METASTOREURIS.varname) == null) {
            LOG.error(
                    "Can't find hive metastore uri to connect: "
                            + "either set {} for paimon table or set hive.metastore.uris "
                            + "in hive-site.xml or hadoop configurations. "
                            + "Will use empty metastore uris, which means we may use a embedded metastore. "
                            + "This may cause unpredictable consensus problem.",
                    IcebergOptions.URI.key());
        }

        this.icebergDatabases =
                IcebergOptions.metastoreDatabases(options, identifier.getDatabaseName());
        String tableValue = options.get(IcebergOptions.METASTORE_TABLE);
        this.icebergTableName =
                tableValue != null && !tableValue.isEmpty()
                        ? tableValue
                        : identifier.getTableName();

        this.clients =
                new CachedClientPool(
                        hiveConf, options, options.getString(IcebergOptions.HIVE_CLIENT_CLASS));
    }

    @Override
    public String identifier() {
        return "hive";
    }

    @Override
    public void commitMetadata(Path newMetadataPath, @Nullable Path baseMetadataPath) {
        try {
            commitMetadataImpl(newMetadataPath, baseMetadataPath);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

    @Override
    public void commitMetadata(
            IcebergMetadata icebergMetadata, @Nullable IcebergMetadata baseIcebergMetadata) {
        throw new UnsupportedOperationException();
    }

    private void commitMetadataImpl(Path newMetadataPath, @Nullable Path baseMetadataPath)
            throws Exception {
        for (String databaseName : icebergDatabases) {
            commitMetadataForTarget(
                    databaseName, icebergTableName, newMetadataPath, baseMetadataPath);
        }
    }

    private void commitMetadataForTarget(
            String databaseName,
            String tableName,
            Path newMetadataPath,
            @Nullable Path baseMetadataPath)
            throws Exception {
        if (!databaseExists(databaseName)) {
            createDatabase(databaseName);
        }

        Table hiveTable;
        if (tableExists(databaseName, tableName)) {
            hiveTable = clients.run(client -> client.getTable(databaseName, tableName));
        } else {
            hiveTable = createTable(databaseName, tableName, newMetadataPath);
        }

        // Iceberg readers (e.g. iceberg.spark.SparkSessionCatalog) only load tables where
        // table_type=ICEBERG, so stamp it on every commit, existing entries created by Paimon's
        // HiveCatalog as table_type=PAIMON are migrated here.
        hiveTable.getParameters().put("table_type", "ICEBERG");
        hiveTable.getParameters().put("metadata_location", newMetadataPath.toString());
        if (baseMetadataPath != null) {
            hiveTable
                    .getParameters()
                    .put("previous_metadata_location", baseMetadataPath.toString());
        }

        StorageDescriptor sd = hiveTable.getSd();
        sd.setCols(
                table.copyWithLatestSchema().schema().fields().stream()
                        .map(this::convertToFieldSchema)
                        .collect(Collectors.toList()));

        Options options = new Options(table.options());
        EnvironmentContext environmentContext = new EnvironmentContext();
        environmentContext.putToProperties(StatsSetupConst.CASCADE, StatsSetupConst.TRUE);

        boolean skipAWSGlueArchive = options.get(IcebergOptions.GLUE_SKIP_ARCHIVE);
        environmentContext.putToProperties(
                "skipAWSGlueArchive", Boolean.toString(skipAWSGlueArchive));

        boolean skipHiveUpdateStats = options.get(IcebergOptions.HIVE_SKIP_UPDATE_STATS);
        environmentContext.putToProperties(
                StatsSetupConst.DO_NOT_UPDATE_STATS, Boolean.toString(skipHiveUpdateStats));

        clients.execute(
                client ->
                        client.alter_table_with_environmentContext(
                                databaseName, tableName, hiveTable, environmentContext));
    }

    private boolean databaseExists(String databaseName) throws Exception {
        try {
            clients.run(client -> client.getDatabase(databaseName));
            return true;
        } catch (NoSuchObjectException ignore) {
            return false;
        }
    }

    private void createDatabase(String databaseName) throws Exception {
        Database database = new Database();
        database.setName(databaseName);
        database.setLocationUri(catalogDatabasePath(table).toString());
        clients.execute(client -> client.createDatabase(database));
    }

    private boolean tableExists(String databaseName, String tableName) throws Exception {
        return clients.run(client -> client.tableExists(databaseName, tableName));
    }

    private Table createTable(String databaseName, String tableName, Path metadataPath)
            throws Exception {
        long currentTimeMillis = System.currentTimeMillis();
        Table hiveTable =
                new Table(
                        tableName,
                        databaseName,
                        // current linux user
                        System.getProperty("user.name"),
                        (int) (currentTimeMillis / 1000),
                        (int) (currentTimeMillis / 1000),
                        Integer.MAX_VALUE,
                        new StorageDescriptor(),
                        Collections.emptyList(),
                        new HashMap<>(),
                        null,
                        null,
                        "EXTERNAL_TABLE");

        Options options = new Options(table.options());
        boolean skipHiveUpdateStats = options.get(IcebergOptions.HIVE_SKIP_UPDATE_STATS);

        hiveTable.getParameters().put("DO_NOT_UPDATE_STATS", Boolean.toString(skipHiveUpdateStats));
        hiveTable.getParameters().put("EXTERNAL", "TRUE");
        hiveTable.getParameters().put("table_type", "ICEBERG");

        StorageDescriptor sd = hiveTable.getSd();
        sd.setLocation(metadataPath.getParent().getParent().toString());
        sd.setCols(
                table.copyWithLatestSchema().schema().fields().stream()
                        .map(this::convertToFieldSchema)
                        .collect(Collectors.toList()));
        sd.setInputFormat("org.apache.hadoop.mapred.FileInputFormat");
        sd.setOutputFormat("org.apache.hadoop.mapred.FileOutputFormat");

        SerDeInfo serDeInfo = new SerDeInfo();
        serDeInfo.setSerializationLib("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe");
        hiveTable.getSd().setSerdeInfo(serDeInfo);

        clients.execute(client -> client.createTable(hiveTable));
        return hiveTable;
    }

    private FieldSchema convertToFieldSchema(DataField dataField) {
        return new FieldSchema(
                dataField.name(),
                HiveTypeUtils.toTypeInfo(dataField.type()).getTypeName(),
                normalizeColumnComment(dataField.description()));
    }

    @VisibleForTesting
    static String normalizeColumnComment(@Nullable String comment) {
        if (comment == null) {
            return comment;
        }

        comment = comment.replace('\n', ' ').replace('\r', ' ');

        if (comment.length() <= HIVE_COLUMN_COMMENT_MAX_LENGTH) {
            return comment;
        }

        return comment.substring(0, HIVE_COLUMN_COMMENT_MAX_LENGTH - TRUNCATION_MARKER.length())
                + TRUNCATION_MARKER;
    }
}
