DataTableProcessor.cs

//------------------------------------------------------------
// Game Framework
// Copyright © 2013-2020 Jiang Yin. All rights reserved.
// Homepage: https://gameframework.cn/
// Feedback: mailto:ellan@gameframework.cn
//------------------------------------------------------------

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using GameFramework;
using GameFramework.FileSystem;
using UGFExtensions;
using UnityEngine;

namespace DE.Editor.DataTableTools
{
    public sealed partial class DataTableProcessor
    {
        private const string CommentLineSeparator = "#";
        private static readonly char[] DataSplitSeparators = { '\t' };
        private static readonly char[] DataTrimSeparators = { '\"' };
        private readonly string[] m_CommentRow;

        private readonly DataProcessor[] m_DataProcessor;
        private readonly string[] m_DefaultValueRow;

        private readonly string[] m_NameRow;
        private readonly string[][] m_RawValues;
        private readonly string[] m_Strings;
        private readonly string[] m_TypeRow;
        private DataTableCodeGenerator m_CodeGenerator;

        private string m_CodeTemplate;


        public DataTableProcessor(string dataTableFileName, Encoding encoding, int nameRow, int typeRow,
            int? defaultValueRow, int? commentRow, int contentStartRow, int idColumn)
        {
            if (string.IsNullOrEmpty(dataTableFileName))
                throw new GameFrameworkException("Data table file name is invalid.");

            if (!dataTableFileName.EndsWith(".txt", StringComparison.Ordinal))
                throw new GameFrameworkException(Utility.Text.Format("Data table file '{0}' is not a txt.",
                    dataTableFileName));

            if (!File.Exists(dataTableFileName))
                throw new GameFrameworkException(Utility.Text.Format("Data table file '{0}' is not exist.",
                    dataTableFileName));

            var lines = File.ReadAllLines(dataTableFileName, encoding);
            var rawRowCount = lines.Length;

            var rawColumnCount = 0;
            var rawValues = new List<string[]>();
            for (var i = 0; i < lines.Length; i++)
            {
                var rawValue = lines[i].Split(DataSplitSeparators);
                for (var j = 0; j < rawValue.Length; j++) rawValue[j] = rawValue[j].Trim(DataTrimSeparators);

                if (i == 0)
                    rawColumnCount = rawValue.Length;
                else if (rawValue.Length != rawColumnCount)
                    throw new GameFrameworkException(Utility.Text.Format(
                        "Data table file '{0}', raw Column is '{2}', but line '{1}' column is '{3}'.",
                        dataTableFileName, i.ToString(), rawColumnCount.ToString(), rawValue.Length.ToString()));

                rawValues.Add(rawValue);
            }

            m_RawValues = rawValues.ToArray();

            if (nameRow < 0)
                throw new GameFrameworkException(Utility.Text.Format("Name row '{0}' is invalid.", nameRow.ToString()));

            if (typeRow < 0)
                throw new GameFrameworkException(Utility.Text.Format("Type row '{0}' is invalid.", typeRow.ToString()));

            if (contentStartRow < 0)
                throw new GameFrameworkException(Utility.Text.Format("Content start row '{0}' is invalid.",
                    contentStartRow.ToString()));

            if (idColumn < 0)
                throw new GameFrameworkException(
                    Utility.Text.Format("Id column '{0}' is invalid.", idColumn.ToString()));

            if (nameRow >= rawRowCount)
                throw new GameFrameworkException(Utility.Text.Format(
                    "Name row '{0}' >= raw row count '{1}' is not allow.", nameRow.ToString(), rawRowCount.ToString()));

            if (typeRow >= rawRowCount)
                throw new GameFrameworkException(Utility.Text.Format(
                    "Type row '{0}' >= raw row count '{1}' is not allow.", typeRow.ToString(), rawRowCount.ToString()));

            if (defaultValueRow.HasValue && defaultValueRow.Value >= rawRowCount)
                throw new GameFrameworkException(Utility.Text.Format(
                    "Default value row '{0}' >= raw row count '{1}' is not allow.", defaultValueRow.Value.ToString(),
                    rawRowCount.ToString()));

            if (commentRow.HasValue && commentRow.Value >= rawRowCount)
                throw new GameFrameworkException(Utility.Text.Format(
                    "Comment row '{0}' >= raw row count '{1}' is not allow.", commentRow.Value.ToString(),
                    rawRowCount.ToString()));

            if (contentStartRow > rawRowCount)
                throw new GameFrameworkException(Utility.Text.Format(
                    "Content start row '{0}' > raw row count '{1}' is not allow.", contentStartRow.ToString(),
                    rawRowCount.ToString()));

            if (idColumn >= rawColumnCount)
                throw new GameFrameworkException(Utility.Text.Format(
                    "Id column '{0}' >= raw column count '{1}' is not allow.", idColumn.ToString(),
                    rawColumnCount.ToString()));

            m_NameRow = m_RawValues[nameRow];
            m_TypeRow = m_RawValues[typeRow];
            m_DefaultValueRow = defaultValueRow.HasValue ? m_RawValues[defaultValueRow.Value] : null;
            m_CommentRow = commentRow.HasValue ? m_RawValues[commentRow.Value] : null;
            ContentStartRow = contentStartRow;
            IdColumn = idColumn;

            m_DataProcessor = new DataProcessor[rawColumnCount];
            for (var i = 0; i < rawColumnCount; i++)
                if (i == IdColumn)
                    m_DataProcessor[i] = DataProcessorUtility.GetDataProcessor("id");
                else
                    m_DataProcessor[i] = DataProcessorUtility.GetDataProcessor(m_TypeRow[i]);

            var strings = new Dictionary<string, int>(StringComparer.Ordinal);
            for (var i = contentStartRow; i < rawRowCount; i++)
            {
                if (IsCommentRow(i)) continue;

                for (var j = 0; j < rawColumnCount; j++)
                {
                    if (m_DataProcessor[j] is ICollectionProcessor collectionProcessor)
                    {
                        if (collectionProcessor.ItemLanguageKeyword != "string") continue;
                    }
                    else
                    {
                        if (m_DataProcessor[j].LanguageKeyword != "string") continue;
                    }

                    var str = m_RawValues[i][j];
                    var values = str.Split(',');
                    foreach (var value in values)
                        if (strings.ContainsKey(value))
                            strings[value]++;
                        else
                            strings[value] = 1;
                }
            }

            m_Strings = strings.OrderBy(value => value.Key).ThenByDescending(value => value.Value)
                .Select(value => value.Key).ToArray();

            m_CodeTemplate = null;
            m_CodeGenerator = null;
        }

        public int RawRowCount => m_RawValues.Length;

        public int RawColumnCount => m_RawValues.Length > 0 ? m_RawValues[0].Length : 0;

        public int StringCount => m_Strings.Length;

        public int ContentStartRow { get; }

        public int IdColumn { get; }

        public bool IsListColumn(int rawColumn)
        {
            if (rawColumn < 0 || rawColumn >= RawColumnCount)
                throw new GameFrameworkException(Utility.Text.Format("Raw column '{0}' is out of range.",
                    rawColumn.ToString()));

            return m_DataProcessor[rawColumn].GetTypeStrings()[0].Equals("List<{0}>");
        }

        public bool IsArrayColumn(int rawColumn)
        {
            if (rawColumn < 0 || rawColumn >= RawColumnCount)
                throw new GameFrameworkException(Utility.Text.Format("Raw column '{0}' is out of range.",
                    rawColumn.ToString()));

            return m_DataProcessor[rawColumn].GetTypeStrings()[0].Equals("{0}[]");
        }

        public bool IsEnumrColumn(int rawColumn)
        {
            if (rawColumn < 0 || rawColumn >= RawColumnCount)
                throw new GameFrameworkException(Utility.Text.Format("Raw column '{0}' is out of range.",
                    rawColumn.ToString()));

            return m_DataProcessor[rawColumn].IsEnum;
        }

        public bool IsDictionaryColumn(int rawColumn)
        {
            if (rawColumn < 0 || rawColumn >= RawColumnCount)
                throw new GameFrameworkException(Utility.Text.Format("Raw column '{0}' is out of range.",
                    rawColumn.ToString()));

            return m_DataProcessor[rawColumn].GetTypeStrings()[0].Equals("Dictionary<{0},{1}>");
        }

        public bool IsIdColumn(int rawColumn)
        {
            if (rawColumn < 0 || rawColumn >= RawColumnCount)
                throw new GameFrameworkException(Utility.Text.Format("Raw column '{0}' is out of range.",
                    rawColumn.ToString()));

            return m_DataProcessor[rawColumn].IsId;
        }

        public bool IsCommentRow(int rawRow)
        {
            if (rawRow < 0 || rawRow >= RawRowCount)
                throw new GameFrameworkException(Utility.Text.Format("Raw row '{0}' is out of range.",
                    rawRow.ToString()));

            return GetValue(rawRow, 0).StartsWith(CommentLineSeparator, StringComparison.Ordinal);
        }

        public bool IsCommentColumn(int rawColumn)
        {
            if (rawColumn < 0 || rawColumn >= RawColumnCount)
                throw new GameFrameworkException(Utility.Text.Format("Raw column '{0}' is out of range.",
                    rawColumn.ToString()));

            return string.IsNullOrEmpty(GetName(rawColumn)) || m_DataProcessor[rawColumn].IsComment;
        }

        public string GetName(int rawColumn)
        {
            if (rawColumn < 0 || rawColumn >= RawColumnCount)
                throw new GameFrameworkException(Utility.Text.Format("Raw column '{0}' is out of range.",
                    rawColumn.ToString()));

            if (IsIdColumn(rawColumn)) return "Id";

            return m_NameRow[rawColumn];
        }

        public bool IsSystem(int rawColumn)
        {
            if (rawColumn < 0 || rawColumn >= RawColumnCount)
                throw new GameFrameworkException(Utility.Text.Format("Raw column '{0}' is out of range.",
                    rawColumn.ToString()));

            return m_DataProcessor[rawColumn].IsSystem;
        }

        public Type GetType(int rawColumn)
        {
            if (rawColumn < 0 || rawColumn >= RawColumnCount)
                throw new GameFrameworkException(Utility.Text.Format("Raw column '{0}' is out of range.",
                    rawColumn.ToString()));

            return m_DataProcessor[rawColumn].Type;
        }

        public string GetLanguageKeyword(int rawColumn)
        {
            if (rawColumn < 0 || rawColumn >= RawColumnCount)
                throw new GameFrameworkException(Utility.Text.Format("Raw column '{0}' is out of range.",
                    rawColumn.ToString()));

            return m_DataProcessor[rawColumn].LanguageKeyword;
        }

        public string[] GetTypeStrings(int rawColumn)
        {
            if (rawColumn < 0 || rawColumn >= RawColumnCount)
                throw new GameFrameworkException(Utility.Text.Format("Raw column '{0}' is out of range.",
                    rawColumn.ToString()));

            return m_DataProcessor[rawColumn].GetTypeStrings();
        }

        public string GetDefaultValue(int rawColumn)
        {
            if (rawColumn < 0 || rawColumn >= RawColumnCount)
                throw new GameFrameworkException(Utility.Text.Format("Raw column '{0}' is out of range.",
                    rawColumn.ToString()));

            return m_DefaultValueRow != null ? m_DefaultValueRow[rawColumn] : null;
        }

        public string GetComment(int rawColumn)
        {
            if (rawColumn < 0 || rawColumn >= RawColumnCount)
                throw new GameFrameworkException(Utility.Text.Format("Raw column '{0}' is out of range.",
                    rawColumn.ToString()));

            return m_CommentRow != null ? m_CommentRow[rawColumn] : null;
        }

        public string GetValue(int rawRow, int rawColumn)
        {
            if (rawRow < 0 || rawRow >= RawRowCount)
                throw new GameFrameworkException(Utility.Text.Format("Raw row '{0}' is out of range.",
                    rawRow.ToString()));

            if (rawColumn < 0 || rawColumn >= RawColumnCount)
                throw new GameFrameworkException(Utility.Text.Format("Raw column '{0}' is out of range.",
                    rawColumn.ToString()));

            return m_RawValues[rawRow][rawColumn];
        }

        public string GetString(int index)
        {
            if (index < 0 || index >= StringCount)
                throw new GameFrameworkException(Utility.Text.Format("String index '{0}' is out of range.",
                    index.ToString()));

            return m_Strings[index];
        }

        public int GetStringIndex(string str)
        {
            for (var i = 0; i < StringCount; i++)
                if (m_Strings[i] == str)
                    return i;

            return -1;
        }

        public bool GenerateDataFile(string outputFileName)
        {
            if (string.IsNullOrEmpty(outputFileName)) throw new GameFrameworkException("Output file name is invalid.");

            try
            {
                using (var fileStream = new FileStream(outputFileName, FileMode.Create, FileAccess.Write))
                {
                    using (var binaryWriter = new BinaryWriter(fileStream, Encoding.UTF8))
                    {
                        for (var rawRow = ContentStartRow; rawRow < RawRowCount; rawRow++)
                        {
                            if (IsCommentRow(rawRow)) continue;

                            var bytes = GetRowBytes(outputFileName, rawRow);
                            binaryWriter.Write7BitEncodedInt32(bytes.Length);
                            binaryWriter.Write(bytes);
                        }
                    }
                }

                Debug.Log(Utility.Text.Format("Parse data table '{0}' success.", outputFileName));
                return true;
            }
            catch (Exception exception)
            {
                Debug.LogError(Utility.Text.Format("Parse data table '{0}' failure, exception is '{1}'.",
                    outputFileName, exception.ToString()));
                return false;
            }
        }

        public bool GenerateFileSystemFile(string outputFileName)
        {
            string tempFile = "~Temp.bytes";
            if (string.IsNullOrEmpty(outputFileName)) throw new GameFrameworkException("Output file name is invalid.");
            try
            {
                using (var tempStream = new FileStream(tempFile, FileMode.Create, FileAccess.ReadWrite))
                {
                    using (var tempWriter = new BinaryWriter(tempStream, Encoding.UTF8))
                    {
                        DataTableRowConfig tableConfig = new DataTableRowConfig();
                        int lastLength = 0;
                        tableConfig.DataTableRowSettings =
                            new Dictionary<int, DataTableRowSetting>(RawRowCount - ContentStartRow);
                        for (var rawRow = ContentStartRow; rawRow < RawRowCount; rawRow++)
                        {
                            if (IsCommentRow(rawRow)) continue;
                            string id = GetValue(rawRow, IdColumn);
                            var bytes = GetRowBytes(outputFileName, rawRow);
                            tableConfig.DataTableRowSettings.Add(Int32.Parse(id),
                                new DataTableRowSetting(lastLength, bytes.Length));
                            lastLength += bytes.Length;
                            tableConfig.Count++;
                            tempWriter.Write(bytes);
                        }

                        using (FileStream fileStream =
                            new FileStream(outputFileName, FileMode.Create, FileAccess.Write))
                        {
                            using (var binaryWriter = new BinaryWriter(fileStream))
                            {
                                var tableConfigBytes = tableConfig.Serialize();
                                binaryWriter.Write7BitEncodedInt32(tableConfigBytes.Length);
                                binaryWriter.Write(tableConfigBytes);
                                tempStream.Seek(0, SeekOrigin.Begin);
                                tempStream.CopyTo(fileStream);
                            }
                        }
                    }
                }

                File.Delete(tempFile);

                Debug.Log(Utility.Text.Format("Parse data table '{0}' success.", outputFileName));
                return true;
            }
            catch (Exception exception)
            {
                Debug.LogError(Utility.Text.Format("Parse data table '{0}' failure, exception is '{1}'.",
                    outputFileName, exception.ToString()));
                return false;
            }
        }

        public bool SetCodeTemplate(string codeTemplateFileName, Encoding encoding)
        {
            try
            {
                m_CodeTemplate = File.ReadAllText(codeTemplateFileName, encoding);
                Debug.Log(Utility.Text.Format("Set code template '{0}' success.", codeTemplateFileName));
                return true;
            }
            catch (Exception exception)
            {
                Debug.LogError(Utility.Text.Format("Set code template '{0}' failure, exception is '{1}'.",
                    codeTemplateFileName, exception.ToString()));
                return false;
            }
        }

        public void SetCodeGenerator(DataTableCodeGenerator codeGenerator)
        {
            m_CodeGenerator = codeGenerator;
        }

        public bool GenerateCodeFile(string outputFileName, Encoding encoding, object userData = null)
        {
            if (string.IsNullOrEmpty(m_CodeTemplate))
                throw new GameFrameworkException("You must set code template first.");

            if (string.IsNullOrEmpty(outputFileName)) throw new GameFrameworkException("Output file name is invalid.");

            try
            {
                var stringBuilder = new StringBuilder(m_CodeTemplate);
                if (m_CodeGenerator != null) m_CodeGenerator(this, stringBuilder, userData);

                using (var fileStream = new FileStream(outputFileName, FileMode.Create, FileAccess.Write))
                {
                    using (var stream = new StreamWriter(fileStream, encoding))
                    {
                        stream.Write(stringBuilder.ToString());
                    }
                }

                Debug.Log(Utility.Text.Format("Generate code file '{0}' success.", outputFileName));
                return true;
            }
            catch (Exception exception)
            {
                Debug.LogError(Utility.Text.Format("Generate code file '{0}' failure, exception is '{1}'.",
                    outputFileName, exception.ToString()));
                return false;
            }
        }

        private byte[] GetRowBytes(string outputFileName, int rawRow)
        {
            using (var memoryStream = new MemoryStream())
            {
                using (var binaryWriter = new BinaryWriter(memoryStream, Encoding.UTF8))
                {
                    for (var rawColumn = 0; rawColumn < RawColumnCount; rawColumn++)
                    {
                        if (IsCommentColumn(rawColumn)) continue;

                        try
                        {
                            m_DataProcessor[rawColumn].WriteToStream(this, binaryWriter, GetValue(rawRow, rawColumn));
                        }
                        catch
                        {
                            if (m_DataProcessor[rawColumn].IsId || string.IsNullOrEmpty(GetDefaultValue(rawColumn)))
                            {
                                Debug.LogError(Utility.Text.Format(
                                    "Parse raw value failure. OutputFileName='{0}' RawRow='{1}' RowColumn='{2}' Name='{3}' Type='{4}' RawValue='{5}'",
                                    outputFileName, rawRow.ToString(), rawColumn.ToString(), GetName(rawColumn),
                                    GetLanguageKeyword(rawColumn), GetValue(rawRow, rawColumn)));
                                return null;
                            }

                            Debug.LogWarning(Utility.Text.Format(
                                "Parse raw value failure, will try default value. OutputFileName='{0}' RawRow='{1}' RowColumn='{2}' Name='{3}' Type='{4}' RawValue='{5}'",
                                outputFileName, rawRow.ToString(), rawColumn.ToString(), GetName(rawColumn),
                                GetLanguageKeyword(rawColumn), GetValue(rawRow, rawColumn)));
                            try
                            {
                                m_DataProcessor[rawColumn]
                                    .WriteToStream(this, binaryWriter, GetDefaultValue(rawColumn));
                            }
                            catch
                            {
                                Debug.LogError(Utility.Text.Format(
                                    "Parse default value failure. OutputFileName='{0}' RawRow='{1}' RowColumn='{2}' Name='{3}' Type='{4}' RawValue='{5}'",
                                    outputFileName, rawRow.ToString(), rawColumn.ToString(), GetName(rawColumn),
                                    GetLanguageKeyword(rawColumn), GetComment(rawColumn)));
                                return null;
                            }
                        }
                    }

                    return memoryStream.ToArray();
                }
            }
        }

        public DataProcessor GetDataProcessor(int rawColumn)
        {
            return m_DataProcessor[rawColumn];
        }

        public string GetNameSpace(int rawColumn)
        {
            return m_DataProcessor[rawColumn].GetType().GetProperty("NameSpace")
                ?.GetValue(m_DataProcessor[rawColumn]) as string;
        }
    }
}