public static bool CookExcelFile(string excelPath)
{
byte[] excelBuffer = null;
try
{
excelBuffer = File.ReadAllBytes(excelPath);
}
catch (Exception ex)
{
ExceptionLogger.LogException(ex);
Console.WriteLine(String.Format("Error reading file {0}", excelPath));
return false;
}
ExcelFile excelFile = new ExcelFile(excelPath);
try
{
excelFile.ParseCSV(excelBuffer, _fileManager);
}
catch (Exception e)
{
Console.WriteLine("Critical Error:\n" + e);
return false;
}
if (excelFile.HasIntegrity == false)
{
Console.WriteLine(String.Format("Failed to parse excel file {0}", excelPath));
return false;
}
Console.WriteLine(String.Format("Cooking {0}", Path.GetFileNameWithoutExtension(excelPath)));
excelBuffer = excelFile.ToByteArray();
if (excelBuffer == null)
{
Console.WriteLine(String.Format("Failed to serialize excel file {0}", excelFile.StringId));
return false;
}
String writeToPath = excelPath + ".cooked";
try
{
File.WriteAllBytes(writeToPath, excelBuffer);
}
catch (Exception ex)
{
ExceptionLogger.LogException(ex);
Console.WriteLine(String.Format("Failed to write cooked file {0} ", writeToPath));
return false;
}
return true;
}
public int _GetExcelRowIndex(ExcelFile excelTable, String value)
{
FieldInfo field = excelTable.Attributes.RowType.GetFields()[0];
bool isStringField = (field.FieldType == typeof(String));
ObjectDelegator excelDelegator = DataFileDelegators[excelTable.StringId];
ObjectDelegator.FieldGetValueDelegate getValue = excelDelegator[field.Name];
int i = 0;
foreach (Object row in excelTable.Rows)
{
if (isStringField)
{
String val = (String)getValue(row);
if (val == value) return i;
}
else // string offset
{
int offset = (int)getValue(row);
String stringVal = excelTable.ReadStringTable(offset);
if (stringVal == value) return i;
}
i++;
}
return -1;
}
/// <summary>
/// Loads all of the available Excel and Strings files to a hashtable.
/// </summary>
/// <returns>Returns true on success.</returns>
public bool LoadTableFiles(bool ignorePatchedOut = false)
{
// want excel files and strings files
foreach (PackFileEntry fileEntry in
FileEntries.Values.Where(fileEntry => fileEntry.Name.EndsWith(ExcelFile.Extension) ||
(fileEntry.Name.EndsWith(StringsFile.Extention) && fileEntry.Path.Contains(Language))))
{
byte[] fileBytes = GetFileBytes(fileEntry, ignorePatchedOut);
if (fileBytes == null)
{
Debug.WriteLine("Warning: Failed to read file bytes in LoadTableFiles(). FileEntry = " + fileEntry.Name);
continue;
}
//if (fileEntry.Name.Contains("stats"))
//{
// int bp = 0;
//}
// parse file data
DataFile dataFile;
try
{
if (fileEntry.Name.EndsWith(ExcelFile.Extension))
{
dataFile = new ExcelFile(fileBytes, fileEntry.Path, ClientVersion);
}
else
{
dataFile = new StringsFile(fileBytes, fileEntry.Path);
}
}
catch (Exceptions.DataFileStringIdNotFound dataFileStringIdNotFound)
{
Debug.WriteLine(dataFileStringIdNotFound.ToString());
ExceptionLogger.LogException(dataFileStringIdNotFound);
continue;
}
catch (Exception ex)
{
Debug.WriteLine(ex.ToString());
ExceptionLogger.LogException(ex);
Console.WriteLine(String.Format("Critical Error: Failed to load data file {0} (ClientVersion = {1})", fileEntry.Name, ClientVersion));
continue;
}
if (!dataFile.HasIntegrity)
{
String failedParse = "Error: Failed to load data file: " + dataFile.StringId;
Debug.WriteLine(failedParse); // Console.WriteLine randomly (more often than not) just doesn't output to the Ouput window in VS during debugging
Console.WriteLine(failedParse);
continue;
}
if (dataFile.Attributes.IsEmpty) continue;
try
{
DataFiles.Add(dataFile.StringId, dataFile);
DataFileDelegators.Add(dataFile.StringId, dataFile.Delegator); // not sure if we need this still...
}
catch (Exception e)
{
Debug.WriteLine("Critical Error: Cannot add table data file to dictionary: " + dataFile + "\n" + e);
Console.WriteLine("Critical Error: Cannot add table data file to dictionary: " + dataFile + "\n" + e);
}
}
Debug.WriteLine("Loaded {0} {1} data files.", DataFiles.Count, ClientVersion);
return true;
}
/// <summary>
/// Extracts all Excel files to their \data\ locations.
/// Primarly a debug function.
/// </summary>
public void ExtractAllExcel(String root = null, bool doCSVAsWell = false)
{
if (root == null) root = HellgatePath;
foreach (PackFileEntry fileEntry in FileEntries.Values)
{
if (!fileEntry.Name.EndsWith(ExcelFile.Extension)) continue;
PackFileEntry extractFileEntry = fileEntry;
//if (fileEntry.Index.ToString().Contains("4580") && fileEntry.Siblings != null)
//{
// extractFileEntry = (from fi in fileEntry.Siblings
// where fi.Index.ToString().Contains("4256")
// select fi).FirstOrDefault();
// if (extractFileEntry == null)
// {
// extractFileEntry = (from fi in fileEntry.Siblings
// where fi.Index.ToString().Contains("000")
// select fi).FirstOrDefault();
// }
// Debug.Assert(extractFileEntry != null);
//}
byte[] fileBytes = GetFileBytes(extractFileEntry, true);
String filePath = Path.Combine(root, extractFileEntry.Path);
Directory.CreateDirectory(Path.GetDirectoryName(filePath));
File.WriteAllBytes(filePath, fileBytes);
if (!doCSVAsWell) continue;
byte[] csvBytes;
try
{
ExcelFile excelFile = new ExcelFile(fileBytes, filePath, ClientVersion);
csvBytes = excelFile.ExportCSV(this);
}
catch (Exception)
{
continue;
}
File.WriteAllBytes(filePath.Replace(ExcelFile.Extension, ExcelFile.ExtensionDeserialised), csvBytes);
}
}
private void _GenerateRelations(ExcelFile excelFile)
{
if ((excelFile == null))
{
return;
}
Type type = excelFile.Attributes.RowType;
int col;
String mainTableName = excelFile.StringId;
DataTable mainDataTable = XlsDataSet.Tables[mainTableName];
// remove all extra generated columns on this table
for (col = 0; col < mainDataTable.Columns.Count; col++)
{
DataColumn dc = mainDataTable.Columns[col];
if (!(dc.ExtendedProperties.Contains(ColumnKeys.IsRelationGenerated)))
{
continue;
}
mainDataTable.Columns.Remove(dc);
}
col = 1;
// regenerate relations
foreach (FieldInfo fieldInfo in type.GetFields(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic))
{
if (fieldInfo.IsPrivate)
{
if (fieldInfo.FieldType == typeof(ExcelFile.RowHeader))
{
col++;
}
continue;
}
OutputAttribute excelOutputAttribute = ExcelFile.GetExcelAttribute(fieldInfo);
if ((excelOutputAttribute == null))
{
col++;
continue;
}
DataColumn dcChild = mainDataTable.Columns[col];
if ((excelOutputAttribute.IsStringIndex))
{
DataTable dtStrings = XlsDataSet.Tables[StringsTableName] ?? _LoadStringsTable();
if (dtStrings != null)
{
DataColumn dcParent = dtStrings.Columns["ReferenceId"];
String relationName = String.Format("{0}_{1}_{2}", excelFile.StringId, dcChild.ColumnName, ColumnKeys.IsStringIndex);
// parent and child are swapped here for StringId relations
DataRelation relation = new DataRelation(relationName, dcChild, dcParent, false);
XlsDataSet.Relations.Add(relation);
DataColumn dcString = mainDataTable.Columns.Add(dcChild.ColumnName + "_string", typeof(String));
dcString.SetOrdinal(col + 1);
dcString.ExtendedProperties.Add(ExcelFile.ColumnTypeKeys.IsRelationGenerated, true);
// need to use MIN (and thus Child) for cases of strings with same reference id (e.g. Items; SINGULAR and PLURAL, etc)
dcString.Expression = "MIN(Child(" + relationName + ").String)";
col++;
}
}
if (excelOutputAttribute.IsTableIndex)
{
String tableStringId = excelOutputAttribute.TableStringId;
DataTable dt = XlsDataSet.Tables[tableStringId] ?? _LoadRelatedTable(tableStringId);
if (dt != null)
{
DataColumn dcParent = dt.Columns["Index"];
String relatedColumn = dt.Columns[2].ColumnName;
// todo
if (dcChild.ExtendedProperties.ContainsKey(ExcelFile.ColumnTypeKeys.IsArray) && (bool)dcChild.ExtendedProperties[ExcelFile.ColumnTypeKeys.IsArray])
{
//if (excelFile.StringId == "ITEMS")
//{
// int bp = 0;
//}
col++;
continue;
}
//String relationNameOld = excelFile.StringId + dcChild.ColumnName + ExcelFile.ColumnTypeKeys.IsTableIndex;
String relationName = String.Format("{0}_{1}_{2}", excelFile.StringId, dcChild.ColumnName, ExcelFile.ColumnTypeKeys.IsTableIndex);
DataRelation relation = new DataRelation(relationName, dcParent, dcChild, false);
//if (relationName == "PLAYERS_paperdollSkill_IsTableIndex")
//{
// int bp = 0;
//}
XlsDataSet.Relations.Add(relation);
DataColumn dcString = mainDataTable.Columns.Add(dcChild.ColumnName + "_string", typeof(String), String.Format("Parent({0}).{1}", relationName, relatedColumn));
dcString.SetOrdinal(col + 1);
dcString.ExtendedProperties.Add(ExcelFile.ColumnTypeKeys.IsRelationGenerated, true);
col++;
}
}
col++;
}
}
/// <summary>
/// This function is to test excel CSV cooking.
/// </summary>
public static void DoCookTest()
{
FileManager fileManager = new FileManager(Config.HglDir);
fileManager.LoadTableFiles();
fileManager.ExtractAllExcel();
ExcelScript.GlobalDebug(true);
foreach (PackFileEntry fileEntry in fileManager.FileEntries.Values)
{
if (!fileEntry.Name.EndsWith(ExcelFile.Extension)) continue;
byte[] fileBytes = fileManager.GetFileBytes(fileEntry, true);
Debug.Assert(fileBytes != null);
String filePath = Path.Combine(Config.HglDir, fileEntry.Path);
ExcelFile excelFile = new ExcelFile(fileBytes, fileEntry.Path);
if (excelFile.Attributes.IsEmpty) continue;
Console.WriteLine("Cooking file: " + fileEntry.Path);
//if (!fileEntry.RelativeFullPathWithoutPatch.Contains("display_item")) continue;
byte[] csvBytes = excelFile.ExportCSV(fileManager);
File.WriteAllBytes(filePath.Replace(ExcelFile.Extension, ExcelFile.ExtensionDeserialised), csvBytes);
ExcelFile excelFileCSV = new ExcelFile(fileEntry.Path);
excelFileCSV.ParseCSV(csvBytes, fileManager);
//byte[] recookedBytes = excelFileCSV.ToByteArray();
byte[] csvBytes2 = excelFileCSV.ExportCSV(fileManager);
//if (!csvBytes.SequenceEqual(csvBytes2))
//{
// int b5p = 0;
//}
//if (excelFile.StringId == "GLOBAL_STRING")
//{
// ExcelFile temp = new ExcelFile(recookedBytes, fileEntry.RelativeFullPathWithoutPatch);
// int bp1 = 0;
//}
//File.WriteAllBytes(filePath, recookedBytes);
}
//int bp = 0;
}
/// <summary>
/// Function to test saving/importing/exported/etc relating to DataTables (i.e. actions from WITHIN Reanimator etc.)
/// </summary>
public static void TestDataTableExportAndImport(bool doTCv4 = false)
{
/*
bytesXls Bytes->Xls
bytesXlsBytes Bytes->Xls->Bytes
bytesXlsCsv Bytes->Xls->CSV
bytesXlsCsvXls Bytes->Xls->CSV->Xls
bytesXlsCsvXlsBytes Bytes->Xls->CSV->Xls->Bytes == bytesXlsTableXlsBytes as single byte is stripped ->CSV
bytesXlsTable Bytes->Xls->Table
bytesXlsTableXls Bytes->Xls->Table->Xls
bytesXlsTableXlsBytes Bytes->Xls->Table->Xls->Bytes != bytesXlsBytes because of single script byte thingy lost to table
bytesXlsTableXlsCsv Bytes->Xls->Table->Xls->CSV
bytesXlsTableXlsCsvXls Bytes->Xls->Table->Xls->CSV->Xls
bytesXlsTableXlsCsvXlsBytes Bytes->Xls->Table->Xls->CSV->Xls->Bytes
byte[] bytesXlsBytes = bytesXls.ToByteArray();
byte[] bytesXlsTableXlsCsvXlsBytes = bytesXlsTableXlsCsvXls.ToByteArray();
*/
String root = @"C:\excel_datatable_debug";
FileManager.ClientVersions clientVersion = FileManager.ClientVersions.SinglePlayer;
if (doTCv4)
{
root = Path.Combine(root, "tcv4");
clientVersion = FileManager.ClientVersions.TestCenter;
}
root += @"\"; // lazy
Directory.CreateDirectory(root);
FileManager fileManager = new FileManager(Config.HglDir, clientVersion);
fileManager.BeginAllDatReadAccess();
fileManager.LoadTableFiles();
List<String> sequenceFailed = new List<String>();
List<String> sequenceChecked = new List<String>();
foreach (KeyValuePair<String, DataFile> keyValuePair in fileManager.DataFiles)
{
String stringId = keyValuePair.Key;
DataFile dataFile = keyValuePair.Value;
if (dataFile.IsStringsFile) continue;
//if (stringId != "WARDROBE_LAYER") continue;
//if (stringId == "WARDROBE_LAYER")
//{
// int bp = 0;
//}
Debug.Write("Checking " + stringId + "... ");
ExcelFile bytesXls = (ExcelFile)dataFile;
byte[] bytes = fileManager.GetFileBytes(bytesXls.FilePath, true);
sequenceChecked.Add(stringId);
sequenceFailed.Add(stringId);
if (stringId == "PROPERTIES") continue; // can't do this atm - need to export Properties script table to DataTable etc.
Debug.Write("DataTable... ");
DataTable bytesXlsTable = fileManager.GetDataTable(stringId);
if (bytesXlsTable == null)
{
Debug.WriteLine("FAILED!");
continue;
}
// parse as DataTable
Debug.Write("new Excel... ");
DataFile bytesXlsTableXls = new ExcelFile(bytesXls.FilePath, fileManager.ClientVersion);
if (!bytesXlsTableXls.ParseDataTable(bytesXlsTable, fileManager))
{
Debug.WriteLine("FAILED!");
continue;
}
// check just data table import/export
byte[] bytesXlsBytes = bytesXls.ToByteArray();
byte[] bytesXlsTableXlsBytes = bytesXlsTableXls.ToByteArray();
if (bytesXls.ScriptBuffer == null && // can't check this as ScriptBuffer[0] is NOT always 0x00 (even though it's unused and means no script normally when pointed to)
!bytesXls.HasStringBuffer) // as above - gets re-arranged etc (only consistant comparint CSV->XLS->DATA
{
Debug.Write("bytesXlsBytes==bytesXlsTableXlsBytes... ");
if (!bytesXlsBytes.SequenceEqual(bytesXlsTableXlsBytes))
{
Debug.WriteLine("FALSE!");
File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes);
File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "2.bytesXlsTableXlsBytes", bytesXlsTableXlsBytes);
continue;
}
}
// export CSV
Debug.Write("ExportCSV... ");
byte[] bytesXlsTableXlsCsv = bytesXlsTableXls.ExportCSV(fileManager);
if (bytesXlsTableXlsCsv == null)
{
Debug.WriteLine("FAILED!");
continue;
}
Debug.Write("bytesXlsCsv==bytesXlsTableXlsCsv... ");
byte[] bytesXlsCsv = bytesXls.ExportCSV(fileManager);
if (!bytesXlsCsv.SequenceEqual(bytesXlsTableXlsCsv))
{
Debug.WriteLine("FALSE!");
File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes);
File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1a.bytesXlsCsv.csv", bytesXlsCsv);
File.WriteAllBytes(root + bytesXls.StringId + "2.bytesXlsTableXlsBytes", bytesXlsTableXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "3.bytesXlsTableXlsCsv.csv", bytesXlsTableXlsCsv);
continue;
}
Debug.Write("bytesXlsCsvXlsBytes==bytesXlsTableXlsBytes... ");
DataFile bytesXlsCsvXls = new ExcelFile(bytesXls.FilePath, fileManager.ClientVersion);
bytesXlsCsvXls.ParseCSV(bytesXlsCsv, fileManager);
byte[] bytesXlsCsvXlsBytes = bytesXlsCsvXls.ToByteArray();
if (!bytesXlsCsvXlsBytes.SequenceEqual(bytesXlsTableXlsBytes) &&
stringId != "ITEMDISPLAY" && stringId != "LEVEL") // ITEMDISPLAY has weird single non-standard ASCII char that is lost (from using MS Work on their part lol)
{ // and LEVEL references blank TREASURE rows and hence rowId references differ on recooks
Debug.WriteLine("FALSE!");
File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes);
File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1a.bytesXlsCsv.csv", bytesXlsCsv);
File.WriteAllBytes(root + bytesXls.StringId + "1b.bytesXlsCsvXlsBytes", bytesXlsCsvXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "2.bytesXlsTableXlsBytes", bytesXlsTableXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "3.bytesXlsTableXlsCsv.csv", bytesXlsTableXlsCsv);
continue;
}
// import CSV
Debug.Write("ParseCSV... ");
DataFile bytesXlsTableXlsCsvXls = new ExcelFile(bytesXls.FilePath, fileManager.ClientVersion);
if (!bytesXlsTableXlsCsvXls.ParseCSV(bytesXlsTableXlsCsv, fileManager))
{
Debug.WriteLine("FAILED!");
continue;
}
// export imported as cooked and do byte compare
Debug.Write("ToByteArray... ");
byte[] bytesXlsTableXlsCsvXlsBytes = bytesXlsTableXlsCsvXls.ToByteArray();
Debug.Write("bytesXlsCsvXlsBytes==bytesXlsTableXlsCsvXlsBytes... ");
if (!bytesXlsCsvXlsBytes.SequenceEqual(bytesXlsTableXlsCsvXlsBytes))
{
Debug.WriteLine("FALSE!");
File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes);
File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "2.bytesXlsTableXlsBytes", bytesXlsTableXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "3.bytesXlsTableXlsCsv", bytesXlsTableXlsCsv);
File.WriteAllBytes(root + bytesXls.StringId + "4.bytesXlsTableXlsCsvXlsBytes", bytesXlsTableXlsCsvXlsBytes);
continue;
}
// yay
Debug.WriteLine("OK!");
sequenceFailed.RemoveAt(sequenceFailed.Count - 1);
}
Debug.WriteLine("Totals: {0} sequence checks, {1} fails. ({2}% failed!)", sequenceChecked.Count, sequenceFailed.Count, (float)sequenceFailed.Count * 100f / sequenceChecked.Count);
Debug.Write("Failed Tables: ");
foreach (String stringIdFailed in sequenceFailed)
{
Debug.Write(stringIdFailed + ", ");
}
}
/// <summary>
/// Initialise private variables and generate excel script functions if required.
/// </summary>
public ExcelScript(FileManager fileManager)
{
if (fileManager == null) throw new ArgumentNullException("fileManager", "File Manager cannot be null.");
if (!fileManager.DataFiles.ContainsKey("STATS")) throw new Exceptions.ScriptNotInitialisedException("The supplied file manager did not have a valid Stats table.");
// these initial counts are required for script compiling when determing if a function is from the .exe or from an Excel table
if (!_haveExcelFunctions)
{
_initCallFunctionsCountSinglePlayer = CallFunctionsSinglePlayer.Count;
_initCallFunctionsTestCenterCount = CallFunctionsTestCenter.Count;
_initCallFunctionsResurrectionCount = CallFunctionsResurrection.Count;
}
_fileManager = fileManager;
if (_fileManager.IsVersionResurrection || _fileManager.IsVersionMod)
{
CallFunctions = CallFunctionsResurrection;
_initCallFunctionsCount = _initCallFunctionsResurrectionCount;
_debugRoot = DebugRootResurrection;
}
else if (_fileManager.IsVersionTestCenter)
{
CallFunctions = CallFunctionsTestCenter;
_initCallFunctionsCount = _initCallFunctionsTestCenterCount;
_debugRoot = DebugRootTestCenter;
}
else
{
CallFunctions = CallFunctionsSinglePlayer;
_initCallFunctionsCount = _initCallFunctionsCountSinglePlayer;
_debugRoot = DebugRoot;
}
_statsTable = (ExcelFile)_fileManager.DataFiles["STATS"];
_statsDelegator = _fileManager.DataFileDelegators["STATS"];
if (_globalDebug) EnableDebug(true);
if (CallFunctions.Count == _initCallFunctionsCount) _GenerateExcelScriptFunctions();
}
// Row from index
public object GetRowFromIndex(Xls.TableCodes tableCode, Int32 index)
{
ExcelFile table = GetExcelTableFromCode(tableCode);
return(index < table.Rows.Count() ? (table.Rows[index]) : null);
}
/// <summary>
/// Loads all of the available Excel and Strings files to a hashtable.
/// </summary>
/// <returns>Returns true on success.</returns>
public bool LoadTableFiles(bool ignorePatchedOut = false)
{
// want excel files and strings files
foreach (PackFileEntry fileEntry in
FileEntries.Values.Where(fileEntry => fileEntry.Name.EndsWith(ExcelFile.Extension) ||
(fileEntry.Name.EndsWith(StringsFile.Extention) && fileEntry.Path.Contains(Language))))
{
byte[] fileBytes = GetFileBytes(fileEntry, ignorePatchedOut);
if (fileBytes == null)
{
Debug.WriteLine("Warning: Failed to read file bytes in LoadTableFiles(). FileEntry = " + fileEntry.Name);
continue;
}
//if (fileEntry.Name.Contains("stats"))
//{
// int bp = 0;
//}
// parse file data
DataFile dataFile;
try
{
if (fileEntry.Name.EndsWith(ExcelFile.Extension))
{
dataFile = new ExcelFile(fileBytes, fileEntry.Path, ClientVersion);
}
else
{
dataFile = new StringsFile(fileBytes, fileEntry.Path);
}
}
catch (Exceptions.DataFileStringIdNotFound dataFileStringIdNotFound)
{
Debug.WriteLine(dataFileStringIdNotFound.ToString());
ExceptionLogger.LogException(dataFileStringIdNotFound);
continue;
}
catch (Exception ex)
{
Debug.WriteLine(ex.ToString());
ExceptionLogger.LogException(ex);
Console.WriteLine(String.Format("Critical Error: Failed to load data file {0} (ClientVersion = {1})", fileEntry.Name, ClientVersion));
continue;
}
if (!dataFile.HasIntegrity)
{
String failedParse = "Error: Failed to load data file: " + dataFile.StringId;
Debug.WriteLine(failedParse); // Console.WriteLine randomly (more often than not) just doesn't output to the Ouput window in VS during debugging
Console.WriteLine(failedParse);
continue;
}
if (dataFile.Attributes.IsEmpty)
{
continue;
}
try
{
DataFiles.Add(dataFile.StringId, dataFile);
DataFileDelegators.Add(dataFile.StringId, dataFile.Delegator); // not sure if we need this still...
}
catch (Exception e)
{
Debug.WriteLine("Critical Error: Cannot add table data file to dictionary: " + dataFile + "\n" + e);
Console.WriteLine("Critical Error: Cannot add table data file to dictionary: " + dataFile + "\n" + e);
}
}
Debug.WriteLine("Loaded {0} {1} data files.", DataFiles.Count, ClientVersion);
return(true);
}
public void ProcessTables()
{
// excel tables
ExcelFile excelTables = (ExcelFile)DataFiles[ExcelTablesStringId];
foreach (ExcelTablesRow excelTableRow in excelTables.Rows)
{
Xls.TableCodes tableCode = (Xls.TableCodes)excelTableRow.code;
ExcelFile excelTable = GetExcelTableFromCode(tableCode);
if (excelTable == null)
{
continue;
}
excelTable.TableCode = tableCode;
}
// stats table
ExcelFile statsTable = (ExcelFile)DataFiles[StatsTableStringId];
foreach (StatsRow stat in statsTable.Rows)
{
int bitCount = _GetTableRowBitMax(stat.valTable, false);
if (bitCount != -1)
{
stat.valbits = bitCount;
}
bitCount = _GetTableRowBitMax(stat.param1Table, true);
if (bitCount != -1)
{
stat.param1Bits = bitCount;
}
if (stat.param1Bits > 0)
{
stat.ParamCount++;
}
bitCount = _GetTableRowBitMax(stat.param2Table, true);
if (bitCount != -1)
{
stat.param2Bits = bitCount;
}
if (stat.param2Bits > 0)
{
stat.ParamCount++;
}
bitCount = _GetTableRowBitMax(stat.param3Table, true);
if (bitCount != -1)
{
stat.param3Bits = bitCount;
}
if (stat.param3Bits > 0)
{
stat.ParamCount++;
}
bitCount = _GetTableRowBitMax(stat.param4Table, true);
if (bitCount != -1)
{
stat.param4Bits = bitCount;
}
if (stat.param4Bits > 0)
{
stat.ParamCount++;
}
if (stat.valTable != -1)
{
stat.ValueExcelTable = GetExcelTableFromIndex(stat.valTable);
}
if (stat.param1Table != -1)
{
stat.Param1ExcelTable = GetExcelTableFromIndex(stat.param1Table);
}
if (stat.param2Table != -1)
{
stat.Param2ExcelTable = GetExcelTableFromIndex(stat.param2Table);
}
if (stat.param3Table != -1)
{
stat.Param3ExcelTable = GetExcelTableFromIndex(stat.param3Table);
}
if (stat.param4Table != -1)
{
stat.Param4ExcelTable = GetExcelTableFromIndex(stat.param4Table);
}
}
// level drlg choice table
ExcelFile levelDrlgChoiceTable = (ExcelFile)DataFiles["LEVEL_DRLG_CHOICE"];
foreach (LevelDrlgChoiceRow levelDrlgChoice in levelDrlgChoiceTable.Rows)
{
if (levelDrlgChoice.nameOffset == -1)
{
continue;
}
levelDrlgChoice.name = levelDrlgChoiceTable.ReadStringTable(levelDrlgChoice.nameOffset);
Debug.Assert(!String.IsNullOrEmpty(levelDrlgChoice.name));
}
}
/// <summary>
/// Converts the Excel file to another Excel type, keeping all but the rows intact.
/// Used for TCv4 -> SP conversion.
/// </summary>
/// <param name="excelFile">An Excel object of type to convert to.</param>
/// <param name="newRows">The rows to replace the old.</param>
public void ConvertType(ExcelFile excelFile, IEnumerable<Object> newRows)
{
Attributes = excelFile.Attributes;
_excelFileHeader = excelFile._excelFileHeader;
Rows = new List<Object>(newRows);
}
private void _ReadStatValue(BitBuffer bitBuffer, int bitCount, ref int value, ExcelFile valueTable, ref Object valueRow, StatContext statContext)
{
if (bitCount <= 0) return;
value = bitBuffer.ReadBits(bitCount);
if (valueTable == null) return;
if (statContext == StatContext.UseRows)
{
if (value >= 0 && value < valueTable.Rows.Count)
{
valueRow = valueTable.Rows[value];
}
if (valueRow == null) Debug.WriteLine(String.Format("Warning: OutOfBounds stat index {0} on table {1}", value, valueTable.TableCode));
}
else
{
valueRow = _fileManager.GetRowFromCode(valueTable.TableCode, (short)value);
if (valueRow == null) Debug.WriteLine(String.Format("Warning: Unknown stat value code {0} (0x{0:X4}) on table {1}", value, valueTable.TableCode));
}
}
private void _WriteStatValue(BitBuffer bitBuffer, int value, Object valueRow, ExcelFile valueTable, int bitCount, StatContext statContext)
{
if (bitCount <= 0) return;
if (valueTable == null)
{
bitBuffer.WriteBits(value, bitCount);
return;
}
if (valueRow == null)
{
bitBuffer.WriteBits(0, bitCount);
return;
}
if (statContext == StatContext.UseRows)
{
int rowIndex = valueTable.Rows.IndexOf(valueRow);
bitBuffer.WriteBits(rowIndex, bitCount);
}
else
{
int code = _fileManager.GetCodeFromRow(valueTable.TableCode, valueRow);
bitBuffer.WriteBits(code, bitCount);
}
}
private String _GetExcelStringFromExcelFile(ExcelFile excelTable, int rowIndex, int colIndex)
{
if (excelTable == null || rowIndex >= excelTable.Rows.Count) return null;
ObjectDelegator tableDelegator = DataFileDelegators[excelTable.StringId];
ObjectDelegator.FieldDelegate fieldDelegate = tableDelegator.GetPublicFieldDelegate(colIndex);
bool isStringField = (fieldDelegate.FieldType == typeof(String));
Object row = excelTable.Rows[rowIndex];
if (isStringField) return (String)fieldDelegate.GetValue(row);
int offset = (int)fieldDelegate.GetValue(row);
String stringVal = excelTable.ReadStringTable(offset);
return stringVal;
}
private static void _CookExcelFiles(IEnumerable<String> excelFilesToCook)
{
Dictionary<String, ExcelFile> excelFiles = new Dictionary<String, ExcelFile>();
Console.WriteLine("Reading Excel CSV content...");
foreach (String excelPath in excelFilesToCook)
{
Console.Write(Path.GetFileName(excelPath) + "... ");
byte[] fileBytes;
try
{
fileBytes = File.ReadAllBytes(excelPath);
}
catch (Exception e)
{
ExceptionLogger.LogException(e);
Console.WriteLine("\nFailed to read file contents!\nIgnore and Continue? [Y/N]: ");
char c = (char)Console.Read();
if (c == 'y' || c == 'Y') continue;
return;
}
ExcelFile excelFile = new ExcelFile(excelPath);
try
{
excelFile.LoadCSV(fileBytes);
}
catch (Exception e)
{
ExceptionLogger.LogException(e);
Console.WriteLine("\nFailed to load CSV contents!\nIgnore and Continue? [Y/N]: ");
char c = (char)Console.Read();
if (c == 'y' || c == 'Y') continue;
return;
}
excelFiles.Add(excelFile.StringId, excelFile);
}
if (excelFiles.Count == 0) return;
Console.WriteLine("\nProcessing Excel CSV content...");
foreach (ExcelFile excelFile in excelFiles.Values)
{
Console.WriteLine("Cooking " + Path.GetFileName(excelFile.FilePath));
try
{
excelFile.ParseCSV(_fileManager, excelFiles);
}
catch (Exception e)
{
ExceptionLogger.LogException(e);
Console.WriteLine("Failed to parse CSV data!\nIgnore and Continue? [Y/N]: ");
char c = (char)Console.Read();
if (c == 'y' || c == 'Y') continue;
return;
}
if (excelFile.HasIntegrity == false)
{
Console.WriteLine("Failed to parse CSV data!\nIgnore and Continue? [Y/N]: ");
char c = (char)Console.Read();
if (c == 'y' || c == 'Y') continue;
return;
}
byte[] cookedFileBytes;
try
{
cookedFileBytes = excelFile.ToByteArray();
}
catch (Exception e)
{
ExceptionLogger.LogException(e);
Console.WriteLine("Failed to serialise CSV data!\nIgnore and Continue? [Y/N]: ");
char c = (char)Console.Read();
if (c == 'y' || c == 'Y') continue;
return;
}
if (cookedFileBytes == null)
{
Console.WriteLine("Failed to serialise CSV data!\nIgnore and Continue? [Y/N]: ");
char c = (char)Console.Read();
if (c == 'y' || c == 'Y') continue;
return;
}
String savePath = excelFile.FilePath.Replace(ExcelFile.ExtensionDeserialised, ExcelFile.Extension);
try
{
File.WriteAllBytes(savePath, cookedFileBytes);
}
catch (Exception e)
{
ExceptionLogger.LogException(e);
Console.WriteLine("Failed to write cooked excel file!\nIgnore and Continue? [Y/N]: ");
char c = (char)Console.Read();
if (c == 'y' || c == 'Y') continue;
return;
}
}
}
/// <summary>
/// Main worker function that parses and appends excel script functions to the global call function array.
/// </summary>
/// <param name="excelFile">The excel file containing the excel script functions to parse.</param>
/// <returns>The number of functions added.</returns>
private void _ParseExcelFunctions(ExcelFile excelFile)
{
const bool debugOutputLikeFunction = true;
foreach (ExcelFile.ExcelFunction excelFunction in excelFile.ExcelFunctions)
{
//if (excelFunction.Parameters[0].Name == "holy_radius_enemies")
//{
// int bp = 0;
//}
// not sure what to do with them - they don't appear to be used
if (excelFunction.Parameters[0].TypeValues.Length <= 4) continue;
Function function = new Function();
List<Argument> arguments = new List<Argument>();
for (int i = 0; i < excelFunction.Parameters.Count; i++)
{
ExcelFile.ExcelFunction.Parameter parameter = excelFunction.Parameters[i];
if (i == 0)
{
function.Name = parameter.Name;
int functionIndex = parameter.TypeValues[4];
if (functionIndex != CallFunctions.Count) throw new Exceptions.ScriptFunctionCountError(functionIndex, CallFunctions.Count);
continue;
}
Argument arg = new Argument
{
Name = parameter.Name,
Type = 0, // I think they're all int
ByteOffset = parameter.TypeValues[3],
};
arguments.Add(arg);
}
String byteCodeString = null;
if (DebugEnabled && debugOutputLikeFunction)
{
int offset = 0;
byteCodeString = FileTools.ByteArrayToInt32Array(excelFunction.ScriptByteCode, ref offset, excelFunction.ScriptByteCode.Length / 4).ToString(",");
}
function.Args = arguments.ToArray();
function.ExcelScript = _Decompile(excelFunction, function, byteCodeString, "PROPERTIES", 0, 0, function.Name);
if (DebugEnabled && debugOutputLikeFunction)
{
String[] code = function.ExcelScript.Split(new[] { "; ", ";", "\n" }, StringSplitOptions.RemoveEmptyEntries);
String codeStr = code.Aggregate(String.Empty, (current, line) => current + ("\t" + line + ";\n"));
String args = String.Empty;
for (int i = 0; i < function.ArgCount; i++)
{
args += "int " + function.Args[i].Name;
if (i != function.ArgCount - 1) args += ", ";
}
String funcString = String.Format("void {0}({1})\n{{\n{2}}}\n", function.Name, args, codeStr);
Debug.WriteLine(funcString);
}
CallFunctions.Add(function);
}
}
private void _DoQuickExcel(ProgressForm progressForm, Object param)
{
FileManager fileManager = (FileManager)param;
String root = _quickExcelDir_textBox.Text;
if (root == "") return;
if (fileManager.IsVersionTestCenter) root = Path.Combine(root, "tcv4");
foreach (IndexFile file in fileManager.IndexFiles) // Open Dats for reading
file.BeginDatReading();
int i = 0;
foreach (PackFileEntry fileEntry in fileManager.FileEntries.Values)
{
if (!fileEntry.Name.EndsWith(ExcelFile.Extension) &&
(!fileEntry.Name.EndsWith(StringsFile.Extention) || !fileEntry.Directory.Contains(fileManager.Language))) continue;
if (i++ % 10 == 0 && progressForm != null) progressForm.SetCurrentItemText(fileEntry.Path);
byte[] fileBytes;
try
{
fileBytes = fileManager.GetFileBytes(fileEntry, true);
}
catch (Exception e)
{
String error = String.Format("Failed to get '{0}' file bytes, continue?\n\n{1}", fileEntry.Path, e);
DialogResult dr = MessageBox.Show(error, "Excel Extration Error", MessageBoxButtons.YesNo, MessageBoxIcon.Exclamation);
if (dr == DialogResult.No) return;
continue;
}
ExcelFile excelFile = null;
if (fileEntry.Name.EndsWith(ExcelFile.Extension))
{
excelFile = new ExcelFile(fileBytes, fileEntry.Path, fileManager.ClientVersion);
if (excelFile.Attributes.IsEmpty) continue;
}
byte[] writeBytes = fileBytes;
if (excelFile != null)
{
String[] columns = null;
if (fileManager.IsVersionTestCenter)
{
DataFile spVersion;
if (_fileManager.DataFiles.TryGetValue(excelFile.StringId, out spVersion))
{
ObjectDelegator objectDelegator = _fileManager.DataFileDelegators[spVersion.StringId];
FieldInfo[] fieldInfos = spVersion.Attributes.RowType.GetFields();
columns = new String[objectDelegator.PublicFieldCount];
int col = 0;
foreach (ObjectDelegator.FieldDelegate fieldDelegate in objectDelegator.FieldDelegatesPublicList)
{
columns[col++] = fieldDelegate.Name;
}
}
}
try
{
writeBytes = excelFile.ExportCSV(fileManager, columns);
}
catch (Exception e)
{
String error = String.Format("Failed to export CSV for '{0}', continue?\n\n{1}", fileEntry.Path, e);
DialogResult dr = MessageBox.Show(error, "Excel Extration Error", MessageBoxButtons.YesNo, MessageBoxIcon.Exclamation);
if (dr == DialogResult.No) return;
continue;
}
}
String filePath = Path.Combine(root, fileEntry.Path).Replace(ExcelFile.Extension, ExcelFile.ExtensionDeserialised);
Directory.CreateDirectory(Path.GetDirectoryName(filePath));
File.WriteAllBytes(filePath, writeBytes);
}
foreach (IndexFile file in fileManager.IndexFiles) // Close Dats
file.EndDatAccess();
}
/// <summary>
/// Decompiles an excel script function from byte codes to human readable text.
/// </summary>
/// <param name="excelScriptFunction">The excel script function to decompile.</param>
/// <param name="excelFunction">The associated excel function to use for agrument reference.</param>
/// <param name="debugScriptByteString">For debugging purposes only.</param>
/// <param name="debugStringId">For debugging purposes only.</param>
/// <param name="debugRow">For debugging purposes only.</param>
/// <param name="debugCol">For debugging purposes only.</param>
/// <param name="debugColName">For debugging purposes only.</param>
/// <returns>Decompiled excel script byte codes as human readable script.</returns>
private String _Decompile(ExcelFile.ExcelFunction excelScriptFunction, Function excelFunction, String debugScriptByteString = null, String debugStringId = null, int debugRow = 0, int debugCol = 0, String debugColName = null)
{
if (excelScriptFunction == null) throw new ArgumentNullException("excelScriptFunction", "The excel script function cannot be null!");
if (excelFunction == null) throw new ArgumentNullException("excelFunction", "The excel function cannot be null!");
_vars = new List<StackObject>();
_stack = new Stack<StackObject>();
_excelScriptFunction = excelScriptFunction;
_excelFunction = excelFunction;
_startOffset = 0;
_offset = 0;
if (DebugEnabled)
{
_debugScriptByteString = debugScriptByteString;
_debugStringId = debugStringId;
_debugRow = debugRow;
_debugCol = debugCol;
_debugColName = debugColName;
}
return _script = _Decompile(_excelScriptFunction.ScriptByteCode, _excelScriptFunction.ScriptByteCode.Length, 0).Value;
}
private DataTable _LoadExcelTable(ExcelFile excelFile, bool doRelations, bool force)
{
String tableName = excelFile.StringId;
DataTable dataTable = XlsDataSet.Tables[tableName];
if (dataTable != null && !force) return dataTable;
if (dataTable != null)
{
XlsDataSet.Relations.Clear();
XlsDataSet.Tables.Remove(tableName);
}
dataTable = XlsDataSet.Tables.Add(tableName);
dataTable.TableName = tableName;
dataTable.ExtendedProperties.Add("FileHeader", excelFile._excelFileHeader.DeepClone());
Type dataType = excelFile.Attributes.RowType;
List<OutputAttribute> outputAttributes = new List<OutputAttribute>();
#region Generate Columns
DataColumn indexColumn = dataTable.Columns.Add("Index");
indexColumn.AutoIncrement = true;
indexColumn.Unique = true;
dataTable.PrimaryKey = new[] { indexColumn };
outputAttributes.Add(null);
FieldInfo[] fieldInfos = dataType.GetFields(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic);
foreach (FieldInfo fieldInfo in fieldInfos)
{
OutputAttribute excelAttribute = ExcelFile.GetExcelAttribute(fieldInfo);
// The only private field we add is the TableHeader
if (fieldInfo.IsPrivate)
{
if (fieldInfo.FieldType != typeof(ExcelFile.RowHeader)) continue;
outputAttributes.Add(null);
dataTable.Columns.Add(fieldInfo.Name, typeof(String));
continue;
}
Type fieldType = fieldInfo.FieldType;
bool isArray = false;
bool isEnum = false;
if (fieldInfo.FieldType.BaseType == typeof(Array))
{
fieldType = typeof(String);
isArray = true;
}
else if (fieldInfo.FieldType.BaseType == typeof(Enum) && excelAttribute == null)
{
fieldType = fieldInfo.FieldType;
isEnum = true;
}
DataColumn dataColumn = dataTable.Columns.Add(fieldInfo.Name, fieldType);
if (isArray)
{
dataColumn.ExtendedProperties.Add(ColumnKeys.IsArray, true);
}
else if (isEnum)
{
dataColumn.ExtendedProperties.Add(ColumnKeys.IsEnum, true);
}
if (excelAttribute == null)
{
outputAttributes.Add(null);
continue;
}
outputAttributes.Add(excelAttribute);
if (excelAttribute.IsStringOffset)
{
dataColumn.DataType = typeof(String);
dataColumn.ExtendedProperties.Add(ColumnKeys.IsStringOffset, true);
dataColumn.DefaultValue = String.Empty;
}
if (excelAttribute.IsScript)
{
dataColumn.DataType = typeof(String);
dataColumn.ExtendedProperties.Add(ColumnKeys.IsScript, true);
dataColumn.DefaultValue = String.Empty;
}
if (excelAttribute.IsSecondaryString)
{
dataColumn.DataType = typeof(String);
dataColumn.ExtendedProperties.Add(ColumnKeys.IsSecondaryString, true);
dataColumn.DefaultValue = String.Empty;
}
if (excelAttribute.IsStringIndex)
{
dataColumn.ExtendedProperties.Add(ColumnKeys.IsStringIndex, true);
// Add new column for the string
DataColumn dataColumnString = dataTable.Columns.Add(fieldInfo.Name + "_string", typeof(String));
dataColumnString.DefaultValue = String.Empty;
outputAttributes.Add(null);
dataColumnString.ExtendedProperties.Add(ColumnKeys.IsRelationGenerated, true);
}
if (excelAttribute.IsTableIndex)
{
dataColumn.ExtendedProperties.Add(ColumnKeys.IsTableIndex, true);
// Add new column for the string
DataColumn dataColumnString = dataTable.Columns.Add(fieldInfo.Name + "_string", typeof(String));
dataColumnString.DefaultValue = String.Empty;
outputAttributes.Add(null);
dataColumnString.ExtendedProperties.Add(ColumnKeys.IsRelationGenerated, true);
}
if (excelAttribute.IsBitmask)
{
dataColumn.ExtendedProperties.Add(ColumnKeys.IsBitmask, true);
}
if (excelAttribute.IsBool)
{
dataColumn.ExtendedProperties.Add(ColumnKeys.IsBool, true);
}
}
if (excelFile.Attributes.HasStats) // items, missiles, monsters, objects, players
{
DataColumn extendedDataColumn = dataTable.Columns.Add("Stats");
extendedDataColumn.DataType = typeof(String);
extendedDataColumn.ExtendedProperties.Add(ExcelFile.ColumnTypeKeys.IsStats, true);
outputAttributes.Add(null);
}
#endregion
#region Generate Rows
int row = 1;
object[] baseRow = new object[outputAttributes.Count];
ObjectDelegator objectDelegator = new ObjectDelegator(fieldInfos);
foreach (Object tableRow in excelFile.Rows)
{
int col = 1;
foreach (FieldInfo fieldInfo in fieldInfos)
{
Object value = objectDelegator[fieldInfo.Name](tableRow);
if (fieldInfo.IsPrivate)
{
if (fieldInfo.FieldType != typeof(ExcelFile.RowHeader)) continue;
baseRow[col++] = FileTools.ObjectToStringGeneric(value, ",");
continue;
}
OutputAttribute excelOutputAttribute = outputAttributes[col];
if (excelOutputAttribute == null)
{
if (value.GetType().BaseType == typeof(Array))
{
value = ((Array)value).ToString(",");
}
baseRow[col++] = value;
continue;
}
if (excelOutputAttribute.IsStringOffset)
{
int valueInt = (int)value;
baseRow[col++] = (valueInt != -1) ? excelFile.ReadStringTable(valueInt) : String.Empty;
continue;
}
if (excelOutputAttribute.IsSecondaryString)
{
int valueInt = (int)value;
baseRow[col++] = (valueInt != -1) ? excelFile.ReadSecondaryStringTable(valueInt) : String.Empty;
continue;
}
if (excelOutputAttribute.IsScript)
{
int scriptOffset = (int)value;
if (scriptOffset == 0)
{
baseRow[col++] = String.Empty;
continue;
}
String script;
if (scriptOffset == 9649 && excelFile.StringId == "SKILLS") // todo: not sure what's with this script...
{
/* Compiled Bytes:
* 26,30,700,6,26,1,399,358,669,616562688,711,26,62,3,17,669,641728512,26,8,711,26,62,3,17,358,669,322961408,26,5,700,6,26,1,399,358,388,0
*
* Ending Stack (FIFO):
* SetStat669('sfx_attack_pct', 'all', 30 * ($sklvl - 1))
* SetStat669('sfx_duration_pct', 'all', get_skill_level(@unit, 'Shield_Mastery'))
* SetStat669('damage_percent_skill', 8 * get_skill_level(@unit, 'Shield_Mastery'))
* SetStat669('damage_percent_skill', 8 * get_skill_level(@unit, 'Shield_Mastery')) + 5 * ($sklvl - 1)
*
* The last SetStat has strange overhang - decompiling wrong?
* Or is it "supposed" to be there?
* i.e. It's actually decompiling correctly, but because I've assumed such scripts to be wrong (as the end +5... segment is useless) we get the Stack exception
*/
int[] scriptCode = excelFile.ReadScriptTable(scriptOffset);
script = scriptCode != null ? FileTools.ArrayToStringGeneric(scriptCode, ",") : "ScriptError";
baseRow[col++] = script;
continue;
}
//if (fieldInfo.Name == "props1" && row == 45)
//{
// int bp = 0;
//}
ExcelScript excelScript = new ExcelScript(this);
try
{
if (ExcelScript.DebugEnabled)
{
int[] scriptCode = excelFile.ReadScriptTable(scriptOffset);
script = scriptCode != null ? FileTools.ArrayToStringGeneric(scriptCode, ",") : String.Empty;
script = excelScript.Decompile(excelFile.ScriptBuffer, scriptOffset, script, excelFile.StringId, row, col, fieldInfo.Name);
}
else
{
script = excelScript.Decompile(excelFile.ScriptBuffer, scriptOffset);
}
//if (script.StartsWith("GetStat666('skill_points_bonus_total', '') > -1;"))
//{
// int bp = 0;
//}
}
catch (Exception)
{
int[] scriptCode = excelFile.ReadScriptTable(scriptOffset);
script = scriptCode != null ? FileTools.ArrayToStringGeneric(scriptCode, ",") : "ScriptError";
}
baseRow[col++] = script;
continue;
}
if (excelOutputAttribute.IsTableIndex || excelOutputAttribute.IsStringIndex)
{
if (value.GetType().BaseType == typeof(Array))
{
value = ((Array)value).ToString(",");
}
else
{
value = (int)value;
}
baseRow[col++] = value;
col++; // for _strings relational column
continue;
}
// Else its something else, ie bitmask, bool, table/string index
baseRow[col++] = value;
}
// stats, only a component of the UnitData row type
if (excelFile.Attributes.HasStats)
{
baseRow[col++] = FileTools.ArrayToStringGeneric(excelFile.ReadStats(row - 1), ",");
}
dataTable.Rows.Add(baseRow);
row++;
}
#endregion
// Generate Relationships as required
if (doRelations) _GenerateRelations(excelFile);
return dataTable;
}
/// <summary>
/// Function to test all excel files cooking/recooking/etc to/from byte arrays and csv types.
/// </summary>
public static void TestExcelCooking(bool doTCv4 = false)
{
/* 0
0 byte[] fileBytes = fileManager.GetFileBytes(excelFile.FilePath, true); Bytes 0-gen bytes
0 1 2 3 4
foreach (DataFile dataFile in fileManager.DataFiles.Values) Bytes->Xls 1-gen Xls file
1 byte[] dataFileBytes = dataFile.ToByteArray(); Bytes->Xls->Bytes 1-gen bytes
fromBytesExcel.ParseData(dataFileBytes); Bytes->Xls->Bytes->Xls 2-gen Xls file
1a byte[] bytesXlsBytesXlsBytes = fromBytesExcel.ToByteArray(); Bytes->Xls->Bytes->Xls->Bytes 2-gen bytes
1ba byte[] bytesXlsCsv = dataFile.ExportCSV(fileManager); Bytes->Xls->CSV 1-gen CSV-bytes
bytesXlsCsvXls.ParseCSV(bytesXlsCsv, fileManager); Bytes->Xls->CSV ->Xls 2-gen Xls file
1bb byte[] bytesXlsCsvXlsBytes = bytesXlsCsvXls.ToByteArray(); Bytes->Xls->CSV ->Xls->Bytes 2-gen CSV-bytes
1b byte[] csvBytes = fromBytesExcel.ExportCSV(fileManager); Bytes->Xls->Bytes->Xls->CSV 2-gen CSV file
csvExcel.ParseCSV(csvBytes, fileManager); Bytes->Xls->Bytes->Xls->CSV->Xls 3-gen Xls file
1c byte[] bytesXlsBytesXlsCsvXlsBytes = csvExcel.ToByteArray(); Bytes->Xls->Bytes->Xls->CSV->Xls->Bytes 3-gen bytes
finalExcel.ParseData(recookedExcelBytes); Bytes->Xls->Bytes->Xls->CSV->Xls->Bytes->Xls 4-gen Xls file
1d byte[] csvCheck = finalExcel.ExportCSV(fileManager); Bytes->Xls->Bytes->Xls->CSV->Xls->Bytes->Xls->CSV 4-gen CSV file
1e byte[] finalCheck = finalExcel.ToByteArray(); Bytes->Xls->Bytes->Xls->CSV->Xls->Bytes->Xls->Bytes 4-gen bytes
*/
String root = @"C:\excel_debug";
FileManager.ClientVersions clientVersion = FileManager.ClientVersions.SinglePlayer;
if (doTCv4)
{
root = Path.Combine(root, "tcv4");
clientVersion = FileManager.ClientVersions.TestCenter;
}
root += @"\"; // lazy
Directory.CreateDirectory(root);
FileManager fileManager = new FileManager(Config.HglDir, clientVersion);
fileManager.BeginAllDatReadAccess();
fileManager.LoadTableFiles();
ExcelFile.EnableDebug = true;
int checkedCount = 0;
List<String> excelFailed = new List<String>();
foreach (DataFile bytesXls in fileManager.DataFiles.Values)
{
ExcelFile excelFile = bytesXls as ExcelFile;
if (excelFile == null) continue;
const String debugStr = "SKILLS";
//if (excelFile.StringId != debugStr) continue;
//if (excelFile.StringId == debugStr)
//{
// int bp = 0;
//}
checkedCount++;
Debug.Write(String.Format("Checking {0}... ", bytesXls.StringId));
byte[] bytes = fileManager.GetFileBytes(excelFile.FilePath, true);
try
{
byte[] bytesXlsBytes = bytesXls.ToByteArray();
if (bytesXls.StringId == "SOUNDS" && false)
{
byte[] csvBytesSounds = excelFile.ExportCSV(fileManager);
ExcelFile soundsCSV = new ExcelFile(excelFile.FilePath);
soundsCSV.ParseCSV(csvBytesSounds, fileManager);
byte[] soundsBytes = soundsCSV.ToByteArray();
//byte[] soundsBytesFromCSV = soundsCSV.ExportCSV();
//ExcelFile soundsCSVFromBytesFromCSV = new ExcelFile(soundsBytesFromCSV, fileEntry.RelativeFullPathWithoutPatch);
// some brute force ftw
byte[][] bytesArrays = new[] { bytes, soundsBytes };
for (int z = 0; z < bytesArrays.Length; z++)
{
byte[] bytesBrute = bytesArrays[z];
int offset = 0x20;
int stringsBytesCount = FileTools.ByteArrayToInt32(bytesBrute, ref offset);
StringWriter stringWriterByteStrings = new StringWriter();
stringWriterByteStrings.WriteLine(stringsBytesCount + " bytes");
List<String> strings = new List<String>();
List<int> offsets = new List<int>();
while (offset < stringsBytesCount + 0x20)
{
String str = FileTools.ByteArrayToStringASCII(bytesBrute, offset);
strings.Add(str);
offsets.Add(offset);
offset += str.Length + 1;
}
String[] sortedStrings = strings.ToArray();
int[] sortedOffsets = offsets.ToArray();
Array.Sort(sortedStrings, sortedOffsets);
stringWriterByteStrings.WriteLine(strings.Count + " strings");
for (int i = 0; i < strings.Count; i++)
{
stringWriterByteStrings.WriteLine(sortedStrings[i] + "\t\t\t" + sortedOffsets[i]);
}
File.WriteAllText(@"C:\excel_debug\strings" + z + ".txt", stringWriterByteStrings.ToString());
}
}
Debug.Write("ToByteArray... ");
if (bytes.Length != bytesXlsBytes.Length && !doTCv4) // some TCv4 tables don't have their sort columns yet
{
Debug.WriteLine("ToByteArray() dataFileBytes has differing length: " + bytesXls.StringId);
File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes);
File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes);
excelFailed.Add(bytesXls.StringId);
continue;
}
ExcelFile bytesXlsBytesXls = new ExcelFile(excelFile.FilePath, fileManager.ClientVersion);
bytesXlsBytesXls.ParseData(bytesXlsBytes);
Debug.Write("new ExcelFile... ");
if (!bytesXlsBytesXls.HasIntegrity)
{
Debug.WriteLine("fromBytesExcel = new Excel from ToByteArray() failed!");
File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes);
File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes);
excelFailed.Add(bytesXls.StringId);
continue;
}
// more checks
Debug.Write("ToByteArray->ToByteArray... ");
byte[] bytesXlsBytesXlsBytes = bytesXlsBytesXls.ToByteArray(); // bytesXlsBytesXlsBytes
if (bytes.Length != bytesXlsBytesXlsBytes.Length && !doTCv4) // some TCv4 tables don't have their sort columns yet
{
Debug.WriteLine("ToByteArray() dataFileBytesFromToByteArray has differing length!");
File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes);
File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1a.toByteArrayFromByteArray", bytesXlsBytesXlsBytes);
excelFailed.Add(bytesXls.StringId);
continue;
}
// check generated sort index arrays);
Debug.Write("IndexSortArrays... ");
if (excelFile.IndexSortArray != null)
{
if (bytesXlsBytesXls.IndexSortArray == null || excelFile.IndexSortArray.Count != bytesXlsBytesXls.IndexSortArray.Count)
{
Debug.WriteLine("fromBytesExcel has not-matching IndexSortArray count!");
excelFailed.Add(bytesXls.StringId);
continue;
}
bool hasError = false;
for (int i = 0; i < excelFile.IndexSortArray.Count; i++)
{
if (excelFile.IndexSortArray[i].SequenceEqual(bytesXlsBytesXls.IndexSortArray[i])) continue;
Debug.WriteLine(String.Format("IndexSortArray[{0}] NOT EQUAL to original!", i));
hasError = true;
}
if (hasError)
{
File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes);
File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1a.bytesXlsBytesXlsBytes", bytesXlsBytesXlsBytes);
excelFailed.Add(bytesXls.StringId);
continue;
}
}
// some csv stuff
Debug.Write("BaseXls->BaseCSV==");
byte[] bytesXlsCsv = bytesXls.ExportCSV(fileManager); // Bytes->Xls->CSV
Debug.Write("ExportCSV... ");
byte[] bytesXlsBytesXlsCsv = bytesXlsBytesXls.ExportCSV(fileManager);
if (!bytesXlsCsv.SequenceEqual(bytesXlsBytesXlsCsv))
{
Debug.WriteLine("FALSE!");
File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes);
File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1a.bytesXlsBytesXlsBytes", bytesXlsBytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1b.bytesXlsBytesXlsCsv.csv", bytesXlsBytesXlsCsv);
File.WriteAllBytes(root + bytesXls.StringId + "1ba.bytesXlsCsv.csv", bytesXlsCsv);
excelFailed.Add(bytesXls.StringId);
continue;
}
ExcelFile bytesXlsCsvXls = new ExcelFile(excelFile.FilePath, fileManager.ClientVersion);
bytesXlsCsvXls.ParseCSV(bytesXlsCsv, fileManager);
byte[] bytesXlsCsvXlsBytes = bytesXlsCsvXls.ToByteArray(); // used later as well
if (excelFile.ScriptBuffer == null && // can only compare here for no-script tables - can't compare to pure-base xls-bytes as xls->csv->xls rearranges script code
!excelFile.HasStringBuffer) // xls->csv->xls also rearranges the strings buffer
{
Debug.Write("BytesXlsBytesXlsBytes==BytesXlsCsvXlsBytes... ");
if (!bytesXlsBytesXlsBytes.SequenceEqual(bytesXlsCsvXlsBytes))
{
Debug.WriteLine("FALSE!");
File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes);
File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1a.bytesXlsBytesXlsBytes", bytesXlsBytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1b.bytesXlsBytesXlsCsv.csv", bytesXlsBytesXlsCsv);
File.WriteAllBytes(root + bytesXls.StringId + "1ba.bytesXlsCsv.csv", bytesXlsCsv);
File.WriteAllBytes(root + bytesXls.StringId + "1bb.bytesXlsCsvXlsBytes", bytesXlsCsvXlsBytes);
excelFailed.Add(bytesXls.StringId);
continue;
}
Debug.Write("BytesXlsBytes==BytesXlsCsvXlsBytes... ");
if (!bytesXlsBytes.SequenceEqual(bytesXlsCsvXlsBytes))
{
Debug.WriteLine("FALSE!");
File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes);
File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1a.bytesXlsBytesXlsBytes", bytesXlsBytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1b.bytesXlsBytesXlsCsv.csv", bytesXlsBytesXlsCsv);
File.WriteAllBytes(root + bytesXls.StringId + "1ba.bytesXlsCsv.csv", bytesXlsCsv);
File.WriteAllBytes(root + bytesXls.StringId + "1bb.bytesXlsCsvXlsBytes", bytesXlsCsvXlsBytes);
excelFailed.Add(bytesXls.StringId);
continue;
}
}
Debug.Write("BytesXlsBytesXlsCsv -> new ExcelFile");
ExcelFile bytesXlsBytesXlsCsvXls = new ExcelFile(excelFile.FilePath, fileManager.ClientVersion);
bytesXlsBytesXlsCsvXls.ParseCSV(bytesXlsBytesXlsCsv, fileManager);
Debug.Write("... ");
if (!bytesXlsBytesXlsCsvXls.HasIntegrity)
{
Debug.WriteLine("Failed!");
File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes);
File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1a.bytesXlsBytesXlsBytes", bytesXlsBytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1b.bytesXlsBytesXlsCsv.csv", bytesXlsBytesXlsCsv);
File.WriteAllBytes(root + bytesXls.StringId + "1ba.bytesXlsCsv.csv", bytesXlsCsv);
excelFailed.Add(bytesXls.StringId);
continue;
}
byte[] bytesXlsBytesXlsCsvXlsBytes = bytesXlsBytesXlsCsvXls.ToByteArray();
Debug.Write("BytesXlsCsvXlsBytes==BytesXlsBytesXlsCsvXlsBytes... ");
if (!bytesXlsCsvXlsBytes.SequenceEqual(bytesXlsBytesXlsCsvXlsBytes))
{
Debug.WriteLine("FALSE!");
File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes);
File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1a.bytesXlsBytesXlsBytes", bytesXlsBytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1b.bytesXlsBytesXlsCsv.csv", bytesXlsBytesXlsCsv);
File.WriteAllBytes(root + bytesXls.StringId + "1ba.bytesXlsCsv.csv", bytesXlsCsv);
File.WriteAllBytes(root + bytesXls.StringId + "1bb.bytesXlsCsvXlsBytes", bytesXlsCsvXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1c.bytesXlsBytesXlsCsvXlsBytes", bytesXlsBytesXlsCsvXlsBytes);
excelFailed.Add(bytesXls.StringId);
continue;
}
if (!fileManager.IsVersionTestCenter)
{
Debug.Write("StructureId... ");
UInt32 structureId = BitConverter.ToUInt32(bytes, 4);
UInt32 fromCSVStructureId = BitConverter.ToUInt32(bytesXlsBytesXlsCsvXlsBytes, 4);
if (structureId != fromCSVStructureId)
{
Debug.WriteLine("Structure Id value do not match: " + structureId + " != " + fromCSVStructureId);
excelFailed.Add(bytesXls.StringId);
continue;
}
}
Debug.Write("Almost Done... ");
ExcelFile bytesXlsBytesXlsCsvXlsBytesXls = new ExcelFile(excelFile.FilePath, fileManager.ClientVersion);
bytesXlsBytesXlsCsvXlsBytesXls.ParseData(bytesXlsBytesXlsCsvXlsBytes);
byte[] bytesXlsBytesXlsCsvXlsBytesXlsCsv = bytesXlsBytesXlsCsvXlsBytesXls.ExportCSV(fileManager);
int recookedLength = bytesXlsBytesXlsCsvXlsBytes.Length;
if (excelFile.StringId == "SKILLS") recookedLength += 12; // 12 bytes in int ptr data not used/referenced at all and are removed/lost in bytes -> csv -> bytes
if (bytes.Length != recookedLength && !doTCv4) // some TCv4 tables don't have their sort columns yet
{
Debug.WriteLine("Recooked Excel file has differing length!");
File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes);
File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1a.bytesXlsBytesXlsBytes", bytesXlsBytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1b.bytesXlsBytesXlsCsv.csv", bytesXlsBytesXlsCsv);
File.WriteAllBytes(root + bytesXls.StringId + "1ba.bytesXlsCsv.csv", bytesXlsCsv);
File.WriteAllBytes(root + bytesXls.StringId + "1c.bytesXlsBytesXlsCsvXlsBytes", bytesXlsBytesXlsCsvXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1d.bytesXlsBytesXlsCsvXlsBytesXlsCsv.csv", bytesXlsBytesXlsCsvXlsBytesXlsCsv);
excelFailed.Add(bytesXls.StringId);
continue;
}
Debug.Assert(bytesXlsBytesXlsCsvXlsBytesXls.HasIntegrity);
byte[] bytesXlsBytesXlsCsvXlsBytesXlsBytes = bytesXlsBytesXlsCsvXlsBytesXls.ToByteArray();
if (excelFile.StringId == "SKILLS") Debug.Assert(bytesXlsBytesXlsCsvXlsBytesXlsBytes.Length + 12 == bytesXlsBytes.Length);
else Debug.Assert(bytesXlsBytesXlsCsvXlsBytesXlsBytes.Length == bytesXlsBytes.Length || doTCv4);
if (!bytesXlsBytesXlsCsv.SequenceEqual(bytesXlsBytesXlsCsvXlsBytesXlsCsv))
{
Debug.WriteLine("csvBytes.SequenceEqual failed!");
File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes);
File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1a.bytesXlsBytesXlsBytes", bytesXlsBytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1b.bytesXlsBytesXlsCsv.csv", bytesXlsBytesXlsCsv);
File.WriteAllBytes(root + bytesXls.StringId + "1ba.bytesXlsCsv.csv", bytesXlsCsv);
File.WriteAllBytes(root + bytesXls.StringId + "1c.bytesXlsBytesXlsCsvXlsBytes", bytesXlsBytesXlsCsvXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1d.bytesXlsBytesXlsCsvXlsBytesXlsCsv.csv", bytesXlsBytesXlsCsvXlsBytesXlsCsv);
File.WriteAllBytes(root + bytesXls.StringId + "1e.bytesXlsBytesXlsCsvXlsBytesXlsBytes", bytesXlsBytesXlsCsvXlsBytesXlsBytes);
excelFailed.Add(bytesXls.StringId);
continue;
}
Debug.Write("\nPerforming deep scan: ");
ObjectDelegator objectDelegator = fileManager.DataFileDelegators[excelFile.StringId];
int lastPercent = 0;
int col = 0;
bool failed = false;
foreach (FieldDelegate fieldDelegate in objectDelegator)
{
int percent = col * 100 / objectDelegator.FieldCount - 1;
int dotCount = percent - lastPercent;
for (int i = 0; i < dotCount; i++) Debug.Write(".");
lastPercent = percent;
ExcelFile.OutputAttribute excelAttribute = ExcelFile.GetExcelAttribute(fieldDelegate.Info);
bool isArray = (fieldDelegate.FieldType.BaseType == typeof(Array));
for (int row = 0; row < excelFile.Rows.Count; row++)
{
Object obj1 = fieldDelegate.GetValue(excelFile.Rows[row]);
Object obj2 = fieldDelegate.GetValue(bytesXlsBytesXlsCsvXlsBytesXls.Rows[row]);
if (isArray)
{
//if (excelFile.StringId == "TREASURE")
//{
// int bp = 0;
//}
IEnumerator arrayEnumator1 = ((Array)obj1).GetEnumerator();
IEnumerator arrayEnumator2 = ((Array)obj2).GetEnumerator();
int index = -1;
while (arrayEnumator1.MoveNext() && arrayEnumator2.MoveNext())
{
index++;
Object elementVal1 = arrayEnumator1.Current;
Object elementVal2 = arrayEnumator2.Current;
if (elementVal1.Equals(elementVal2)) continue;
Debug.WriteLine(String.Format("Array Element '{0}' != '{1}' on col({2}) = '{3}', row({4}), index({5})", elementVal1, elementVal2, col, fieldDelegate.Name, row, index));
failed = true;
break;
}
if (failed) break;
continue;
}
if (obj1.Equals(obj2)) continue;
if (excelAttribute != null)
{
if (excelAttribute.IsStringOffset)
{
int offset1 = (int)obj1;
int offset2 = (int)obj2;
String str1 = excelFile.ReadStringTable(offset1);
String str2 = bytesXlsBytesXlsCsvXlsBytesXls.ReadStringTable(offset2);
if (str1 == str2) continue;
obj1 = str1;
obj2 = str2;
}
if (excelAttribute.IsScript)
{
int offset1 = (int)obj1;
int offset2 = (int)obj2;
Int32[] script1 = excelFile.ReadScriptTable(offset1);
Int32[] script2 = bytesXlsBytesXlsCsvXlsBytesXls.ReadScriptTable(offset2);
if (script1.SequenceEqual(script2)) continue;
obj1 = script1.ToString(",");
obj2 = script2.ToString(",");
}
}
String str = obj1 as String;
if ((str != null && str.StartsWith("+N%-N% base damage as [elem]")) || // "+N%-N% base damage as [elem]…" != "+N%-N% base damage as [elem]?" on col(2) = 'exampleDescription', row(132) // the '?' at end
fieldDelegate.Name == "bonusStartingTreasure") // refernces multiple different blank rows
continue;
Debug.WriteLine(String.Format("'{0}' != '{1}' on col({2}) = '{3}', row({4})", obj1, obj2, col, fieldDelegate.Name, row));
failed = true;
break;
}
if (failed) break;
col++;
}
if (failed)
{
File.WriteAllBytes(root + bytesXls.StringId + ".bytesXlsBytesXlsCsv.csv", bytesXlsBytesXlsCsv);
File.WriteAllBytes(root + bytesXls.StringId + ".bytesXlsBytesXlsCsvXlsBytesXlsCsv.csv", bytesXlsBytesXlsCsvXlsBytesXlsCsv);
excelFailed.Add(bytesXls.StringId);
continue;
}
Debug.WriteLine("OK\n");
}
catch (Exception e)
{
Debug.WriteLine("Excel file Exception: " + bytesXls.StringId + "\n" + e);
}
}
Debug.WriteLine(checkedCount + " excel files checked.");
if (excelFailed.Count <= 0) return;
Debug.WriteLine(excelFailed.Count + " failed excel checks! (" + ((float)excelFailed.Count / checkedCount * 100) + "%)");
foreach (String str in excelFailed) Debug.Write(str + ", ");
}
private void _GenerateRelations(ExcelFile excelFile)
{
if ((excelFile == null)) return;
Type type = excelFile.Attributes.RowType;
int col;
String mainTableName = excelFile.StringId;
DataTable mainDataTable = XlsDataSet.Tables[mainTableName];
// remove all extra generated columns on this table
for (col = 0; col < mainDataTable.Columns.Count; col++)
{
DataColumn dc = mainDataTable.Columns[col];
if (!(dc.ExtendedProperties.Contains(ColumnKeys.IsRelationGenerated))) continue;
mainDataTable.Columns.Remove(dc);
}
col = 1;
// regenerate relations
foreach (FieldInfo fieldInfo in type.GetFields(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic))
{
if (fieldInfo.IsPrivate)
{
if (fieldInfo.FieldType == typeof(ExcelFile.RowHeader))
{
col++;
}
continue;
}
OutputAttribute excelOutputAttribute = ExcelFile.GetExcelAttribute(fieldInfo);
if ((excelOutputAttribute == null))
{
col++;
continue;
}
DataColumn dcChild = mainDataTable.Columns[col];
if ((excelOutputAttribute.IsStringIndex))
{
DataTable dtStrings = XlsDataSet.Tables[StringsTableName] ?? _LoadStringsTable();
if (dtStrings != null)
{
DataColumn dcParent = dtStrings.Columns["ReferenceId"];
String relationName = String.Format("{0}_{1}_{2}", excelFile.StringId, dcChild.ColumnName, ColumnKeys.IsStringIndex);
// parent and child are swapped here for StringId relations
DataRelation relation = new DataRelation(relationName, dcChild, dcParent, false);
XlsDataSet.Relations.Add(relation);
DataColumn dcString = mainDataTable.Columns.Add(dcChild.ColumnName + "_string", typeof(String));
dcString.SetOrdinal(col + 1);
dcString.ExtendedProperties.Add(ExcelFile.ColumnTypeKeys.IsRelationGenerated, true);
// need to use MIN (and thus Child) for cases of strings with same reference id (e.g. Items; SINGULAR and PLURAL, etc)
dcString.Expression = "MIN(Child(" + relationName + ").String)";
col++;
}
}
if (excelOutputAttribute.IsTableIndex)
{
String tableStringId = excelOutputAttribute.TableStringId;
DataTable dt = XlsDataSet.Tables[tableStringId] ?? _LoadRelatedTable(tableStringId);
if (dt != null)
{
DataColumn dcParent = dt.Columns["Index"];
String relatedColumn = dt.Columns[2].ColumnName;
// todo
if (dcChild.ExtendedProperties.ContainsKey(ExcelFile.ColumnTypeKeys.IsArray) && (bool)dcChild.ExtendedProperties[ExcelFile.ColumnTypeKeys.IsArray])
{
//if (excelFile.StringId == "ITEMS")
//{
// int bp = 0;
//}
col++;
continue;
}
//String relationNameOld = excelFile.StringId + dcChild.ColumnName + ExcelFile.ColumnTypeKeys.IsTableIndex;
String relationName = String.Format("{0}_{1}_{2}", excelFile.StringId, dcChild.ColumnName, ExcelFile.ColumnTypeKeys.IsTableIndex);
DataRelation relation = new DataRelation(relationName, dcParent, dcChild, false);
//if (relationName == "PLAYERS_paperdollSkill_IsTableIndex")
//{
// int bp = 0;
//}
XlsDataSet.Relations.Add(relation);
DataColumn dcString = mainDataTable.Columns.Add(dcChild.ColumnName + "_string", typeof(String), String.Format("Parent({0}).{1}", relationName, relatedColumn));
dcString.SetOrdinal(col + 1);
dcString.ExtendedProperties.Add(ExcelFile.ColumnTypeKeys.IsRelationGenerated, true);
col++;
}
}
col++;
}
}
/// <summary>
/// This function checks every row/col of every excel file, determining if it needs to be visible/public or not.
/// Any element non-zero for all rows, needs to be public (or add new OutputAttribute for "const" values?)
/// </summary>
public static void ExcelValuesDeepScan()
{
FileManager fileManager = new FileManager(Config.HglDir);
fileManager.ExtractAllExcel();
Dictionary<uint, uint> rowTypeCounts = new Dictionary<uint, uint>();
Dictionary<String, uint[]> outputMessages = new Dictionary<String, uint[]>();
Dictionary<String, ObjectDelegator> objectDelegators = new Dictionary<String, ObjectDelegator>();
foreach (PackFileEntry fileEntry in fileManager.FileEntries.Values)
{
if (!fileEntry.Name.EndsWith(ExcelFile.Extension)) continue;
byte[] fileBytes = fileManager.GetFileBytes(fileEntry, true);
Debug.Assert(fileBytes != null);
ExcelFile excelFile = new ExcelFile(fileBytes, fileEntry.Path);
if (excelFile.Attributes.IsEmpty) continue;
Debug.WriteLine("Checking file: " + fileEntry.Path);
uint structureId = excelFile.Attributes.StructureId;
uint structureUseCount = 0;
if (rowTypeCounts.TryGetValue(structureId, out structureUseCount))
{
rowTypeCounts[structureId] = ++structureUseCount;
}
else
{
rowTypeCounts.Add(structureId, 1);
}
ObjectDelegator excelDelegator;
Type rowType = excelFile.Attributes.RowType;
FieldInfo[] fieldInfos = rowType.GetFields(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance);
int rowCount = excelFile.Rows.Count;
// create delegates
if (!objectDelegators.TryGetValue(excelFile.StringId, out excelDelegator))
{
excelDelegator = new ObjectDelegator(fieldInfos);
objectDelegators.Add(excelFile.StringId, excelDelegator);
}
// check by column, by row
int col = -1;
foreach (FieldInfo fieldInfo in fieldInfos)
{
if (++col == 0) continue; // row header
ObjectDelegator.FieldGetValueDelegate getValue = excelDelegator.GetFieldGetDelegate(fieldInfo.Name);
ExcelFile.OutputAttribute outputAttribute = ExcelFile.GetExcelAttribute(fieldInfo);
bool isArray = false;
bool allEqual = true;
Object firstValue = null;
Array firstValueArray = null;
int arrayIndexFirstDifferent = -1;
int row = -1;
String message = null;
foreach (Object value in excelFile.Rows.Select(rowObject => getValue(rowObject)))
{
arrayIndexFirstDifferent = -1;
if (++row == 0)
{
if (fieldInfo.FieldType.BaseType == typeof(Array))
{
firstValueArray = (Array)value;
isArray = true;
IEnumerator enumerator = firstValueArray.GetEnumerator();
enumerator.MoveNext();
firstValue = enumerator.Current;
}
else
{
firstValue = value;
}
continue;
}
if (isArray)
{
Array objArray = (Array)value;
Debug.Assert(firstValueArray != null);
bool arrayEqual = true;
IEnumerator enumeratorFirst = firstValueArray.GetEnumerator();
IEnumerator enumeratorCurrent = objArray.GetEnumerator();
while (enumeratorFirst.MoveNext())
{
enumeratorCurrent.MoveNext();
arrayIndexFirstDifferent++;
Object objFirst = enumeratorFirst.Current;
Object objCurrent = enumeratorCurrent.Current;
if (firstValue == null) firstValue = objFirst;
if (objFirst.Equals(objCurrent)) continue;
arrayEqual = false;
break;
}
if (arrayEqual) continue;
allEqual = false;
break;
}
Debug.Assert(firstValue != null);
if (firstValue.Equals(value)) continue;
allEqual = false;
break;
}
Debug.Assert(firstValue != null);
Object constValue = null;
bool ignoreConstCheck = false;
if (outputAttribute != null)
{
constValue = outputAttribute.ConstantValue;
ignoreConstCheck = outputAttribute.DebugIgnoreConstantCheck;
}
if (fieldInfo.IsPrivate)
{
if (!allEqual)
{
message = String.Format("Warning: Private column (Type = {1}, StructureId = 0x{2}) \"{0}\" has per-row differing values",
fieldInfo.Name, fieldInfo.FieldType, structureId.ToString("X8")); // need to add structureId to ensure message is unique per structure type
if (isArray)
{
message += ", first index differing: " + arrayIndexFirstDifferent;
}
}
else // allEqual && IsPrivate
{
if (isArray)
{
if (fieldInfo.FieldType == typeof(Int32[]))
{
if (firstValue.Equals((Int32)0)) continue;
}
else if (fieldInfo.FieldType == typeof(byte[]))
{
if (firstValue.Equals((byte)0x00)) continue;
}
else
{
Debug.Assert(false, "Unexpected Array Type: " + fieldInfo.FieldType);
}
if (outputAttribute != null && firstValue.Equals(outputAttribute.ConstantValue)) continue;
message = String.Format("Warning: Private column (Type = {2}, StructureId = 0x{3}) \"{0}\" has constant value not zero \"{1}\"",
fieldInfo.Name, firstValue, fieldInfo.FieldType, structureId.ToString("X8"));
}
else // not array
{
if (firstValue.Equals(constValue)) continue;
if (fieldInfo.FieldType == typeof(Int32) &&
((Int32)firstValue == 0 && (firstValue.Equals(constValue) || constValue == null))) continue;
if (fieldInfo.FieldType == typeof(Int16) &&
((Int16)firstValue == 0 && (firstValue.Equals(constValue) || constValue == null))) continue;
message = String.Format("Warning: Private column (Type = {2}, StructureId = 0x{3}) \"{0}\" has constant value not zero \"{1}\"",
fieldInfo.Name, firstValue, fieldInfo.FieldType, structureId.ToString("X8"));
if (outputAttribute != null && outputAttribute.ConstantValue != null) message += " (outputAttribute.ConstantValue is set)";
}
}
}
else // IsPublic
{
if (ignoreConstCheck) continue;
if (allEqual && fieldInfo.IsPublic && rowCount != 1 && (outputAttribute == null || outputAttribute.ConstantValue == null))
{
message = String.Format("Notice: Public column (Type = {2}, StructureId = 0x{3}) \"{0}\" has constant value \"{1}\"",
fieldInfo.Name, firstValue, fieldInfo.FieldType, structureId.ToString("X8"));
}
}
if (message == null) continue;
uint[] messageCounts;
if (outputMessages.TryGetValue(message, out messageCounts))
{
outputMessages[message] = new[] { messageCounts[0] + 1, structureId };
}
else
{
outputMessages.Add(message, new uint[] { 1, structureId });
}
}
//int bp3 = 0;
}
String previousStringId = null;
StringWriter stringWriter = new StringWriter();
foreach (KeyValuePair<String, uint[]> message in outputMessages)
{
String msg = message.Key;
uint msgCount = message.Value[0];
uint forStructureId = message.Value[1];
uint structureCount = rowTypeCounts[forStructureId];
if (structureCount != msgCount) continue; // if not equal, then we have a message in one table, but in another table it's not applicable
String[] stringIds1 = (from dataTableAttribute in DataFile.DataFileMap
where dataTableAttribute.Value.StructureId == forStructureId
select dataTableAttribute.Key).ToArray();
String[] stringIds2 = (from dataTableAttribute in DataFile.DataFileMapTestCenter
where dataTableAttribute.Value.StructureId == forStructureId
select dataTableAttribute.Key).ToArray();
String[] stringIds3 = (from dataTableAttribute in DataFile.DataFileMapResurrection
where dataTableAttribute.Value.StructureId == forStructureId
select dataTableAttribute.Key).ToArray();
String[] stringIds4 = (from dataTableAttribute in DataFile.DataFileMapMod
where dataTableAttribute.Value.StructureId == forStructureId
select dataTableAttribute.Key).ToArray();
String stringIdPrepend = String.Join(",", stringIds1) + String.Join(",", stringIds2) + String.Join(",", stringIds3) + String.Join(",", stringIds4);
if (previousStringId != stringIdPrepend)
{
//Console.WriteLine(stringIdPrepend);
stringWriter.WriteLine(stringIdPrepend);
previousStringId = stringIdPrepend;
}
//Console.WriteLine(msg);
stringWriter.WriteLine(msg);
}
File.WriteAllText(@"C:\asdf.txt", stringWriter.ToString());
}
private DataTable _LoadExcelTable(ExcelFile excelFile, bool doRelations, bool force)
{
String tableName = excelFile.StringId;
DataTable dataTable = XlsDataSet.Tables[tableName];
if (dataTable != null && !force)
{
return(dataTable);
}
if (dataTable != null)
{
XlsDataSet.Relations.Clear();
XlsDataSet.Tables.Remove(tableName);
}
dataTable = XlsDataSet.Tables.Add(tableName);
dataTable.TableName = tableName;
dataTable.ExtendedProperties.Add("FileHeader", excelFile._excelFileHeader.DeepClone());
Type dataType = excelFile.Attributes.RowType;
List <OutputAttribute> outputAttributes = new List <OutputAttribute>();
#region Generate Columns
DataColumn indexColumn = dataTable.Columns.Add("Index");
indexColumn.AutoIncrement = true;
indexColumn.Unique = true;
dataTable.PrimaryKey = new[] { indexColumn };
outputAttributes.Add(null);
FieldInfo[] fieldInfos = dataType.GetFields(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic);
foreach (FieldInfo fieldInfo in fieldInfos)
{
OutputAttribute excelAttribute = ExcelFile.GetExcelAttribute(fieldInfo);
// The only private field we add is the TableHeader
if (fieldInfo.IsPrivate)
{
if (fieldInfo.FieldType != typeof(ExcelFile.RowHeader))
{
continue;
}
outputAttributes.Add(null);
dataTable.Columns.Add(fieldInfo.Name, typeof(String));
continue;
}
Type fieldType = fieldInfo.FieldType;
bool isArray = false;
bool isEnum = false;
if (fieldInfo.FieldType.BaseType == typeof(Array))
{
fieldType = typeof(String);
isArray = true;
}
else if (fieldInfo.FieldType.BaseType == typeof(Enum) && excelAttribute == null)
{
fieldType = fieldInfo.FieldType;
isEnum = true;
}
DataColumn dataColumn = dataTable.Columns.Add(fieldInfo.Name, fieldType);
if (isArray)
{
dataColumn.ExtendedProperties.Add(ColumnKeys.IsArray, true);
}
else if (isEnum)
{
dataColumn.ExtendedProperties.Add(ColumnKeys.IsEnum, true);
}
if (excelAttribute == null)
{
outputAttributes.Add(null);
continue;
}
outputAttributes.Add(excelAttribute);
if (excelAttribute.IsStringOffset)
{
dataColumn.DataType = typeof(String);
dataColumn.ExtendedProperties.Add(ColumnKeys.IsStringOffset, true);
dataColumn.DefaultValue = String.Empty;
}
if (excelAttribute.IsScript)
{
dataColumn.DataType = typeof(String);
dataColumn.ExtendedProperties.Add(ColumnKeys.IsScript, true);
dataColumn.DefaultValue = String.Empty;
}
if (excelAttribute.IsSecondaryString)
{
dataColumn.DataType = typeof(String);
dataColumn.ExtendedProperties.Add(ColumnKeys.IsSecondaryString, true);
dataColumn.DefaultValue = String.Empty;
}
if (excelAttribute.IsStringIndex)
{
dataColumn.ExtendedProperties.Add(ColumnKeys.IsStringIndex, true);
// Add new column for the string
DataColumn dataColumnString = dataTable.Columns.Add(fieldInfo.Name + "_string", typeof(String));
dataColumnString.DefaultValue = String.Empty;
outputAttributes.Add(null);
dataColumnString.ExtendedProperties.Add(ColumnKeys.IsRelationGenerated, true);
}
if (excelAttribute.IsTableIndex)
{
dataColumn.ExtendedProperties.Add(ColumnKeys.IsTableIndex, true);
// Add new column for the string
DataColumn dataColumnString = dataTable.Columns.Add(fieldInfo.Name + "_string", typeof(String));
dataColumnString.DefaultValue = String.Empty;
outputAttributes.Add(null);
dataColumnString.ExtendedProperties.Add(ColumnKeys.IsRelationGenerated, true);
}
if (excelAttribute.IsBitmask)
{
dataColumn.ExtendedProperties.Add(ColumnKeys.IsBitmask, true);
}
if (excelAttribute.IsBool)
{
dataColumn.ExtendedProperties.Add(ColumnKeys.IsBool, true);
}
}
if (excelFile.Attributes.HasStats) // items, missiles, monsters, objects, players
{
DataColumn extendedDataColumn = dataTable.Columns.Add("Stats");
extendedDataColumn.DataType = typeof(String);
extendedDataColumn.ExtendedProperties.Add(ExcelFile.ColumnTypeKeys.IsStats, true);
outputAttributes.Add(null);
}
#endregion
#region Generate Rows
int row = 1;
object[] baseRow = new object[outputAttributes.Count];
ObjectDelegator objectDelegator = new ObjectDelegator(fieldInfos);
foreach (Object tableRow in excelFile.Rows)
{
int col = 1;
foreach (FieldInfo fieldInfo in fieldInfos)
{
Object value = objectDelegator[fieldInfo.Name](tableRow);
if (fieldInfo.IsPrivate)
{
if (fieldInfo.FieldType != typeof(ExcelFile.RowHeader))
{
continue;
}
baseRow[col++] = FileTools.ObjectToStringGeneric(value, ",");
continue;
}
OutputAttribute excelOutputAttribute = outputAttributes[col];
if (excelOutputAttribute == null)
{
if (value.GetType().BaseType == typeof(Array))
{
value = ((Array)value).ToString(",");
}
baseRow[col++] = value;
continue;
}
if (excelOutputAttribute.IsStringOffset)
{
int valueInt = (int)value;
baseRow[col++] = (valueInt != -1) ? excelFile.ReadStringTable(valueInt) : String.Empty;
continue;
}
if (excelOutputAttribute.IsSecondaryString)
{
int valueInt = (int)value;
baseRow[col++] = (valueInt != -1) ? excelFile.ReadSecondaryStringTable(valueInt) : String.Empty;
continue;
}
if (excelOutputAttribute.IsScript)
{
int scriptOffset = (int)value;
if (scriptOffset == 0)
{
baseRow[col++] = String.Empty;
continue;
}
String script;
if (scriptOffset == 9649 && excelFile.StringId == "SKILLS") // todo: not sure what's with this script...
{
/* Compiled Bytes:
* 26,30,700,6,26,1,399,358,669,616562688,711,26,62,3,17,669,641728512,26,8,711,26,62,3,17,358,669,322961408,26,5,700,6,26,1,399,358,388,0
*
* Ending Stack (FIFO):
* SetStat669('sfx_attack_pct', 'all', 30 * ($sklvl - 1))
* SetStat669('sfx_duration_pct', 'all', get_skill_level(@unit, 'Shield_Mastery'))
* SetStat669('damage_percent_skill', 8 * get_skill_level(@unit, 'Shield_Mastery'))
* SetStat669('damage_percent_skill', 8 * get_skill_level(@unit, 'Shield_Mastery')) + 5 * ($sklvl - 1)
*
* The last SetStat has strange overhang - decompiling wrong?
* Or is it "supposed" to be there?
* i.e. It's actually decompiling correctly, but because I've assumed such scripts to be wrong (as the end +5... segment is useless) we get the Stack exception
*/
int[] scriptCode = excelFile.ReadScriptTable(scriptOffset);
script = scriptCode != null?FileTools.ArrayToStringGeneric(scriptCode, ",") : "ScriptError";
baseRow[col++] = script;
continue;
}
//if (fieldInfo.Name == "props1" && row == 45)
//{
// int bp = 0;
//}
ExcelScript excelScript = new ExcelScript(this);
try
{
if (ExcelScript.DebugEnabled)
{
int[] scriptCode = excelFile.ReadScriptTable(scriptOffset);
script = scriptCode != null?FileTools.ArrayToStringGeneric(scriptCode, ",") : String.Empty;
script = excelScript.Decompile(excelFile.ScriptBuffer, scriptOffset, script, excelFile.StringId, row, col, fieldInfo.Name);
}
else
{
script = excelScript.Decompile(excelFile.ScriptBuffer, scriptOffset);
}
//if (script.StartsWith("GetStat666('skill_points_bonus_total', '') > -1;"))
//{
// int bp = 0;
//}
}
catch (Exception)
{
int[] scriptCode = excelFile.ReadScriptTable(scriptOffset);
script = scriptCode != null?FileTools.ArrayToStringGeneric(scriptCode, ",") : "ScriptError";
}
baseRow[col++] = script;
continue;
}
if (excelOutputAttribute.IsTableIndex || excelOutputAttribute.IsStringIndex)
{
if (value.GetType().BaseType == typeof(Array))
{
value = ((Array)value).ToString(",");
}
else
{
value = (int)value;
}
baseRow[col++] = value;
col++; // for _strings relational column
continue;
}
// Else its something else, ie bitmask, bool, table/string index
baseRow[col++] = value;
}
// stats, only a component of the UnitData row type
if (excelFile.Attributes.HasStats)
{
baseRow[col++] = FileTools.ArrayToStringGeneric(excelFile.ReadStats(row - 1), ",");
}
dataTable.Rows.Add(baseRow);
row++;
}
#endregion
// Generate Relationships as required
if (doRelations)
{
_GenerateRelations(excelFile);
}
return(dataTable);
}