public static bool CookExcelFile(string excelPath) { byte[] excelBuffer = null; try { excelBuffer = File.ReadAllBytes(excelPath); } catch (Exception ex) { ExceptionLogger.LogException(ex); Console.WriteLine(String.Format("Error reading file {0}", excelPath)); return false; } ExcelFile excelFile = new ExcelFile(excelPath); try { excelFile.ParseCSV(excelBuffer, _fileManager); } catch (Exception e) { Console.WriteLine("Critical Error:\n" + e); return false; } if (excelFile.HasIntegrity == false) { Console.WriteLine(String.Format("Failed to parse excel file {0}", excelPath)); return false; } Console.WriteLine(String.Format("Cooking {0}", Path.GetFileNameWithoutExtension(excelPath))); excelBuffer = excelFile.ToByteArray(); if (excelBuffer == null) { Console.WriteLine(String.Format("Failed to serialize excel file {0}", excelFile.StringId)); return false; } String writeToPath = excelPath + ".cooked"; try { File.WriteAllBytes(writeToPath, excelBuffer); } catch (Exception ex) { ExceptionLogger.LogException(ex); Console.WriteLine(String.Format("Failed to write cooked file {0} ", writeToPath)); return false; } return true; }
public int _GetExcelRowIndex(ExcelFile excelTable, String value) { FieldInfo field = excelTable.Attributes.RowType.GetFields()[0]; bool isStringField = (field.FieldType == typeof(String)); ObjectDelegator excelDelegator = DataFileDelegators[excelTable.StringId]; ObjectDelegator.FieldGetValueDelegate getValue = excelDelegator[field.Name]; int i = 0; foreach (Object row in excelTable.Rows) { if (isStringField) { String val = (String)getValue(row); if (val == value) return i; } else // string offset { int offset = (int)getValue(row); String stringVal = excelTable.ReadStringTable(offset); if (stringVal == value) return i; } i++; } return -1; }
/// <summary> /// Loads all of the available Excel and Strings files to a hashtable. /// </summary> /// <returns>Returns true on success.</returns> public bool LoadTableFiles(bool ignorePatchedOut = false) { // want excel files and strings files foreach (PackFileEntry fileEntry in FileEntries.Values.Where(fileEntry => fileEntry.Name.EndsWith(ExcelFile.Extension) || (fileEntry.Name.EndsWith(StringsFile.Extention) && fileEntry.Path.Contains(Language)))) { byte[] fileBytes = GetFileBytes(fileEntry, ignorePatchedOut); if (fileBytes == null) { Debug.WriteLine("Warning: Failed to read file bytes in LoadTableFiles(). FileEntry = " + fileEntry.Name); continue; } //if (fileEntry.Name.Contains("stats")) //{ // int bp = 0; //} // parse file data DataFile dataFile; try { if (fileEntry.Name.EndsWith(ExcelFile.Extension)) { dataFile = new ExcelFile(fileBytes, fileEntry.Path, ClientVersion); } else { dataFile = new StringsFile(fileBytes, fileEntry.Path); } } catch (Exceptions.DataFileStringIdNotFound dataFileStringIdNotFound) { Debug.WriteLine(dataFileStringIdNotFound.ToString()); ExceptionLogger.LogException(dataFileStringIdNotFound); continue; } catch (Exception ex) { Debug.WriteLine(ex.ToString()); ExceptionLogger.LogException(ex); Console.WriteLine(String.Format("Critical Error: Failed to load data file {0} (ClientVersion = {1})", fileEntry.Name, ClientVersion)); continue; } if (!dataFile.HasIntegrity) { String failedParse = "Error: Failed to load data file: " + dataFile.StringId; Debug.WriteLine(failedParse); // Console.WriteLine randomly (more often than not) just doesn't output to the Ouput window in VS during debugging Console.WriteLine(failedParse); continue; } if (dataFile.Attributes.IsEmpty) continue; try { DataFiles.Add(dataFile.StringId, dataFile); DataFileDelegators.Add(dataFile.StringId, dataFile.Delegator); // not sure if we need this still... } catch (Exception e) { Debug.WriteLine("Critical Error: Cannot add table data file to dictionary: " + dataFile + "\n" + e); Console.WriteLine("Critical Error: Cannot add table data file to dictionary: " + dataFile + "\n" + e); } } Debug.WriteLine("Loaded {0} {1} data files.", DataFiles.Count, ClientVersion); return true; }
/// <summary> /// Extracts all Excel files to their \data\ locations. /// Primarly a debug function. /// </summary> public void ExtractAllExcel(String root = null, bool doCSVAsWell = false) { if (root == null) root = HellgatePath; foreach (PackFileEntry fileEntry in FileEntries.Values) { if (!fileEntry.Name.EndsWith(ExcelFile.Extension)) continue; PackFileEntry extractFileEntry = fileEntry; //if (fileEntry.Index.ToString().Contains("4580") && fileEntry.Siblings != null) //{ // extractFileEntry = (from fi in fileEntry.Siblings // where fi.Index.ToString().Contains("4256") // select fi).FirstOrDefault(); // if (extractFileEntry == null) // { // extractFileEntry = (from fi in fileEntry.Siblings // where fi.Index.ToString().Contains("000") // select fi).FirstOrDefault(); // } // Debug.Assert(extractFileEntry != null); //} byte[] fileBytes = GetFileBytes(extractFileEntry, true); String filePath = Path.Combine(root, extractFileEntry.Path); Directory.CreateDirectory(Path.GetDirectoryName(filePath)); File.WriteAllBytes(filePath, fileBytes); if (!doCSVAsWell) continue; byte[] csvBytes; try { ExcelFile excelFile = new ExcelFile(fileBytes, filePath, ClientVersion); csvBytes = excelFile.ExportCSV(this); } catch (Exception) { continue; } File.WriteAllBytes(filePath.Replace(ExcelFile.Extension, ExcelFile.ExtensionDeserialised), csvBytes); } }
private void _GenerateRelations(ExcelFile excelFile) { if ((excelFile == null)) { return; } Type type = excelFile.Attributes.RowType; int col; String mainTableName = excelFile.StringId; DataTable mainDataTable = XlsDataSet.Tables[mainTableName]; // remove all extra generated columns on this table for (col = 0; col < mainDataTable.Columns.Count; col++) { DataColumn dc = mainDataTable.Columns[col]; if (!(dc.ExtendedProperties.Contains(ColumnKeys.IsRelationGenerated))) { continue; } mainDataTable.Columns.Remove(dc); } col = 1; // regenerate relations foreach (FieldInfo fieldInfo in type.GetFields(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic)) { if (fieldInfo.IsPrivate) { if (fieldInfo.FieldType == typeof(ExcelFile.RowHeader)) { col++; } continue; } OutputAttribute excelOutputAttribute = ExcelFile.GetExcelAttribute(fieldInfo); if ((excelOutputAttribute == null)) { col++; continue; } DataColumn dcChild = mainDataTable.Columns[col]; if ((excelOutputAttribute.IsStringIndex)) { DataTable dtStrings = XlsDataSet.Tables[StringsTableName] ?? _LoadStringsTable(); if (dtStrings != null) { DataColumn dcParent = dtStrings.Columns["ReferenceId"]; String relationName = String.Format("{0}_{1}_{2}", excelFile.StringId, dcChild.ColumnName, ColumnKeys.IsStringIndex); // parent and child are swapped here for StringId relations DataRelation relation = new DataRelation(relationName, dcChild, dcParent, false); XlsDataSet.Relations.Add(relation); DataColumn dcString = mainDataTable.Columns.Add(dcChild.ColumnName + "_string", typeof(String)); dcString.SetOrdinal(col + 1); dcString.ExtendedProperties.Add(ExcelFile.ColumnTypeKeys.IsRelationGenerated, true); // need to use MIN (and thus Child) for cases of strings with same reference id (e.g. Items; SINGULAR and PLURAL, etc) dcString.Expression = "MIN(Child(" + relationName + ").String)"; col++; } } if (excelOutputAttribute.IsTableIndex) { String tableStringId = excelOutputAttribute.TableStringId; DataTable dt = XlsDataSet.Tables[tableStringId] ?? _LoadRelatedTable(tableStringId); if (dt != null) { DataColumn dcParent = dt.Columns["Index"]; String relatedColumn = dt.Columns[2].ColumnName; // todo if (dcChild.ExtendedProperties.ContainsKey(ExcelFile.ColumnTypeKeys.IsArray) && (bool)dcChild.ExtendedProperties[ExcelFile.ColumnTypeKeys.IsArray]) { //if (excelFile.StringId == "ITEMS") //{ // int bp = 0; //} col++; continue; } //String relationNameOld = excelFile.StringId + dcChild.ColumnName + ExcelFile.ColumnTypeKeys.IsTableIndex; String relationName = String.Format("{0}_{1}_{2}", excelFile.StringId, dcChild.ColumnName, ExcelFile.ColumnTypeKeys.IsTableIndex); DataRelation relation = new DataRelation(relationName, dcParent, dcChild, false); //if (relationName == "PLAYERS_paperdollSkill_IsTableIndex") //{ // int bp = 0; //} XlsDataSet.Relations.Add(relation); DataColumn dcString = mainDataTable.Columns.Add(dcChild.ColumnName + "_string", typeof(String), String.Format("Parent({0}).{1}", relationName, relatedColumn)); dcString.SetOrdinal(col + 1); dcString.ExtendedProperties.Add(ExcelFile.ColumnTypeKeys.IsRelationGenerated, true); col++; } } col++; } }
/// <summary> /// This function is to test excel CSV cooking. /// </summary> public static void DoCookTest() { FileManager fileManager = new FileManager(Config.HglDir); fileManager.LoadTableFiles(); fileManager.ExtractAllExcel(); ExcelScript.GlobalDebug(true); foreach (PackFileEntry fileEntry in fileManager.FileEntries.Values) { if (!fileEntry.Name.EndsWith(ExcelFile.Extension)) continue; byte[] fileBytes = fileManager.GetFileBytes(fileEntry, true); Debug.Assert(fileBytes != null); String filePath = Path.Combine(Config.HglDir, fileEntry.Path); ExcelFile excelFile = new ExcelFile(fileBytes, fileEntry.Path); if (excelFile.Attributes.IsEmpty) continue; Console.WriteLine("Cooking file: " + fileEntry.Path); //if (!fileEntry.RelativeFullPathWithoutPatch.Contains("display_item")) continue; byte[] csvBytes = excelFile.ExportCSV(fileManager); File.WriteAllBytes(filePath.Replace(ExcelFile.Extension, ExcelFile.ExtensionDeserialised), csvBytes); ExcelFile excelFileCSV = new ExcelFile(fileEntry.Path); excelFileCSV.ParseCSV(csvBytes, fileManager); //byte[] recookedBytes = excelFileCSV.ToByteArray(); byte[] csvBytes2 = excelFileCSV.ExportCSV(fileManager); //if (!csvBytes.SequenceEqual(csvBytes2)) //{ // int b5p = 0; //} //if (excelFile.StringId == "GLOBAL_STRING") //{ // ExcelFile temp = new ExcelFile(recookedBytes, fileEntry.RelativeFullPathWithoutPatch); // int bp1 = 0; //} //File.WriteAllBytes(filePath, recookedBytes); } //int bp = 0; }
/// <summary> /// Function to test saving/importing/exported/etc relating to DataTables (i.e. actions from WITHIN Reanimator etc.) /// </summary> public static void TestDataTableExportAndImport(bool doTCv4 = false) { /* bytesXls Bytes->Xls bytesXlsBytes Bytes->Xls->Bytes bytesXlsCsv Bytes->Xls->CSV bytesXlsCsvXls Bytes->Xls->CSV->Xls bytesXlsCsvXlsBytes Bytes->Xls->CSV->Xls->Bytes == bytesXlsTableXlsBytes as single byte is stripped ->CSV bytesXlsTable Bytes->Xls->Table bytesXlsTableXls Bytes->Xls->Table->Xls bytesXlsTableXlsBytes Bytes->Xls->Table->Xls->Bytes != bytesXlsBytes because of single script byte thingy lost to table bytesXlsTableXlsCsv Bytes->Xls->Table->Xls->CSV bytesXlsTableXlsCsvXls Bytes->Xls->Table->Xls->CSV->Xls bytesXlsTableXlsCsvXlsBytes Bytes->Xls->Table->Xls->CSV->Xls->Bytes byte[] bytesXlsBytes = bytesXls.ToByteArray(); byte[] bytesXlsTableXlsCsvXlsBytes = bytesXlsTableXlsCsvXls.ToByteArray(); */ String root = @"C:\excel_datatable_debug"; FileManager.ClientVersions clientVersion = FileManager.ClientVersions.SinglePlayer; if (doTCv4) { root = Path.Combine(root, "tcv4"); clientVersion = FileManager.ClientVersions.TestCenter; } root += @"\"; // lazy Directory.CreateDirectory(root); FileManager fileManager = new FileManager(Config.HglDir, clientVersion); fileManager.BeginAllDatReadAccess(); fileManager.LoadTableFiles(); List<String> sequenceFailed = new List<String>(); List<String> sequenceChecked = new List<String>(); foreach (KeyValuePair<String, DataFile> keyValuePair in fileManager.DataFiles) { String stringId = keyValuePair.Key; DataFile dataFile = keyValuePair.Value; if (dataFile.IsStringsFile) continue; //if (stringId != "WARDROBE_LAYER") continue; //if (stringId == "WARDROBE_LAYER") //{ // int bp = 0; //} Debug.Write("Checking " + stringId + "... "); ExcelFile bytesXls = (ExcelFile)dataFile; byte[] bytes = fileManager.GetFileBytes(bytesXls.FilePath, true); sequenceChecked.Add(stringId); sequenceFailed.Add(stringId); if (stringId == "PROPERTIES") continue; // can't do this atm - need to export Properties script table to DataTable etc. Debug.Write("DataTable... "); DataTable bytesXlsTable = fileManager.GetDataTable(stringId); if (bytesXlsTable == null) { Debug.WriteLine("FAILED!"); continue; } // parse as DataTable Debug.Write("new Excel... "); DataFile bytesXlsTableXls = new ExcelFile(bytesXls.FilePath, fileManager.ClientVersion); if (!bytesXlsTableXls.ParseDataTable(bytesXlsTable, fileManager)) { Debug.WriteLine("FAILED!"); continue; } // check just data table import/export byte[] bytesXlsBytes = bytesXls.ToByteArray(); byte[] bytesXlsTableXlsBytes = bytesXlsTableXls.ToByteArray(); if (bytesXls.ScriptBuffer == null && // can't check this as ScriptBuffer[0] is NOT always 0x00 (even though it's unused and means no script normally when pointed to) !bytesXls.HasStringBuffer) // as above - gets re-arranged etc (only consistant comparint CSV->XLS->DATA { Debug.Write("bytesXlsBytes==bytesXlsTableXlsBytes... "); if (!bytesXlsBytes.SequenceEqual(bytesXlsTableXlsBytes)) { Debug.WriteLine("FALSE!"); File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes); File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes); File.WriteAllBytes(root + bytesXls.StringId + "2.bytesXlsTableXlsBytes", bytesXlsTableXlsBytes); continue; } } // export CSV Debug.Write("ExportCSV... "); byte[] bytesXlsTableXlsCsv = bytesXlsTableXls.ExportCSV(fileManager); if (bytesXlsTableXlsCsv == null) { Debug.WriteLine("FAILED!"); continue; } Debug.Write("bytesXlsCsv==bytesXlsTableXlsCsv... "); byte[] bytesXlsCsv = bytesXls.ExportCSV(fileManager); if (!bytesXlsCsv.SequenceEqual(bytesXlsTableXlsCsv)) { Debug.WriteLine("FALSE!"); File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes); File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes); File.WriteAllBytes(root + bytesXls.StringId + "1a.bytesXlsCsv.csv", bytesXlsCsv); File.WriteAllBytes(root + bytesXls.StringId + "2.bytesXlsTableXlsBytes", bytesXlsTableXlsBytes); File.WriteAllBytes(root + bytesXls.StringId + "3.bytesXlsTableXlsCsv.csv", bytesXlsTableXlsCsv); continue; } Debug.Write("bytesXlsCsvXlsBytes==bytesXlsTableXlsBytes... "); DataFile bytesXlsCsvXls = new ExcelFile(bytesXls.FilePath, fileManager.ClientVersion); bytesXlsCsvXls.ParseCSV(bytesXlsCsv, fileManager); byte[] bytesXlsCsvXlsBytes = bytesXlsCsvXls.ToByteArray(); if (!bytesXlsCsvXlsBytes.SequenceEqual(bytesXlsTableXlsBytes) && stringId != "ITEMDISPLAY" && stringId != "LEVEL") // ITEMDISPLAY has weird single non-standard ASCII char that is lost (from using MS Work on their part lol) { // and LEVEL references blank TREASURE rows and hence rowId references differ on recooks Debug.WriteLine("FALSE!"); File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes); File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes); File.WriteAllBytes(root + bytesXls.StringId + "1a.bytesXlsCsv.csv", bytesXlsCsv); File.WriteAllBytes(root + bytesXls.StringId + "1b.bytesXlsCsvXlsBytes", bytesXlsCsvXlsBytes); File.WriteAllBytes(root + bytesXls.StringId + "2.bytesXlsTableXlsBytes", bytesXlsTableXlsBytes); File.WriteAllBytes(root + bytesXls.StringId + "3.bytesXlsTableXlsCsv.csv", bytesXlsTableXlsCsv); continue; } // import CSV Debug.Write("ParseCSV... "); DataFile bytesXlsTableXlsCsvXls = new ExcelFile(bytesXls.FilePath, fileManager.ClientVersion); if (!bytesXlsTableXlsCsvXls.ParseCSV(bytesXlsTableXlsCsv, fileManager)) { Debug.WriteLine("FAILED!"); continue; } // export imported as cooked and do byte compare Debug.Write("ToByteArray... "); byte[] bytesXlsTableXlsCsvXlsBytes = bytesXlsTableXlsCsvXls.ToByteArray(); Debug.Write("bytesXlsCsvXlsBytes==bytesXlsTableXlsCsvXlsBytes... "); if (!bytesXlsCsvXlsBytes.SequenceEqual(bytesXlsTableXlsCsvXlsBytes)) { Debug.WriteLine("FALSE!"); File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes); File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes); File.WriteAllBytes(root + bytesXls.StringId + "2.bytesXlsTableXlsBytes", bytesXlsTableXlsBytes); File.WriteAllBytes(root + bytesXls.StringId + "3.bytesXlsTableXlsCsv", bytesXlsTableXlsCsv); File.WriteAllBytes(root + bytesXls.StringId + "4.bytesXlsTableXlsCsvXlsBytes", bytesXlsTableXlsCsvXlsBytes); continue; } // yay Debug.WriteLine("OK!"); sequenceFailed.RemoveAt(sequenceFailed.Count - 1); } Debug.WriteLine("Totals: {0} sequence checks, {1} fails. ({2}% failed!)", sequenceChecked.Count, sequenceFailed.Count, (float)sequenceFailed.Count * 100f / sequenceChecked.Count); Debug.Write("Failed Tables: "); foreach (String stringIdFailed in sequenceFailed) { Debug.Write(stringIdFailed + ", "); } }
/// <summary> /// Initialise private variables and generate excel script functions if required. /// </summary> public ExcelScript(FileManager fileManager) { if (fileManager == null) throw new ArgumentNullException("fileManager", "File Manager cannot be null."); if (!fileManager.DataFiles.ContainsKey("STATS")) throw new Exceptions.ScriptNotInitialisedException("The supplied file manager did not have a valid Stats table."); // these initial counts are required for script compiling when determing if a function is from the .exe or from an Excel table if (!_haveExcelFunctions) { _initCallFunctionsCountSinglePlayer = CallFunctionsSinglePlayer.Count; _initCallFunctionsTestCenterCount = CallFunctionsTestCenter.Count; _initCallFunctionsResurrectionCount = CallFunctionsResurrection.Count; } _fileManager = fileManager; if (_fileManager.IsVersionResurrection || _fileManager.IsVersionMod) { CallFunctions = CallFunctionsResurrection; _initCallFunctionsCount = _initCallFunctionsResurrectionCount; _debugRoot = DebugRootResurrection; } else if (_fileManager.IsVersionTestCenter) { CallFunctions = CallFunctionsTestCenter; _initCallFunctionsCount = _initCallFunctionsTestCenterCount; _debugRoot = DebugRootTestCenter; } else { CallFunctions = CallFunctionsSinglePlayer; _initCallFunctionsCount = _initCallFunctionsCountSinglePlayer; _debugRoot = DebugRoot; } _statsTable = (ExcelFile)_fileManager.DataFiles["STATS"]; _statsDelegator = _fileManager.DataFileDelegators["STATS"]; if (_globalDebug) EnableDebug(true); if (CallFunctions.Count == _initCallFunctionsCount) _GenerateExcelScriptFunctions(); }
// Row from index public object GetRowFromIndex(Xls.TableCodes tableCode, Int32 index) { ExcelFile table = GetExcelTableFromCode(tableCode); return(index < table.Rows.Count() ? (table.Rows[index]) : null); }
/// <summary> /// Loads all of the available Excel and Strings files to a hashtable. /// </summary> /// <returns>Returns true on success.</returns> public bool LoadTableFiles(bool ignorePatchedOut = false) { // want excel files and strings files foreach (PackFileEntry fileEntry in FileEntries.Values.Where(fileEntry => fileEntry.Name.EndsWith(ExcelFile.Extension) || (fileEntry.Name.EndsWith(StringsFile.Extention) && fileEntry.Path.Contains(Language)))) { byte[] fileBytes = GetFileBytes(fileEntry, ignorePatchedOut); if (fileBytes == null) { Debug.WriteLine("Warning: Failed to read file bytes in LoadTableFiles(). FileEntry = " + fileEntry.Name); continue; } //if (fileEntry.Name.Contains("stats")) //{ // int bp = 0; //} // parse file data DataFile dataFile; try { if (fileEntry.Name.EndsWith(ExcelFile.Extension)) { dataFile = new ExcelFile(fileBytes, fileEntry.Path, ClientVersion); } else { dataFile = new StringsFile(fileBytes, fileEntry.Path); } } catch (Exceptions.DataFileStringIdNotFound dataFileStringIdNotFound) { Debug.WriteLine(dataFileStringIdNotFound.ToString()); ExceptionLogger.LogException(dataFileStringIdNotFound); continue; } catch (Exception ex) { Debug.WriteLine(ex.ToString()); ExceptionLogger.LogException(ex); Console.WriteLine(String.Format("Critical Error: Failed to load data file {0} (ClientVersion = {1})", fileEntry.Name, ClientVersion)); continue; } if (!dataFile.HasIntegrity) { String failedParse = "Error: Failed to load data file: " + dataFile.StringId; Debug.WriteLine(failedParse); // Console.WriteLine randomly (more often than not) just doesn't output to the Ouput window in VS during debugging Console.WriteLine(failedParse); continue; } if (dataFile.Attributes.IsEmpty) { continue; } try { DataFiles.Add(dataFile.StringId, dataFile); DataFileDelegators.Add(dataFile.StringId, dataFile.Delegator); // not sure if we need this still... } catch (Exception e) { Debug.WriteLine("Critical Error: Cannot add table data file to dictionary: " + dataFile + "\n" + e); Console.WriteLine("Critical Error: Cannot add table data file to dictionary: " + dataFile + "\n" + e); } } Debug.WriteLine("Loaded {0} {1} data files.", DataFiles.Count, ClientVersion); return(true); }
public void ProcessTables() { // excel tables ExcelFile excelTables = (ExcelFile)DataFiles[ExcelTablesStringId]; foreach (ExcelTablesRow excelTableRow in excelTables.Rows) { Xls.TableCodes tableCode = (Xls.TableCodes)excelTableRow.code; ExcelFile excelTable = GetExcelTableFromCode(tableCode); if (excelTable == null) { continue; } excelTable.TableCode = tableCode; } // stats table ExcelFile statsTable = (ExcelFile)DataFiles[StatsTableStringId]; foreach (StatsRow stat in statsTable.Rows) { int bitCount = _GetTableRowBitMax(stat.valTable, false); if (bitCount != -1) { stat.valbits = bitCount; } bitCount = _GetTableRowBitMax(stat.param1Table, true); if (bitCount != -1) { stat.param1Bits = bitCount; } if (stat.param1Bits > 0) { stat.ParamCount++; } bitCount = _GetTableRowBitMax(stat.param2Table, true); if (bitCount != -1) { stat.param2Bits = bitCount; } if (stat.param2Bits > 0) { stat.ParamCount++; } bitCount = _GetTableRowBitMax(stat.param3Table, true); if (bitCount != -1) { stat.param3Bits = bitCount; } if (stat.param3Bits > 0) { stat.ParamCount++; } bitCount = _GetTableRowBitMax(stat.param4Table, true); if (bitCount != -1) { stat.param4Bits = bitCount; } if (stat.param4Bits > 0) { stat.ParamCount++; } if (stat.valTable != -1) { stat.ValueExcelTable = GetExcelTableFromIndex(stat.valTable); } if (stat.param1Table != -1) { stat.Param1ExcelTable = GetExcelTableFromIndex(stat.param1Table); } if (stat.param2Table != -1) { stat.Param2ExcelTable = GetExcelTableFromIndex(stat.param2Table); } if (stat.param3Table != -1) { stat.Param3ExcelTable = GetExcelTableFromIndex(stat.param3Table); } if (stat.param4Table != -1) { stat.Param4ExcelTable = GetExcelTableFromIndex(stat.param4Table); } } // level drlg choice table ExcelFile levelDrlgChoiceTable = (ExcelFile)DataFiles["LEVEL_DRLG_CHOICE"]; foreach (LevelDrlgChoiceRow levelDrlgChoice in levelDrlgChoiceTable.Rows) { if (levelDrlgChoice.nameOffset == -1) { continue; } levelDrlgChoice.name = levelDrlgChoiceTable.ReadStringTable(levelDrlgChoice.nameOffset); Debug.Assert(!String.IsNullOrEmpty(levelDrlgChoice.name)); } }
/// <summary> /// Converts the Excel file to another Excel type, keeping all but the rows intact. /// Used for TCv4 -> SP conversion. /// </summary> /// <param name="excelFile">An Excel object of type to convert to.</param> /// <param name="newRows">The rows to replace the old.</param> public void ConvertType(ExcelFile excelFile, IEnumerable<Object> newRows) { Attributes = excelFile.Attributes; _excelFileHeader = excelFile._excelFileHeader; Rows = new List<Object>(newRows); }
private void _ReadStatValue(BitBuffer bitBuffer, int bitCount, ref int value, ExcelFile valueTable, ref Object valueRow, StatContext statContext) { if (bitCount <= 0) return; value = bitBuffer.ReadBits(bitCount); if (valueTable == null) return; if (statContext == StatContext.UseRows) { if (value >= 0 && value < valueTable.Rows.Count) { valueRow = valueTable.Rows[value]; } if (valueRow == null) Debug.WriteLine(String.Format("Warning: OutOfBounds stat index {0} on table {1}", value, valueTable.TableCode)); } else { valueRow = _fileManager.GetRowFromCode(valueTable.TableCode, (short)value); if (valueRow == null) Debug.WriteLine(String.Format("Warning: Unknown stat value code {0} (0x{0:X4}) on table {1}", value, valueTable.TableCode)); } }
private void _WriteStatValue(BitBuffer bitBuffer, int value, Object valueRow, ExcelFile valueTable, int bitCount, StatContext statContext) { if (bitCount <= 0) return; if (valueTable == null) { bitBuffer.WriteBits(value, bitCount); return; } if (valueRow == null) { bitBuffer.WriteBits(0, bitCount); return; } if (statContext == StatContext.UseRows) { int rowIndex = valueTable.Rows.IndexOf(valueRow); bitBuffer.WriteBits(rowIndex, bitCount); } else { int code = _fileManager.GetCodeFromRow(valueTable.TableCode, valueRow); bitBuffer.WriteBits(code, bitCount); } }
private String _GetExcelStringFromExcelFile(ExcelFile excelTable, int rowIndex, int colIndex) { if (excelTable == null || rowIndex >= excelTable.Rows.Count) return null; ObjectDelegator tableDelegator = DataFileDelegators[excelTable.StringId]; ObjectDelegator.FieldDelegate fieldDelegate = tableDelegator.GetPublicFieldDelegate(colIndex); bool isStringField = (fieldDelegate.FieldType == typeof(String)); Object row = excelTable.Rows[rowIndex]; if (isStringField) return (String)fieldDelegate.GetValue(row); int offset = (int)fieldDelegate.GetValue(row); String stringVal = excelTable.ReadStringTable(offset); return stringVal; }
private static void _CookExcelFiles(IEnumerable<String> excelFilesToCook) { Dictionary<String, ExcelFile> excelFiles = new Dictionary<String, ExcelFile>(); Console.WriteLine("Reading Excel CSV content..."); foreach (String excelPath in excelFilesToCook) { Console.Write(Path.GetFileName(excelPath) + "... "); byte[] fileBytes; try { fileBytes = File.ReadAllBytes(excelPath); } catch (Exception e) { ExceptionLogger.LogException(e); Console.WriteLine("\nFailed to read file contents!\nIgnore and Continue? [Y/N]: "); char c = (char)Console.Read(); if (c == 'y' || c == 'Y') continue; return; } ExcelFile excelFile = new ExcelFile(excelPath); try { excelFile.LoadCSV(fileBytes); } catch (Exception e) { ExceptionLogger.LogException(e); Console.WriteLine("\nFailed to load CSV contents!\nIgnore and Continue? [Y/N]: "); char c = (char)Console.Read(); if (c == 'y' || c == 'Y') continue; return; } excelFiles.Add(excelFile.StringId, excelFile); } if (excelFiles.Count == 0) return; Console.WriteLine("\nProcessing Excel CSV content..."); foreach (ExcelFile excelFile in excelFiles.Values) { Console.WriteLine("Cooking " + Path.GetFileName(excelFile.FilePath)); try { excelFile.ParseCSV(_fileManager, excelFiles); } catch (Exception e) { ExceptionLogger.LogException(e); Console.WriteLine("Failed to parse CSV data!\nIgnore and Continue? [Y/N]: "); char c = (char)Console.Read(); if (c == 'y' || c == 'Y') continue; return; } if (excelFile.HasIntegrity == false) { Console.WriteLine("Failed to parse CSV data!\nIgnore and Continue? [Y/N]: "); char c = (char)Console.Read(); if (c == 'y' || c == 'Y') continue; return; } byte[] cookedFileBytes; try { cookedFileBytes = excelFile.ToByteArray(); } catch (Exception e) { ExceptionLogger.LogException(e); Console.WriteLine("Failed to serialise CSV data!\nIgnore and Continue? [Y/N]: "); char c = (char)Console.Read(); if (c == 'y' || c == 'Y') continue; return; } if (cookedFileBytes == null) { Console.WriteLine("Failed to serialise CSV data!\nIgnore and Continue? [Y/N]: "); char c = (char)Console.Read(); if (c == 'y' || c == 'Y') continue; return; } String savePath = excelFile.FilePath.Replace(ExcelFile.ExtensionDeserialised, ExcelFile.Extension); try { File.WriteAllBytes(savePath, cookedFileBytes); } catch (Exception e) { ExceptionLogger.LogException(e); Console.WriteLine("Failed to write cooked excel file!\nIgnore and Continue? [Y/N]: "); char c = (char)Console.Read(); if (c == 'y' || c == 'Y') continue; return; } } }
/// <summary> /// Main worker function that parses and appends excel script functions to the global call function array. /// </summary> /// <param name="excelFile">The excel file containing the excel script functions to parse.</param> /// <returns>The number of functions added.</returns> private void _ParseExcelFunctions(ExcelFile excelFile) { const bool debugOutputLikeFunction = true; foreach (ExcelFile.ExcelFunction excelFunction in excelFile.ExcelFunctions) { //if (excelFunction.Parameters[0].Name == "holy_radius_enemies") //{ // int bp = 0; //} // not sure what to do with them - they don't appear to be used if (excelFunction.Parameters[0].TypeValues.Length <= 4) continue; Function function = new Function(); List<Argument> arguments = new List<Argument>(); for (int i = 0; i < excelFunction.Parameters.Count; i++) { ExcelFile.ExcelFunction.Parameter parameter = excelFunction.Parameters[i]; if (i == 0) { function.Name = parameter.Name; int functionIndex = parameter.TypeValues[4]; if (functionIndex != CallFunctions.Count) throw new Exceptions.ScriptFunctionCountError(functionIndex, CallFunctions.Count); continue; } Argument arg = new Argument { Name = parameter.Name, Type = 0, // I think they're all int ByteOffset = parameter.TypeValues[3], }; arguments.Add(arg); } String byteCodeString = null; if (DebugEnabled && debugOutputLikeFunction) { int offset = 0; byteCodeString = FileTools.ByteArrayToInt32Array(excelFunction.ScriptByteCode, ref offset, excelFunction.ScriptByteCode.Length / 4).ToString(","); } function.Args = arguments.ToArray(); function.ExcelScript = _Decompile(excelFunction, function, byteCodeString, "PROPERTIES", 0, 0, function.Name); if (DebugEnabled && debugOutputLikeFunction) { String[] code = function.ExcelScript.Split(new[] { "; ", ";", "\n" }, StringSplitOptions.RemoveEmptyEntries); String codeStr = code.Aggregate(String.Empty, (current, line) => current + ("\t" + line + ";\n")); String args = String.Empty; for (int i = 0; i < function.ArgCount; i++) { args += "int " + function.Args[i].Name; if (i != function.ArgCount - 1) args += ", "; } String funcString = String.Format("void {0}({1})\n{{\n{2}}}\n", function.Name, args, codeStr); Debug.WriteLine(funcString); } CallFunctions.Add(function); } }
private void _DoQuickExcel(ProgressForm progressForm, Object param) { FileManager fileManager = (FileManager)param; String root = _quickExcelDir_textBox.Text; if (root == "") return; if (fileManager.IsVersionTestCenter) root = Path.Combine(root, "tcv4"); foreach (IndexFile file in fileManager.IndexFiles) // Open Dats for reading file.BeginDatReading(); int i = 0; foreach (PackFileEntry fileEntry in fileManager.FileEntries.Values) { if (!fileEntry.Name.EndsWith(ExcelFile.Extension) && (!fileEntry.Name.EndsWith(StringsFile.Extention) || !fileEntry.Directory.Contains(fileManager.Language))) continue; if (i++ % 10 == 0 && progressForm != null) progressForm.SetCurrentItemText(fileEntry.Path); byte[] fileBytes; try { fileBytes = fileManager.GetFileBytes(fileEntry, true); } catch (Exception e) { String error = String.Format("Failed to get '{0}' file bytes, continue?\n\n{1}", fileEntry.Path, e); DialogResult dr = MessageBox.Show(error, "Excel Extration Error", MessageBoxButtons.YesNo, MessageBoxIcon.Exclamation); if (dr == DialogResult.No) return; continue; } ExcelFile excelFile = null; if (fileEntry.Name.EndsWith(ExcelFile.Extension)) { excelFile = new ExcelFile(fileBytes, fileEntry.Path, fileManager.ClientVersion); if (excelFile.Attributes.IsEmpty) continue; } byte[] writeBytes = fileBytes; if (excelFile != null) { String[] columns = null; if (fileManager.IsVersionTestCenter) { DataFile spVersion; if (_fileManager.DataFiles.TryGetValue(excelFile.StringId, out spVersion)) { ObjectDelegator objectDelegator = _fileManager.DataFileDelegators[spVersion.StringId]; FieldInfo[] fieldInfos = spVersion.Attributes.RowType.GetFields(); columns = new String[objectDelegator.PublicFieldCount]; int col = 0; foreach (ObjectDelegator.FieldDelegate fieldDelegate in objectDelegator.FieldDelegatesPublicList) { columns[col++] = fieldDelegate.Name; } } } try { writeBytes = excelFile.ExportCSV(fileManager, columns); } catch (Exception e) { String error = String.Format("Failed to export CSV for '{0}', continue?\n\n{1}", fileEntry.Path, e); DialogResult dr = MessageBox.Show(error, "Excel Extration Error", MessageBoxButtons.YesNo, MessageBoxIcon.Exclamation); if (dr == DialogResult.No) return; continue; } } String filePath = Path.Combine(root, fileEntry.Path).Replace(ExcelFile.Extension, ExcelFile.ExtensionDeserialised); Directory.CreateDirectory(Path.GetDirectoryName(filePath)); File.WriteAllBytes(filePath, writeBytes); } foreach (IndexFile file in fileManager.IndexFiles) // Close Dats file.EndDatAccess(); }
/// <summary> /// Decompiles an excel script function from byte codes to human readable text. /// </summary> /// <param name="excelScriptFunction">The excel script function to decompile.</param> /// <param name="excelFunction">The associated excel function to use for agrument reference.</param> /// <param name="debugScriptByteString">For debugging purposes only.</param> /// <param name="debugStringId">For debugging purposes only.</param> /// <param name="debugRow">For debugging purposes only.</param> /// <param name="debugCol">For debugging purposes only.</param> /// <param name="debugColName">For debugging purposes only.</param> /// <returns>Decompiled excel script byte codes as human readable script.</returns> private String _Decompile(ExcelFile.ExcelFunction excelScriptFunction, Function excelFunction, String debugScriptByteString = null, String debugStringId = null, int debugRow = 0, int debugCol = 0, String debugColName = null) { if (excelScriptFunction == null) throw new ArgumentNullException("excelScriptFunction", "The excel script function cannot be null!"); if (excelFunction == null) throw new ArgumentNullException("excelFunction", "The excel function cannot be null!"); _vars = new List<StackObject>(); _stack = new Stack<StackObject>(); _excelScriptFunction = excelScriptFunction; _excelFunction = excelFunction; _startOffset = 0; _offset = 0; if (DebugEnabled) { _debugScriptByteString = debugScriptByteString; _debugStringId = debugStringId; _debugRow = debugRow; _debugCol = debugCol; _debugColName = debugColName; } return _script = _Decompile(_excelScriptFunction.ScriptByteCode, _excelScriptFunction.ScriptByteCode.Length, 0).Value; }
private DataTable _LoadExcelTable(ExcelFile excelFile, bool doRelations, bool force) { String tableName = excelFile.StringId; DataTable dataTable = XlsDataSet.Tables[tableName]; if (dataTable != null && !force) return dataTable; if (dataTable != null) { XlsDataSet.Relations.Clear(); XlsDataSet.Tables.Remove(tableName); } dataTable = XlsDataSet.Tables.Add(tableName); dataTable.TableName = tableName; dataTable.ExtendedProperties.Add("FileHeader", excelFile._excelFileHeader.DeepClone()); Type dataType = excelFile.Attributes.RowType; List<OutputAttribute> outputAttributes = new List<OutputAttribute>(); #region Generate Columns DataColumn indexColumn = dataTable.Columns.Add("Index"); indexColumn.AutoIncrement = true; indexColumn.Unique = true; dataTable.PrimaryKey = new[] { indexColumn }; outputAttributes.Add(null); FieldInfo[] fieldInfos = dataType.GetFields(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); foreach (FieldInfo fieldInfo in fieldInfos) { OutputAttribute excelAttribute = ExcelFile.GetExcelAttribute(fieldInfo); // The only private field we add is the TableHeader if (fieldInfo.IsPrivate) { if (fieldInfo.FieldType != typeof(ExcelFile.RowHeader)) continue; outputAttributes.Add(null); dataTable.Columns.Add(fieldInfo.Name, typeof(String)); continue; } Type fieldType = fieldInfo.FieldType; bool isArray = false; bool isEnum = false; if (fieldInfo.FieldType.BaseType == typeof(Array)) { fieldType = typeof(String); isArray = true; } else if (fieldInfo.FieldType.BaseType == typeof(Enum) && excelAttribute == null) { fieldType = fieldInfo.FieldType; isEnum = true; } DataColumn dataColumn = dataTable.Columns.Add(fieldInfo.Name, fieldType); if (isArray) { dataColumn.ExtendedProperties.Add(ColumnKeys.IsArray, true); } else if (isEnum) { dataColumn.ExtendedProperties.Add(ColumnKeys.IsEnum, true); } if (excelAttribute == null) { outputAttributes.Add(null); continue; } outputAttributes.Add(excelAttribute); if (excelAttribute.IsStringOffset) { dataColumn.DataType = typeof(String); dataColumn.ExtendedProperties.Add(ColumnKeys.IsStringOffset, true); dataColumn.DefaultValue = String.Empty; } if (excelAttribute.IsScript) { dataColumn.DataType = typeof(String); dataColumn.ExtendedProperties.Add(ColumnKeys.IsScript, true); dataColumn.DefaultValue = String.Empty; } if (excelAttribute.IsSecondaryString) { dataColumn.DataType = typeof(String); dataColumn.ExtendedProperties.Add(ColumnKeys.IsSecondaryString, true); dataColumn.DefaultValue = String.Empty; } if (excelAttribute.IsStringIndex) { dataColumn.ExtendedProperties.Add(ColumnKeys.IsStringIndex, true); // Add new column for the string DataColumn dataColumnString = dataTable.Columns.Add(fieldInfo.Name + "_string", typeof(String)); dataColumnString.DefaultValue = String.Empty; outputAttributes.Add(null); dataColumnString.ExtendedProperties.Add(ColumnKeys.IsRelationGenerated, true); } if (excelAttribute.IsTableIndex) { dataColumn.ExtendedProperties.Add(ColumnKeys.IsTableIndex, true); // Add new column for the string DataColumn dataColumnString = dataTable.Columns.Add(fieldInfo.Name + "_string", typeof(String)); dataColumnString.DefaultValue = String.Empty; outputAttributes.Add(null); dataColumnString.ExtendedProperties.Add(ColumnKeys.IsRelationGenerated, true); } if (excelAttribute.IsBitmask) { dataColumn.ExtendedProperties.Add(ColumnKeys.IsBitmask, true); } if (excelAttribute.IsBool) { dataColumn.ExtendedProperties.Add(ColumnKeys.IsBool, true); } } if (excelFile.Attributes.HasStats) // items, missiles, monsters, objects, players { DataColumn extendedDataColumn = dataTable.Columns.Add("Stats"); extendedDataColumn.DataType = typeof(String); extendedDataColumn.ExtendedProperties.Add(ExcelFile.ColumnTypeKeys.IsStats, true); outputAttributes.Add(null); } #endregion #region Generate Rows int row = 1; object[] baseRow = new object[outputAttributes.Count]; ObjectDelegator objectDelegator = new ObjectDelegator(fieldInfos); foreach (Object tableRow in excelFile.Rows) { int col = 1; foreach (FieldInfo fieldInfo in fieldInfos) { Object value = objectDelegator[fieldInfo.Name](tableRow); if (fieldInfo.IsPrivate) { if (fieldInfo.FieldType != typeof(ExcelFile.RowHeader)) continue; baseRow[col++] = FileTools.ObjectToStringGeneric(value, ","); continue; } OutputAttribute excelOutputAttribute = outputAttributes[col]; if (excelOutputAttribute == null) { if (value.GetType().BaseType == typeof(Array)) { value = ((Array)value).ToString(","); } baseRow[col++] = value; continue; } if (excelOutputAttribute.IsStringOffset) { int valueInt = (int)value; baseRow[col++] = (valueInt != -1) ? excelFile.ReadStringTable(valueInt) : String.Empty; continue; } if (excelOutputAttribute.IsSecondaryString) { int valueInt = (int)value; baseRow[col++] = (valueInt != -1) ? excelFile.ReadSecondaryStringTable(valueInt) : String.Empty; continue; } if (excelOutputAttribute.IsScript) { int scriptOffset = (int)value; if (scriptOffset == 0) { baseRow[col++] = String.Empty; continue; } String script; if (scriptOffset == 9649 && excelFile.StringId == "SKILLS") // todo: not sure what's with this script... { /* Compiled Bytes: * 26,30,700,6,26,1,399,358,669,616562688,711,26,62,3,17,669,641728512,26,8,711,26,62,3,17,358,669,322961408,26,5,700,6,26,1,399,358,388,0 * * Ending Stack (FIFO): * SetStat669('sfx_attack_pct', 'all', 30 * ($sklvl - 1)) * SetStat669('sfx_duration_pct', 'all', get_skill_level(@unit, 'Shield_Mastery')) * SetStat669('damage_percent_skill', 8 * get_skill_level(@unit, 'Shield_Mastery')) * SetStat669('damage_percent_skill', 8 * get_skill_level(@unit, 'Shield_Mastery')) + 5 * ($sklvl - 1) * * The last SetStat has strange overhang - decompiling wrong? * Or is it "supposed" to be there? * i.e. It's actually decompiling correctly, but because I've assumed such scripts to be wrong (as the end +5... segment is useless) we get the Stack exception */ int[] scriptCode = excelFile.ReadScriptTable(scriptOffset); script = scriptCode != null ? FileTools.ArrayToStringGeneric(scriptCode, ",") : "ScriptError"; baseRow[col++] = script; continue; } //if (fieldInfo.Name == "props1" && row == 45) //{ // int bp = 0; //} ExcelScript excelScript = new ExcelScript(this); try { if (ExcelScript.DebugEnabled) { int[] scriptCode = excelFile.ReadScriptTable(scriptOffset); script = scriptCode != null ? FileTools.ArrayToStringGeneric(scriptCode, ",") : String.Empty; script = excelScript.Decompile(excelFile.ScriptBuffer, scriptOffset, script, excelFile.StringId, row, col, fieldInfo.Name); } else { script = excelScript.Decompile(excelFile.ScriptBuffer, scriptOffset); } //if (script.StartsWith("GetStat666('skill_points_bonus_total', '') > -1;")) //{ // int bp = 0; //} } catch (Exception) { int[] scriptCode = excelFile.ReadScriptTable(scriptOffset); script = scriptCode != null ? FileTools.ArrayToStringGeneric(scriptCode, ",") : "ScriptError"; } baseRow[col++] = script; continue; } if (excelOutputAttribute.IsTableIndex || excelOutputAttribute.IsStringIndex) { if (value.GetType().BaseType == typeof(Array)) { value = ((Array)value).ToString(","); } else { value = (int)value; } baseRow[col++] = value; col++; // for _strings relational column continue; } // Else its something else, ie bitmask, bool, table/string index baseRow[col++] = value; } // stats, only a component of the UnitData row type if (excelFile.Attributes.HasStats) { baseRow[col++] = FileTools.ArrayToStringGeneric(excelFile.ReadStats(row - 1), ","); } dataTable.Rows.Add(baseRow); row++; } #endregion // Generate Relationships as required if (doRelations) _GenerateRelations(excelFile); return dataTable; }
/// <summary> /// Function to test all excel files cooking/recooking/etc to/from byte arrays and csv types. /// </summary> public static void TestExcelCooking(bool doTCv4 = false) { /* 0 0 byte[] fileBytes = fileManager.GetFileBytes(excelFile.FilePath, true); Bytes 0-gen bytes 0 1 2 3 4 foreach (DataFile dataFile in fileManager.DataFiles.Values) Bytes->Xls 1-gen Xls file 1 byte[] dataFileBytes = dataFile.ToByteArray(); Bytes->Xls->Bytes 1-gen bytes fromBytesExcel.ParseData(dataFileBytes); Bytes->Xls->Bytes->Xls 2-gen Xls file 1a byte[] bytesXlsBytesXlsBytes = fromBytesExcel.ToByteArray(); Bytes->Xls->Bytes->Xls->Bytes 2-gen bytes 1ba byte[] bytesXlsCsv = dataFile.ExportCSV(fileManager); Bytes->Xls->CSV 1-gen CSV-bytes bytesXlsCsvXls.ParseCSV(bytesXlsCsv, fileManager); Bytes->Xls->CSV ->Xls 2-gen Xls file 1bb byte[] bytesXlsCsvXlsBytes = bytesXlsCsvXls.ToByteArray(); Bytes->Xls->CSV ->Xls->Bytes 2-gen CSV-bytes 1b byte[] csvBytes = fromBytesExcel.ExportCSV(fileManager); Bytes->Xls->Bytes->Xls->CSV 2-gen CSV file csvExcel.ParseCSV(csvBytes, fileManager); Bytes->Xls->Bytes->Xls->CSV->Xls 3-gen Xls file 1c byte[] bytesXlsBytesXlsCsvXlsBytes = csvExcel.ToByteArray(); Bytes->Xls->Bytes->Xls->CSV->Xls->Bytes 3-gen bytes finalExcel.ParseData(recookedExcelBytes); Bytes->Xls->Bytes->Xls->CSV->Xls->Bytes->Xls 4-gen Xls file 1d byte[] csvCheck = finalExcel.ExportCSV(fileManager); Bytes->Xls->Bytes->Xls->CSV->Xls->Bytes->Xls->CSV 4-gen CSV file 1e byte[] finalCheck = finalExcel.ToByteArray(); Bytes->Xls->Bytes->Xls->CSV->Xls->Bytes->Xls->Bytes 4-gen bytes */ String root = @"C:\excel_debug"; FileManager.ClientVersions clientVersion = FileManager.ClientVersions.SinglePlayer; if (doTCv4) { root = Path.Combine(root, "tcv4"); clientVersion = FileManager.ClientVersions.TestCenter; } root += @"\"; // lazy Directory.CreateDirectory(root); FileManager fileManager = new FileManager(Config.HglDir, clientVersion); fileManager.BeginAllDatReadAccess(); fileManager.LoadTableFiles(); ExcelFile.EnableDebug = true; int checkedCount = 0; List<String> excelFailed = new List<String>(); foreach (DataFile bytesXls in fileManager.DataFiles.Values) { ExcelFile excelFile = bytesXls as ExcelFile; if (excelFile == null) continue; const String debugStr = "SKILLS"; //if (excelFile.StringId != debugStr) continue; //if (excelFile.StringId == debugStr) //{ // int bp = 0; //} checkedCount++; Debug.Write(String.Format("Checking {0}... ", bytesXls.StringId)); byte[] bytes = fileManager.GetFileBytes(excelFile.FilePath, true); try { byte[] bytesXlsBytes = bytesXls.ToByteArray(); if (bytesXls.StringId == "SOUNDS" && false) { byte[] csvBytesSounds = excelFile.ExportCSV(fileManager); ExcelFile soundsCSV = new ExcelFile(excelFile.FilePath); soundsCSV.ParseCSV(csvBytesSounds, fileManager); byte[] soundsBytes = soundsCSV.ToByteArray(); //byte[] soundsBytesFromCSV = soundsCSV.ExportCSV(); //ExcelFile soundsCSVFromBytesFromCSV = new ExcelFile(soundsBytesFromCSV, fileEntry.RelativeFullPathWithoutPatch); // some brute force ftw byte[][] bytesArrays = new[] { bytes, soundsBytes }; for (int z = 0; z < bytesArrays.Length; z++) { byte[] bytesBrute = bytesArrays[z]; int offset = 0x20; int stringsBytesCount = FileTools.ByteArrayToInt32(bytesBrute, ref offset); StringWriter stringWriterByteStrings = new StringWriter(); stringWriterByteStrings.WriteLine(stringsBytesCount + " bytes"); List<String> strings = new List<String>(); List<int> offsets = new List<int>(); while (offset < stringsBytesCount + 0x20) { String str = FileTools.ByteArrayToStringASCII(bytesBrute, offset); strings.Add(str); offsets.Add(offset); offset += str.Length + 1; } String[] sortedStrings = strings.ToArray(); int[] sortedOffsets = offsets.ToArray(); Array.Sort(sortedStrings, sortedOffsets); stringWriterByteStrings.WriteLine(strings.Count + " strings"); for (int i = 0; i < strings.Count; i++) { stringWriterByteStrings.WriteLine(sortedStrings[i] + "\t\t\t" + sortedOffsets[i]); } File.WriteAllText(@"C:\excel_debug\strings" + z + ".txt", stringWriterByteStrings.ToString()); } } Debug.Write("ToByteArray... "); if (bytes.Length != bytesXlsBytes.Length && !doTCv4) // some TCv4 tables don't have their sort columns yet { Debug.WriteLine("ToByteArray() dataFileBytes has differing length: " + bytesXls.StringId); File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes); File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes); excelFailed.Add(bytesXls.StringId); continue; } ExcelFile bytesXlsBytesXls = new ExcelFile(excelFile.FilePath, fileManager.ClientVersion); bytesXlsBytesXls.ParseData(bytesXlsBytes); Debug.Write("new ExcelFile... "); if (!bytesXlsBytesXls.HasIntegrity) { Debug.WriteLine("fromBytesExcel = new Excel from ToByteArray() failed!"); File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes); File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes); excelFailed.Add(bytesXls.StringId); continue; } // more checks Debug.Write("ToByteArray->ToByteArray... "); byte[] bytesXlsBytesXlsBytes = bytesXlsBytesXls.ToByteArray(); // bytesXlsBytesXlsBytes if (bytes.Length != bytesXlsBytesXlsBytes.Length && !doTCv4) // some TCv4 tables don't have their sort columns yet { Debug.WriteLine("ToByteArray() dataFileBytesFromToByteArray has differing length!"); File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes); File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes); File.WriteAllBytes(root + bytesXls.StringId + "1a.toByteArrayFromByteArray", bytesXlsBytesXlsBytes); excelFailed.Add(bytesXls.StringId); continue; } // check generated sort index arrays); Debug.Write("IndexSortArrays... "); if (excelFile.IndexSortArray != null) { if (bytesXlsBytesXls.IndexSortArray == null || excelFile.IndexSortArray.Count != bytesXlsBytesXls.IndexSortArray.Count) { Debug.WriteLine("fromBytesExcel has not-matching IndexSortArray count!"); excelFailed.Add(bytesXls.StringId); continue; } bool hasError = false; for (int i = 0; i < excelFile.IndexSortArray.Count; i++) { if (excelFile.IndexSortArray[i].SequenceEqual(bytesXlsBytesXls.IndexSortArray[i])) continue; Debug.WriteLine(String.Format("IndexSortArray[{0}] NOT EQUAL to original!", i)); hasError = true; } if (hasError) { File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes); File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes); File.WriteAllBytes(root + bytesXls.StringId + "1a.bytesXlsBytesXlsBytes", bytesXlsBytesXlsBytes); excelFailed.Add(bytesXls.StringId); continue; } } // some csv stuff Debug.Write("BaseXls->BaseCSV=="); byte[] bytesXlsCsv = bytesXls.ExportCSV(fileManager); // Bytes->Xls->CSV Debug.Write("ExportCSV... "); byte[] bytesXlsBytesXlsCsv = bytesXlsBytesXls.ExportCSV(fileManager); if (!bytesXlsCsv.SequenceEqual(bytesXlsBytesXlsCsv)) { Debug.WriteLine("FALSE!"); File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes); File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes); File.WriteAllBytes(root + bytesXls.StringId + "1a.bytesXlsBytesXlsBytes", bytesXlsBytesXlsBytes); File.WriteAllBytes(root + bytesXls.StringId + "1b.bytesXlsBytesXlsCsv.csv", bytesXlsBytesXlsCsv); File.WriteAllBytes(root + bytesXls.StringId + "1ba.bytesXlsCsv.csv", bytesXlsCsv); excelFailed.Add(bytesXls.StringId); continue; } ExcelFile bytesXlsCsvXls = new ExcelFile(excelFile.FilePath, fileManager.ClientVersion); bytesXlsCsvXls.ParseCSV(bytesXlsCsv, fileManager); byte[] bytesXlsCsvXlsBytes = bytesXlsCsvXls.ToByteArray(); // used later as well if (excelFile.ScriptBuffer == null && // can only compare here for no-script tables - can't compare to pure-base xls-bytes as xls->csv->xls rearranges script code !excelFile.HasStringBuffer) // xls->csv->xls also rearranges the strings buffer { Debug.Write("BytesXlsBytesXlsBytes==BytesXlsCsvXlsBytes... "); if (!bytesXlsBytesXlsBytes.SequenceEqual(bytesXlsCsvXlsBytes)) { Debug.WriteLine("FALSE!"); File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes); File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes); File.WriteAllBytes(root + bytesXls.StringId + "1a.bytesXlsBytesXlsBytes", bytesXlsBytesXlsBytes); File.WriteAllBytes(root + bytesXls.StringId + "1b.bytesXlsBytesXlsCsv.csv", bytesXlsBytesXlsCsv); File.WriteAllBytes(root + bytesXls.StringId + "1ba.bytesXlsCsv.csv", bytesXlsCsv); File.WriteAllBytes(root + bytesXls.StringId + "1bb.bytesXlsCsvXlsBytes", bytesXlsCsvXlsBytes); excelFailed.Add(bytesXls.StringId); continue; } Debug.Write("BytesXlsBytes==BytesXlsCsvXlsBytes... "); if (!bytesXlsBytes.SequenceEqual(bytesXlsCsvXlsBytes)) { Debug.WriteLine("FALSE!"); File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes); File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes); File.WriteAllBytes(root + bytesXls.StringId + "1a.bytesXlsBytesXlsBytes", bytesXlsBytesXlsBytes); File.WriteAllBytes(root + bytesXls.StringId + "1b.bytesXlsBytesXlsCsv.csv", bytesXlsBytesXlsCsv); File.WriteAllBytes(root + bytesXls.StringId + "1ba.bytesXlsCsv.csv", bytesXlsCsv); File.WriteAllBytes(root + bytesXls.StringId + "1bb.bytesXlsCsvXlsBytes", bytesXlsCsvXlsBytes); excelFailed.Add(bytesXls.StringId); continue; } } Debug.Write("BytesXlsBytesXlsCsv -> new ExcelFile"); ExcelFile bytesXlsBytesXlsCsvXls = new ExcelFile(excelFile.FilePath, fileManager.ClientVersion); bytesXlsBytesXlsCsvXls.ParseCSV(bytesXlsBytesXlsCsv, fileManager); Debug.Write("... "); if (!bytesXlsBytesXlsCsvXls.HasIntegrity) { Debug.WriteLine("Failed!"); File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes); File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes); File.WriteAllBytes(root + bytesXls.StringId + "1a.bytesXlsBytesXlsBytes", bytesXlsBytesXlsBytes); File.WriteAllBytes(root + bytesXls.StringId + "1b.bytesXlsBytesXlsCsv.csv", bytesXlsBytesXlsCsv); File.WriteAllBytes(root + bytesXls.StringId + "1ba.bytesXlsCsv.csv", bytesXlsCsv); excelFailed.Add(bytesXls.StringId); continue; } byte[] bytesXlsBytesXlsCsvXlsBytes = bytesXlsBytesXlsCsvXls.ToByteArray(); Debug.Write("BytesXlsCsvXlsBytes==BytesXlsBytesXlsCsvXlsBytes... "); if (!bytesXlsCsvXlsBytes.SequenceEqual(bytesXlsBytesXlsCsvXlsBytes)) { Debug.WriteLine("FALSE!"); File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes); File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes); File.WriteAllBytes(root + bytesXls.StringId + "1a.bytesXlsBytesXlsBytes", bytesXlsBytesXlsBytes); File.WriteAllBytes(root + bytesXls.StringId + "1b.bytesXlsBytesXlsCsv.csv", bytesXlsBytesXlsCsv); File.WriteAllBytes(root + bytesXls.StringId + "1ba.bytesXlsCsv.csv", bytesXlsCsv); File.WriteAllBytes(root + bytesXls.StringId + "1bb.bytesXlsCsvXlsBytes", bytesXlsCsvXlsBytes); File.WriteAllBytes(root + bytesXls.StringId + "1c.bytesXlsBytesXlsCsvXlsBytes", bytesXlsBytesXlsCsvXlsBytes); excelFailed.Add(bytesXls.StringId); continue; } if (!fileManager.IsVersionTestCenter) { Debug.Write("StructureId... "); UInt32 structureId = BitConverter.ToUInt32(bytes, 4); UInt32 fromCSVStructureId = BitConverter.ToUInt32(bytesXlsBytesXlsCsvXlsBytes, 4); if (structureId != fromCSVStructureId) { Debug.WriteLine("Structure Id value do not match: " + structureId + " != " + fromCSVStructureId); excelFailed.Add(bytesXls.StringId); continue; } } Debug.Write("Almost Done... "); ExcelFile bytesXlsBytesXlsCsvXlsBytesXls = new ExcelFile(excelFile.FilePath, fileManager.ClientVersion); bytesXlsBytesXlsCsvXlsBytesXls.ParseData(bytesXlsBytesXlsCsvXlsBytes); byte[] bytesXlsBytesXlsCsvXlsBytesXlsCsv = bytesXlsBytesXlsCsvXlsBytesXls.ExportCSV(fileManager); int recookedLength = bytesXlsBytesXlsCsvXlsBytes.Length; if (excelFile.StringId == "SKILLS") recookedLength += 12; // 12 bytes in int ptr data not used/referenced at all and are removed/lost in bytes -> csv -> bytes if (bytes.Length != recookedLength && !doTCv4) // some TCv4 tables don't have their sort columns yet { Debug.WriteLine("Recooked Excel file has differing length!"); File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes); File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes); File.WriteAllBytes(root + bytesXls.StringId + "1a.bytesXlsBytesXlsBytes", bytesXlsBytesXlsBytes); File.WriteAllBytes(root + bytesXls.StringId + "1b.bytesXlsBytesXlsCsv.csv", bytesXlsBytesXlsCsv); File.WriteAllBytes(root + bytesXls.StringId + "1ba.bytesXlsCsv.csv", bytesXlsCsv); File.WriteAllBytes(root + bytesXls.StringId + "1c.bytesXlsBytesXlsCsvXlsBytes", bytesXlsBytesXlsCsvXlsBytes); File.WriteAllBytes(root + bytesXls.StringId + "1d.bytesXlsBytesXlsCsvXlsBytesXlsCsv.csv", bytesXlsBytesXlsCsvXlsBytesXlsCsv); excelFailed.Add(bytesXls.StringId); continue; } Debug.Assert(bytesXlsBytesXlsCsvXlsBytesXls.HasIntegrity); byte[] bytesXlsBytesXlsCsvXlsBytesXlsBytes = bytesXlsBytesXlsCsvXlsBytesXls.ToByteArray(); if (excelFile.StringId == "SKILLS") Debug.Assert(bytesXlsBytesXlsCsvXlsBytesXlsBytes.Length + 12 == bytesXlsBytes.Length); else Debug.Assert(bytesXlsBytesXlsCsvXlsBytesXlsBytes.Length == bytesXlsBytes.Length || doTCv4); if (!bytesXlsBytesXlsCsv.SequenceEqual(bytesXlsBytesXlsCsvXlsBytesXlsCsv)) { Debug.WriteLine("csvBytes.SequenceEqual failed!"); File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes); File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes); File.WriteAllBytes(root + bytesXls.StringId + "1a.bytesXlsBytesXlsBytes", bytesXlsBytesXlsBytes); File.WriteAllBytes(root + bytesXls.StringId + "1b.bytesXlsBytesXlsCsv.csv", bytesXlsBytesXlsCsv); File.WriteAllBytes(root + bytesXls.StringId + "1ba.bytesXlsCsv.csv", bytesXlsCsv); File.WriteAllBytes(root + bytesXls.StringId + "1c.bytesXlsBytesXlsCsvXlsBytes", bytesXlsBytesXlsCsvXlsBytes); File.WriteAllBytes(root + bytesXls.StringId + "1d.bytesXlsBytesXlsCsvXlsBytesXlsCsv.csv", bytesXlsBytesXlsCsvXlsBytesXlsCsv); File.WriteAllBytes(root + bytesXls.StringId + "1e.bytesXlsBytesXlsCsvXlsBytesXlsBytes", bytesXlsBytesXlsCsvXlsBytesXlsBytes); excelFailed.Add(bytesXls.StringId); continue; } Debug.Write("\nPerforming deep scan: "); ObjectDelegator objectDelegator = fileManager.DataFileDelegators[excelFile.StringId]; int lastPercent = 0; int col = 0; bool failed = false; foreach (FieldDelegate fieldDelegate in objectDelegator) { int percent = col * 100 / objectDelegator.FieldCount - 1; int dotCount = percent - lastPercent; for (int i = 0; i < dotCount; i++) Debug.Write("."); lastPercent = percent; ExcelFile.OutputAttribute excelAttribute = ExcelFile.GetExcelAttribute(fieldDelegate.Info); bool isArray = (fieldDelegate.FieldType.BaseType == typeof(Array)); for (int row = 0; row < excelFile.Rows.Count; row++) { Object obj1 = fieldDelegate.GetValue(excelFile.Rows[row]); Object obj2 = fieldDelegate.GetValue(bytesXlsBytesXlsCsvXlsBytesXls.Rows[row]); if (isArray) { //if (excelFile.StringId == "TREASURE") //{ // int bp = 0; //} IEnumerator arrayEnumator1 = ((Array)obj1).GetEnumerator(); IEnumerator arrayEnumator2 = ((Array)obj2).GetEnumerator(); int index = -1; while (arrayEnumator1.MoveNext() && arrayEnumator2.MoveNext()) { index++; Object elementVal1 = arrayEnumator1.Current; Object elementVal2 = arrayEnumator2.Current; if (elementVal1.Equals(elementVal2)) continue; Debug.WriteLine(String.Format("Array Element '{0}' != '{1}' on col({2}) = '{3}', row({4}), index({5})", elementVal1, elementVal2, col, fieldDelegate.Name, row, index)); failed = true; break; } if (failed) break; continue; } if (obj1.Equals(obj2)) continue; if (excelAttribute != null) { if (excelAttribute.IsStringOffset) { int offset1 = (int)obj1; int offset2 = (int)obj2; String str1 = excelFile.ReadStringTable(offset1); String str2 = bytesXlsBytesXlsCsvXlsBytesXls.ReadStringTable(offset2); if (str1 == str2) continue; obj1 = str1; obj2 = str2; } if (excelAttribute.IsScript) { int offset1 = (int)obj1; int offset2 = (int)obj2; Int32[] script1 = excelFile.ReadScriptTable(offset1); Int32[] script2 = bytesXlsBytesXlsCsvXlsBytesXls.ReadScriptTable(offset2); if (script1.SequenceEqual(script2)) continue; obj1 = script1.ToString(","); obj2 = script2.ToString(","); } } String str = obj1 as String; if ((str != null && str.StartsWith("+N%-N% base damage as [elem]")) || // "+N%-N% base damage as [elem]…" != "+N%-N% base damage as [elem]?" on col(2) = 'exampleDescription', row(132) // the '?' at end fieldDelegate.Name == "bonusStartingTreasure") // refernces multiple different blank rows continue; Debug.WriteLine(String.Format("'{0}' != '{1}' on col({2}) = '{3}', row({4})", obj1, obj2, col, fieldDelegate.Name, row)); failed = true; break; } if (failed) break; col++; } if (failed) { File.WriteAllBytes(root + bytesXls.StringId + ".bytesXlsBytesXlsCsv.csv", bytesXlsBytesXlsCsv); File.WriteAllBytes(root + bytesXls.StringId + ".bytesXlsBytesXlsCsvXlsBytesXlsCsv.csv", bytesXlsBytesXlsCsvXlsBytesXlsCsv); excelFailed.Add(bytesXls.StringId); continue; } Debug.WriteLine("OK\n"); } catch (Exception e) { Debug.WriteLine("Excel file Exception: " + bytesXls.StringId + "\n" + e); } } Debug.WriteLine(checkedCount + " excel files checked."); if (excelFailed.Count <= 0) return; Debug.WriteLine(excelFailed.Count + " failed excel checks! (" + ((float)excelFailed.Count / checkedCount * 100) + "%)"); foreach (String str in excelFailed) Debug.Write(str + ", "); }
private void _GenerateRelations(ExcelFile excelFile) { if ((excelFile == null)) return; Type type = excelFile.Attributes.RowType; int col; String mainTableName = excelFile.StringId; DataTable mainDataTable = XlsDataSet.Tables[mainTableName]; // remove all extra generated columns on this table for (col = 0; col < mainDataTable.Columns.Count; col++) { DataColumn dc = mainDataTable.Columns[col]; if (!(dc.ExtendedProperties.Contains(ColumnKeys.IsRelationGenerated))) continue; mainDataTable.Columns.Remove(dc); } col = 1; // regenerate relations foreach (FieldInfo fieldInfo in type.GetFields(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic)) { if (fieldInfo.IsPrivate) { if (fieldInfo.FieldType == typeof(ExcelFile.RowHeader)) { col++; } continue; } OutputAttribute excelOutputAttribute = ExcelFile.GetExcelAttribute(fieldInfo); if ((excelOutputAttribute == null)) { col++; continue; } DataColumn dcChild = mainDataTable.Columns[col]; if ((excelOutputAttribute.IsStringIndex)) { DataTable dtStrings = XlsDataSet.Tables[StringsTableName] ?? _LoadStringsTable(); if (dtStrings != null) { DataColumn dcParent = dtStrings.Columns["ReferenceId"]; String relationName = String.Format("{0}_{1}_{2}", excelFile.StringId, dcChild.ColumnName, ColumnKeys.IsStringIndex); // parent and child are swapped here for StringId relations DataRelation relation = new DataRelation(relationName, dcChild, dcParent, false); XlsDataSet.Relations.Add(relation); DataColumn dcString = mainDataTable.Columns.Add(dcChild.ColumnName + "_string", typeof(String)); dcString.SetOrdinal(col + 1); dcString.ExtendedProperties.Add(ExcelFile.ColumnTypeKeys.IsRelationGenerated, true); // need to use MIN (and thus Child) for cases of strings with same reference id (e.g. Items; SINGULAR and PLURAL, etc) dcString.Expression = "MIN(Child(" + relationName + ").String)"; col++; } } if (excelOutputAttribute.IsTableIndex) { String tableStringId = excelOutputAttribute.TableStringId; DataTable dt = XlsDataSet.Tables[tableStringId] ?? _LoadRelatedTable(tableStringId); if (dt != null) { DataColumn dcParent = dt.Columns["Index"]; String relatedColumn = dt.Columns[2].ColumnName; // todo if (dcChild.ExtendedProperties.ContainsKey(ExcelFile.ColumnTypeKeys.IsArray) && (bool)dcChild.ExtendedProperties[ExcelFile.ColumnTypeKeys.IsArray]) { //if (excelFile.StringId == "ITEMS") //{ // int bp = 0; //} col++; continue; } //String relationNameOld = excelFile.StringId + dcChild.ColumnName + ExcelFile.ColumnTypeKeys.IsTableIndex; String relationName = String.Format("{0}_{1}_{2}", excelFile.StringId, dcChild.ColumnName, ExcelFile.ColumnTypeKeys.IsTableIndex); DataRelation relation = new DataRelation(relationName, dcParent, dcChild, false); //if (relationName == "PLAYERS_paperdollSkill_IsTableIndex") //{ // int bp = 0; //} XlsDataSet.Relations.Add(relation); DataColumn dcString = mainDataTable.Columns.Add(dcChild.ColumnName + "_string", typeof(String), String.Format("Parent({0}).{1}", relationName, relatedColumn)); dcString.SetOrdinal(col + 1); dcString.ExtendedProperties.Add(ExcelFile.ColumnTypeKeys.IsRelationGenerated, true); col++; } } col++; } }
/// <summary> /// This function checks every row/col of every excel file, determining if it needs to be visible/public or not. /// Any element non-zero for all rows, needs to be public (or add new OutputAttribute for "const" values?) /// </summary> public static void ExcelValuesDeepScan() { FileManager fileManager = new FileManager(Config.HglDir); fileManager.ExtractAllExcel(); Dictionary<uint, uint> rowTypeCounts = new Dictionary<uint, uint>(); Dictionary<String, uint[]> outputMessages = new Dictionary<String, uint[]>(); Dictionary<String, ObjectDelegator> objectDelegators = new Dictionary<String, ObjectDelegator>(); foreach (PackFileEntry fileEntry in fileManager.FileEntries.Values) { if (!fileEntry.Name.EndsWith(ExcelFile.Extension)) continue; byte[] fileBytes = fileManager.GetFileBytes(fileEntry, true); Debug.Assert(fileBytes != null); ExcelFile excelFile = new ExcelFile(fileBytes, fileEntry.Path); if (excelFile.Attributes.IsEmpty) continue; Debug.WriteLine("Checking file: " + fileEntry.Path); uint structureId = excelFile.Attributes.StructureId; uint structureUseCount = 0; if (rowTypeCounts.TryGetValue(structureId, out structureUseCount)) { rowTypeCounts[structureId] = ++structureUseCount; } else { rowTypeCounts.Add(structureId, 1); } ObjectDelegator excelDelegator; Type rowType = excelFile.Attributes.RowType; FieldInfo[] fieldInfos = rowType.GetFields(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance); int rowCount = excelFile.Rows.Count; // create delegates if (!objectDelegators.TryGetValue(excelFile.StringId, out excelDelegator)) { excelDelegator = new ObjectDelegator(fieldInfos); objectDelegators.Add(excelFile.StringId, excelDelegator); } // check by column, by row int col = -1; foreach (FieldInfo fieldInfo in fieldInfos) { if (++col == 0) continue; // row header ObjectDelegator.FieldGetValueDelegate getValue = excelDelegator.GetFieldGetDelegate(fieldInfo.Name); ExcelFile.OutputAttribute outputAttribute = ExcelFile.GetExcelAttribute(fieldInfo); bool isArray = false; bool allEqual = true; Object firstValue = null; Array firstValueArray = null; int arrayIndexFirstDifferent = -1; int row = -1; String message = null; foreach (Object value in excelFile.Rows.Select(rowObject => getValue(rowObject))) { arrayIndexFirstDifferent = -1; if (++row == 0) { if (fieldInfo.FieldType.BaseType == typeof(Array)) { firstValueArray = (Array)value; isArray = true; IEnumerator enumerator = firstValueArray.GetEnumerator(); enumerator.MoveNext(); firstValue = enumerator.Current; } else { firstValue = value; } continue; } if (isArray) { Array objArray = (Array)value; Debug.Assert(firstValueArray != null); bool arrayEqual = true; IEnumerator enumeratorFirst = firstValueArray.GetEnumerator(); IEnumerator enumeratorCurrent = objArray.GetEnumerator(); while (enumeratorFirst.MoveNext()) { enumeratorCurrent.MoveNext(); arrayIndexFirstDifferent++; Object objFirst = enumeratorFirst.Current; Object objCurrent = enumeratorCurrent.Current; if (firstValue == null) firstValue = objFirst; if (objFirst.Equals(objCurrent)) continue; arrayEqual = false; break; } if (arrayEqual) continue; allEqual = false; break; } Debug.Assert(firstValue != null); if (firstValue.Equals(value)) continue; allEqual = false; break; } Debug.Assert(firstValue != null); Object constValue = null; bool ignoreConstCheck = false; if (outputAttribute != null) { constValue = outputAttribute.ConstantValue; ignoreConstCheck = outputAttribute.DebugIgnoreConstantCheck; } if (fieldInfo.IsPrivate) { if (!allEqual) { message = String.Format("Warning: Private column (Type = {1}, StructureId = 0x{2}) \"{0}\" has per-row differing values", fieldInfo.Name, fieldInfo.FieldType, structureId.ToString("X8")); // need to add structureId to ensure message is unique per structure type if (isArray) { message += ", first index differing: " + arrayIndexFirstDifferent; } } else // allEqual && IsPrivate { if (isArray) { if (fieldInfo.FieldType == typeof(Int32[])) { if (firstValue.Equals((Int32)0)) continue; } else if (fieldInfo.FieldType == typeof(byte[])) { if (firstValue.Equals((byte)0x00)) continue; } else { Debug.Assert(false, "Unexpected Array Type: " + fieldInfo.FieldType); } if (outputAttribute != null && firstValue.Equals(outputAttribute.ConstantValue)) continue; message = String.Format("Warning: Private column (Type = {2}, StructureId = 0x{3}) \"{0}\" has constant value not zero \"{1}\"", fieldInfo.Name, firstValue, fieldInfo.FieldType, structureId.ToString("X8")); } else // not array { if (firstValue.Equals(constValue)) continue; if (fieldInfo.FieldType == typeof(Int32) && ((Int32)firstValue == 0 && (firstValue.Equals(constValue) || constValue == null))) continue; if (fieldInfo.FieldType == typeof(Int16) && ((Int16)firstValue == 0 && (firstValue.Equals(constValue) || constValue == null))) continue; message = String.Format("Warning: Private column (Type = {2}, StructureId = 0x{3}) \"{0}\" has constant value not zero \"{1}\"", fieldInfo.Name, firstValue, fieldInfo.FieldType, structureId.ToString("X8")); if (outputAttribute != null && outputAttribute.ConstantValue != null) message += " (outputAttribute.ConstantValue is set)"; } } } else // IsPublic { if (ignoreConstCheck) continue; if (allEqual && fieldInfo.IsPublic && rowCount != 1 && (outputAttribute == null || outputAttribute.ConstantValue == null)) { message = String.Format("Notice: Public column (Type = {2}, StructureId = 0x{3}) \"{0}\" has constant value \"{1}\"", fieldInfo.Name, firstValue, fieldInfo.FieldType, structureId.ToString("X8")); } } if (message == null) continue; uint[] messageCounts; if (outputMessages.TryGetValue(message, out messageCounts)) { outputMessages[message] = new[] { messageCounts[0] + 1, structureId }; } else { outputMessages.Add(message, new uint[] { 1, structureId }); } } //int bp3 = 0; } String previousStringId = null; StringWriter stringWriter = new StringWriter(); foreach (KeyValuePair<String, uint[]> message in outputMessages) { String msg = message.Key; uint msgCount = message.Value[0]; uint forStructureId = message.Value[1]; uint structureCount = rowTypeCounts[forStructureId]; if (structureCount != msgCount) continue; // if not equal, then we have a message in one table, but in another table it's not applicable String[] stringIds1 = (from dataTableAttribute in DataFile.DataFileMap where dataTableAttribute.Value.StructureId == forStructureId select dataTableAttribute.Key).ToArray(); String[] stringIds2 = (from dataTableAttribute in DataFile.DataFileMapTestCenter where dataTableAttribute.Value.StructureId == forStructureId select dataTableAttribute.Key).ToArray(); String[] stringIds3 = (from dataTableAttribute in DataFile.DataFileMapResurrection where dataTableAttribute.Value.StructureId == forStructureId select dataTableAttribute.Key).ToArray(); String[] stringIds4 = (from dataTableAttribute in DataFile.DataFileMapMod where dataTableAttribute.Value.StructureId == forStructureId select dataTableAttribute.Key).ToArray(); String stringIdPrepend = String.Join(",", stringIds1) + String.Join(",", stringIds2) + String.Join(",", stringIds3) + String.Join(",", stringIds4); if (previousStringId != stringIdPrepend) { //Console.WriteLine(stringIdPrepend); stringWriter.WriteLine(stringIdPrepend); previousStringId = stringIdPrepend; } //Console.WriteLine(msg); stringWriter.WriteLine(msg); } File.WriteAllText(@"C:\asdf.txt", stringWriter.ToString()); }
private DataTable _LoadExcelTable(ExcelFile excelFile, bool doRelations, bool force) { String tableName = excelFile.StringId; DataTable dataTable = XlsDataSet.Tables[tableName]; if (dataTable != null && !force) { return(dataTable); } if (dataTable != null) { XlsDataSet.Relations.Clear(); XlsDataSet.Tables.Remove(tableName); } dataTable = XlsDataSet.Tables.Add(tableName); dataTable.TableName = tableName; dataTable.ExtendedProperties.Add("FileHeader", excelFile._excelFileHeader.DeepClone()); Type dataType = excelFile.Attributes.RowType; List <OutputAttribute> outputAttributes = new List <OutputAttribute>(); #region Generate Columns DataColumn indexColumn = dataTable.Columns.Add("Index"); indexColumn.AutoIncrement = true; indexColumn.Unique = true; dataTable.PrimaryKey = new[] { indexColumn }; outputAttributes.Add(null); FieldInfo[] fieldInfos = dataType.GetFields(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); foreach (FieldInfo fieldInfo in fieldInfos) { OutputAttribute excelAttribute = ExcelFile.GetExcelAttribute(fieldInfo); // The only private field we add is the TableHeader if (fieldInfo.IsPrivate) { if (fieldInfo.FieldType != typeof(ExcelFile.RowHeader)) { continue; } outputAttributes.Add(null); dataTable.Columns.Add(fieldInfo.Name, typeof(String)); continue; } Type fieldType = fieldInfo.FieldType; bool isArray = false; bool isEnum = false; if (fieldInfo.FieldType.BaseType == typeof(Array)) { fieldType = typeof(String); isArray = true; } else if (fieldInfo.FieldType.BaseType == typeof(Enum) && excelAttribute == null) { fieldType = fieldInfo.FieldType; isEnum = true; } DataColumn dataColumn = dataTable.Columns.Add(fieldInfo.Name, fieldType); if (isArray) { dataColumn.ExtendedProperties.Add(ColumnKeys.IsArray, true); } else if (isEnum) { dataColumn.ExtendedProperties.Add(ColumnKeys.IsEnum, true); } if (excelAttribute == null) { outputAttributes.Add(null); continue; } outputAttributes.Add(excelAttribute); if (excelAttribute.IsStringOffset) { dataColumn.DataType = typeof(String); dataColumn.ExtendedProperties.Add(ColumnKeys.IsStringOffset, true); dataColumn.DefaultValue = String.Empty; } if (excelAttribute.IsScript) { dataColumn.DataType = typeof(String); dataColumn.ExtendedProperties.Add(ColumnKeys.IsScript, true); dataColumn.DefaultValue = String.Empty; } if (excelAttribute.IsSecondaryString) { dataColumn.DataType = typeof(String); dataColumn.ExtendedProperties.Add(ColumnKeys.IsSecondaryString, true); dataColumn.DefaultValue = String.Empty; } if (excelAttribute.IsStringIndex) { dataColumn.ExtendedProperties.Add(ColumnKeys.IsStringIndex, true); // Add new column for the string DataColumn dataColumnString = dataTable.Columns.Add(fieldInfo.Name + "_string", typeof(String)); dataColumnString.DefaultValue = String.Empty; outputAttributes.Add(null); dataColumnString.ExtendedProperties.Add(ColumnKeys.IsRelationGenerated, true); } if (excelAttribute.IsTableIndex) { dataColumn.ExtendedProperties.Add(ColumnKeys.IsTableIndex, true); // Add new column for the string DataColumn dataColumnString = dataTable.Columns.Add(fieldInfo.Name + "_string", typeof(String)); dataColumnString.DefaultValue = String.Empty; outputAttributes.Add(null); dataColumnString.ExtendedProperties.Add(ColumnKeys.IsRelationGenerated, true); } if (excelAttribute.IsBitmask) { dataColumn.ExtendedProperties.Add(ColumnKeys.IsBitmask, true); } if (excelAttribute.IsBool) { dataColumn.ExtendedProperties.Add(ColumnKeys.IsBool, true); } } if (excelFile.Attributes.HasStats) // items, missiles, monsters, objects, players { DataColumn extendedDataColumn = dataTable.Columns.Add("Stats"); extendedDataColumn.DataType = typeof(String); extendedDataColumn.ExtendedProperties.Add(ExcelFile.ColumnTypeKeys.IsStats, true); outputAttributes.Add(null); } #endregion #region Generate Rows int row = 1; object[] baseRow = new object[outputAttributes.Count]; ObjectDelegator objectDelegator = new ObjectDelegator(fieldInfos); foreach (Object tableRow in excelFile.Rows) { int col = 1; foreach (FieldInfo fieldInfo in fieldInfos) { Object value = objectDelegator[fieldInfo.Name](tableRow); if (fieldInfo.IsPrivate) { if (fieldInfo.FieldType != typeof(ExcelFile.RowHeader)) { continue; } baseRow[col++] = FileTools.ObjectToStringGeneric(value, ","); continue; } OutputAttribute excelOutputAttribute = outputAttributes[col]; if (excelOutputAttribute == null) { if (value.GetType().BaseType == typeof(Array)) { value = ((Array)value).ToString(","); } baseRow[col++] = value; continue; } if (excelOutputAttribute.IsStringOffset) { int valueInt = (int)value; baseRow[col++] = (valueInt != -1) ? excelFile.ReadStringTable(valueInt) : String.Empty; continue; } if (excelOutputAttribute.IsSecondaryString) { int valueInt = (int)value; baseRow[col++] = (valueInt != -1) ? excelFile.ReadSecondaryStringTable(valueInt) : String.Empty; continue; } if (excelOutputAttribute.IsScript) { int scriptOffset = (int)value; if (scriptOffset == 0) { baseRow[col++] = String.Empty; continue; } String script; if (scriptOffset == 9649 && excelFile.StringId == "SKILLS") // todo: not sure what's with this script... { /* Compiled Bytes: * 26,30,700,6,26,1,399,358,669,616562688,711,26,62,3,17,669,641728512,26,8,711,26,62,3,17,358,669,322961408,26,5,700,6,26,1,399,358,388,0 * * Ending Stack (FIFO): * SetStat669('sfx_attack_pct', 'all', 30 * ($sklvl - 1)) * SetStat669('sfx_duration_pct', 'all', get_skill_level(@unit, 'Shield_Mastery')) * SetStat669('damage_percent_skill', 8 * get_skill_level(@unit, 'Shield_Mastery')) * SetStat669('damage_percent_skill', 8 * get_skill_level(@unit, 'Shield_Mastery')) + 5 * ($sklvl - 1) * * The last SetStat has strange overhang - decompiling wrong? * Or is it "supposed" to be there? * i.e. It's actually decompiling correctly, but because I've assumed such scripts to be wrong (as the end +5... segment is useless) we get the Stack exception */ int[] scriptCode = excelFile.ReadScriptTable(scriptOffset); script = scriptCode != null?FileTools.ArrayToStringGeneric(scriptCode, ",") : "ScriptError"; baseRow[col++] = script; continue; } //if (fieldInfo.Name == "props1" && row == 45) //{ // int bp = 0; //} ExcelScript excelScript = new ExcelScript(this); try { if (ExcelScript.DebugEnabled) { int[] scriptCode = excelFile.ReadScriptTable(scriptOffset); script = scriptCode != null?FileTools.ArrayToStringGeneric(scriptCode, ",") : String.Empty; script = excelScript.Decompile(excelFile.ScriptBuffer, scriptOffset, script, excelFile.StringId, row, col, fieldInfo.Name); } else { script = excelScript.Decompile(excelFile.ScriptBuffer, scriptOffset); } //if (script.StartsWith("GetStat666('skill_points_bonus_total', '') > -1;")) //{ // int bp = 0; //} } catch (Exception) { int[] scriptCode = excelFile.ReadScriptTable(scriptOffset); script = scriptCode != null?FileTools.ArrayToStringGeneric(scriptCode, ",") : "ScriptError"; } baseRow[col++] = script; continue; } if (excelOutputAttribute.IsTableIndex || excelOutputAttribute.IsStringIndex) { if (value.GetType().BaseType == typeof(Array)) { value = ((Array)value).ToString(","); } else { value = (int)value; } baseRow[col++] = value; col++; // for _strings relational column continue; } // Else its something else, ie bitmask, bool, table/string index baseRow[col++] = value; } // stats, only a component of the UnitData row type if (excelFile.Attributes.HasStats) { baseRow[col++] = FileTools.ArrayToStringGeneric(excelFile.ReadStats(row - 1), ","); } dataTable.Rows.Add(baseRow); row++; } #endregion // Generate Relationships as required if (doRelations) { _GenerateRelations(excelFile); } return(dataTable); }