/// <summary>
/// Function to test saving/importing/exported/etc relating to DataTables (i.e. actions from WITHIN Reanimator etc.)
/// </summary>
public static void TestDataTableExportAndImport(bool doTCv4 = false)
{
/*
bytesXls Bytes->Xls
bytesXlsBytes Bytes->Xls->Bytes
bytesXlsCsv Bytes->Xls->CSV
bytesXlsCsvXls Bytes->Xls->CSV->Xls
bytesXlsCsvXlsBytes Bytes->Xls->CSV->Xls->Bytes == bytesXlsTableXlsBytes as single byte is stripped ->CSV
bytesXlsTable Bytes->Xls->Table
bytesXlsTableXls Bytes->Xls->Table->Xls
bytesXlsTableXlsBytes Bytes->Xls->Table->Xls->Bytes != bytesXlsBytes because of single script byte thingy lost to table
bytesXlsTableXlsCsv Bytes->Xls->Table->Xls->CSV
bytesXlsTableXlsCsvXls Bytes->Xls->Table->Xls->CSV->Xls
bytesXlsTableXlsCsvXlsBytes Bytes->Xls->Table->Xls->CSV->Xls->Bytes
byte[] bytesXlsBytes = bytesXls.ToByteArray();
byte[] bytesXlsTableXlsCsvXlsBytes = bytesXlsTableXlsCsvXls.ToByteArray();
*/
String root = @"C:\excel_datatable_debug";
FileManager.ClientVersions clientVersion = FileManager.ClientVersions.SinglePlayer;
if (doTCv4)
{
root = Path.Combine(root, "tcv4");
clientVersion = FileManager.ClientVersions.TestCenter;
}
root += @"\"; // lazy
Directory.CreateDirectory(root);
FileManager fileManager = new FileManager(Config.HglDir, clientVersion);
fileManager.BeginAllDatReadAccess();
fileManager.LoadTableFiles();
List<String> sequenceFailed = new List<String>();
List<String> sequenceChecked = new List<String>();
foreach (KeyValuePair<String, DataFile> keyValuePair in fileManager.DataFiles)
{
String stringId = keyValuePair.Key;
DataFile dataFile = keyValuePair.Value;
if (dataFile.IsStringsFile) continue;
//if (stringId != "WARDROBE_LAYER") continue;
//if (stringId == "WARDROBE_LAYER")
//{
// int bp = 0;
//}
Debug.Write("Checking " + stringId + "... ");
ExcelFile bytesXls = (ExcelFile)dataFile;
byte[] bytes = fileManager.GetFileBytes(bytesXls.FilePath, true);
sequenceChecked.Add(stringId);
sequenceFailed.Add(stringId);
if (stringId == "PROPERTIES") continue; // can't do this atm - need to export Properties script table to DataTable etc.
Debug.Write("DataTable... ");
DataTable bytesXlsTable = fileManager.GetDataTable(stringId);
if (bytesXlsTable == null)
{
Debug.WriteLine("FAILED!");
continue;
}
// parse as DataTable
Debug.Write("new Excel... ");
DataFile bytesXlsTableXls = new ExcelFile(bytesXls.FilePath, fileManager.ClientVersion);
if (!bytesXlsTableXls.ParseDataTable(bytesXlsTable, fileManager))
{
Debug.WriteLine("FAILED!");
continue;
}
// check just data table import/export
byte[] bytesXlsBytes = bytesXls.ToByteArray();
byte[] bytesXlsTableXlsBytes = bytesXlsTableXls.ToByteArray();
if (bytesXls.ScriptBuffer == null && // can't check this as ScriptBuffer[0] is NOT always 0x00 (even though it's unused and means no script normally when pointed to)
!bytesXls.HasStringBuffer) // as above - gets re-arranged etc (only consistant comparint CSV->XLS->DATA
{
Debug.Write("bytesXlsBytes==bytesXlsTableXlsBytes... ");
if (!bytesXlsBytes.SequenceEqual(bytesXlsTableXlsBytes))
{
Debug.WriteLine("FALSE!");
File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes);
File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "2.bytesXlsTableXlsBytes", bytesXlsTableXlsBytes);
continue;
}
}
// export CSV
Debug.Write("ExportCSV... ");
byte[] bytesXlsTableXlsCsv = bytesXlsTableXls.ExportCSV(fileManager);
if (bytesXlsTableXlsCsv == null)
{
Debug.WriteLine("FAILED!");
continue;
}
Debug.Write("bytesXlsCsv==bytesXlsTableXlsCsv... ");
byte[] bytesXlsCsv = bytesXls.ExportCSV(fileManager);
if (!bytesXlsCsv.SequenceEqual(bytesXlsTableXlsCsv))
{
Debug.WriteLine("FALSE!");
File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes);
File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1a.bytesXlsCsv.csv", bytesXlsCsv);
File.WriteAllBytes(root + bytesXls.StringId + "2.bytesXlsTableXlsBytes", bytesXlsTableXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "3.bytesXlsTableXlsCsv.csv", bytesXlsTableXlsCsv);
continue;
}
Debug.Write("bytesXlsCsvXlsBytes==bytesXlsTableXlsBytes... ");
DataFile bytesXlsCsvXls = new ExcelFile(bytesXls.FilePath, fileManager.ClientVersion);
bytesXlsCsvXls.ParseCSV(bytesXlsCsv, fileManager);
byte[] bytesXlsCsvXlsBytes = bytesXlsCsvXls.ToByteArray();
if (!bytesXlsCsvXlsBytes.SequenceEqual(bytesXlsTableXlsBytes) &&
stringId != "ITEMDISPLAY" && stringId != "LEVEL") // ITEMDISPLAY has weird single non-standard ASCII char that is lost (from using MS Work on their part lol)
{ // and LEVEL references blank TREASURE rows and hence rowId references differ on recooks
Debug.WriteLine("FALSE!");
File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes);
File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1a.bytesXlsCsv.csv", bytesXlsCsv);
File.WriteAllBytes(root + bytesXls.StringId + "1b.bytesXlsCsvXlsBytes", bytesXlsCsvXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "2.bytesXlsTableXlsBytes", bytesXlsTableXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "3.bytesXlsTableXlsCsv.csv", bytesXlsTableXlsCsv);
continue;
}
// import CSV
Debug.Write("ParseCSV... ");
DataFile bytesXlsTableXlsCsvXls = new ExcelFile(bytesXls.FilePath, fileManager.ClientVersion);
if (!bytesXlsTableXlsCsvXls.ParseCSV(bytesXlsTableXlsCsv, fileManager))
{
Debug.WriteLine("FAILED!");
continue;
}
// export imported as cooked and do byte compare
Debug.Write("ToByteArray... ");
byte[] bytesXlsTableXlsCsvXlsBytes = bytesXlsTableXlsCsvXls.ToByteArray();
Debug.Write("bytesXlsCsvXlsBytes==bytesXlsTableXlsCsvXlsBytes... ");
if (!bytesXlsCsvXlsBytes.SequenceEqual(bytesXlsTableXlsCsvXlsBytes))
{
Debug.WriteLine("FALSE!");
File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes);
File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "2.bytesXlsTableXlsBytes", bytesXlsTableXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "3.bytesXlsTableXlsCsv", bytesXlsTableXlsCsv);
File.WriteAllBytes(root + bytesXls.StringId + "4.bytesXlsTableXlsCsvXlsBytes", bytesXlsTableXlsCsvXlsBytes);
continue;
}
// yay
Debug.WriteLine("OK!");
sequenceFailed.RemoveAt(sequenceFailed.Count - 1);
}
Debug.WriteLine("Totals: {0} sequence checks, {1} fails. ({2}% failed!)", sequenceChecked.Count, sequenceFailed.Count, (float)sequenceFailed.Count * 100f / sequenceChecked.Count);
Debug.Write("Failed Tables: ");
foreach (String stringIdFailed in sequenceFailed)
{
Debug.Write(stringIdFailed + ", ");
}
}
/// <summary>
/// Function to test all excel files cooking/recooking/etc to/from byte arrays and csv types.
/// </summary>
public static void TestExcelCooking(bool doTCv4 = false)
{
/* 0
0 byte[] fileBytes = fileManager.GetFileBytes(excelFile.FilePath, true); Bytes 0-gen bytes
0 1 2 3 4
foreach (DataFile dataFile in fileManager.DataFiles.Values) Bytes->Xls 1-gen Xls file
1 byte[] dataFileBytes = dataFile.ToByteArray(); Bytes->Xls->Bytes 1-gen bytes
fromBytesExcel.ParseData(dataFileBytes); Bytes->Xls->Bytes->Xls 2-gen Xls file
1a byte[] bytesXlsBytesXlsBytes = fromBytesExcel.ToByteArray(); Bytes->Xls->Bytes->Xls->Bytes 2-gen bytes
1ba byte[] bytesXlsCsv = dataFile.ExportCSV(fileManager); Bytes->Xls->CSV 1-gen CSV-bytes
bytesXlsCsvXls.ParseCSV(bytesXlsCsv, fileManager); Bytes->Xls->CSV ->Xls 2-gen Xls file
1bb byte[] bytesXlsCsvXlsBytes = bytesXlsCsvXls.ToByteArray(); Bytes->Xls->CSV ->Xls->Bytes 2-gen CSV-bytes
1b byte[] csvBytes = fromBytesExcel.ExportCSV(fileManager); Bytes->Xls->Bytes->Xls->CSV 2-gen CSV file
csvExcel.ParseCSV(csvBytes, fileManager); Bytes->Xls->Bytes->Xls->CSV->Xls 3-gen Xls file
1c byte[] bytesXlsBytesXlsCsvXlsBytes = csvExcel.ToByteArray(); Bytes->Xls->Bytes->Xls->CSV->Xls->Bytes 3-gen bytes
finalExcel.ParseData(recookedExcelBytes); Bytes->Xls->Bytes->Xls->CSV->Xls->Bytes->Xls 4-gen Xls file
1d byte[] csvCheck = finalExcel.ExportCSV(fileManager); Bytes->Xls->Bytes->Xls->CSV->Xls->Bytes->Xls->CSV 4-gen CSV file
1e byte[] finalCheck = finalExcel.ToByteArray(); Bytes->Xls->Bytes->Xls->CSV->Xls->Bytes->Xls->Bytes 4-gen bytes
*/
String root = @"C:\excel_debug";
FileManager.ClientVersions clientVersion = FileManager.ClientVersions.SinglePlayer;
if (doTCv4)
{
root = Path.Combine(root, "tcv4");
clientVersion = FileManager.ClientVersions.TestCenter;
}
root += @"\"; // lazy
Directory.CreateDirectory(root);
FileManager fileManager = new FileManager(Config.HglDir, clientVersion);
fileManager.BeginAllDatReadAccess();
fileManager.LoadTableFiles();
ExcelFile.EnableDebug = true;
int checkedCount = 0;
List<String> excelFailed = new List<String>();
foreach (DataFile bytesXls in fileManager.DataFiles.Values)
{
ExcelFile excelFile = bytesXls as ExcelFile;
if (excelFile == null) continue;
const String debugStr = "SKILLS";
//if (excelFile.StringId != debugStr) continue;
//if (excelFile.StringId == debugStr)
//{
// int bp = 0;
//}
checkedCount++;
Debug.Write(String.Format("Checking {0}... ", bytesXls.StringId));
byte[] bytes = fileManager.GetFileBytes(excelFile.FilePath, true);
try
{
byte[] bytesXlsBytes = bytesXls.ToByteArray();
if (bytesXls.StringId == "SOUNDS" && false)
{
byte[] csvBytesSounds = excelFile.ExportCSV(fileManager);
ExcelFile soundsCSV = new ExcelFile(excelFile.FilePath);
soundsCSV.ParseCSV(csvBytesSounds, fileManager);
byte[] soundsBytes = soundsCSV.ToByteArray();
//byte[] soundsBytesFromCSV = soundsCSV.ExportCSV();
//ExcelFile soundsCSVFromBytesFromCSV = new ExcelFile(soundsBytesFromCSV, fileEntry.RelativeFullPathWithoutPatch);
// some brute force ftw
byte[][] bytesArrays = new[] { bytes, soundsBytes };
for (int z = 0; z < bytesArrays.Length; z++)
{
byte[] bytesBrute = bytesArrays[z];
int offset = 0x20;
int stringsBytesCount = FileTools.ByteArrayToInt32(bytesBrute, ref offset);
StringWriter stringWriterByteStrings = new StringWriter();
stringWriterByteStrings.WriteLine(stringsBytesCount + " bytes");
List<String> strings = new List<String>();
List<int> offsets = new List<int>();
while (offset < stringsBytesCount + 0x20)
{
String str = FileTools.ByteArrayToStringASCII(bytesBrute, offset);
strings.Add(str);
offsets.Add(offset);
offset += str.Length + 1;
}
String[] sortedStrings = strings.ToArray();
int[] sortedOffsets = offsets.ToArray();
Array.Sort(sortedStrings, sortedOffsets);
stringWriterByteStrings.WriteLine(strings.Count + " strings");
for (int i = 0; i < strings.Count; i++)
{
stringWriterByteStrings.WriteLine(sortedStrings[i] + "\t\t\t" + sortedOffsets[i]);
}
File.WriteAllText(@"C:\excel_debug\strings" + z + ".txt", stringWriterByteStrings.ToString());
}
}
Debug.Write("ToByteArray... ");
if (bytes.Length != bytesXlsBytes.Length && !doTCv4) // some TCv4 tables don't have their sort columns yet
{
Debug.WriteLine("ToByteArray() dataFileBytes has differing length: " + bytesXls.StringId);
File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes);
File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes);
excelFailed.Add(bytesXls.StringId);
continue;
}
ExcelFile bytesXlsBytesXls = new ExcelFile(excelFile.FilePath, fileManager.ClientVersion);
bytesXlsBytesXls.ParseData(bytesXlsBytes);
Debug.Write("new ExcelFile... ");
if (!bytesXlsBytesXls.HasIntegrity)
{
Debug.WriteLine("fromBytesExcel = new Excel from ToByteArray() failed!");
File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes);
File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes);
excelFailed.Add(bytesXls.StringId);
continue;
}
// more checks
Debug.Write("ToByteArray->ToByteArray... ");
byte[] bytesXlsBytesXlsBytes = bytesXlsBytesXls.ToByteArray(); // bytesXlsBytesXlsBytes
if (bytes.Length != bytesXlsBytesXlsBytes.Length && !doTCv4) // some TCv4 tables don't have their sort columns yet
{
Debug.WriteLine("ToByteArray() dataFileBytesFromToByteArray has differing length!");
File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes);
File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1a.toByteArrayFromByteArray", bytesXlsBytesXlsBytes);
excelFailed.Add(bytesXls.StringId);
continue;
}
// check generated sort index arrays);
Debug.Write("IndexSortArrays... ");
if (excelFile.IndexSortArray != null)
{
if (bytesXlsBytesXls.IndexSortArray == null || excelFile.IndexSortArray.Count != bytesXlsBytesXls.IndexSortArray.Count)
{
Debug.WriteLine("fromBytesExcel has not-matching IndexSortArray count!");
excelFailed.Add(bytesXls.StringId);
continue;
}
bool hasError = false;
for (int i = 0; i < excelFile.IndexSortArray.Count; i++)
{
if (excelFile.IndexSortArray[i].SequenceEqual(bytesXlsBytesXls.IndexSortArray[i])) continue;
Debug.WriteLine(String.Format("IndexSortArray[{0}] NOT EQUAL to original!", i));
hasError = true;
}
if (hasError)
{
File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes);
File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1a.bytesXlsBytesXlsBytes", bytesXlsBytesXlsBytes);
excelFailed.Add(bytesXls.StringId);
continue;
}
}
// some csv stuff
Debug.Write("BaseXls->BaseCSV==");
byte[] bytesXlsCsv = bytesXls.ExportCSV(fileManager); // Bytes->Xls->CSV
Debug.Write("ExportCSV... ");
byte[] bytesXlsBytesXlsCsv = bytesXlsBytesXls.ExportCSV(fileManager);
if (!bytesXlsCsv.SequenceEqual(bytesXlsBytesXlsCsv))
{
Debug.WriteLine("FALSE!");
File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes);
File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1a.bytesXlsBytesXlsBytes", bytesXlsBytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1b.bytesXlsBytesXlsCsv.csv", bytesXlsBytesXlsCsv);
File.WriteAllBytes(root + bytesXls.StringId + "1ba.bytesXlsCsv.csv", bytesXlsCsv);
excelFailed.Add(bytesXls.StringId);
continue;
}
ExcelFile bytesXlsCsvXls = new ExcelFile(excelFile.FilePath, fileManager.ClientVersion);
bytesXlsCsvXls.ParseCSV(bytesXlsCsv, fileManager);
byte[] bytesXlsCsvXlsBytes = bytesXlsCsvXls.ToByteArray(); // used later as well
if (excelFile.ScriptBuffer == null && // can only compare here for no-script tables - can't compare to pure-base xls-bytes as xls->csv->xls rearranges script code
!excelFile.HasStringBuffer) // xls->csv->xls also rearranges the strings buffer
{
Debug.Write("BytesXlsBytesXlsBytes==BytesXlsCsvXlsBytes... ");
if (!bytesXlsBytesXlsBytes.SequenceEqual(bytesXlsCsvXlsBytes))
{
Debug.WriteLine("FALSE!");
File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes);
File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1a.bytesXlsBytesXlsBytes", bytesXlsBytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1b.bytesXlsBytesXlsCsv.csv", bytesXlsBytesXlsCsv);
File.WriteAllBytes(root + bytesXls.StringId + "1ba.bytesXlsCsv.csv", bytesXlsCsv);
File.WriteAllBytes(root + bytesXls.StringId + "1bb.bytesXlsCsvXlsBytes", bytesXlsCsvXlsBytes);
excelFailed.Add(bytesXls.StringId);
continue;
}
Debug.Write("BytesXlsBytes==BytesXlsCsvXlsBytes... ");
if (!bytesXlsBytes.SequenceEqual(bytesXlsCsvXlsBytes))
{
Debug.WriteLine("FALSE!");
File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes);
File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1a.bytesXlsBytesXlsBytes", bytesXlsBytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1b.bytesXlsBytesXlsCsv.csv", bytesXlsBytesXlsCsv);
File.WriteAllBytes(root + bytesXls.StringId + "1ba.bytesXlsCsv.csv", bytesXlsCsv);
File.WriteAllBytes(root + bytesXls.StringId + "1bb.bytesXlsCsvXlsBytes", bytesXlsCsvXlsBytes);
excelFailed.Add(bytesXls.StringId);
continue;
}
}
Debug.Write("BytesXlsBytesXlsCsv -> new ExcelFile");
ExcelFile bytesXlsBytesXlsCsvXls = new ExcelFile(excelFile.FilePath, fileManager.ClientVersion);
bytesXlsBytesXlsCsvXls.ParseCSV(bytesXlsBytesXlsCsv, fileManager);
Debug.Write("... ");
if (!bytesXlsBytesXlsCsvXls.HasIntegrity)
{
Debug.WriteLine("Failed!");
File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes);
File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1a.bytesXlsBytesXlsBytes", bytesXlsBytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1b.bytesXlsBytesXlsCsv.csv", bytesXlsBytesXlsCsv);
File.WriteAllBytes(root + bytesXls.StringId + "1ba.bytesXlsCsv.csv", bytesXlsCsv);
excelFailed.Add(bytesXls.StringId);
continue;
}
byte[] bytesXlsBytesXlsCsvXlsBytes = bytesXlsBytesXlsCsvXls.ToByteArray();
Debug.Write("BytesXlsCsvXlsBytes==BytesXlsBytesXlsCsvXlsBytes... ");
if (!bytesXlsCsvXlsBytes.SequenceEqual(bytesXlsBytesXlsCsvXlsBytes))
{
Debug.WriteLine("FALSE!");
File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes);
File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1a.bytesXlsBytesXlsBytes", bytesXlsBytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1b.bytesXlsBytesXlsCsv.csv", bytesXlsBytesXlsCsv);
File.WriteAllBytes(root + bytesXls.StringId + "1ba.bytesXlsCsv.csv", bytesXlsCsv);
File.WriteAllBytes(root + bytesXls.StringId + "1bb.bytesXlsCsvXlsBytes", bytesXlsCsvXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1c.bytesXlsBytesXlsCsvXlsBytes", bytesXlsBytesXlsCsvXlsBytes);
excelFailed.Add(bytesXls.StringId);
continue;
}
if (!fileManager.IsVersionTestCenter)
{
Debug.Write("StructureId... ");
UInt32 structureId = BitConverter.ToUInt32(bytes, 4);
UInt32 fromCSVStructureId = BitConverter.ToUInt32(bytesXlsBytesXlsCsvXlsBytes, 4);
if (structureId != fromCSVStructureId)
{
Debug.WriteLine("Structure Id value do not match: " + structureId + " != " + fromCSVStructureId);
excelFailed.Add(bytesXls.StringId);
continue;
}
}
Debug.Write("Almost Done... ");
ExcelFile bytesXlsBytesXlsCsvXlsBytesXls = new ExcelFile(excelFile.FilePath, fileManager.ClientVersion);
bytesXlsBytesXlsCsvXlsBytesXls.ParseData(bytesXlsBytesXlsCsvXlsBytes);
byte[] bytesXlsBytesXlsCsvXlsBytesXlsCsv = bytesXlsBytesXlsCsvXlsBytesXls.ExportCSV(fileManager);
int recookedLength = bytesXlsBytesXlsCsvXlsBytes.Length;
if (excelFile.StringId == "SKILLS") recookedLength += 12; // 12 bytes in int ptr data not used/referenced at all and are removed/lost in bytes -> csv -> bytes
if (bytes.Length != recookedLength && !doTCv4) // some TCv4 tables don't have their sort columns yet
{
Debug.WriteLine("Recooked Excel file has differing length!");
File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes);
File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1a.bytesXlsBytesXlsBytes", bytesXlsBytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1b.bytesXlsBytesXlsCsv.csv", bytesXlsBytesXlsCsv);
File.WriteAllBytes(root + bytesXls.StringId + "1ba.bytesXlsCsv.csv", bytesXlsCsv);
File.WriteAllBytes(root + bytesXls.StringId + "1c.bytesXlsBytesXlsCsvXlsBytes", bytesXlsBytesXlsCsvXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1d.bytesXlsBytesXlsCsvXlsBytesXlsCsv.csv", bytesXlsBytesXlsCsvXlsBytesXlsCsv);
excelFailed.Add(bytesXls.StringId);
continue;
}
Debug.Assert(bytesXlsBytesXlsCsvXlsBytesXls.HasIntegrity);
byte[] bytesXlsBytesXlsCsvXlsBytesXlsBytes = bytesXlsBytesXlsCsvXlsBytesXls.ToByteArray();
if (excelFile.StringId == "SKILLS") Debug.Assert(bytesXlsBytesXlsCsvXlsBytesXlsBytes.Length + 12 == bytesXlsBytes.Length);
else Debug.Assert(bytesXlsBytesXlsCsvXlsBytesXlsBytes.Length == bytesXlsBytes.Length || doTCv4);
if (!bytesXlsBytesXlsCsv.SequenceEqual(bytesXlsBytesXlsCsvXlsBytesXlsCsv))
{
Debug.WriteLine("csvBytes.SequenceEqual failed!");
File.WriteAllBytes(root + bytesXls.StringId + "0.bytes", bytes);
File.WriteAllBytes(root + bytesXls.StringId + "1.bytesXlsBytes", bytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1a.bytesXlsBytesXlsBytes", bytesXlsBytesXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1b.bytesXlsBytesXlsCsv.csv", bytesXlsBytesXlsCsv);
File.WriteAllBytes(root + bytesXls.StringId + "1ba.bytesXlsCsv.csv", bytesXlsCsv);
File.WriteAllBytes(root + bytesXls.StringId + "1c.bytesXlsBytesXlsCsvXlsBytes", bytesXlsBytesXlsCsvXlsBytes);
File.WriteAllBytes(root + bytesXls.StringId + "1d.bytesXlsBytesXlsCsvXlsBytesXlsCsv.csv", bytesXlsBytesXlsCsvXlsBytesXlsCsv);
File.WriteAllBytes(root + bytesXls.StringId + "1e.bytesXlsBytesXlsCsvXlsBytesXlsBytes", bytesXlsBytesXlsCsvXlsBytesXlsBytes);
excelFailed.Add(bytesXls.StringId);
continue;
}
Debug.Write("\nPerforming deep scan: ");
ObjectDelegator objectDelegator = fileManager.DataFileDelegators[excelFile.StringId];
int lastPercent = 0;
int col = 0;
bool failed = false;
foreach (FieldDelegate fieldDelegate in objectDelegator)
{
int percent = col * 100 / objectDelegator.FieldCount - 1;
int dotCount = percent - lastPercent;
for (int i = 0; i < dotCount; i++) Debug.Write(".");
lastPercent = percent;
ExcelFile.OutputAttribute excelAttribute = ExcelFile.GetExcelAttribute(fieldDelegate.Info);
bool isArray = (fieldDelegate.FieldType.BaseType == typeof(Array));
for (int row = 0; row < excelFile.Rows.Count; row++)
{
Object obj1 = fieldDelegate.GetValue(excelFile.Rows[row]);
Object obj2 = fieldDelegate.GetValue(bytesXlsBytesXlsCsvXlsBytesXls.Rows[row]);
if (isArray)
{
//if (excelFile.StringId == "TREASURE")
//{
// int bp = 0;
//}
IEnumerator arrayEnumator1 = ((Array)obj1).GetEnumerator();
IEnumerator arrayEnumator2 = ((Array)obj2).GetEnumerator();
int index = -1;
while (arrayEnumator1.MoveNext() && arrayEnumator2.MoveNext())
{
index++;
Object elementVal1 = arrayEnumator1.Current;
Object elementVal2 = arrayEnumator2.Current;
if (elementVal1.Equals(elementVal2)) continue;
Debug.WriteLine(String.Format("Array Element '{0}' != '{1}' on col({2}) = '{3}', row({4}), index({5})", elementVal1, elementVal2, col, fieldDelegate.Name, row, index));
failed = true;
break;
}
if (failed) break;
continue;
}
if (obj1.Equals(obj2)) continue;
if (excelAttribute != null)
{
if (excelAttribute.IsStringOffset)
{
int offset1 = (int)obj1;
int offset2 = (int)obj2;
String str1 = excelFile.ReadStringTable(offset1);
String str2 = bytesXlsBytesXlsCsvXlsBytesXls.ReadStringTable(offset2);
if (str1 == str2) continue;
obj1 = str1;
obj2 = str2;
}
if (excelAttribute.IsScript)
{
int offset1 = (int)obj1;
int offset2 = (int)obj2;
Int32[] script1 = excelFile.ReadScriptTable(offset1);
Int32[] script2 = bytesXlsBytesXlsCsvXlsBytesXls.ReadScriptTable(offset2);
if (script1.SequenceEqual(script2)) continue;
obj1 = script1.ToString(",");
obj2 = script2.ToString(",");
}
}
String str = obj1 as String;
if ((str != null && str.StartsWith("+N%-N% base damage as [elem]")) || // "+N%-N% base damage as [elem]…" != "+N%-N% base damage as [elem]?" on col(2) = 'exampleDescription', row(132) // the '?' at end
fieldDelegate.Name == "bonusStartingTreasure") // refernces multiple different blank rows
continue;
Debug.WriteLine(String.Format("'{0}' != '{1}' on col({2}) = '{3}', row({4})", obj1, obj2, col, fieldDelegate.Name, row));
failed = true;
break;
}
if (failed) break;
col++;
}
if (failed)
{
File.WriteAllBytes(root + bytesXls.StringId + ".bytesXlsBytesXlsCsv.csv", bytesXlsBytesXlsCsv);
File.WriteAllBytes(root + bytesXls.StringId + ".bytesXlsBytesXlsCsvXlsBytesXlsCsv.csv", bytesXlsBytesXlsCsvXlsBytesXlsCsv);
excelFailed.Add(bytesXls.StringId);
continue;
}
Debug.WriteLine("OK\n");
}
catch (Exception e)
{
Debug.WriteLine("Excel file Exception: " + bytesXls.StringId + "\n" + e);
}
}
Debug.WriteLine(checkedCount + " excel files checked.");
if (excelFailed.Count <= 0) return;
Debug.WriteLine(excelFailed.Count + " failed excel checks! (" + ((float)excelFailed.Count / checkedCount * 100) + "%)");
foreach (String str in excelFailed) Debug.Write(str + ", ");
}
public static void TestAllXml(bool doTCv4 = false)
{
//System.Security.Cryptography.MD5CryptoServiceProvider md5 = new System.Security.Cryptography.MD5CryptoServiceProvider();
//const String root = @"D:\Games\Hellgate\Data\";
//const String root = @"D:\Games\Hellgate London\data\mp_hellgate_1.10.180.3416_1.0.86.4580\";
//const String root = @"D:\Games\Hellgate London\data\mp_hellgate_1.10.180.3416_1.0.86.4580\data\background\";
//List<String> xmlFiles = new List<String>(Directory.GetFiles(root, "*.xml.cooked", SearchOption.AllDirectories));
String debugPath = @"C:\xml_debug\";
FileManager.ClientVersions clientVersion = FileManager.ClientVersions.SinglePlayer;
if (doTCv4)
{
debugPath = Path.Combine(debugPath, "tcv4");
clientVersion = FileManager.ClientVersions.TestCenter;
}
debugPath += @"\"; // lazy
Directory.CreateDirectory(debugPath);
FileManager fileManager = new FileManager(Config.HglDir, clientVersion);
fileManager.BeginAllDatReadAccess();
fileManager.LoadTableFiles();
int count = 0;
List<XmlCookedFile> excelStringWarnings = new List<XmlCookedFile>();
List<String> testCentreWarnings = new List<String>();
List<String> resurrectionWarnings = new List<String>();
int i = 0;
foreach (PackFileEntry fileEntry in fileManager.FileEntries.Values)
{
if (!fileEntry.Path.EndsWith(XmlCookedFile.Extension)) continue;
String xmlFilePath = fileEntry.Path;
//const String debugStr = "buildintx_inside_env";
//if (!xmlFilePath.Contains(debugStr)) continue;
//if (xmlFilePath.Contains(debugStr))
//{
// int bp = 0;
//}
//foreach (String xmlFilePath in xmlFiles)
//{
//bool skip = ((i++ % 23) > 0);
//if (skip)
//{
// if (xmlFilePath.Contains("\\Data\\colorsets.xml")) continue;
// if (xmlFilePath.Contains("\\Data\\ai\\")) continue;
// if (xmlFilePath.Contains("\\Data\\background\\")) continue;
// if (xmlFilePath.Contains("\\Data\\demolevel\\")) continue;
// if (xmlFilePath.Contains("\\Data\\lights\\")) continue;
// if (xmlFilePath.Contains("\\Data\\materials\\")) continue;
// if (xmlFilePath.Contains("\\Data\\particles\\")) continue;
// if (xmlFilePath.Contains("\\Data\\screenfx\\")) continue;
// if (xmlFilePath.Contains("\\Data\\skills\\")) continue;
// if (xmlFilePath.Contains("\\Data\\sounds\\")) continue;
// if (xmlFilePath.Contains("\\Data\\states\\")) continue;
// if (xmlFilePath.Contains("\\Data\\units\\items\\")) continue;
// if (xmlFilePath.Contains("\\Data\\units\\missiles\\")) continue;
// if (xmlFilePath.Contains("\\Data\\units\\monsters\\")) continue;
// if (xmlFilePath.Contains("\\Data\\units\\npc\\")) continue;
// if (xmlFilePath.Contains("\\Data\\units\\objects\\")) continue;
//}
String path = xmlFilePath;
String fileName = Path.GetFileName(path);
Debug.Assert(!String.IsNullOrEmpty(fileName));
//path = @"D:\Games\Hellgate London\data\mp_hellgate_1.10.180.3416_1.0.86.4580\data\background\city\treasury\cap_path.xml.cooked";
//path = "D:\\Games\\Hellgate\\Data\\background\\_environments\\outdoor_redhellcow_env.xml.cooked";
//path = "D:\\Games\\Hellgate\\Data\\particles\\background\\t_background\\tokyo_dark_smoke_c_large_01.xml.cooked";
//if (path == @"D:\Games\Hellgate\Data\background\cans and boxes.xml.cooked")
//{
// int bp = 0;
//}
XmlCookedFile xmlCookedFile = new XmlCookedFile(fileManager, fileName) { CookExcludeTestCentre = !doTCv4 };
byte[] data = fileManager.GetFileBytes(fileEntry);
Debug.Assert(data != null && data.Length > 0);
Console.Write("Uncooking " + fileName + "... ");
try
{
xmlCookedFile.ParseFileBytes(data, true);
}
catch (Exception e)
{
File.WriteAllBytes(debugPath + fileName, data);
Console.WriteLine("Failed to uncooked file \"" + path + "\"\n" + e + "\n");
continue;
}
byte[] newXmlBytes = xmlCookedFile.ExportAsDocument();
count++;
if (xmlCookedFile.HasExcelStringsMissing) excelStringWarnings.Add(xmlCookedFile);
if (xmlCookedFile.HasTestCentreElements) testCentreWarnings.Add(Path.GetFileName(fileName));
if (xmlCookedFile.HasResurrectionElements) resurrectionWarnings.Add(Path.GetFileName(fileName));
//if (xmlCookedFile.HasExcelStringsMissing || xmlCookedFile.HasTestCentreElements || xmlCookedFile.HasResurrectionElements) continue;
XmlCookedFile recookedXmlFile = new XmlCookedFile(fileManager, fileName) { CookExcludeTestCentre = !doTCv4 };
byte[] newXmlCookedBytes = recookedXmlFile.ParseFileBytes(newXmlBytes);
// check if *cooking method* is working. i.e. is the cook from the *new* XML format == the original cooked
bool identicalNew = data.SequenceEqual(newXmlCookedBytes);
// if file passes byte-byte test, then continue
if (identicalNew)
{
Console.WriteLine("OK!");
continue;
}
if (
path.Contains("female_3p_appearance.xml") || // this file has some weird bytes in a string element
path.Contains("focus_item12_mesh_appearance.xml") || // this file has non-zeroed flag base masks (all differing)
path.Contains("focus_item10_mesh_appearance.xml") || // as above // (all 3 probably from not zeroing a ptr at original cooking)
path.Contains("dof_test.xml") || // as above
path.Contains("motionblur.xml") || // as above
path.Contains("player dead.xml")
)
{
Console.WriteLine("OK!");
continue;
}
File.WriteAllBytes(debugPath + fileName, data);
File.WriteAllBytes(debugPath + fileName + "recooked", newXmlCookedBytes);
File.WriteAllBytes(debugPath + fileName.Replace(".cooked", ""), newXmlBytes);
Console.WriteLine("FAILED!");
}
TextWriter consoleOut = Console.Out;
TextWriter textWriter = new StreamWriter("uncook_results.txt");
Console.SetOut(textWriter);
Console.WriteLine("XML Files Uncooked: " + count);
if (excelStringWarnings.Count > 0)
{
Console.WriteLine("Warning: " + excelStringWarnings.Count + " files had excel strings missing:");
foreach (XmlCookedFile xmlCookedFile in excelStringWarnings)
{
Console.WriteLine("\t" + xmlCookedFile.FileName);
foreach (String str in xmlCookedFile.ExcelStringsMissing) Console.WriteLine("\t\t- \"" + str + "\"");
}
}
if (testCentreWarnings.Count > 0)
{
Console.WriteLine("Warning: " + testCentreWarnings.Count + " files had TestCentre-specific elements:");
foreach (String str in testCentreWarnings) Console.WriteLine("\t" + str);
}
if (resurrectionWarnings.Count > 0)
{
Console.WriteLine("Warning: " + resurrectionWarnings.Count + " files had Resurrection-specific elements:");
foreach (String str in resurrectionWarnings) Console.WriteLine("\t" + str);
}
textWriter.Close();
Console.SetOut(consoleOut);
}
public static void TestAllLevelRules(bool doTCv4=false)
{
//const String root = @"D:\Games\Hellgate London\data\background\";
//List<String> drlFiles = new List<String>(Directory.GetFiles(root, "*.drl", SearchOption.AllDirectories));
String debugPath = @"C:\drl_debug\";
FileManager.ClientVersions clientVersion = FileManager.ClientVersions.SinglePlayer;
if (doTCv4)
{
debugPath = Path.Combine(debugPath, "tcv4");
clientVersion = FileManager.ClientVersions.TestCenter;
}
debugPath += @"\"; // lazy
Directory.CreateDirectory(debugPath);
FileManager fileManager = new FileManager(Config.HglDir, clientVersion);
fileManager.BeginAllDatReadAccess();
fileManager.LoadTableFiles();
int tested = 0;
int failed = 0;
//foreach (String drlFilePath in drlFiles)
foreach (PackFileEntry fileEntry in fileManager.FileEntries.Values)
{
if (!fileEntry.Path.EndsWith(LevelRulesFile.Extension)) continue;
tested++;
failed++;
//String path = drlFilePath;
String path = fileEntry.Path;
//path = @"D:\Games\Hellgate London\data\background\catacombs\ct_rule_100.drl";
//path = @"D:\Games\Hellgate London\data\background\city\rule_pmt02.drl";
//byte[] levelRulesBytes = File.ReadAllBytes(path);
byte[] bytes = fileManager.GetFileBytes(fileEntry);
LevelRulesFile bytesObj = new LevelRulesFile();
//String fileName = path.Replace(@"D:\Games\Hellgate London\data\background\", "");
String fileName = Path.GetFileName(path);
String xmlPath = path.Replace(LevelRulesFile.Extension, LevelRulesFile.ExtensionDeserialised);
Console.WriteLine("Loading: " + fileName);
try
{
bytesObj.ParseFileBytes(bytes);
byte[] bytesObjXml = bytesObj.ExportAsDocument();
MemoryStream ms = new MemoryStream(bytesObjXml);
//File.WriteAllBytes(xmlPath, xmlBytes);
XmlDocument xmlDocument = new XmlDocument();
xmlDocument.Load(ms);//.Load(xmlPath);
LevelRulesFile bytesObjXmlObj = new LevelRulesFile();
bytesObjXmlObj.ParseXmlDocument(xmlDocument);
byte[] bytesObjXmlObjXml = bytesObjXmlObj.ExportAsDocument();
byte[] bytesObjXmlObjXmlBytes = bytesObjXmlObj.ToByteArray();
if (!bytesObjXml.SequenceEqual(bytesObjXmlObjXml))
{
File.WriteAllBytes(debugPath + fileName + "0.bytesObjXml.xml", bytesObjXml);
File.WriteAllBytes(debugPath + fileName + "1.bytesObjXmlObjXml.xml", bytesObjXmlObjXml);
}
// note: ct_rule_100.drl has unreferenced rules (only one found out of all files)
if (bytes.Length != bytesObjXmlObjXmlBytes.Length && !path.Contains("ct_rule_100"))
{
File.WriteAllBytes(debugPath + fileName + "0.bytesObjXml.xml", bytesObjXml);
File.WriteAllBytes(debugPath + fileName + "1.bytesObjXmlObjXml.xml", bytesObjXmlObjXml);
File.WriteAllBytes(debugPath + fileName + "2.bytes", bytes);
File.WriteAllBytes(debugPath + fileName + "3.bytesObjXmlObjXmlBytes", bytesObjXmlObjXmlBytes);
}
failed--;
}
catch (Exception e)
{
Console.WriteLine("Failed to load file!\n" + e);
continue;
}
}
Console.WriteLine("Tested: " + tested);
Console.WriteLine("Failed: " + failed);
}
public static void TestAllExcelScripts()
{
ExcelScript.GlobalDebug(true);
FileManager fileManager = new FileManager(Config.HglDir);
fileManager.BeginAllDatReadAccess();
fileManager.LoadTableFiles();
fileManager.EndAllDatAccess();
StringWriter results = new StringWriter();
Dictionary<String, int> excelScriptDecompileFails = new Dictionary<String, int>();
Dictionary<String, int> excelScriptRecompileFails = new Dictionary<String, int>();
Dictionary<String, int> excelScriptComparisonFails = new Dictionary<String, int>();
int grandTotalScripts = 0;
int grandTotalScriptsDecompiled = 0;
int grandTotalScriptsFailedDecompilation = 0;
int grandTotalScriptsRecompiled = 0;
int grandTotalScriptsFailedRecompilation = 0;
int grandTotalScriptsCompared = 0;
int grandTotalScriptsFailedComparison = 0;
foreach (ExcelFile excelFile in fileManager.DataFiles.Values.Where(dataFile => dataFile.IsExcelFile))
{
String excelStringId = String.Format("{0}...", excelFile.StringId);
Debug.Write(excelStringId);
results.Write(excelStringId);
//if (excelFile.StringId != "SKILLS") continue;
if (excelFile.Delegator == null || excelFile.ScriptCode == null || excelFile.ScriptCode.Length == 0)
{
Debug.WriteLine(" No script data");
results.WriteLine(" No script data");
continue;
}
String scriptIntValues = String.Format(" {0} script int values...", excelFile.ScriptCode.Length);
Debug.WriteLine(scriptIntValues);
results.WriteLine(scriptIntValues);
StringWriter excelFileResults = new StringWriter();
int colIndex = -1;
int totalScriptsFound = 0;
int totalScriptsDecompiled = 0;
int totalScriptsFailedDecompilation = 0;
int totalScriptsRecompiled = 0;
int totalScriptsFailedRecompilation = 0;
int totalScriptsCompared = 0;
int totalScriptsFailedComparison = 0;
foreach (FieldDelegate fieldDelegate in excelFile.Delegator)
{
colIndex++;
ExcelAttributes excelAtributes = (ExcelAttributes)fieldDelegate.Info.GetCustomAttributes(typeof(ExcelAttributes), true).FirstOrDefault();
if (excelAtributes == null || !excelAtributes.IsScript) continue;
String currentColumn = String.Format("\tColumn[{0}] = {1}", colIndex, fieldDelegate.Name);
Debug.WriteLine(currentColumn);
results.WriteLine(currentColumn);
excelFileResults.WriteLine(currentColumn);
int rowIndex = -1;
int prevOffset = 0;
int scriptCount = 0;
int scriptsDecompiled = 0;
int scriptsFailedDecompilation = 0;
int scriptsRecompiled = 0;
int scriptsFailedRecompilation = 0;
int scriptsCompared = 0;
int scriptsFailedComparison = 0;
foreach (Object row in excelFile.Rows)
{
rowIndex++;
int byteOffset = (int)fieldDelegate.GetValue(row);
if (byteOffset == 0) continue;
Debug.Assert(byteOffset > prevOffset);
prevOffset = byteOffset;
if (byteOffset == 9649 && excelFile.StringId == "SKILLS") // todo: not sure what's with this script...
{
continue;
}
excelFileResults.WriteLine(String.Format("\t\tColumn({0}) = {3}, Row({1}), ByteOffset({2}):", colIndex, rowIndex, byteOffset, fieldDelegate.Name));
ExcelScript excelScript = new ExcelScript(fileManager);
scriptCount++;
String script;
int[] scriptCodes = excelFile.ReadScriptTable(byteOffset);
String scriptCodesStr = scriptCodes.ToString(",");
String semiDecompiledScript = _ReadScriptTable(excelScript, scriptCodes);
//if (scriptCodesStr == "707,0,714,666,0,3,201,673,603979779,26,5,2,284,0,0")
//{
// int bp = 0;
//}
// test decompiling
try
{
script = excelScript.Decompile(excelFile.ScriptBuffer, byteOffset, scriptCodesStr, excelFile.StringId, rowIndex, colIndex, fieldDelegate.Name);
scriptsDecompiled++;
}
catch (Exception e)
{
if (!excelScriptDecompileFails.ContainsKey(excelFile.StringId))
{
excelScriptDecompileFails.Add(excelFile.StringId, 1);
}
else
{
excelScriptDecompileFails[excelFile.StringId]++;
}
scriptsFailedDecompilation++;
excelFileResults.WriteLine("Script Decompile Failed:\n" + semiDecompiledScript + "\n" + e + "\n");
continue;
}
// test compiling of decompiled script
int[] recompiledScriptCode;
try
{
ExcelScript excelScriptCompile = new ExcelScript(fileManager);
recompiledScriptCode = excelScriptCompile.Compile(script, scriptCodesStr, excelFile.StringId, rowIndex, colIndex, fieldDelegate.Name);
scriptsRecompiled++;
}
catch (Exception e)
{
if (!excelScriptRecompileFails.ContainsKey(excelFile.StringId))
{
excelScriptRecompileFails.Add(excelFile.StringId, 1);
}
else
{
excelScriptRecompileFails[excelFile.StringId]++;
}
scriptsFailedRecompilation++;
excelFileResults.WriteLine("Script Recompile Failed:\n" + semiDecompiledScript + "\n" + e + "\n");
continue;
}
// check recompiled fidelity
scriptsCompared++;
if (!scriptCodes.SequenceEqual(recompiledScriptCode))
{
if (!excelScriptComparisonFails.ContainsKey(excelFile.StringId))
{
excelScriptComparisonFails.Add(excelFile.StringId, 1);
}
else
{
excelScriptComparisonFails[excelFile.StringId]++;
}
scriptsFailedComparison++;
String compareFailed = String.Format("Script Comparison Failed:\nOriginal Script: {0} = {1}\nRecompiled Script: {2}\nSemi-Decompiled:\n{3}", scriptCodesStr, script, recompiledScriptCode.ToString(","), semiDecompiledScript);
excelFileResults.WriteLine(compareFailed);
continue;
}
excelFileResults.WriteLine(script + "\n");
}
totalScriptsFound += scriptCount;
totalScriptsDecompiled += scriptsDecompiled;
totalScriptsFailedDecompilation += scriptsFailedDecompilation;
totalScriptsRecompiled += scriptsRecompiled;
totalScriptsFailedRecompilation += scriptsFailedRecompilation;
totalScriptsCompared += scriptsCompared;
totalScriptsFailedComparison += scriptsFailedComparison;
String columnStats = String.Format("\t\t{0} scripts found, {1} scripts decompiled, {2} scripts failed to decompile.\n", scriptCount, scriptsDecompiled, scriptsFailedDecompilation);
columnStats += String.Format("\t\t\t{0} scripts recompiled, {1} scripts failed to recompile, {2} scripts compared, {3} scripts failed comparison",
scriptsRecompiled, scriptsFailedRecompilation, scriptsCompared, scriptsFailedComparison);
Debug.WriteLine(columnStats);
excelFileResults.WriteLine(columnStats);
}
grandTotalScripts += totalScriptsFound;
grandTotalScriptsDecompiled += totalScriptsDecompiled;
grandTotalScriptsFailedDecompilation += totalScriptsFailedDecompilation;
grandTotalScriptsRecompiled += totalScriptsRecompiled;
grandTotalScriptsFailedRecompilation += totalScriptsFailedRecompilation;
grandTotalScriptsCompared += totalScriptsCompared;
grandTotalScriptsFailedComparison += totalScriptsFailedComparison;
String totalStats = String.Format("Totals: {0} scripts found, {1} scripts decompiled, {2} scripts failed to decompile.\n", totalScriptsFound, totalScriptsDecompiled, totalScriptsFailedDecompilation);
totalStats += String.Format("\t{0} scripts recompiled, {1} scripts failed to recompile, {2} scripts compared, {3} scripts failed comparison",
totalScriptsRecompiled, totalScriptsFailedRecompilation, totalScriptsCompared, totalScriptsFailedComparison);
Debug.WriteLine(totalStats);
results.WriteLine(totalStats);
excelFileResults.WriteLine(totalStats);
File.WriteAllText(@"C:\TestScripts\excelScripts_" + excelFile.StringId + ".txt", excelFileResults.ToString());
excelFileResults.Close();
}
foreach (KeyValuePair<String, int> keyValuePair in excelScriptDecompileFails)
{
String excelFileFails = String.Format("{0} had {1} failed decompilation.", keyValuePair.Key, keyValuePair.Value);
Debug.WriteLine(excelFileFails);
results.WriteLine(excelFileFails);
}
String grandTotalStats = String.Format("Grand Totals: {0} scripts found, {1} scripts decompiled, {2} scripts failed to decompile.\n", grandTotalScripts, grandTotalScriptsDecompiled, grandTotalScriptsFailedDecompilation);
grandTotalStats += String.Format("\t{0} scripts recompiled, {1} scripts failed to recompile, {2} scripts compared, {3} scripts failed comparison",
grandTotalScriptsRecompiled, grandTotalScriptsFailedRecompilation, grandTotalScriptsCompared, grandTotalScriptsFailedComparison);
Debug.WriteLine(grandTotalStats);
results.WriteLine(grandTotalStats);
File.WriteAllText(@"C:\TestScripts\excelScripts_Results.txt", results.ToString());
}
static void Main(string[] args)
{
string hellgatePath = Config.HglDir;
string currentDir = Directory.GetCurrentDirectory();
string dataDir = Path.Combine(currentDir, Hellgate.Common.DataPath);
string dataCommonDir = Path.Combine(currentDir, Hellgate.Common.DataCommonPath);
bool doCookTxt = false;
bool doCookXml = false;
bool doPackDat = false; // do pack any referenced files into a hellgate london dat/idx
bool doSearchCd = false; // search the current directory for files to cook and pack
bool doExcludeRaw = false; // do not pack source files, only cooked versions.
bool doLevelRules = false;
bool doRoomDefinitions = false;
List<string> filesToPack = new List<string>();
List<string> excelFilesToCook = new List<string>();
List<string> stringFilesToCook = new List<string>();
List<string> xmlFilesToCook = new List<string>();
List<string> levelRulesFilesToSerialize = new List<string>();
List<string> roomDefinitionFilesSerialize = new List<string>();
#region alexs_stuff
if (false)
{
//_fileManager = new FileManager(@"D:\Games\Hellgate London");
//_fileManager.LoadTableFiles();
////byte[] buffer = fileManager.DataFiles["SOUNDS"].ExportCSV();
////return;
////File.WriteAllBytes(@"D:\levels_rules.txt", buffer);
//foreach (DataFile dataFile in _fileManager.DataFiles.Values)
//{
// if (dataFile.IsStringsFile) continue;
// ExcelFile excelFile = (ExcelFile) dataFile;
// Console.WriteLine(dataFile.FileName);
// String dir = Path.GetDirectoryName(dataFile.FilePath);
// if (Directory.Exists(dir) == false) Directory.CreateDirectory(dir);
// byte[] ebuffer = excelFile.ExportCSV(_fileManager);
// //for (int i = 0; i < 10; i++)
// //{
// // Stopwatch stopwatch = new Stopwatch();
// // stopwatch.Start();
// // ebuffer = dataFile.ExportCSV();
// // stopwatch.Stop();
// // Console.WriteLine("Elapsed: {0}", stopwatch.Elapsed);
// //}
// File.WriteAllBytes(dataFile.FilePath.Replace(".cooked", ""), ebuffer);
//}
//Stopwatch stopwatch = new Stopwatch();
//stopwatch.Start();
//const String pathExcel = @"data\excel\items.txt";
//byte[] excelBytes = File.ReadAllBytes(pathExcel);
//ExcelFile excelFile = new ExcelFile(excelBytes, pathExcel);
//byte[] excelCsvBytes = excelFile.ToByteArray();
//stopwatch.Stop();
//Console.WriteLine("Elapsed: {0}", stopwatch.Elapsed);
//const String path = @"data\excel\strings\english\strings_revival.xls.uni2";
//byte[] cookedBytes = File.ReadAllBytes(path);
//StringsFile stringsFile = new StringsFile(cookedBytes, path);
//byte[] csvBytes = stringsFile.ExportCSV();
//if (!cookedBytes.SequenceEqual(csvBytes))
//{
// File.WriteAllBytes(path + "2", csvBytes);
// int bp = 0;
//}
//doSearchCd = true;
//doCookTxt = true;
//if (doSearchCd && doCookTxt)
//{
// excelFilesToCook.AddRange(SearchForExcelFiles(currentDir));
// stringFilesToCook.AddRange(SearchForStringFiles(currentDir));
//}
//if (doCookTxt)
//{
// CookExcelFiles(excelFilesToCook.ToArray());
// CookStringFiles(stringFilesToCook.ToArray());
//}
//return;
}
#endregion
#region Model conversion testing
if (false)
{
//_fileManager = new FileManager(@"D:\Games\Hellgate London");
//string filePath = @"data\background\tubestations\charingcross\cc_southbound.m";
//byte[] buffer = _fileManager.GetFileBytes(filePath);
//if (buffer == null)
//{
// Console.WriteLine("Could not read specified file.");
// return;
//}
//string modelId = Path.GetFileNameWithoutExtension(filePath);
//Model model = new Model(buffer, modelId);
//buffer = model.ExportCollada();
//File.WriteAllBytes(modelId + ".xml", buffer);
//return;
}
#endregion
#region SQL test
if (false)
{
//_fileManager = new FileManager(@"D:\Games\Hellgate London", true);
//_fileManager.LoadTableFiles();
//byte[] sqlBuffer = new byte[1024];
//int sqlOffset = 0;
////string createDB = "CREATE DATABASE hellgate_london;\nUSE hellgate_london;\n";
////byte[] createDBArray = FileTools.StringToASCIIByteArray(createDB);
////FileTools.WriteToBuffer(ref sqlBuffer, ref sqlOffset, createDBArray);
//foreach (DataFile dataFile in _fileManager.DataFiles.Values)
//{
// if (dataFile.IsStringsFile) continue;
// ExcelFile excelFile = dataFile as ExcelFile;
// byte[] buffer = excelFile.ExportSQL(_fileManager, "hgl");
// //if (dataFile.IsExcelFile) continue;
// //byte[] buffer = dataFile.ExportSQL("hgl_tcv4_");
// Console.WriteLine(dataFile.FileName);
// FileTools.WriteToBuffer(ref sqlBuffer, ref sqlOffset, buffer);
// if (true)
// {
// String dir = Path.GetDirectoryName(dataFile.FilePath);
// if (Directory.Exists(dir) == false) Directory.CreateDirectory(dir);
// string fileName = dataFile.FilePath;
// fileName = fileName.Replace(".txt.cooked", ".sql");
// fileName = fileName.Replace(".xls.uni.cooked", ".sql");
// File.WriteAllBytes(fileName, buffer);
// }
//}
//Array.Resize(ref sqlBuffer, sqlOffset);
//File.WriteAllBytes("hellgate_london_db.sql", sqlBuffer);
//return;
}
#endregion
Console.WriteLine("Hellpack - the Hellgate London compiler.\nWritten by the Revival Team, 2012\nhttp://www.hellgateaus.net");
Console.WriteLine(String.Empty);
#region Command Line Arugements
if (args.Length == 0)
{
// No arguments defined - this is the default program
doCookTxt = true;
doCookXml = true;
doPackDat = true;
doSearchCd = true;
doExcludeRaw = true;
doLevelRules = true;
doRoomDefinitions = true;
}
else
{
foreach (string arg in args)
{
switch (arg)
{
case "/t":
doCookTxt = true;
break;
case "/x":
doCookXml = true;
break;
case "/p":
doPackDat = true;
break;
case "/s":
doSearchCd = true;
break;
case "/e":
doExcludeRaw = true;
break;
case "/lr":
doLevelRules = true;
break;
case "/rd":
doRoomDefinitions = true;
break;
case "/?":
case "/help":
Console.WriteLine(UsageMsg);
return;
default:
if (arg.StartsWith("/p:"))
{
_defaultDat = arg.Replace("/p:", "");
// Trim in case someone has appended the extention
if (_defaultDat.EndsWith(".idx"))
_defaultDat = _defaultDat.Replace(".idx", "");
if (_defaultDat.EndsWith(".dat"))
_defaultDat = _defaultDat.Replace(".dat", "");
break;
}
if (arg.StartsWith("/h:"))
{
hellgatePath = arg.Replace("/h:", "");
break;
}
if (arg.EndsWith(ExcelFile.ExtensionDeserialised))
{
excelFilesToCook.Add(arg);
doCookTxt = true;
break;
}
if (arg.EndsWith(StringsFile.ExtensionDeserialised))
{
stringFilesToCook.Add(arg);
doCookTxt = true;
break;
}
if (arg.EndsWith(LevelRulesFile.ExtensionDeserialised))
{
levelRulesFilesToSerialize.Add(arg);
doLevelRules = true;
break;
}
if (arg.EndsWith(RoomDefinitionFile.ExtensionDeserialised))
{
roomDefinitionFilesSerialize.Add(arg);
doRoomDefinitions = true;
break;
}
if (arg.EndsWith(XmlCookedFile.ExtensionDeserialised))
{
xmlFilesToCook.Add(arg);
doCookXml = true;
break;
}
else
{
Console.WriteLine(String.Format("Incorrect argument given: {0}", arg));
Console.WriteLine(UsageMsg);
return;
}
}
}
}
#endregion
#region Program Functions
// Search for Txt files to cook
if (doSearchCd && doCookTxt)
{
string[] result = SearchForExcelFiles(currentDir);
if (result != null) excelFilesToCook.AddRange(result);
result = SearchForStringFiles(currentDir);
if (result != null) stringFilesToCook.AddRange(result);
}
// Search for Xml files to cook
if (doSearchCd && doCookXml)
{
String[] xmlToCook = SearchForXmlFiles(currentDir);
if (xmlToCook != null) xmlFilesToCook.AddRange(xmlToCook);
}
// Search for .drl Level Rules files to cook
if (doSearchCd && doLevelRules)
{
IEnumerable<String> drlXmlToCompile = _SearchForDrlXmlFiles(currentDir);
if (drlXmlToCompile != null) levelRulesFilesToSerialize.AddRange(drlXmlToCompile);
}
// Search for .rom Level Rules files to cook
if (doSearchCd && doRoomDefinitions)
{
// todo
}
// need for code/name -> row index lookups
Console.WriteLine("Loading FileManager...");
_fileManager = new FileManager(hellgatePath);
_fileManager.BeginAllDatReadAccess();
Console.WriteLine("Loading strings and tables...");
_fileManager.LoadTableFiles();
_fileManager.EndAllDatAccess();
// Cook Txt files)
if (doCookTxt)
{
_CookExcelFiles(excelFilesToCook.ToArray());
CookStringFiles(stringFilesToCook.ToArray());
}
// Cook Xml files
if (doCookXml)
{
// ensure we have the correct hellgate installation path
if (Directory.Exists(hellgatePath))
{
if (_fileManager.HasIntegrity == false)
{
Console.WriteLine("Warning: XML could not be cooked - fileManager.Integrity = false");
}
else
{
_CookXmlFiles(xmlFilesToCook.ToArray(), _fileManager);
}
}
else
{
Console.WriteLine("Warning: Can not cook XML, Hellgate London directory missing.");
}
}
// cook .drl Level Rules files
if (doLevelRules)
{
CookLevelRulesFiles(levelRulesFilesToSerialize);
}
// cook .rom Room Definition files
if (doRoomDefinitions)
{
CookRoomDefinitionFiles(roomDefinitionFilesSerialize);
}
// Files to pack
if (doPackDat)
{
filesToPack.AddRange(SearchForFilesToPack(currentDir, doExcludeRaw));
PackDatFile(filesToPack.ToArray(), Path.Combine(dataDir, _defaultDat + ".idx"), false);
}
#endregion
return;
}
static void Main(string[] args)
{
var manager = new FileManager(@"C:\Hellgate");
manager.BeginAllDatReadAccess();
manager.LoadTableFiles();
manager.EndAllDatAccess();
//TODO: hp/power regen is messed up when displaying the full formula because I guess the itemdisplay formula isn't used
//note: "display dmg" is ilvl multi*item dmg*dmg% (before "increments" are applied), where dmg% is the first or second argument in dmg_elec(100,100)
//TODO: weapon ranges
//TODO: add affix names to properties
//TODO: sword attack rates
//TODO: wtf is up with ilvls that don't match any of the listed numbers (maxlevel, fixedlevel, level)?
//long term TODO: assign unit types so "isa" functions (among other things) work
args = new[] { "ITEMS" };
//new Items(manager).WriteAllUniqueLegendaryItemPages();
//return;
string sqlStatement;
WikiScript script;
foreach (string arg in args)
{
switch (arg)
{
case "ACHIEVEMENTS":
script = new Achievements(manager);
break;
case "AFFIXES":
script = new Affixes(manager);
break;
case "BASEWAVES":
script = new BaseDefenseWaves(manager);
break;
case "MONSTERAFFIXES":
script = new MonsterAffixes(manager);
break;
case "ARMORAFFIXES":
script = new ArmorAffixes(manager);
break;
case "ITEMLEVELS":
script = new ItemLevels(manager);
break;
case "LEVELS":
script = new Levels(manager);
break;
case "MONSTERS":
script = new Monsters(manager);
break;
case "PVPRANKS":
script = new PVPRanks(manager);
break;
case "LEVELSCALING":
script = new LevelScaling(manager);
break;
case "ITEMS":
script = new Items(manager);
break;
case "ITEM_QUALITY":
script = new ItemQuality(manager);
break;
case "TREASURE":
script = new NewTreasure(manager);
break;
case "MONSTER_QUALITY":
script = new MonsterQuality(manager);
break;
case "RECIPES":
script = new Recipes(manager);
break;
case "ITEM_SETS":
script = new Sets(manager);
break;
default:
throw new Exception("Unknown WikiScript: " + arg);
}
sqlStatement = script.ExportTableInsertScript();
File.WriteAllText(arg.ToLower() + ".sql", sqlStatement);
}
return;
}
