public void TestScript()
{
GlobalTest.Init();
using (var biopProEar = MEPackageHandler.OpenMEPackage(Path.Combine(GlobalTest.GetTestPackagesDirectory(), "PC", "ME3", "BioP_ProEar.pcc")))
{
var biopProEarLib = new FileLib(biopProEar);
bool fileLibInitialized = biopProEarLib.Initialize().Result;
Assert.IsTrue(fileLibInitialized, "ME3 Script failed to compile BioP_ProEar class definitions!");
foreach (ExportEntry funcExport in biopProEar.Exports.Where(exp => exp.ClassName == "Function"))
{
(ASTNode astNode, string text) = UnrealScriptCompiler.DecompileExport(funcExport, biopProEarLib);
Assert.IsInstanceOfType(astNode, typeof(Function), $"#{funcExport.UIndex} {funcExport.InstancedFullPath} in BioP_ProEar did not decompile!");
(_, MessageLog log) = UnrealScriptCompiler.CompileFunction(funcExport, text, biopProEarLib);
if (log.AllErrors.Any())
{
Assert.Fail($"#{funcExport.UIndex} {funcExport.InstancedFullPath} in BioP_ProEar did not recompile!");
}
}
}
}
public void TestParallelCRC()
{
GlobalTest.Init();
// Loads compressed packages and attempts to enumerate every object's properties.
var filesPath = GlobalTest.GetTestCRCDirectory();
var files = Directory.GetFiles(filesPath, "*.*", SearchOption.AllDirectories);
foreach (var f in files)
{
var expectedCrc = Path.GetFileNameWithoutExtension(f);
var calculatedCrc = ParallelCRC.Compute(File.ReadAllBytes(f));
Assert.AreEqual(expectedCrc, calculatedCrc.ToString("x8"));
}
}
public void TestBinaryConverters()
{
GlobalTest.Init();
var packagesPath = GlobalTest.GetTestPackagesDirectory();
var packages = Directory.GetFiles(packagesPath, "*.*", SearchOption.AllDirectories);
foreach (var p in packages)
{
if (p.RepresentsPackageFilePath())
{
// Do not use package caching in tests
Console.WriteLine($"Opening package {p}");
var(game, platform) = GlobalTest.GetExpectedTypes(p);
if (platform == MEPackage.GamePlatform.PC) // Will expand in future, but not now.
{
var originalLoadedPackage = MEPackageHandler.OpenMEPackage(p, forceLoadFromDisk: true);
foreach (var export in originalLoadedPackage.Exports)
{
PropertyCollection props = export.GetProperties();
ObjectBinary bin = ObjectBinary.From(export) ?? export.GetBinaryData();
if (game == MEGame.UDK)
{
continue; // No point testing converting things to UDK in this fashion
}
byte[] original = export.Data;
export.WritePropertiesAndBinary(props, bin);
byte[] changed = export.Data;
Assert.AreEqual(original.Length, changed.Length,
$"Reserialization of export {export.UIndex} {export.InstancedFullPath} produced a different sized byte array than the input. Original size: {original.Length}, reserialized: {changed.Length}, difference: 0x{(changed.Length - original.Length):X8} bytes. File: {p}");
Assert.IsTrue(original.SequenceEqual(changed),
$"Reserialization of export {export.UIndex} {export.InstancedFullPath} produced a different byte array than the input. File: {p}");
bin.GetNames(game);
var uindexes = bin.GetUIndexes(game);
foreach (var uindex in uindexes)
{
if (uindex.Item1 != 0)
{
originalLoadedPackage.GetEntry(uindex.Item1);
}
}
}
}
}
}
}
public void TestTextureOperations()
{
GlobalTest.Init();
var packagesPath = GlobalTest.GetTestTexturesDirectory();
var packages = Directory.GetFiles(packagesPath, "*.*", SearchOption.AllDirectories);
foreach (var p in packages)
{
if (p.RepresentsPackageFilePath())
{
// Do not use package caching in tests
Console.WriteLine($"Opening package {p}");
(var game, var platform) = GlobalTest.GetExpectedTypes(p);
if (platform == MEPackage.GamePlatform.PC)
{
var loadedPackage = MEPackageHandler.OpenMEPackage(p, forceLoadFromDisk: true);
foreach (var textureExp in loadedPackage.Exports.Where(x => x.IsTexture()))
{
Texture2D.GetTextureCRC(textureExp);
var t2d = new Texture2D(textureExp);
var mips = Texture2D.GetTexture2DMipInfos(textureExp, t2d.GetTopMip().TextureCacheName);
foreach (var v in t2d.Mips)
{
var displayStr = v.MipDisplayString;
var texCache = v.TextureCacheName;
var textureData = Texture2D.GetTextureData(v, v.Export.Game);
var imageDataFromInternal = t2d.GetImageBytesForMip(v, v.Export.Game, false);
if (!textureData.SequenceEqual(imageDataFromInternal))
{
Assert.Fail($"Texture data accessed using wrapper and internal method did not match! Export: {textureExp.InstancedFullPath} in {p}. Static size: {textureData.Length} Instance size: {imageDataFromInternal.Length}");
}
}
t2d.RemoveEmptyMipsFromMipList();
t2d.SerializeNewData();
}
}
}
}
}
public void TestMounts()
{
GlobalTest.Init();
// Loads compressed packages and attempts to enumerate every object's properties.
var packagesPath = GlobalTest.GetTestMountsDirectory();
var mounts = Directory.GetFiles(packagesPath, "*.dlc", SearchOption.AllDirectories);
foreach (var mountFilePath in mounts)
{
// Do not use package caching in tests
Debug.WriteLine($"Opening mount file {mountFilePath}");
(MEGame expectedGame, MEPackage.GamePlatform expectedPlatform) = GlobalTest.GetExpectedTypes(mountFilePath);
var mountFName = Path.GetFileNameWithoutExtension(mountFilePath);
var mountProps = mountFName.Split('-');
var mountName = mountProps[0];
var expectedMountPriority = ushort.Parse(mountProps[1]);
var expectedMountTLK = int.Parse(mountProps[2]);
EMountFileFlag expectedMountFlag = (EMountFileFlag)byte.Parse(mountProps[3], NumberStyles.HexNumber);
string me2DlcName = expectedGame == MEGame.ME2 ? mountProps[4] : null;
string me2HRDlcName = expectedGame == MEGame.ME2 ? mountProps[5] : null;
MountFile mf = new MountFile(mountFilePath);
Assert.AreEqual(expectedGame == MEGame.ME2, mf.IsME2, $"Mount file {mountName} parsed to the wrong game");
Assert.AreEqual(expectedMountPriority, mf.MountPriority, $"Mount file {mountName} has wrong parsed mount priority");
Assert.AreEqual(expectedMountTLK, mf.TLKID, $"Mount file {mountName} has wrong parsed mount TLKID");
Assert.AreEqual(expectedMountFlag, mf.MountFlag, $"Mount file {mountName} has wrong parsed mount flag");
if (expectedGame == MEGame.ME2)
{
Assert.AreEqual(me2DlcName, mf.ME2Only_DLCFolderName, $"Mount file {mountName} has wrong DLC folder name");
Assert.AreEqual(me2HRDlcName, mf.ME2Only_DLCHumanName, $"Mount file {mountName} has wrong human name");
}
}
}
public void TestPlatformsGamesAndProperties()
{
GlobalTest.Init();
// Loads compressed packages and attempts to enumerate every object's properties.
var packagesPath = GlobalTest.GetTestPackagesDirectory();
var packages = Directory.GetFiles(packagesPath, "*.*", SearchOption.AllDirectories);
foreach (var p in packages)
{
if (p.RepresentsPackageFilePath())
{
// Do not use package caching in tests
Console.WriteLine($"Opening package {p}");
(MEGame expectedGame, MEPackage.GamePlatform expectedPlatform) = GlobalTest.GetExpectedTypes(p);
var package = MEPackageHandler.OpenMEPackage(p, forceLoadFromDisk: true);
Assert.AreEqual(expectedGame, package.Game,
"The expected game and the resolved game do not match!");
Assert.AreEqual(expectedPlatform, package.Platform,
"The expected platform and the resolved platform do not match!");
Console.WriteLine($" > Enumerating all exports for properties");
foreach (var exp in package.Exports)
{
if (exp.ClassName != "Class")
{
var props = exp.GetProperties(forceReload: true, includeNoneProperties: true);
Assert.IsInstanceOfType(props.LastOrDefault(), typeof(NoneProperty),
$"Error parsing properties on export {exp.UIndex} {exp.InstancedFullPath} in file {exp.FileRef.FilePath}");
}
}
}
}
}
public void TestSFARParsing()
{
GlobalTest.Init();
// Loads compressed packages and attempts to enumerate every object's properties.
var sfarsPath = GlobalTest.GetTestSFARsDirectory();
var sfars = Directory.GetFiles(sfarsPath, "*.sfar", SearchOption.AllDirectories);
foreach (var s in sfars)
{
var expectedCompressionType = Path.GetFileNameWithoutExtension(s);
Console.WriteLine($"Opening SFAR {s}");
DLCPackage dlc = new DLCPackage(s);
Assert.AreEqual(expectedCompressionType, dlc.Header.CompressionScheme);
// Enumerate files
foreach (var f in dlc.Files)
{
var lookup = dlc.FindFileEntry(Path.GetFileName(f.FileName));
Assert.AreEqual(dlc.Files[lookup], f);
var de = dlc.DecompressEntry(lookup);
Assert.AreEqual(f.RealUncompressedSize, de.Length);
}
}
}
public void TestCompression()
{
GlobalTest.Init();
// Loads compressed packages, save them uncompressed. Load package, save re-compressed, compare results
var packagesPath = GlobalTest.GetTestPackagesDirectory();
//var packages = Directory.GetFiles(packagesPath, "*.*", SearchOption.AllDirectories);
var packages = Directory.GetFiles(packagesPath, "*.*", SearchOption.AllDirectories);
foreach (var p in packages)
{
if (p.RepresentsPackageFilePath())
{
// Do not use package caching in tests
Console.WriteLine($"Opening package {p}");
var originalLoadedPackage = MEPackageHandler.OpenMEPackage(p, forceLoadFromDisk: true);
if (originalLoadedPackage.Platform != MEPackage.GamePlatform.PC)
{
Assert.ThrowsException <Exception>(() => { originalLoadedPackage.SaveToStream(true); },
"Non-PC platform package should not be saveable. An exception should have been thrown to stop this!");
continue;
}
// Is PC
var uncompressedPS = originalLoadedPackage.SaveToStream(false);
var compressedPS = originalLoadedPackage.SaveToStream(true);
uncompressedPS.Position = compressedPS.Position = 0;
var reopenedUCP = MEPackageHandler.OpenMEPackageFromStream(uncompressedPS);
var reopenedCCP = MEPackageHandler.OpenMEPackageFromStream(compressedPS);
Assert.AreEqual(reopenedCCP.NameCount, reopenedUCP.NameCount,
$"Name count is not identical between compressed/uncompressed packages");
Assert.AreEqual(reopenedCCP.ImportCount, reopenedUCP.ImportCount,
$"Import count is not identical between compressed/uncompressed packages");
Assert.AreEqual(reopenedCCP.ExportCount, reopenedUCP.ExportCount,
$"Export count is not identical between compressed/uncompressed packages");
for (int i = 0; i < reopenedCCP.NameCount; i++)
{
var nameCCP = reopenedCCP.Names[i];
var nameUCP = reopenedUCP.Names[i];
Assert.AreEqual(nameCCP, nameUCP,
$"Names are not identical between compressed/uncompressed packages, name index {i}");
}
for (int i = 0; i < reopenedCCP.ImportCount; i++)
{
var importCCP = reopenedCCP.Imports[i];
var importUCP = reopenedUCP.Imports[i];
Assert.IsTrue(importCCP.Header.SequenceEqual(importUCP.Header),
$"Header data for import {-(i + 1)} are not identical between compressed/uncompressed packages");
}
for (int i = 0; i < reopenedCCP.ExportCount; i++)
{
var exportCCP = reopenedCCP.Exports[i];
var exportUCP = reopenedUCP.Exports[i];
Assert.IsTrue(exportCCP.Header.SequenceEqual(exportUCP.Header),
$"Header data for xport {i + 1} are not identical between compressed/uncompressed packages");
}
}
}
}
public void TestNameOperations()
{
GlobalTest.Init();
Random random = new Random();
// Loads compressed packages, save them uncompressed. Load package, save re-compressed, compare results
var packagesPath = GlobalTest.GetTestPackagesDirectory();
//var packages = Directory.GetFiles(packagesPath, "*.*", SearchOption.AllDirectories);
var packages = Directory.GetFiles(packagesPath, "*.*", SearchOption.AllDirectories);
foreach (var p in packages)
{
if (p.RepresentsPackageFilePath())
{
// Do not use package caching in tests
Console.WriteLine($"Opening package {p}");
(var game, var platform) = GlobalTest.GetExpectedTypes(p);
if (platform == MEPackage.GamePlatform.PC) // Will expand in future, but not now.
{
var loadedPackage = MEPackageHandler.OpenMEPackage(p, forceLoadFromDisk: true);
var afterLoadNameCount = loadedPackage.NameCount;
for (int i = 0; i < afterLoadNameCount; i++)
{
var existingName = loadedPackage.Names[i];
var existingNameIndex = loadedPackage.FindNameOrAdd(existingName);
Assert.IsTrue(existingNameIndex == i,
"An existing name was added when it shouldn't have been!");
}
// Test adding
for (int i = 0; i < 20; i++)
{
var expectedNameIndex = loadedPackage.NameCount;
var newName =
RandomString(random,
35); // If we have a same-collision on 35 char random strings, let Mgamerz know he should buy a lottery ticket
var newNameIndex = loadedPackage.FindNameOrAdd(newName);
Assert.AreEqual(expectedNameIndex, newNameIndex,
"A name was added, but the index lookup was wrong!");
}
// Test changing
for (int i = 0; i < 20; i++)
{
var existingIndex = random.Next(loadedPackage.NameCount);
var newName = RandomString(random, 38); //even more entropy
loadedPackage.replaceName(existingIndex, newName);
// Check it's correct
var calculatedIndex = loadedPackage.FindNameOrAdd(newName);
Assert.AreEqual(existingIndex, calculatedIndex,
"A name was replaced, but the index of the replaced name was wrong when looked up via FindNameOrAdd()!");
var checkedNameGet = loadedPackage.GetNameEntry(calculatedIndex);
var checkedNameAccessor = loadedPackage.GetNameEntry(calculatedIndex);
Assert.AreEqual(newName, checkedNameGet,
"A name was replaced, but the GetNameEntry() for the replaced name returned the wrong name!");
Assert.AreEqual(newName, checkedNameAccessor,
"A name was replaced, but the Names[] array accessor for the replaced name returned the wrong name!");
}
}
}
}
}
public void TestFunctionParsing()
{
// This method is essentially test of the BytecodeEditor parser with the actual ui logic all removed
GlobalTest.Init();
// Loads compressed packages, save them uncompressed. Load package, save re-compressed, compare results
var packagesPath = GlobalTest.GetTestPackagesDirectory();
//var packages = Directory.GetFiles(packagesPath, "*.*", SearchOption.AllDirectories);
var packages = Directory.GetFiles(packagesPath, "*.*", SearchOption.AllDirectories);
foreach (var p in packages)
{
if (p.RepresentsPackageFilePath())
{
(var game, var platform) = GlobalTest.GetExpectedTypes(p);
// Use to skip
//if (platform != MEPackage.GamePlatform.Xenon) continue;
//if (game != MEGame.ME1) continue;
Console.WriteLine($"Opening package {p}");
// Do not use package caching in tests
var originalLoadedPackage = MEPackageHandler.OpenMEPackage(p, forceLoadFromDisk: true);
foreach (var export in originalLoadedPackage.Exports.Where(x => x.ClassName == "Function" || x.ClassName == "State"))
{
//Console.WriteLine($" >> Decompiling {export.InstancedFullPath}");
var data = export.Data;
var funcBin = ObjectBinary.From <UFunction>(export); //parse it out
if (export.FileRef.Game == MEGame.ME3 || export.FileRef.Platform == MEPackage.GamePlatform.PS3)
{
var func = new Function(data, export);
func.ParseFunction();
func.GetSignature();
if (export.ClassName == "Function")
{
var nativeBackOffset = export.FileRef.Game < MEGame.ME3 ? 7 : 6;
var pos = data.Length - nativeBackOffset;
string flagStr = func.GetFlags();
var nativeIndex = EndianReader.ToInt16(data, pos, export.FileRef.Endian);
pos += 2;
var flags = EndianReader.ToInt16(data, pos, export.FileRef.Endian);
}
else
{
//State
//parse remaining
var footerstartpos = 0x20 + funcBin.ScriptStorageSize;
var footerdata = data.Slice(footerstartpos, (int)data.Length - (footerstartpos));
var fpos = 0;
//ScriptFooterBlocks.Add(new ScriptHeaderItem("Probemask?", "??", fpos + footerstartpos) { length = 8 });
fpos += 0x8;
//ScriptFooterBlocks.Add(new ScriptHeaderItem("Unknown 8 FF's", "??", fpos + footerstartpos) { length = 8 });
fpos += 0x8;
//ScriptFooterBlocks.Add(new ScriptHeaderItem("Label Table Offset", "??", fpos + footerstartpos) { length = 2 });
fpos += 0x2;
var stateFlagsBytes = footerdata.Slice(fpos, 0x4);
var stateFlags = (StateFlags)EndianReader.ToInt32(stateFlagsBytes, 0, export.FileRef.Endian);
var names = stateFlags.ToString().Split(new[] { ',', ' ' }, StringSplitOptions.RemoveEmptyEntries);
fpos += 0x4;
var numMappedFunctions = EndianReader.ToInt32(footerdata, fpos, export.FileRef.Endian);
fpos += 4;
for (int i = 0; i < numMappedFunctions; i++)
{
var name = EndianReader.ToInt32(footerdata, fpos, export.FileRef.Endian);
var uindex = EndianReader.ToInt32(footerdata, fpos + 8, export.FileRef.Endian);
var funcMapText = $"{export.FileRef.GetNameEntry(name)} => {export.FileRef.GetEntry(uindex)?.FullPath}()";
fpos += 12;
}
}
}
else if (export.FileRef.Game == MEGame.ME1 || export.FileRef.Game == MEGame.ME2)
{
//Header
int pos = 16;
var nextItemCompilingChain = EndianReader.ToInt32(data, pos, export.FileRef.Endian);
//ScriptHeaderBlocks.Add(new ScriptHeaderItem("Next item in loading chain", nextItemCompilingChain, pos, nextItemCompilingChain > 0 ? export : null));
pos += 8;
nextItemCompilingChain = EndianReader.ToInt32(data, pos, export.FileRef.Endian);
//ScriptHeaderBlocks.Add(new ScriptHeaderItem("Children Probe Start", nextItemCompilingChain, pos, nextItemCompilingChain > 0 ? export : null));
pos += 8;
var line = EndianReader.ToInt32(data, pos, export.FileRef.Endian);
//ScriptHeaderBlocks.Add(new ScriptHeaderItem("Line", EndianReader.ToInt32(data, pos, export.FileRef.Endian), pos));
pos += 4;
//EndianReader.ToInt32(data, pos, export.FileRef.Endian)
//ScriptHeaderBlocks.Add(new ScriptHeaderItem("TextPos", EndianReader.ToInt32(data, pos, export.FileRef.Endian), pos));
pos += 4;
int scriptSize = EndianReader.ToInt32(data, pos, export.FileRef.Endian);
//ScriptHeaderBlocks.Add(new ScriptHeaderItem("Script Size", scriptSize, pos));
pos += 4;
var BytecodeStart = pos;
var func = export.ClassName == "State" ? UE3FunctionReader.ReadState(export, data) : UE3FunctionReader.ReadFunction(export, data);
func.Decompile(new TextBuilder(), false); //parse bytecode
bool defined = func.HasFlag("Defined");
//if (defined)
//{
// DecompiledScriptBlocks.Add(func.FunctionSignature + " {");
//}
//else
//{
// //DecompiledScriptBlocks.Add(func.FunctionSignature);
//}
for (int i = 0; i < func.Statements.statements.Count; i++)
{
Statement s = func.Statements.statements[i];
s.SetPaddingForScriptSize(scriptSize);
if (s.Reader != null && i == 0)
{
//Add tokens read from statement. All of them point to the same reader, so just do only the first one.
s.Reader.ReadTokens.Select(x => x.ToBytecodeSingularToken(pos)).OrderBy(x => x.StartPos);
}
}
//if (defined)
//{
// DecompiledScriptBlocks.Add("}");
//}
//Footer
pos = data.Length - (func.HasFlag("Net") ? 17 : 15);
string flagStr = func.GetFlags();
//ScriptFooterBlocks.Add(new ScriptHeaderItem("Native Index", EndianReader.ToInt16(data, pos, export.FileRef.Endian), pos));
pos += 2;
//ScriptFooterBlocks.Add(new ScriptHeaderItem("Operator Precedence", data[pos], pos));
pos++;
int functionFlags = EndianReader.ToInt32(data, pos, export.FileRef.Endian);
//ScriptFooterBlocks.Add(new ScriptHeaderItem("Flags", $"0x{functionFlags:X8} {flagStr}", pos));
pos += 4;
//if ((functionFlags & func._flagSet.GetMask("Net")) != 0)
//{
//ScriptFooterBlocks.Add(new ScriptHeaderItem("Unknown 1 (RepOffset?)", EndianReader.ToInt16(data, pos, export.FileRef.Endian), pos));
//pos += 2;
//}
int friendlyNameIndex = EndianReader.ToInt32(data, pos, export.FileRef.Endian);
var friendlyName = export.FileRef.GetNameEntry(friendlyNameIndex);
//ScriptFooterBlocks.Add(new ScriptHeaderItem("Friendly Name", Pcc.GetNameEntry(friendlyNameIndex), pos) { length = 8 });
pos += 8;
//ME1Explorer.Unreal.Classes.Function func = new ME1Explorer.Unreal.Classes.Function(data, export.FileRef as ME1Package);
//try
//{
// Function_TextBox.Text = func.ToRawText();
//}
//catch (Exception e)
//{
// Function_TextBox.Text = "Error parsing function: " + e.Message;
//}
}
else
{
//Function_TextBox.Text = "Parsing UnrealScript Functions for this game is not supported.";
}
}
//}
}
}
}
public void TestTLKs()
{
GlobalTest.Init();
// Loads compressed packages and attempts to enumerate every object's properties.
var tlkDataPath = GlobalTest.GetTestTLKDirectory();
//ME1
var packages = Directory.GetFiles(tlkDataPath, "*.*", SearchOption.AllDirectories)
.Where(x => x.RepresentsPackageFilePath());
foreach (var p in packages)
{
Console.WriteLine($"Opening package {p}");
(MEGame expectedGame, MEPackage.GamePlatform expectedPlatform) = GlobalTest.GetExpectedTypes(p);
var package = MEPackageHandler.OpenMEPackage(p, forceLoadFromDisk: true);
foreach (var export in package.Exports.Where(x => x.ClassName == "BioTlkFile"))
{
ME1TalkFile me1Tf = new ME1TalkFile(export);
foreach (var stringId in me1Tf.StringRefs)
{
var expected = stringId.Data;
var found = me1Tf.findDataById(stringId.StringID);
// Strip single pair of quotes off. Trim() does multiple so if string ends with " it ruins it
if (found.StartsWith('\"'))
{
found = found.Substring(1);
}
if (found.EndsWith('\"'))
{
found = found.Substring(0, found.Length - 1);
}
Assert.AreEqual(string.IsNullOrEmpty(expected) ? "" : expected, found);
}
}
}
// ME2/ME3
var tlks = Directory.GetFiles(tlkDataPath, "*.tlk", SearchOption.AllDirectories);
foreach (var tlkFilePath in tlks)
{
// Do not use package caching in tests
Debug.WriteLine($"Opening TLK file {tlkFilePath}");
(MEGame expectedGame, MEPackage.GamePlatform expectedPlatform) = GlobalTest.GetExpectedTypes(tlkFilePath);
TalkFile tf = new TalkFile();
tf.LoadTlkData(tlkFilePath);
foreach (var stringId in tf.StringRefs)
{
var expected = stringId.Data;
var found = tf.findDataById(stringId.StringID);
var testcache = found;
// Strip single pair of quotes off. Trim() does multiple so if string ends with " it ruins it
if (found.StartsWith('\"'))
{
found = found.Substring(1);
}
if (found.EndsWith('\"'))
{
found = found.Substring(0, found.Length - 1);
}
if (expected == "Female")
{
continue; //It seems we don't have a way to query female strings.
}
Assert.AreEqual(string.IsNullOrEmpty(expected) ? "" : expected, found);
}
}
}
