public void FunctionParametersTypeNone()
{
byte[] b = new byte[]
{
0x50, 0x45, 0x52, 0x4B, 0x3C, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x24, 0xE1, 0x0B, 0x00, 0x00, 0x00,
0x00, 0x00, 0x2C, 0x00, 0x0C, 0x00, 0x45, 0x44, 0x49, 0x44, 0x34, 0x00, 0x4F, 0x52, 0x44, 0x5F, 0x50, 0x69,
0x63, 0x30, 0x30, 0x5F, 0x50, 0x69, 0x63, 0x6B, 0x70, 0x6F, 0x63, 0x6B, 0x65, 0x74, 0x4D, 0x61, 0x73, 0x74,
0x65, 0x72, 0x79, 0x5F, 0x30, 0x30, 0x5F, 0x50, 0x65, 0x72, 0x6B, 0x5F, 0x57, 0x61, 0x73, 0x4C, 0x69, 0x67,
0x68, 0x74, 0x46, 0x69, 0x6E, 0x67, 0x65, 0x72, 0x73, 0x00, 0x46, 0x55, 0x4C, 0x4C, 0x13, 0x00, 0x50, 0x69,
0x63, 0x6B, 0x70, 0x6F, 0x63, 0x6B, 0x65, 0x74, 0x20, 0x4D, 0x61, 0x73, 0x74, 0x65, 0x72, 0x79, 0x00, 0x44,
0x45, 0x53, 0x43, 0x5E, 0x00, 0x41, 0x64, 0x64, 0x73, 0x20, 0x31, 0x25, 0x20, 0x74, 0x6F, 0x20, 0x79, 0x6F,
0x75, 0x72, 0x20, 0x70, 0x69, 0x63, 0x6B, 0x70, 0x6F, 0x63, 0x6B, 0x65, 0x74, 0x20, 0x63, 0x68, 0x61, 0x6E,
0x63, 0x65, 0x20, 0x70, 0x65, 0x72, 0x20, 0x6C, 0x65, 0x76, 0x65, 0x6C, 0x20, 0x6F, 0x66, 0x20, 0x50, 0x69,
0x63, 0x6B, 0x70, 0x6F, 0x63, 0x6B, 0x65, 0x74, 0x2E, 0x20, 0x41, 0x6C, 0x73, 0x6F, 0x20, 0x69, 0x6E, 0x63,
0x72, 0x65, 0x61, 0x73, 0x65, 0x73, 0x20, 0x43, 0x61, 0x72, 0x72, 0x79, 0x20, 0x57, 0x65, 0x69, 0x67, 0x68,
0x74, 0x20, 0x62, 0x79, 0x20, 0x32, 0x30, 0x2E, 0x00, 0x44, 0x41, 0x54, 0x41, 0x05, 0x00, 0x00, 0x00, 0x01,
0x01, 0x00, 0x50, 0x52, 0x4B, 0x45, 0x03, 0x00, 0x01, 0x00, 0xC3, 0x44, 0x41, 0x54, 0x41, 0x04, 0x00, 0xDD,
0x85, 0x07, 0x03, 0x45, 0x50, 0x46, 0x54, 0x01, 0x00, 0x00, 0x50, 0x52, 0x4B, 0x46, 0x00, 0x00, 0x50, 0x52,
0x4B, 0x45, 0x03, 0x00, 0x02, 0x00, 0xC8, 0x44, 0x41, 0x54, 0x41, 0x03, 0x00, 0x38, 0x0E, 0x03, 0x50, 0x52,
0x4B, 0x43, 0x01, 0x00, 0x00, 0x43, 0x54, 0x44, 0x41, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0xC0, 0x01, 0x00, 0x00, 0x6A, 0x8E, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0x45, 0x50, 0x46, 0x54, 0x01, 0x00, 0x02, 0x45, 0x50, 0x46, 0x44,
0x08, 0x00, 0x00, 0x00, 0x50, 0x41, 0x0A, 0xD7, 0x23, 0x3C, 0x50, 0x52, 0x4B, 0x46, 0x00, 0x00
};
var reader = new MutagenBinaryReadStream(new MemoryStream(b), ModKey.Null, GameRelease.SkyrimSE, dispose: false);
var frame = new MutagenFrame(reader);
var perk = Perk.CreateFromBinary(frame);
perk.Effects.Should().HaveCount(2);
}
private byte GetNumMasters()
{
using var stream = new MutagenBinaryReadStream(SourcePath, GameRelease);
var modFrame = stream.ReadModHeaderFrame();
return(checked ((byte)modFrame.Masters().Count()));
}
public void StreamModKeyCtorDoesNotDisposeStream()
{
var stream = new DisposeTesterWrapStream(File.OpenRead(TestDataPathing.SkyrimOverrideMod));
var mutaStream = new MutagenBinaryReadStream(
stream,
ModKey.FromNameAndExtension("Skyrim.esm"),
GameRelease.SkyrimSE);
stream.Disposed.Should().BeFalse();
}
private static void AlignGroupsByRules(
MutagenBinaryReadStream inputStream,
MutagenWriter writer,
AlignmentRules alignmentRules,
RecordLocatorResults fileLocs)
{
while (!inputStream.Complete)
{
// Import until next listed major record
long noRecordLength;
if (fileLocs.GrupLocations.TryGetInDirection(
inputStream.Position,
higher: true,
result: out var nextRec))
{
noRecordLength = nextRec.Value.Location.Min - inputStream.Position;
}
else
{
noRecordLength = inputStream.Remaining;
}
inputStream.WriteTo(writer.BaseStream, (int)noRecordLength);
// If complete overall, return
if (inputStream.Complete)
{
break;
}
var groupMeta = inputStream.GetGroup();
if (!groupMeta.IsGroup)
{
throw new ArgumentException();
}
writer.Write(inputStream.ReadSpan(groupMeta.HeaderLength));
if (!alignmentRules.GroupAlignment.TryGetValue(groupMeta.GroupType, out var groupRules))
{
continue;
}
var storage = new Dictionary <RecordType, List <ReadOnlyMemorySlice <byte> > >();
var rest = new List <ReadOnlyMemorySlice <byte> >();
using (var frame = MutagenFrame.ByLength(inputStream, groupMeta.ContentLength))
{
while (!frame.Complete)
{
var majorMeta = inputStream.GetMajorRecord();
var bytes = inputStream.ReadMemory(checked ((int)majorMeta.TotalLength));
var type = majorMeta.RecordType;
if (groupRules.Contains(type))
{
storage.GetOrAdd(type).Add(bytes);
}
else
{
rest.Add(bytes);
}
}
}
foreach (var rule in groupRules)
{
if (storage.TryGetValue(rule, out var storageBytes))
{
foreach (var item in storageBytes)
{
writer.Write(item);
}
}
}
foreach (var item in rest)
{
writer.Write(item);
}
}
}
public static void Align(
ModPath inputPath,
FilePath outputPath,
GameRelease gameMode,
AlignmentRules alignmentRules,
DirectoryPath temp)
{
var interest = new RecordInterest(alignmentRules.Alignments.Keys)
{
EmptyMeansInterested = false
};
// Always interested in parent record types
interest.InterestingTypes.Add("CELL");
interest.InterestingTypes.Add("WRLD");
var fileLocs = RecordLocator.GetLocations(inputPath, gameMode, interest);
if (gameMode == GameRelease.Oblivion)
{
var alignedMajorRecordsFile = new ModPath(inputPath.ModKey, Path.Combine(temp, "alignedRules"));
using (var inputStream = new MutagenBinaryReadStream(inputPath, gameMode))
{
using var writer = new MutagenWriter(new FileStream(alignedMajorRecordsFile, FileMode.Create), gameMode);
AlignMajorRecordsByRules(inputStream, writer, alignmentRules, fileLocs);
}
var alignedGroupsFile = new ModPath(inputPath.ModKey, Path.Combine(temp, "alignedGroups"));
using (var inputStream = new MutagenBinaryReadStream(alignedMajorRecordsFile, gameMode))
{
using var writer = new MutagenWriter(new FileStream(alignedGroupsFile, FileMode.Create), gameMode);
AlignGroupsByRules(inputStream, writer, alignmentRules, fileLocs);
}
fileLocs = RecordLocator.GetLocations(alignedGroupsFile, gameMode, interest);
var alignedCellsFile = new ModPath(inputPath.ModKey, Path.Combine(temp, "alignedCells"));
using (var mutaReader = new BinaryReadStream(alignedGroupsFile))
{
using var writer = new MutagenWriter(alignedCellsFile, gameMode);
foreach (var grup in fileLocs.GrupLocations.Keys)
{
if (grup <= mutaReader.Position)
{
continue;
}
var noRecordLength = grup - mutaReader.Position;
mutaReader.WriteTo(writer.BaseStream, (int)noRecordLength);
// If complete overall, return
if (mutaReader.Complete)
{
break;
}
mutaReader.WriteTo(writer.BaseStream, 12);
var grupType = mutaReader.ReadInt32();
writer.Write(grupType);
if (writer.MetaData.Constants.GroupConstants.Cell.TopGroupType == grupType)
{
AlignCellChildren(mutaReader, writer);
}
}
mutaReader.WriteTo(writer.BaseStream, checked ((int)mutaReader.Remaining));
}
fileLocs = RecordLocator.GetLocations(alignedCellsFile, gameMode, interest);
using (var mutaReader = new MutagenBinaryReadStream(alignedCellsFile, gameMode))
{
using var writer = new MutagenWriter(outputPath.Path, gameMode);
foreach (var grup in fileLocs.GrupLocations.Keys)
{
if (grup <= mutaReader.Position)
{
continue;
}
var noRecordLength = grup - mutaReader.Position;
mutaReader.WriteTo(writer.BaseStream, (int)noRecordLength);
// If complete overall, return
if (mutaReader.Complete)
{
break;
}
mutaReader.WriteTo(writer.BaseStream, 12);
var grupType = mutaReader.ReadInt32();
writer.Write(grupType);
if (writer.MetaData.Constants.GroupConstants.World.TopGroupType == grupType)
{
AlignWorldChildren(mutaReader, writer);
}
}
mutaReader.WriteTo(writer.BaseStream, checked ((int)mutaReader.Remaining));
}
}
else
{
var alignedMajorRecordsFile = new ModPath(inputPath.ModKey, Path.Combine(temp, "alignedRules"));
using (var inputStream = new MutagenBinaryReadStream(inputPath, gameMode))
{
using var writer = new MutagenWriter(alignedMajorRecordsFile, gameMode);
AlignMajorRecordsByRules(inputStream, writer, alignmentRules, fileLocs);
}
var alignedGroupsFile = Path.Combine(temp, "alignedGroups");
using (var inputStream = new MutagenBinaryReadStream(alignedMajorRecordsFile, gameMode))
{
using var writer = new MutagenWriter(new FileStream(outputPath.Path, FileMode.Create), gameMode);
AlignGroupsByRules(inputStream, writer, alignmentRules, fileLocs);
}
}
}
public Test BinaryPassthroughTest()
{
(TempFolder tmp, Test processedTest) = SetupProcessedFiles();
var outputPath = Path.Combine(tmp.Dir.Path, $"{this.Nickname}_NormalExport");
var processedPath = ProcessedPath(tmp);
var orderedPath = Path.Combine(tmp.Dir.Path, $"{this.Nickname}_Ordered");
var binaryOverlayPath = Path.Combine(tmp.Dir.Path, $"{this.Nickname}_BinaryOverlay");
var copyInPath = Path.Combine(tmp.Dir.Path, $"{this.Nickname}_CopyIn");
var strsProcessedPath = Path.Combine(tmp.Dir.Path, "Strings/Processed");
var masterRefs = MasterReferenceReader.FromPath(new ModPath(ModKey, this.FilePath.Path), GameRelease);
// Do normal
if (Settings.TestNormal)
{
var strsWriteDir = Path.Combine(tmp.Dir.Path, "Strings", $"{this.Nickname}_Normal");
bool doStrings = false;
var passthrough = TestBattery.RunTest(
"Binary Normal Passthrough",
this.GameRelease,
this.Target,
parallel: Settings.Parallel,
toDo: async(o) =>
{
o.OnNext(FilePath.ToString());
var mod = await ImportBinary(this.FilePath.Path);
doStrings = mod.UsingLocalization;
foreach (var record in mod.EnumerateMajorRecords())
{
record.IsCompressed = false;
}
var writeParam = GetWriteParam(masterRefs, doStrings ? new StringsWriter(mod.ModKey, strsWriteDir) : null);
mod.WriteToBinary(outputPath, writeParam);
GC.Collect();
using var stream = new MutagenBinaryReadStream(processedPath, this.GameRelease);
writeParam.StringsWriter?.Dispose();
AssertFilesEqual(
stream,
outputPath,
amountToReport: 15);
});
processedTest.AddAsChild(passthrough);
if (doStrings)
{
foreach (var item in AssertStringsEqual(
"Binary Normal",
strsProcessedPath,
strsWriteDir))
{
passthrough.AddAsChild(item);
}
}
}
if (Settings.TestBinaryOverlay)
{
var strsWriteDir = Path.Combine(tmp.Dir.Path, "Strings", $"{this.Nickname}_Overlay");
bool doStrings = false;
var passthrough = TestBattery.RunTest(
"Binary Overlay Passthrough",
this.GameRelease,
this.Target,
parallel: Settings.Parallel,
toDo: async(o) =>
{
o.OnNext(FilePath.ToString());
using (var wrapper = await ImportBinaryOverlay(this.FilePath.Path))
{
doStrings = wrapper.UsingLocalization;
var writeParam = GetWriteParam(masterRefs, doStrings ? new StringsWriter(wrapper.ModKey, strsWriteDir) : null);
wrapper.WriteToBinary(binaryOverlayPath, writeParam);
writeParam.StringsWriter?.Dispose();
}
using var stream = new MutagenBinaryReadStream(processedPath, this.GameRelease);
PassthroughTest.AssertFilesEqual(
stream,
binaryOverlayPath,
amountToReport: 15);
});
processedTest.AddAsChild(passthrough);
if (doStrings)
{
foreach (var item in AssertStringsEqual(
"Binary Overlay",
strsProcessedPath,
strsWriteDir))
{
passthrough.AddAsChild(item);
}
}
}
if (Settings.TestCopyIn)
{
var strsWriteDir = Path.Combine(tmp.Dir.Path, "Strings", $"{this.Nickname}_CopyIn");
bool doStrings = false;
var passthrough = TestBattery.RunTest(
"Copy In Passthrough",
this.GameRelease,
this.Target,
parallel: Settings.Parallel,
toDo: async(o) =>
{
o.OnNext(FilePath.ToString());
var copyIn = await ImportCopyIn(this.FilePath.Path);
doStrings = copyIn.UsingLocalization;
var writeParam = GetWriteParam(masterRefs, doStrings ? new StringsWriter(copyIn.ModKey, strsWriteDir) : null);
copyIn.WriteToBinary(copyInPath, writeParam);
writeParam.StringsWriter?.Dispose();
using var stream = new MutagenBinaryReadStream(processedPath, this.GameRelease);
PassthroughTest.AssertFilesEqual(
stream,
copyInPath,
amountToReport: 15);
});
processedTest.AddAsChild(passthrough);
if (doStrings)
{
foreach (var item in AssertStringsEqual(
"Copy In",
strsProcessedPath,
strsWriteDir))
{
passthrough.AddAsChild(item);
}
}
}
return(processedTest);
}
public static void Align(
ModPath inputPath,
FilePath outputPath,
GameRelease release,
AlignmentRules alignmentRules,
TempFolder?temp = null)
{
var interest = new RecordInterest(alignmentRules.Alignments.Keys)
{
EmptyMeansInterested = false
};
var parsingBundle = new ParsingBundle(GameConstants.Get(release), MasterReferenceReader.FromPath(inputPath, release));
var fileLocs = RecordLocator.GetFileLocations(inputPath.Path, release, interest);
temp ??= new TempFolder();
using (temp)
{
var alignedMajorRecordsFile = Path.Combine(temp.Dir.Path, "alignedRules");
using (var inputStream = new MutagenBinaryReadStream(inputPath.Path, parsingBundle))
{
using var writer = new MutagenWriter(new FileStream(alignedMajorRecordsFile, FileMode.Create), release);
AlignMajorRecordsByRules(inputStream, writer, alignmentRules, fileLocs);
}
var alignedGroupsFile = Path.Combine(temp.Dir.Path, "alignedGroups");
using (var inputStream = new MutagenBinaryReadStream(alignedMajorRecordsFile, parsingBundle))
{
using var writer = new MutagenWriter(new FileStream(alignedGroupsFile, FileMode.Create), release);
AlignGroupsByRules(inputStream, writer, alignmentRules, fileLocs);
}
fileLocs = RecordLocator.GetFileLocations(alignedGroupsFile, release, interest);
var alignedCellsFile = Path.Combine(temp.Dir.Path, "alignedCells");
using (var mutaReader = new BinaryReadStream(alignedGroupsFile))
{
using var writer = new MutagenWriter(alignedCellsFile, release);
foreach (var grup in fileLocs.GrupLocations)
{
if (grup <= mutaReader.Position)
{
continue;
}
var noRecordLength = grup - mutaReader.Position;
mutaReader.WriteTo(writer.BaseStream, (int)noRecordLength);
// If complete overall, return
if (mutaReader.Complete)
{
break;
}
mutaReader.WriteTo(writer.BaseStream, 12);
var grupType = (GroupTypeEnum)mutaReader.ReadUInt32();
writer.Write((int)grupType);
switch (grupType)
{
case GroupTypeEnum.CellChildren:
AlignCellChildren(mutaReader, writer);
break;
default:
break;
}
}
mutaReader.WriteTo(writer.BaseStream, checked ((int)mutaReader.Remaining));
}
fileLocs = RecordLocator.GetFileLocations(alignedCellsFile, release, interest);
using (var mutaReader = new MutagenBinaryReadStream(alignedCellsFile, parsingBundle))
{
using var writer = new MutagenWriter(outputPath.Path, GameConstants.Get(release));
foreach (var grup in fileLocs.GrupLocations)
{
if (grup <= mutaReader.Position)
{
continue;
}
var noRecordLength = grup - mutaReader.Position;
mutaReader.WriteTo(writer.BaseStream, (int)noRecordLength);
// If complete overall, return
if (mutaReader.Complete)
{
break;
}
mutaReader.WriteTo(writer.BaseStream, 12);
var grupType = (GroupTypeEnum)mutaReader.ReadUInt32();
writer.Write((int)grupType);
switch (grupType)
{
case GroupTypeEnum.WorldChildren:
AlignWorldChildren(mutaReader, writer);
break;
default:
break;
}
}
mutaReader.WriteTo(writer.BaseStream, checked ((int)mutaReader.Remaining));
}
}
}
protected override IWorldspaceGetter Get(IFileSystem fileSystem, ModPath path)
{
using var reader = new MutagenBinaryReadStream(fileSystem.File.OpenRead(path), path.ModKey, GameRelease.SkyrimSE);
return(Worldspace.CreateFromBinary(new MutagenFrame(reader)));
}
/// <summary>
/// Decompresses mod stream into an output.
/// Will open up two input streams, so a Func factory is given as input.
/// </summary>
/// <param name="streamCreator">A func to create an input stream</param>
/// <param name="outputStream">Stream to write output to</param>
/// <param name="gameMode">Type of game the mod stream is reading</param>
/// <param name="interest">Optional specification of which record types to process</param>
public static void Decompress(
Func <Stream> streamCreator,
Stream outputStream,
GameMode gameMode,
RecordInterest?interest = null)
{
var meta = new ParsingBundle(GameConstants.Get(gameMode));
using var inputStream = new MutagenBinaryReadStream(streamCreator(), meta);
using var inputStreamJumpback = new MutagenBinaryReadStream(streamCreator(), meta);
using var writer = new System.IO.BinaryWriter(outputStream, Encoding.Default, leaveOpen: true);
long runningDiff = 0;
var fileLocs = RecordLocator.GetFileLocations(
inputStream,
interest: interest,
additionalCriteria: (stream, recType, len) =>
{
return(stream.GetMajorRecord().IsCompressed);
});
// Construct group length container for later use
Dictionary <long, (uint Length, long Offset)> grupMeta = new Dictionary <long, (uint Length, long Offset)>();
inputStream.Position = 0;
while (!inputStream.Complete)
{
// Import until next listed major record
long noRecordLength;
if (fileLocs.ListedRecords.TryGetInDirection(
inputStream.Position,
higher: true,
result: out var nextRec))
{
var recordLocation = fileLocs.ListedRecords.Keys[nextRec.Key];
noRecordLength = recordLocation - inputStream.Position;
}
else
{
noRecordLength = inputStream.Length - inputStream.Position;
}
inputStream.WriteTo(outputStream, (int)noRecordLength);
// If complete overall, return
if (inputStream.Complete)
{
break;
}
var majorMeta = inputStream.ReadMajorRecord();
var len = majorMeta.ContentLength;
using (var frame = MutagenFrame.ByLength(
reader: inputStream,
length: len))
{
// Decompress
var decompressed = frame.Decompress();
var decompressedLen = decompressed.TotalLength;
var lengthDiff = decompressedLen - len;
var majorMetaSpan = majorMeta.Span.ToArray();
// Write major Meta
var writableMajorMeta = meta.Constants.MajorRecordWritable(majorMetaSpan.AsSpan());
writableMajorMeta.IsCompressed = false;
writableMajorMeta.RecordLength = (uint)(len + lengthDiff);
writer.Write(majorMetaSpan);
writer.Write(decompressed.ReadRemainingSpan(readSafe: false));
// If no difference in lengths, move on
if (lengthDiff == 0)
{
continue;
}
// Modify parent group lengths
foreach (var grupLoc in fileLocs.GetContainingGroupLocations(nextRec.Value.FormID))
{
if (!grupMeta.TryGetValue(grupLoc, out var loc))
{
loc.Offset = runningDiff;
inputStreamJumpback.Position = grupLoc + 4;
loc.Length = inputStreamJumpback.ReadUInt32();
}
grupMeta[grupLoc] = ((uint)(loc.Length + lengthDiff), loc.Offset);
}
runningDiff += lengthDiff;
}
}
foreach (var item in grupMeta)
{
var grupLoc = item.Key;
outputStream.Position = grupLoc + 4 + item.Value.Offset;
writer.Write(item.Value.Length);
}
}
public static void Sort(
Func <Stream> streamCreator,
Stream outputStream,
GameMode gameMode)
{
var meta = new ParsingBundle(GameConstants.Get(gameMode));
using var inputStream = new MutagenBinaryReadStream(streamCreator(), meta);
using var locatorStream = new MutagenBinaryReadStream(streamCreator(), meta);
using var writer = new MutagenWriter(outputStream, gameMode, dispose: false);
while (!inputStream.Complete)
{
long noRecordLength;
foreach (var grupLoc in RecordLocator.IterateBaseGroupLocations(locatorStream))
{
noRecordLength = grupLoc.Value - inputStream.Position;
inputStream.WriteTo(writer.BaseStream, (int)noRecordLength);
// If complete overall, return
if (inputStream.Complete)
{
return;
}
var groupMeta = inputStream.GetGroup();
if (!groupMeta.IsGroup)
{
throw new ArgumentException();
}
var storage = new Dictionary <FormID, List <ReadOnlyMemorySlice <byte> > >();
using (var grupFrame = new MutagenFrame(inputStream).SpawnWithLength(groupMeta.TotalLength))
{
inputStream.WriteTo(writer.BaseStream, meta.Constants.GroupConstants.HeaderLength);
locatorStream.Position = grupLoc.Value;
foreach (var rec in RecordLocator.ParseTopLevelGRUP(locatorStream))
{
MajorRecordHeader majorMeta = inputStream.GetMajorRecord();
storage.TryCreateValue(rec.FormID).Add(inputStream.ReadMemory(checked ((int)majorMeta.TotalLength), readSafe: true));
if (grupFrame.Complete)
{
continue;
}
GroupHeader subGroupMeta = inputStream.GetGroup();
if (subGroupMeta.IsGroup)
{
storage.TryCreateValue(rec.FormID).Add(inputStream.ReadMemory(checked ((int)subGroupMeta.TotalLength), readSafe: true));
}
}
}
foreach (var item in storage.OrderBy((i) => i.Key.ID))
{
foreach (var bytes in item.Value)
{
writer.Write(bytes);
}
}
}
inputStream.WriteTo(writer.BaseStream, (int)inputStream.Remaining);
}
}
