public void FunctionParametersTypeNone() { byte[] b = new byte[] { 0x50, 0x45, 0x52, 0x4B, 0x3C, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x24, 0xE1, 0x0B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2C, 0x00, 0x0C, 0x00, 0x45, 0x44, 0x49, 0x44, 0x34, 0x00, 0x4F, 0x52, 0x44, 0x5F, 0x50, 0x69, 0x63, 0x30, 0x30, 0x5F, 0x50, 0x69, 0x63, 0x6B, 0x70, 0x6F, 0x63, 0x6B, 0x65, 0x74, 0x4D, 0x61, 0x73, 0x74, 0x65, 0x72, 0x79, 0x5F, 0x30, 0x30, 0x5F, 0x50, 0x65, 0x72, 0x6B, 0x5F, 0x57, 0x61, 0x73, 0x4C, 0x69, 0x67, 0x68, 0x74, 0x46, 0x69, 0x6E, 0x67, 0x65, 0x72, 0x73, 0x00, 0x46, 0x55, 0x4C, 0x4C, 0x13, 0x00, 0x50, 0x69, 0x63, 0x6B, 0x70, 0x6F, 0x63, 0x6B, 0x65, 0x74, 0x20, 0x4D, 0x61, 0x73, 0x74, 0x65, 0x72, 0x79, 0x00, 0x44, 0x45, 0x53, 0x43, 0x5E, 0x00, 0x41, 0x64, 0x64, 0x73, 0x20, 0x31, 0x25, 0x20, 0x74, 0x6F, 0x20, 0x79, 0x6F, 0x75, 0x72, 0x20, 0x70, 0x69, 0x63, 0x6B, 0x70, 0x6F, 0x63, 0x6B, 0x65, 0x74, 0x20, 0x63, 0x68, 0x61, 0x6E, 0x63, 0x65, 0x20, 0x70, 0x65, 0x72, 0x20, 0x6C, 0x65, 0x76, 0x65, 0x6C, 0x20, 0x6F, 0x66, 0x20, 0x50, 0x69, 0x63, 0x6B, 0x70, 0x6F, 0x63, 0x6B, 0x65, 0x74, 0x2E, 0x20, 0x41, 0x6C, 0x73, 0x6F, 0x20, 0x69, 0x6E, 0x63, 0x72, 0x65, 0x61, 0x73, 0x65, 0x73, 0x20, 0x43, 0x61, 0x72, 0x72, 0x79, 0x20, 0x57, 0x65, 0x69, 0x67, 0x68, 0x74, 0x20, 0x62, 0x79, 0x20, 0x32, 0x30, 0x2E, 0x00, 0x44, 0x41, 0x54, 0x41, 0x05, 0x00, 0x00, 0x00, 0x01, 0x01, 0x00, 0x50, 0x52, 0x4B, 0x45, 0x03, 0x00, 0x01, 0x00, 0xC3, 0x44, 0x41, 0x54, 0x41, 0x04, 0x00, 0xDD, 0x85, 0x07, 0x03, 0x45, 0x50, 0x46, 0x54, 0x01, 0x00, 0x00, 0x50, 0x52, 0x4B, 0x46, 0x00, 0x00, 0x50, 0x52, 0x4B, 0x45, 0x03, 0x00, 0x02, 0x00, 0xC8, 0x44, 0x41, 0x54, 0x41, 0x03, 0x00, 0x38, 0x0E, 0x03, 0x50, 0x52, 0x4B, 0x43, 0x01, 0x00, 0x00, 0x43, 0x54, 0x44, 0x41, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xC0, 0x01, 0x00, 0x00, 0x6A, 0x8E, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0x45, 0x50, 0x46, 0x54, 0x01, 0x00, 0x02, 0x45, 0x50, 0x46, 0x44, 0x08, 0x00, 0x00, 0x00, 0x50, 0x41, 0x0A, 0xD7, 0x23, 0x3C, 0x50, 0x52, 0x4B, 0x46, 0x00, 0x00 }; var reader = new MutagenBinaryReadStream(new MemoryStream(b), ModKey.Null, GameRelease.SkyrimSE, dispose: false); var frame = new MutagenFrame(reader); var perk = Perk.CreateFromBinary(frame); perk.Effects.Should().HaveCount(2); }
private byte GetNumMasters() { using var stream = new MutagenBinaryReadStream(SourcePath, GameRelease); var modFrame = stream.ReadModHeaderFrame(); return(checked ((byte)modFrame.Masters().Count())); }
public void StreamModKeyCtorDoesNotDisposeStream() { var stream = new DisposeTesterWrapStream(File.OpenRead(TestDataPathing.SkyrimOverrideMod)); var mutaStream = new MutagenBinaryReadStream( stream, ModKey.FromNameAndExtension("Skyrim.esm"), GameRelease.SkyrimSE); stream.Disposed.Should().BeFalse(); }
private static void AlignGroupsByRules( MutagenBinaryReadStream inputStream, MutagenWriter writer, AlignmentRules alignmentRules, RecordLocatorResults fileLocs) { while (!inputStream.Complete) { // Import until next listed major record long noRecordLength; if (fileLocs.GrupLocations.TryGetInDirection( inputStream.Position, higher: true, result: out var nextRec)) { noRecordLength = nextRec.Value.Location.Min - inputStream.Position; } else { noRecordLength = inputStream.Remaining; } inputStream.WriteTo(writer.BaseStream, (int)noRecordLength); // If complete overall, return if (inputStream.Complete) { break; } var groupMeta = inputStream.GetGroup(); if (!groupMeta.IsGroup) { throw new ArgumentException(); } writer.Write(inputStream.ReadSpan(groupMeta.HeaderLength)); if (!alignmentRules.GroupAlignment.TryGetValue(groupMeta.GroupType, out var groupRules)) { continue; } var storage = new Dictionary <RecordType, List <ReadOnlyMemorySlice <byte> > >(); var rest = new List <ReadOnlyMemorySlice <byte> >(); using (var frame = MutagenFrame.ByLength(inputStream, groupMeta.ContentLength)) { while (!frame.Complete) { var majorMeta = inputStream.GetMajorRecord(); var bytes = inputStream.ReadMemory(checked ((int)majorMeta.TotalLength)); var type = majorMeta.RecordType; if (groupRules.Contains(type)) { storage.GetOrAdd(type).Add(bytes); } else { rest.Add(bytes); } } } foreach (var rule in groupRules) { if (storage.TryGetValue(rule, out var storageBytes)) { foreach (var item in storageBytes) { writer.Write(item); } } } foreach (var item in rest) { writer.Write(item); } } }
public static void Align( ModPath inputPath, FilePath outputPath, GameRelease gameMode, AlignmentRules alignmentRules, DirectoryPath temp) { var interest = new RecordInterest(alignmentRules.Alignments.Keys) { EmptyMeansInterested = false }; // Always interested in parent record types interest.InterestingTypes.Add("CELL"); interest.InterestingTypes.Add("WRLD"); var fileLocs = RecordLocator.GetLocations(inputPath, gameMode, interest); if (gameMode == GameRelease.Oblivion) { var alignedMajorRecordsFile = new ModPath(inputPath.ModKey, Path.Combine(temp, "alignedRules")); using (var inputStream = new MutagenBinaryReadStream(inputPath, gameMode)) { using var writer = new MutagenWriter(new FileStream(alignedMajorRecordsFile, FileMode.Create), gameMode); AlignMajorRecordsByRules(inputStream, writer, alignmentRules, fileLocs); } var alignedGroupsFile = new ModPath(inputPath.ModKey, Path.Combine(temp, "alignedGroups")); using (var inputStream = new MutagenBinaryReadStream(alignedMajorRecordsFile, gameMode)) { using var writer = new MutagenWriter(new FileStream(alignedGroupsFile, FileMode.Create), gameMode); AlignGroupsByRules(inputStream, writer, alignmentRules, fileLocs); } fileLocs = RecordLocator.GetLocations(alignedGroupsFile, gameMode, interest); var alignedCellsFile = new ModPath(inputPath.ModKey, Path.Combine(temp, "alignedCells")); using (var mutaReader = new BinaryReadStream(alignedGroupsFile)) { using var writer = new MutagenWriter(alignedCellsFile, gameMode); foreach (var grup in fileLocs.GrupLocations.Keys) { if (grup <= mutaReader.Position) { continue; } var noRecordLength = grup - mutaReader.Position; mutaReader.WriteTo(writer.BaseStream, (int)noRecordLength); // If complete overall, return if (mutaReader.Complete) { break; } mutaReader.WriteTo(writer.BaseStream, 12); var grupType = mutaReader.ReadInt32(); writer.Write(grupType); if (writer.MetaData.Constants.GroupConstants.Cell.TopGroupType == grupType) { AlignCellChildren(mutaReader, writer); } } mutaReader.WriteTo(writer.BaseStream, checked ((int)mutaReader.Remaining)); } fileLocs = RecordLocator.GetLocations(alignedCellsFile, gameMode, interest); using (var mutaReader = new MutagenBinaryReadStream(alignedCellsFile, gameMode)) { using var writer = new MutagenWriter(outputPath.Path, gameMode); foreach (var grup in fileLocs.GrupLocations.Keys) { if (grup <= mutaReader.Position) { continue; } var noRecordLength = grup - mutaReader.Position; mutaReader.WriteTo(writer.BaseStream, (int)noRecordLength); // If complete overall, return if (mutaReader.Complete) { break; } mutaReader.WriteTo(writer.BaseStream, 12); var grupType = mutaReader.ReadInt32(); writer.Write(grupType); if (writer.MetaData.Constants.GroupConstants.World.TopGroupType == grupType) { AlignWorldChildren(mutaReader, writer); } } mutaReader.WriteTo(writer.BaseStream, checked ((int)mutaReader.Remaining)); } } else { var alignedMajorRecordsFile = new ModPath(inputPath.ModKey, Path.Combine(temp, "alignedRules")); using (var inputStream = new MutagenBinaryReadStream(inputPath, gameMode)) { using var writer = new MutagenWriter(alignedMajorRecordsFile, gameMode); AlignMajorRecordsByRules(inputStream, writer, alignmentRules, fileLocs); } var alignedGroupsFile = Path.Combine(temp, "alignedGroups"); using (var inputStream = new MutagenBinaryReadStream(alignedMajorRecordsFile, gameMode)) { using var writer = new MutagenWriter(new FileStream(outputPath.Path, FileMode.Create), gameMode); AlignGroupsByRules(inputStream, writer, alignmentRules, fileLocs); } } }
public Test BinaryPassthroughTest() { (TempFolder tmp, Test processedTest) = SetupProcessedFiles(); var outputPath = Path.Combine(tmp.Dir.Path, $"{this.Nickname}_NormalExport"); var processedPath = ProcessedPath(tmp); var orderedPath = Path.Combine(tmp.Dir.Path, $"{this.Nickname}_Ordered"); var binaryOverlayPath = Path.Combine(tmp.Dir.Path, $"{this.Nickname}_BinaryOverlay"); var copyInPath = Path.Combine(tmp.Dir.Path, $"{this.Nickname}_CopyIn"); var strsProcessedPath = Path.Combine(tmp.Dir.Path, "Strings/Processed"); var masterRefs = MasterReferenceReader.FromPath(new ModPath(ModKey, this.FilePath.Path), GameRelease); // Do normal if (Settings.TestNormal) { var strsWriteDir = Path.Combine(tmp.Dir.Path, "Strings", $"{this.Nickname}_Normal"); bool doStrings = false; var passthrough = TestBattery.RunTest( "Binary Normal Passthrough", this.GameRelease, this.Target, parallel: Settings.Parallel, toDo: async(o) => { o.OnNext(FilePath.ToString()); var mod = await ImportBinary(this.FilePath.Path); doStrings = mod.UsingLocalization; foreach (var record in mod.EnumerateMajorRecords()) { record.IsCompressed = false; } var writeParam = GetWriteParam(masterRefs, doStrings ? new StringsWriter(mod.ModKey, strsWriteDir) : null); mod.WriteToBinary(outputPath, writeParam); GC.Collect(); using var stream = new MutagenBinaryReadStream(processedPath, this.GameRelease); writeParam.StringsWriter?.Dispose(); AssertFilesEqual( stream, outputPath, amountToReport: 15); }); processedTest.AddAsChild(passthrough); if (doStrings) { foreach (var item in AssertStringsEqual( "Binary Normal", strsProcessedPath, strsWriteDir)) { passthrough.AddAsChild(item); } } } if (Settings.TestBinaryOverlay) { var strsWriteDir = Path.Combine(tmp.Dir.Path, "Strings", $"{this.Nickname}_Overlay"); bool doStrings = false; var passthrough = TestBattery.RunTest( "Binary Overlay Passthrough", this.GameRelease, this.Target, parallel: Settings.Parallel, toDo: async(o) => { o.OnNext(FilePath.ToString()); using (var wrapper = await ImportBinaryOverlay(this.FilePath.Path)) { doStrings = wrapper.UsingLocalization; var writeParam = GetWriteParam(masterRefs, doStrings ? new StringsWriter(wrapper.ModKey, strsWriteDir) : null); wrapper.WriteToBinary(binaryOverlayPath, writeParam); writeParam.StringsWriter?.Dispose(); } using var stream = new MutagenBinaryReadStream(processedPath, this.GameRelease); PassthroughTest.AssertFilesEqual( stream, binaryOverlayPath, amountToReport: 15); }); processedTest.AddAsChild(passthrough); if (doStrings) { foreach (var item in AssertStringsEqual( "Binary Overlay", strsProcessedPath, strsWriteDir)) { passthrough.AddAsChild(item); } } } if (Settings.TestCopyIn) { var strsWriteDir = Path.Combine(tmp.Dir.Path, "Strings", $"{this.Nickname}_CopyIn"); bool doStrings = false; var passthrough = TestBattery.RunTest( "Copy In Passthrough", this.GameRelease, this.Target, parallel: Settings.Parallel, toDo: async(o) => { o.OnNext(FilePath.ToString()); var copyIn = await ImportCopyIn(this.FilePath.Path); doStrings = copyIn.UsingLocalization; var writeParam = GetWriteParam(masterRefs, doStrings ? new StringsWriter(copyIn.ModKey, strsWriteDir) : null); copyIn.WriteToBinary(copyInPath, writeParam); writeParam.StringsWriter?.Dispose(); using var stream = new MutagenBinaryReadStream(processedPath, this.GameRelease); PassthroughTest.AssertFilesEqual( stream, copyInPath, amountToReport: 15); }); processedTest.AddAsChild(passthrough); if (doStrings) { foreach (var item in AssertStringsEqual( "Copy In", strsProcessedPath, strsWriteDir)) { passthrough.AddAsChild(item); } } } return(processedTest); }
public static void Align( ModPath inputPath, FilePath outputPath, GameRelease release, AlignmentRules alignmentRules, TempFolder?temp = null) { var interest = new RecordInterest(alignmentRules.Alignments.Keys) { EmptyMeansInterested = false }; var parsingBundle = new ParsingBundle(GameConstants.Get(release), MasterReferenceReader.FromPath(inputPath, release)); var fileLocs = RecordLocator.GetFileLocations(inputPath.Path, release, interest); temp ??= new TempFolder(); using (temp) { var alignedMajorRecordsFile = Path.Combine(temp.Dir.Path, "alignedRules"); using (var inputStream = new MutagenBinaryReadStream(inputPath.Path, parsingBundle)) { using var writer = new MutagenWriter(new FileStream(alignedMajorRecordsFile, FileMode.Create), release); AlignMajorRecordsByRules(inputStream, writer, alignmentRules, fileLocs); } var alignedGroupsFile = Path.Combine(temp.Dir.Path, "alignedGroups"); using (var inputStream = new MutagenBinaryReadStream(alignedMajorRecordsFile, parsingBundle)) { using var writer = new MutagenWriter(new FileStream(alignedGroupsFile, FileMode.Create), release); AlignGroupsByRules(inputStream, writer, alignmentRules, fileLocs); } fileLocs = RecordLocator.GetFileLocations(alignedGroupsFile, release, interest); var alignedCellsFile = Path.Combine(temp.Dir.Path, "alignedCells"); using (var mutaReader = new BinaryReadStream(alignedGroupsFile)) { using var writer = new MutagenWriter(alignedCellsFile, release); foreach (var grup in fileLocs.GrupLocations) { if (grup <= mutaReader.Position) { continue; } var noRecordLength = grup - mutaReader.Position; mutaReader.WriteTo(writer.BaseStream, (int)noRecordLength); // If complete overall, return if (mutaReader.Complete) { break; } mutaReader.WriteTo(writer.BaseStream, 12); var grupType = (GroupTypeEnum)mutaReader.ReadUInt32(); writer.Write((int)grupType); switch (grupType) { case GroupTypeEnum.CellChildren: AlignCellChildren(mutaReader, writer); break; default: break; } } mutaReader.WriteTo(writer.BaseStream, checked ((int)mutaReader.Remaining)); } fileLocs = RecordLocator.GetFileLocations(alignedCellsFile, release, interest); using (var mutaReader = new MutagenBinaryReadStream(alignedCellsFile, parsingBundle)) { using var writer = new MutagenWriter(outputPath.Path, GameConstants.Get(release)); foreach (var grup in fileLocs.GrupLocations) { if (grup <= mutaReader.Position) { continue; } var noRecordLength = grup - mutaReader.Position; mutaReader.WriteTo(writer.BaseStream, (int)noRecordLength); // If complete overall, return if (mutaReader.Complete) { break; } mutaReader.WriteTo(writer.BaseStream, 12); var grupType = (GroupTypeEnum)mutaReader.ReadUInt32(); writer.Write((int)grupType); switch (grupType) { case GroupTypeEnum.WorldChildren: AlignWorldChildren(mutaReader, writer); break; default: break; } } mutaReader.WriteTo(writer.BaseStream, checked ((int)mutaReader.Remaining)); } } }
protected override IWorldspaceGetter Get(IFileSystem fileSystem, ModPath path) { using var reader = new MutagenBinaryReadStream(fileSystem.File.OpenRead(path), path.ModKey, GameRelease.SkyrimSE); return(Worldspace.CreateFromBinary(new MutagenFrame(reader))); }
/// <summary> /// Decompresses mod stream into an output. /// Will open up two input streams, so a Func factory is given as input. /// </summary> /// <param name="streamCreator">A func to create an input stream</param> /// <param name="outputStream">Stream to write output to</param> /// <param name="gameMode">Type of game the mod stream is reading</param> /// <param name="interest">Optional specification of which record types to process</param> public static void Decompress( Func <Stream> streamCreator, Stream outputStream, GameMode gameMode, RecordInterest?interest = null) { var meta = new ParsingBundle(GameConstants.Get(gameMode)); using var inputStream = new MutagenBinaryReadStream(streamCreator(), meta); using var inputStreamJumpback = new MutagenBinaryReadStream(streamCreator(), meta); using var writer = new System.IO.BinaryWriter(outputStream, Encoding.Default, leaveOpen: true); long runningDiff = 0; var fileLocs = RecordLocator.GetFileLocations( inputStream, interest: interest, additionalCriteria: (stream, recType, len) => { return(stream.GetMajorRecord().IsCompressed); }); // Construct group length container for later use Dictionary <long, (uint Length, long Offset)> grupMeta = new Dictionary <long, (uint Length, long Offset)>(); inputStream.Position = 0; while (!inputStream.Complete) { // Import until next listed major record long noRecordLength; if (fileLocs.ListedRecords.TryGetInDirection( inputStream.Position, higher: true, result: out var nextRec)) { var recordLocation = fileLocs.ListedRecords.Keys[nextRec.Key]; noRecordLength = recordLocation - inputStream.Position; } else { noRecordLength = inputStream.Length - inputStream.Position; } inputStream.WriteTo(outputStream, (int)noRecordLength); // If complete overall, return if (inputStream.Complete) { break; } var majorMeta = inputStream.ReadMajorRecord(); var len = majorMeta.ContentLength; using (var frame = MutagenFrame.ByLength( reader: inputStream, length: len)) { // Decompress var decompressed = frame.Decompress(); var decompressedLen = decompressed.TotalLength; var lengthDiff = decompressedLen - len; var majorMetaSpan = majorMeta.Span.ToArray(); // Write major Meta var writableMajorMeta = meta.Constants.MajorRecordWritable(majorMetaSpan.AsSpan()); writableMajorMeta.IsCompressed = false; writableMajorMeta.RecordLength = (uint)(len + lengthDiff); writer.Write(majorMetaSpan); writer.Write(decompressed.ReadRemainingSpan(readSafe: false)); // If no difference in lengths, move on if (lengthDiff == 0) { continue; } // Modify parent group lengths foreach (var grupLoc in fileLocs.GetContainingGroupLocations(nextRec.Value.FormID)) { if (!grupMeta.TryGetValue(grupLoc, out var loc)) { loc.Offset = runningDiff; inputStreamJumpback.Position = grupLoc + 4; loc.Length = inputStreamJumpback.ReadUInt32(); } grupMeta[grupLoc] = ((uint)(loc.Length + lengthDiff), loc.Offset); } runningDiff += lengthDiff; } } foreach (var item in grupMeta) { var grupLoc = item.Key; outputStream.Position = grupLoc + 4 + item.Value.Offset; writer.Write(item.Value.Length); } }
public static void Sort( Func <Stream> streamCreator, Stream outputStream, GameMode gameMode) { var meta = new ParsingBundle(GameConstants.Get(gameMode)); using var inputStream = new MutagenBinaryReadStream(streamCreator(), meta); using var locatorStream = new MutagenBinaryReadStream(streamCreator(), meta); using var writer = new MutagenWriter(outputStream, gameMode, dispose: false); while (!inputStream.Complete) { long noRecordLength; foreach (var grupLoc in RecordLocator.IterateBaseGroupLocations(locatorStream)) { noRecordLength = grupLoc.Value - inputStream.Position; inputStream.WriteTo(writer.BaseStream, (int)noRecordLength); // If complete overall, return if (inputStream.Complete) { return; } var groupMeta = inputStream.GetGroup(); if (!groupMeta.IsGroup) { throw new ArgumentException(); } var storage = new Dictionary <FormID, List <ReadOnlyMemorySlice <byte> > >(); using (var grupFrame = new MutagenFrame(inputStream).SpawnWithLength(groupMeta.TotalLength)) { inputStream.WriteTo(writer.BaseStream, meta.Constants.GroupConstants.HeaderLength); locatorStream.Position = grupLoc.Value; foreach (var rec in RecordLocator.ParseTopLevelGRUP(locatorStream)) { MajorRecordHeader majorMeta = inputStream.GetMajorRecord(); storage.TryCreateValue(rec.FormID).Add(inputStream.ReadMemory(checked ((int)majorMeta.TotalLength), readSafe: true)); if (grupFrame.Complete) { continue; } GroupHeader subGroupMeta = inputStream.GetGroup(); if (subGroupMeta.IsGroup) { storage.TryCreateValue(rec.FormID).Add(inputStream.ReadMemory(checked ((int)subGroupMeta.TotalLength), readSafe: true)); } } } foreach (var item in storage.OrderBy((i) => i.Key.ID)) { foreach (var bytes in item.Value) { writer.Write(bytes); } } } inputStream.WriteTo(writer.BaseStream, (int)inputStream.Remaining); } }