private static void AlignGroupsByRules(
            MutagenBinaryReadStream inputStream,
            MutagenWriter writer,
            AlignmentRules alignmentRules,
            RecordLocatorResults fileLocs)
        {
            while (!inputStream.Complete)
            {
                // Import until next listed major record
                long noRecordLength;
                if (fileLocs.GrupLocations.TryGetInDirection(
                        inputStream.Position,
                        higher: true,
                        result: out var nextRec))
                {
                    noRecordLength = nextRec.Value.Location.Min - inputStream.Position;
                }
                else
                {
                    noRecordLength = inputStream.Remaining;
                }
                inputStream.WriteTo(writer.BaseStream, (int)noRecordLength);

                // If complete overall, return
                if (inputStream.Complete)
                {
                    break;
                }
                var groupMeta = inputStream.GetGroup();
                if (!groupMeta.IsGroup)
                {
                    throw new ArgumentException();
                }
                writer.Write(inputStream.ReadSpan(groupMeta.HeaderLength));

                if (!alignmentRules.GroupAlignment.TryGetValue(groupMeta.GroupType, out var groupRules))
                {
                    continue;
                }

                var storage = new Dictionary <RecordType, List <ReadOnlyMemorySlice <byte> > >();
                var rest    = new List <ReadOnlyMemorySlice <byte> >();
                using (var frame = MutagenFrame.ByLength(inputStream, groupMeta.ContentLength))
                {
                    while (!frame.Complete)
                    {
                        var majorMeta = inputStream.GetMajorRecord();
                        var bytes     = inputStream.ReadMemory(checked ((int)majorMeta.TotalLength));
                        var type      = majorMeta.RecordType;
                        if (groupRules.Contains(type))
                        {
                            storage.GetOrAdd(type).Add(bytes);
                        }
                        else
                        {
                            rest.Add(bytes);
                        }
                    }
                }
                foreach (var rule in groupRules)
                {
                    if (storage.TryGetValue(rule, out var storageBytes))
                    {
                        foreach (var item in storageBytes)
                        {
                            writer.Write(item);
                        }
                    }
                }
                foreach (var item in rest)
                {
                    writer.Write(item);
                }
            }
        }
        public static void Align(
            ModPath inputPath,
            FilePath outputPath,
            GameRelease gameMode,
            AlignmentRules alignmentRules,
            DirectoryPath temp)
        {
            var interest = new RecordInterest(alignmentRules.Alignments.Keys)
            {
                EmptyMeansInterested = false
            };

            // Always interested in parent record types
            interest.InterestingTypes.Add("CELL");
            interest.InterestingTypes.Add("WRLD");
            var fileLocs = RecordLocator.GetLocations(inputPath, gameMode, interest);

            if (gameMode == GameRelease.Oblivion)
            {
                var alignedMajorRecordsFile = new ModPath(inputPath.ModKey, Path.Combine(temp, "alignedRules"));
                using (var inputStream = new MutagenBinaryReadStream(inputPath, gameMode))
                {
                    using var writer = new MutagenWriter(new FileStream(alignedMajorRecordsFile, FileMode.Create), gameMode);
                    AlignMajorRecordsByRules(inputStream, writer, alignmentRules, fileLocs);
                }

                var alignedGroupsFile = new ModPath(inputPath.ModKey, Path.Combine(temp, "alignedGroups"));
                using (var inputStream = new MutagenBinaryReadStream(alignedMajorRecordsFile, gameMode))
                {
                    using var writer = new MutagenWriter(new FileStream(alignedGroupsFile, FileMode.Create), gameMode);
                    AlignGroupsByRules(inputStream, writer, alignmentRules, fileLocs);
                }

                fileLocs = RecordLocator.GetLocations(alignedGroupsFile, gameMode, interest);
                var alignedCellsFile = new ModPath(inputPath.ModKey, Path.Combine(temp, "alignedCells"));
                using (var mutaReader = new BinaryReadStream(alignedGroupsFile))
                {
                    using var writer = new MutagenWriter(alignedCellsFile, gameMode);
                    foreach (var grup in fileLocs.GrupLocations.Keys)
                    {
                        if (grup <= mutaReader.Position)
                        {
                            continue;
                        }
                        var noRecordLength = grup - mutaReader.Position;
                        mutaReader.WriteTo(writer.BaseStream, (int)noRecordLength);

                        // If complete overall, return
                        if (mutaReader.Complete)
                        {
                            break;
                        }

                        mutaReader.WriteTo(writer.BaseStream, 12);
                        var grupType = mutaReader.ReadInt32();
                        writer.Write(grupType);
                        if (writer.MetaData.Constants.GroupConstants.Cell.TopGroupType == grupType)
                        {
                            AlignCellChildren(mutaReader, writer);
                        }
                    }
                    mutaReader.WriteTo(writer.BaseStream, checked ((int)mutaReader.Remaining));
                }

                fileLocs = RecordLocator.GetLocations(alignedCellsFile, gameMode, interest);
                using (var mutaReader = new MutagenBinaryReadStream(alignedCellsFile, gameMode))
                {
                    using var writer = new MutagenWriter(outputPath.Path, gameMode);
                    foreach (var grup in fileLocs.GrupLocations.Keys)
                    {
                        if (grup <= mutaReader.Position)
                        {
                            continue;
                        }
                        var noRecordLength = grup - mutaReader.Position;
                        mutaReader.WriteTo(writer.BaseStream, (int)noRecordLength);

                        // If complete overall, return
                        if (mutaReader.Complete)
                        {
                            break;
                        }

                        mutaReader.WriteTo(writer.BaseStream, 12);
                        var grupType = mutaReader.ReadInt32();
                        writer.Write(grupType);
                        if (writer.MetaData.Constants.GroupConstants.World.TopGroupType == grupType)
                        {
                            AlignWorldChildren(mutaReader, writer);
                        }
                    }
                    mutaReader.WriteTo(writer.BaseStream, checked ((int)mutaReader.Remaining));
                }
            }
            else
            {
                var alignedMajorRecordsFile = new ModPath(inputPath.ModKey, Path.Combine(temp, "alignedRules"));
                using (var inputStream = new MutagenBinaryReadStream(inputPath, gameMode))
                {
                    using var writer = new MutagenWriter(alignedMajorRecordsFile, gameMode);
                    AlignMajorRecordsByRules(inputStream, writer, alignmentRules, fileLocs);
                }

                var alignedGroupsFile = Path.Combine(temp, "alignedGroups");
                using (var inputStream = new MutagenBinaryReadStream(alignedMajorRecordsFile, gameMode))
                {
                    using var writer = new MutagenWriter(new FileStream(outputPath.Path, FileMode.Create), gameMode);
                    AlignGroupsByRules(inputStream, writer, alignmentRules, fileLocs);
                }
            }
        }
Example #3
0
        /// <summary>
        /// Decompresses mod stream into an output.
        /// Will open up two input streams, so a Func factory is given as input.
        /// </summary>
        /// <param name="streamCreator">A func to create an input stream</param>
        /// <param name="outputStream">Stream to write output to</param>
        /// <param name="gameMode">Type of game the mod stream is reading</param>
        /// <param name="interest">Optional specification of which record types to process</param>
        public static void Decompress(
            Func <Stream> streamCreator,
            Stream outputStream,
            GameMode gameMode,
            RecordInterest?interest = null)
        {
            var meta = new ParsingBundle(GameConstants.Get(gameMode));

            using var inputStream         = new MutagenBinaryReadStream(streamCreator(), meta);
            using var inputStreamJumpback = new MutagenBinaryReadStream(streamCreator(), meta);
            using var writer = new System.IO.BinaryWriter(outputStream, Encoding.Default, leaveOpen: true);

            long runningDiff = 0;
            var  fileLocs    = RecordLocator.GetFileLocations(
                inputStream,
                interest: interest,
                additionalCriteria: (stream, recType, len) =>
            {
                return(stream.GetMajorRecord().IsCompressed);
            });

            // Construct group length container for later use
            Dictionary <long, (uint Length, long Offset)> grupMeta = new Dictionary <long, (uint Length, long Offset)>();

            inputStream.Position = 0;
            while (!inputStream.Complete)
            {
                // Import until next listed major record
                long noRecordLength;
                if (fileLocs.ListedRecords.TryGetInDirection(
                        inputStream.Position,
                        higher: true,
                        result: out var nextRec))
                {
                    var recordLocation = fileLocs.ListedRecords.Keys[nextRec.Key];
                    noRecordLength = recordLocation - inputStream.Position;
                }
                else
                {
                    noRecordLength = inputStream.Length - inputStream.Position;
                }
                inputStream.WriteTo(outputStream, (int)noRecordLength);

                // If complete overall, return
                if (inputStream.Complete)
                {
                    break;
                }
                var majorMeta = inputStream.ReadMajorRecord();
                var len       = majorMeta.ContentLength;
                using (var frame = MutagenFrame.ByLength(
                           reader: inputStream,
                           length: len))
                {
                    // Decompress
                    var decompressed    = frame.Decompress();
                    var decompressedLen = decompressed.TotalLength;
                    var lengthDiff      = decompressedLen - len;
                    var majorMetaSpan   = majorMeta.Span.ToArray();

                    // Write major Meta
                    var writableMajorMeta = meta.Constants.MajorRecordWritable(majorMetaSpan.AsSpan());
                    writableMajorMeta.IsCompressed = false;
                    writableMajorMeta.RecordLength = (uint)(len + lengthDiff);
                    writer.Write(majorMetaSpan);
                    writer.Write(decompressed.ReadRemainingSpan(readSafe: false));

                    // If no difference in lengths, move on
                    if (lengthDiff == 0)
                    {
                        continue;
                    }

                    // Modify parent group lengths
                    foreach (var grupLoc in fileLocs.GetContainingGroupLocations(nextRec.Value.FormID))
                    {
                        if (!grupMeta.TryGetValue(grupLoc, out var loc))
                        {
                            loc.Offset = runningDiff;
                            inputStreamJumpback.Position = grupLoc + 4;
                            loc.Length = inputStreamJumpback.ReadUInt32();
                        }
                        grupMeta[grupLoc] = ((uint)(loc.Length + lengthDiff), loc.Offset);
                    }
                    runningDiff += lengthDiff;
                }
            }

            foreach (var item in grupMeta)
            {
                var grupLoc = item.Key;
                outputStream.Position = grupLoc + 4 + item.Value.Offset;
                writer.Write(item.Value.Length);
            }
        }
Example #4
0
        public static void Align(
            ModPath inputPath,
            FilePath outputPath,
            GameRelease release,
            AlignmentRules alignmentRules,
            TempFolder?temp = null)
        {
            var interest = new RecordInterest(alignmentRules.Alignments.Keys)
            {
                EmptyMeansInterested = false
            };
            var parsingBundle = new ParsingBundle(GameConstants.Get(release), MasterReferenceReader.FromPath(inputPath, release));
            var fileLocs      = RecordLocator.GetFileLocations(inputPath.Path, release, interest);

            temp ??= new TempFolder();
            using (temp)
            {
                var alignedMajorRecordsFile = Path.Combine(temp.Dir.Path, "alignedRules");
                using (var inputStream = new MutagenBinaryReadStream(inputPath.Path, parsingBundle))
                {
                    using var writer = new MutagenWriter(new FileStream(alignedMajorRecordsFile, FileMode.Create), release);
                    AlignMajorRecordsByRules(inputStream, writer, alignmentRules, fileLocs);
                }

                var alignedGroupsFile = Path.Combine(temp.Dir.Path, "alignedGroups");
                using (var inputStream = new MutagenBinaryReadStream(alignedMajorRecordsFile, parsingBundle))
                {
                    using var writer = new MutagenWriter(new FileStream(alignedGroupsFile, FileMode.Create), release);
                    AlignGroupsByRules(inputStream, writer, alignmentRules, fileLocs);
                }

                fileLocs = RecordLocator.GetFileLocations(alignedGroupsFile, release, interest);
                var alignedCellsFile = Path.Combine(temp.Dir.Path, "alignedCells");
                using (var mutaReader = new BinaryReadStream(alignedGroupsFile))
                {
                    using var writer = new MutagenWriter(alignedCellsFile, release);
                    foreach (var grup in fileLocs.GrupLocations)
                    {
                        if (grup <= mutaReader.Position)
                        {
                            continue;
                        }
                        var noRecordLength = grup - mutaReader.Position;
                        mutaReader.WriteTo(writer.BaseStream, (int)noRecordLength);

                        // If complete overall, return
                        if (mutaReader.Complete)
                        {
                            break;
                        }

                        mutaReader.WriteTo(writer.BaseStream, 12);
                        var grupType = (GroupTypeEnum)mutaReader.ReadUInt32();
                        writer.Write((int)grupType);
                        switch (grupType)
                        {
                        case GroupTypeEnum.CellChildren:
                            AlignCellChildren(mutaReader, writer);
                            break;

                        default:
                            break;
                        }
                    }
                    mutaReader.WriteTo(writer.BaseStream, checked ((int)mutaReader.Remaining));
                }

                fileLocs = RecordLocator.GetFileLocations(alignedCellsFile, release, interest);
                using (var mutaReader = new MutagenBinaryReadStream(alignedCellsFile, parsingBundle))
                {
                    using var writer = new MutagenWriter(outputPath.Path, GameConstants.Get(release));
                    foreach (var grup in fileLocs.GrupLocations)
                    {
                        if (grup <= mutaReader.Position)
                        {
                            continue;
                        }
                        var noRecordLength = grup - mutaReader.Position;
                        mutaReader.WriteTo(writer.BaseStream, (int)noRecordLength);

                        // If complete overall, return
                        if (mutaReader.Complete)
                        {
                            break;
                        }

                        mutaReader.WriteTo(writer.BaseStream, 12);
                        var grupType = (GroupTypeEnum)mutaReader.ReadUInt32();
                        writer.Write((int)grupType);
                        switch (grupType)
                        {
                        case GroupTypeEnum.WorldChildren:
                            AlignWorldChildren(mutaReader, writer);
                            break;

                        default:
                            break;
                        }
                    }
                    mutaReader.WriteTo(writer.BaseStream, checked ((int)mutaReader.Remaining));
                }
            }
        }
Example #5
0
        public static void Sort(
            Func <Stream> streamCreator,
            Stream outputStream,
            GameMode gameMode)
        {
            var meta = new ParsingBundle(GameConstants.Get(gameMode));

            using var inputStream   = new MutagenBinaryReadStream(streamCreator(), meta);
            using var locatorStream = new MutagenBinaryReadStream(streamCreator(), meta);
            using var writer        = new MutagenWriter(outputStream, gameMode, dispose: false);
            while (!inputStream.Complete)
            {
                long noRecordLength;
                foreach (var grupLoc in RecordLocator.IterateBaseGroupLocations(locatorStream))
                {
                    noRecordLength = grupLoc.Value - inputStream.Position;
                    inputStream.WriteTo(writer.BaseStream, (int)noRecordLength);

                    // If complete overall, return
                    if (inputStream.Complete)
                    {
                        return;
                    }

                    var groupMeta = inputStream.GetGroup();
                    if (!groupMeta.IsGroup)
                    {
                        throw new ArgumentException();
                    }

                    var storage = new Dictionary <FormID, List <ReadOnlyMemorySlice <byte> > >();
                    using (var grupFrame = new MutagenFrame(inputStream).SpawnWithLength(groupMeta.TotalLength))
                    {
                        inputStream.WriteTo(writer.BaseStream, meta.Constants.GroupConstants.HeaderLength);
                        locatorStream.Position = grupLoc.Value;
                        foreach (var rec in RecordLocator.ParseTopLevelGRUP(locatorStream))
                        {
                            MajorRecordHeader majorMeta = inputStream.GetMajorRecord();
                            storage.TryCreateValue(rec.FormID).Add(inputStream.ReadMemory(checked ((int)majorMeta.TotalLength), readSafe: true));
                            if (grupFrame.Complete)
                            {
                                continue;
                            }
                            GroupHeader subGroupMeta = inputStream.GetGroup();
                            if (subGroupMeta.IsGroup)
                            {
                                storage.TryCreateValue(rec.FormID).Add(inputStream.ReadMemory(checked ((int)subGroupMeta.TotalLength), readSafe: true));
                            }
                        }
                    }

                    foreach (var item in storage.OrderBy((i) => i.Key.ID))
                    {
                        foreach (var bytes in item.Value)
                        {
                            writer.Write(bytes);
                        }
                    }
                }

                inputStream.WriteTo(writer.BaseStream, (int)inputStream.Remaining);
            }
        }