/// <summary>
/// Constructor that opens a read stream to a path
/// </summary>
/// <param name="path">Path to read from</param>
/// <param name="release">Game Release the stream is for</param>
/// <param name="bufferSize">Size of internal buffer</param>
/// <param name="offsetReference">Optional offset reference position to use</param>
public MutagenBinaryReadStream(
ModPath path,
GameRelease release,
int bufferSize = 4096,
long offsetReference = 0)
: base(path, bufferSize)
{
this._path = path;
this.MetaData = new ParsingBundle(release, MasterReferenceReader.FromPath(path, release))
{
ModKey = path.ModKey
};
this.OffsetReference = offsetReference;
}
/// <summary>
/// Constructor that opens a read stream to a path
/// </summary>
/// <param name="path">Path to read from</param>
/// <param name="release">Game Release the stream is for</param>
/// <param name="bufferSize">Size of internal buffer</param>
/// <param name="offsetReference">Optional offset reference position to use</param>
/// <param name="fileSystem">FileSystem to read from</param>
public MutagenBinaryReadStream(
ModPath path,
GameRelease release,
int bufferSize = 4096,
long offsetReference = 0,
IFileSystem?fileSystem = null)
: base(fileSystem.GetOrDefault().File.OpenRead(path), bufferSize)
{
this._path = path;
this.MetaData = new ParsingBundle(release, MasterReferenceReader.FromPath(path, release, fileSystem: fileSystem))
{
ModKey = path.ModKey
};
this.OffsetReference = offsetReference;
}
public static void AddImplicitMasters(RunSynthesisMutagenPatcher settings, ExtendedList <LoadOrderListing> loadOrderListing)
{
HashSet <ModKey> referencedMasters = new HashSet <ModKey>();
foreach (var item in loadOrderListing.OnlyEnabled())
{
MasterReferenceReader reader = MasterReferenceReader.FromPath(Path.Combine(settings.DataFolderPath, item.ModKey.FileName), settings.GameRelease);
referencedMasters.Add(reader.Masters.Select(m => m.Master));
}
for (int i = 0; i < loadOrderListing.Count; i++)
{
var listing = loadOrderListing[i];
if (!listing.Enabled && referencedMasters.Contains(listing.ModKey))
{
loadOrderListing[i] = new LoadOrderListing(listing.ModKey, enabled: true);
}
}
}
public IMasterReferenceReader FromPath(ModPath path)
{
return(MasterReferenceReader.FromPath(path, _gameReleaseContext.Release, fileSystem: _fileSystem));
}
public Test BinaryPassthroughTest()
{
(TempFolder tmp, Test processedTest) = SetupProcessedFiles();
var outputPath = Path.Combine(tmp.Dir.Path, $"{this.Nickname}_NormalExport");
var processedPath = ProcessedPath(tmp);
var orderedPath = Path.Combine(tmp.Dir.Path, $"{this.Nickname}_Ordered");
var binaryOverlayPath = Path.Combine(tmp.Dir.Path, $"{this.Nickname}_BinaryOverlay");
var copyInPath = Path.Combine(tmp.Dir.Path, $"{this.Nickname}_CopyIn");
var strsProcessedPath = Path.Combine(tmp.Dir.Path, "Strings/Processed");
var masterRefs = MasterReferenceReader.FromPath(new ModPath(ModKey, this.FilePath.Path), GameRelease);
// Do normal
if (Settings.TestNormal)
{
var strsWriteDir = Path.Combine(tmp.Dir.Path, "Strings", $"{this.Nickname}_Normal");
bool doStrings = false;
var passthrough = TestBattery.RunTest(
"Binary Normal Passthrough",
this.GameRelease,
this.Target,
parallel: Settings.Parallel,
toDo: async(o) =>
{
o.OnNext(FilePath.ToString());
var mod = await ImportBinary(this.FilePath.Path);
doStrings = mod.UsingLocalization;
foreach (var record in mod.EnumerateMajorRecords())
{
record.IsCompressed = false;
}
var writeParam = GetWriteParam(masterRefs, doStrings ? new StringsWriter(mod.ModKey, strsWriteDir) : null);
mod.WriteToBinary(outputPath, writeParam);
GC.Collect();
using var stream = new MutagenBinaryReadStream(processedPath, this.GameRelease);
writeParam.StringsWriter?.Dispose();
AssertFilesEqual(
stream,
outputPath,
amountToReport: 15);
});
processedTest.AddAsChild(passthrough);
if (doStrings)
{
foreach (var item in AssertStringsEqual(
"Binary Normal",
strsProcessedPath,
strsWriteDir))
{
passthrough.AddAsChild(item);
}
}
}
if (Settings.TestBinaryOverlay)
{
var strsWriteDir = Path.Combine(tmp.Dir.Path, "Strings", $"{this.Nickname}_Overlay");
bool doStrings = false;
var passthrough = TestBattery.RunTest(
"Binary Overlay Passthrough",
this.GameRelease,
this.Target,
parallel: Settings.Parallel,
toDo: async(o) =>
{
o.OnNext(FilePath.ToString());
using (var wrapper = await ImportBinaryOverlay(this.FilePath.Path))
{
doStrings = wrapper.UsingLocalization;
var writeParam = GetWriteParam(masterRefs, doStrings ? new StringsWriter(wrapper.ModKey, strsWriteDir) : null);
wrapper.WriteToBinary(binaryOverlayPath, writeParam);
writeParam.StringsWriter?.Dispose();
}
using var stream = new MutagenBinaryReadStream(processedPath, this.GameRelease);
PassthroughTest.AssertFilesEqual(
stream,
binaryOverlayPath,
amountToReport: 15);
});
processedTest.AddAsChild(passthrough);
if (doStrings)
{
foreach (var item in AssertStringsEqual(
"Binary Overlay",
strsProcessedPath,
strsWriteDir))
{
passthrough.AddAsChild(item);
}
}
}
if (Settings.TestCopyIn)
{
var strsWriteDir = Path.Combine(tmp.Dir.Path, "Strings", $"{this.Nickname}_CopyIn");
bool doStrings = false;
var passthrough = TestBattery.RunTest(
"Copy In Passthrough",
this.GameRelease,
this.Target,
parallel: Settings.Parallel,
toDo: async(o) =>
{
o.OnNext(FilePath.ToString());
var copyIn = await ImportCopyIn(this.FilePath.Path);
doStrings = copyIn.UsingLocalization;
var writeParam = GetWriteParam(masterRefs, doStrings ? new StringsWriter(copyIn.ModKey, strsWriteDir) : null);
copyIn.WriteToBinary(copyInPath, writeParam);
writeParam.StringsWriter?.Dispose();
using var stream = new MutagenBinaryReadStream(processedPath, this.GameRelease);
PassthroughTest.AssertFilesEqual(
stream,
copyInPath,
amountToReport: 15);
});
processedTest.AddAsChild(passthrough);
if (doStrings)
{
foreach (var item in AssertStringsEqual(
"Copy In",
strsProcessedPath,
strsWriteDir))
{
passthrough.AddAsChild(item);
}
}
}
return(processedTest);
}
public static void Align(
ModPath inputPath,
FilePath outputPath,
GameRelease release,
AlignmentRules alignmentRules,
TempFolder?temp = null)
{
var interest = new RecordInterest(alignmentRules.Alignments.Keys)
{
EmptyMeansInterested = false
};
var parsingBundle = new ParsingBundle(GameConstants.Get(release), MasterReferenceReader.FromPath(inputPath, release));
var fileLocs = RecordLocator.GetFileLocations(inputPath.Path, release, interest);
temp ??= new TempFolder();
using (temp)
{
var alignedMajorRecordsFile = Path.Combine(temp.Dir.Path, "alignedRules");
using (var inputStream = new MutagenBinaryReadStream(inputPath.Path, parsingBundle))
{
using var writer = new MutagenWriter(new FileStream(alignedMajorRecordsFile, FileMode.Create), release);
AlignMajorRecordsByRules(inputStream, writer, alignmentRules, fileLocs);
}
var alignedGroupsFile = Path.Combine(temp.Dir.Path, "alignedGroups");
using (var inputStream = new MutagenBinaryReadStream(alignedMajorRecordsFile, parsingBundle))
{
using var writer = new MutagenWriter(new FileStream(alignedGroupsFile, FileMode.Create), release);
AlignGroupsByRules(inputStream, writer, alignmentRules, fileLocs);
}
fileLocs = RecordLocator.GetFileLocations(alignedGroupsFile, release, interest);
var alignedCellsFile = Path.Combine(temp.Dir.Path, "alignedCells");
using (var mutaReader = new BinaryReadStream(alignedGroupsFile))
{
using var writer = new MutagenWriter(alignedCellsFile, release);
foreach (var grup in fileLocs.GrupLocations)
{
if (grup <= mutaReader.Position)
{
continue;
}
var noRecordLength = grup - mutaReader.Position;
mutaReader.WriteTo(writer.BaseStream, (int)noRecordLength);
// If complete overall, return
if (mutaReader.Complete)
{
break;
}
mutaReader.WriteTo(writer.BaseStream, 12);
var grupType = (GroupTypeEnum)mutaReader.ReadUInt32();
writer.Write((int)grupType);
switch (grupType)
{
case GroupTypeEnum.CellChildren:
AlignCellChildren(mutaReader, writer);
break;
default:
break;
}
}
mutaReader.WriteTo(writer.BaseStream, checked ((int)mutaReader.Remaining));
}
fileLocs = RecordLocator.GetFileLocations(alignedCellsFile, release, interest);
using (var mutaReader = new MutagenBinaryReadStream(alignedCellsFile, parsingBundle))
{
using var writer = new MutagenWriter(outputPath.Path, GameConstants.Get(release));
foreach (var grup in fileLocs.GrupLocations)
{
if (grup <= mutaReader.Position)
{
continue;
}
var noRecordLength = grup - mutaReader.Position;
mutaReader.WriteTo(writer.BaseStream, (int)noRecordLength);
// If complete overall, return
if (mutaReader.Complete)
{
break;
}
mutaReader.WriteTo(writer.BaseStream, 12);
var grupType = (GroupTypeEnum)mutaReader.ReadUInt32();
writer.Write((int)grupType);
switch (grupType)
{
case GroupTypeEnum.WorldChildren:
AlignWorldChildren(mutaReader, writer);
break;
default:
break;
}
}
mutaReader.WriteTo(writer.BaseStream, checked ((int)mutaReader.Remaining));
}
}
}
public async Task Process(
TempFolder tmpFolder,
Subject <string> logging,
ModPath sourcePath,
string preprocessedPath,
string outputPath)
{
this.Logging = logging;
this.TempFolder = tmpFolder;
this.SourcePath = sourcePath;
this.Masters = MasterReferenceReader.FromPath(SourcePath, GameRelease);
this.Bundle = new ParsingBundle(GameRelease, Masters);
this._NumMasters = GetNumMasters();
this._AlignedFileLocs = RecordLocator.GetFileLocations(new ModPath(ModKey, preprocessedPath), this.GameRelease);
var preprocessedBytes = File.ReadAllBytes(preprocessedPath);
IMutagenReadStream streamGetter() => new MutagenMemoryReadStream(preprocessedBytes, Bundle);
using (var stream = streamGetter())
{
lock (_lengthTracker)
{
foreach (var grup in this._AlignedFileLocs.GrupLocations.And(this._AlignedFileLocs.ListedRecords.Keys))
{
stream.Position = grup + 4;
this._lengthTracker[grup] = stream.ReadUInt32();
}
}
await this.PreProcessorJobs(streamGetter);
await Task.WhenAll(ExtraJobs(streamGetter));
this.AddDynamicProcessorInstructions();
Parallel.ForEach(this.DynamicProcessors.Keys
.And(this.DynamicStreamProcessors.Keys)
.And(RecordType.Null)
.Distinct(),
ParallelOptions,
type => ProcessDynamicType(type, streamGetter));
lock (_lengthTracker)
{
foreach (var grup in this._lengthTracker)
{
stream.Position = grup.Key + 4;
if (grup.Value == stream.ReadUInt32())
{
continue;
}
this._Instructions.SetSubstitution(
loc: grup.Key + 4,
sub: BitConverter.GetBytes(grup.Value));
}
}
}
var config = this._Instructions.GetConfig();
using (var processor = new BinaryFileProcessor(
new FileStream(preprocessedPath, FileMode.Open, FileAccess.Read),
config))
{
try
{
using var outStream = new FileStream(outputPath, FileMode.Create, FileAccess.Write);
processor.CopyTo(outStream);
}
catch (Exception)
{
if (File.Exists(outputPath))
{
File.Delete(outputPath);
}
throw;
}
}
}
