private void CompileIntoGame()
{
NamedBackgroundWorker nbw = new NamedBackgroundWorker(@"MixinManager CompileIntoGameThread");
List <string> failedApplications = new List <string>();
nbw.DoWork += (a, b) =>
{
BottomLeftMessage = M3L.GetString(M3L.string_compilingMixins);
OperationInProgress = true;
//DEBUG STUFF
#if DEBUG
int numCoresToApplyWith = 1;
#else
var numCoresToApplyWith = Environment.ProcessorCount;
if (numCoresToApplyWith > 4)
{
numCoresToApplyWith = 4; //no more than 4 as this uses a lot of memory
}
#endif
var mixins = AvailableOfficialMixins.Where(x => x.UISelectedForUse).ToList();
MixinHandler.LoadPatchDataForMixins(mixins); //before dynamic
void failedApplicationCallback(string str)
{
failedApplications.Add(str);
}
var compilingListsPerModule = MixinHandler.GetMixinApplicationList(mixins, failedApplicationCallback);
if (Enumerable.Any(failedApplications))
{
//Error building list
Log.Information(@"Aborting mixin install due to incompatible selection of mixins");
return;
}
ProgressBarMax = mixins.Count();
ProgressBarValue = 0;
int numdone = 0;
void completedSingleApplicationCallback()
{
var val = Interlocked.Increment(ref numdone);
ProgressBarValue = val;
}
//Mixins are ready to be applied
Parallel.ForEach(compilingListsPerModule,
new ParallelOptions
{
MaxDegreeOfParallelism = Environment.ProcessorCount > numCoresToApplyWith
? numCoresToApplyWith
: Environment.ProcessorCount
}, mapping =>
{
var dlcFolderName = ModMakerCompiler.ModmakerChunkNameToDLCFoldername(mapping.Key.ToString());
//var outdir = Path.Combine(modpath, ModMakerCompiler.HeaderToDefaultFoldername(mapping.Key), @"CookedPCConsole");
//Directory.CreateDirectory(outdir);
if (mapping.Key == ModJob.JobHeader.BASEGAME)
{
//basegame
foreach (var file in mapping.Value)
{
try
{
using var vanillaPackageAsStream = VanillaDatabaseService.FetchBasegameFile(MEGame.ME3, Path.GetFileName(file.Key));
//packageAsStream.WriteToFile(@"C:\users\dev\desktop\compressed.pcc");
using var decompressedStream = MEPackage.GetDecompressedPackageStream(vanillaPackageAsStream, false, true);
decompressedStream.Position = 0;
var vanillaPackage = MEPackageHandler.OpenMEPackageFromStream(decompressedStream, $@"Vanilla - {Path.GetFileName(file.Key)}");
//decompressedStream.WriteToFile(@"C:\users\dev\desktop\decompressed.pcc");
using var mixinModifiedStream = MixinHandler.ApplyMixins(decompressedStream, file.Value, true,
completedSingleApplicationCallback, failedApplicationCallback);
mixinModifiedStream.Position = 0;
var modifiedPackage = MEPackageHandler.OpenMEPackageFromStream(mixinModifiedStream, $@"Mixin Modified - {Path.GetFileName(file.Key)}");
// three way merge: get target stream
var targetFile = Path.Combine(M3Directories.GetCookedPath(SelectedInstallTarget), Path.GetFileName(file.Key));
var targetPackage = MEPackageHandler.OpenMEPackage(targetFile);
var merged = ThreeWayPackageMerge.AttemptMerge(vanillaPackage, modifiedPackage, targetPackage);
if (merged)
{
targetPackage.Save(compress: true);
Log.Information(@"Three way merge succeeded for " + targetFile);
}
else
{
Log.Error(@"Could not merge three way merge into " + targetFile);
}
//var outfile = Path.Combine(outdir, Path.GetFileName(file.Key));
//package.save(outfile, false); // don't compress
//finalStream.WriteToFile(outfile);
//File.WriteAllBytes(outfile, finalStream.ToArray());
}
catch (Exception e)
{
var mixinsStr = string.Join(@", ", file.Value.Select(x => x.PatchName));
Log.Error($@"Error in mixin application for file {file.Key}: {e.Message}");
failedApplicationCallback(M3L.GetString(M3L.string_interp_errorApplyingMixinsForFile, mixinsStr, file.Key, e.Message));
}
}
}
else
{
//dlc
var dlcPackage = VanillaDatabaseService.FetchVanillaSFAR(dlcFolderName); //do not have to open file multiple times.
var targetCookedPCDir = Path.Combine(M3Directories.GetDLCPath(SelectedInstallTarget), dlcFolderName, @"CookedPCConsole");
var sfar = mapping.Key == ModJob.JobHeader.TESTPATCH ? M3Directories.GetTestPatchSFARPath(SelectedInstallTarget) : Path.Combine(targetCookedPCDir, @"Default.sfar");
bool unpacked = new FileInfo(sfar).Length == 32;
DLCPackage targetDLCPackage = unpacked ? null : new DLCPackage(sfar); //cache SFAR target
foreach (var file in mapping.Value)
{
try
{
using var vanillaPackageAsStream = VanillaDatabaseService.FetchFileFromVanillaSFAR(dlcFolderName, file.Key, forcedDLC: dlcPackage);
using var decompressedStream = MEPackage.GetDecompressedPackageStream(vanillaPackageAsStream);
decompressedStream.Position = 0;
var vanillaPackage = MEPackageHandler.OpenMEPackageFromStream(decompressedStream, $@"VanillaDLC - {Path.GetFileName(file.Key)}");
using var mixinModifiedStream = MixinHandler.ApplyMixins(decompressedStream, file.Value, true, completedSingleApplicationCallback, failedApplicationCallback);
mixinModifiedStream.Position = 0;
var modifiedPackage = MEPackageHandler.OpenMEPackageFromStream(mixinModifiedStream, $@"Mixin Modified - {Path.GetFileName(file.Key)}");
// three way merge: get target stream
// must see if DLC is unpacked first
MemoryStream targetFileStream = null;
//Packed
if (unpacked)
{
targetFileStream = new MemoryStream(File.ReadAllBytes(Path.Combine(targetCookedPCDir, file.Key)));
}
else
{
targetFileStream = VanillaDatabaseService.FetchFileFromVanillaSFAR(dlcFolderName, Path.GetFileName(file.Key), forcedDLC: targetDLCPackage);
}
var targetPackage = MEPackageHandler.OpenMEPackageFromStream(targetFileStream, $@"Target package {dlcFolderName} - {file.Key}, from SFAR: {unpacked}");
var merged = ThreeWayPackageMerge.AttemptMerge(vanillaPackage, modifiedPackage, targetPackage);
if (merged)
{
if (unpacked)
{
targetPackage.Save(Path.Combine(targetCookedPCDir, file.Key));
Log.Information(@"Three way merge succeeded for " + targetPackage.FilePath);
}
else
{
var finalSTream = targetPackage.SaveToStream(false); // No compress. Not sure if we should support doing that though.
targetDLCPackage.ReplaceEntry(finalSTream.ToArray(), targetDLCPackage.FindFileEntry(Path.GetFileName(file.Key)));
Log.Information(@"Three way merge succeeded for " + targetPackage.FilePath);
}
}
else
{
Log.Error(@"Could not three way merge into: " + targetFileStream);
}
}
catch (Exception e)
{
var mixinsStr = string.Join(@", ", file.Value.Select(x => x.PatchName));
Log.Error($@"Error in mixin application for file {file.Key}: {e.Message}");
failedApplicationCallback(M3L.GetString(M3L.string_interp_errorApplyingMixinsForFile, mixinsStr, file.Key, e.Message));
}
//finalStream.WriteToFile(outfile);
}
}
});
MixinHandler.FreeME3TweaksPatchData();
var percent = 0; //this is used to save a localization
BottomLeftMessage = M3L.GetString(M3L.string_interp_runningAutoTOCOnGamePercentX, percent);
//Run autotoc
void tocingUpdate(int percent)
{
BottomLeftMessage = M3L.GetString(M3L.string_interp_runningAutoTOCOnGamePercentX, percent);
}
AutoTOC.RunTOCOnGameTarget(SelectedInstallTarget, tocingUpdate);
//Generate moddesc
//IniData ini = new IniData();
//ini[@"ModManager"][@"cmmver"] = App.HighestSupportedModDesc.ToString(CultureInfo.InvariantCulture); //prevent commas
//ini[@"ModInfo"][@"game"] = @"ME3";
//ini[@"ModInfo"][@"modname"] = modname;
//ini[@"ModInfo"][@"moddev"] = App.AppVersionHR;
//ini[@"ModInfo"][@"moddesc"] = M3L.GetString(M3L.string_compiledFromTheFollowingMixins);
//ini[@"ModInfo"][@"modver"] = @"1.0";
//generateRepaceFilesMapping(ini, modpath);
//File.WriteAllText(Path.Combine(modpath, @"moddesc.ini"), ini.ToString());
};
nbw.RunWorkerCompleted += (a, b) =>
{
if (b.Error != null)
{
Log.Error($@"Exception occurred in {nbw.Name} thread: {b.Error.Message}");
}
OperationInProgress = false;
ClearMixinHandler();
if (failedApplications.Count > 0)
{
var ld = new ListDialog(failedApplications, M3L.GetString(M3L.string_failedToApplyAllMixins), M3L.GetString(M3L.string_theFollowingMixinsFailedToApply), mainwindow);
ld.ShowDialog();
}
/*if (modpath != null)
* {
* OnClosing(new DataEventArgs(modpath));
* }
* else
* {*/
BottomLeftMessage = M3L.GetString(M3L.string_mixinsInstalledMaybe);
//}
};
CompilePanelButton.IsOpen = false;
nbw.RunWorkerAsync();
}
private void CompileAsNewMod()
{
NamedBackgroundWorker nbw = new NamedBackgroundWorker(@"MixinManager CompileAsNewModThread");
List <string> failedApplications = new List <string>();
var modname = NewModName;
var modpath = Path.Combine(Utilities.GetME3ModsDirectory(), Utilities.SanitizePath(modname));
var result = M3L.ShowDialog(mainwindow, M3L.GetString(M3L.string_interp_dialogCreatingNewModWithExistingName, NewModName, modpath), M3L.GetString(M3L.string_modAlreadyExists), MessageBoxButton.YesNo, MessageBoxImage.Warning, MessageBoxResult.No);
if (result == MessageBoxResult.No)
{
Log.Information(@"User has aborted mixin compilation due to same-named mod existing");
return; //abort.
}
nbw.DoWork += (a, b) =>
{
BottomLeftMessage = M3L.GetString(M3L.string_compilingMixins);
OperationInProgress = true;
//DEBUG STUFF
#if DEBUG
int numCoresToApplyWith = 1;
#else
var numCoresToApplyWith = Environment.ProcessorCount;
if (numCoresToApplyWith > 4)
{
numCoresToApplyWith = 4; //no more than 4 as this uses a lot of memory
}
#endif
var mixins = AvailableOfficialMixins.Where(x => x.UISelectedForUse).ToList();
MixinHandler.LoadPatchDataForMixins(mixins); //before dynamic
void failedApplicationCallback(string str)
{
failedApplications.Add(str);
}
var compilingListsPerModule = MixinHandler.GetMixinApplicationList(mixins, failedApplicationCallback);
if (failedApplications.Any())
{
//Error building list
modpath = null;
Log.Information(@"Aborting mixin compiling due to incompatible selection of mixins");
return;
}
if (Directory.Exists(modpath))
{
Utilities.DeleteFilesAndFoldersRecursively(modpath);
}
ProgressBarMax = mixins.Count();
ProgressBarValue = 0;
int numdone = 0;
void completedSingleApplicationCallback()
{
var val = Interlocked.Increment(ref numdone);
ProgressBarValue = val;
}
//Mixins are ready to be applied
Parallel.ForEach(compilingListsPerModule,
new ParallelOptions
{
MaxDegreeOfParallelism = Environment.ProcessorCount > numCoresToApplyWith
? numCoresToApplyWith
: Environment.ProcessorCount
}, mapping =>
{
var dlcFolderName = ModMakerCompiler.ModmakerChunkNameToDLCFoldername(mapping.Key.ToString());
var outdir = Path.Combine(modpath, ModMakerCompiler.HeaderToDefaultFoldername(mapping.Key), @"CookedPCConsole");
Directory.CreateDirectory(outdir);
if (mapping.Key == ModJob.JobHeader.BASEGAME)
{
//basegame
foreach (var file in mapping.Value)
{
try
{
using var packageAsStream =
VanillaDatabaseService.FetchBasegameFile(Mod.MEGame.ME3,
Path.GetFileName(file.Key));
//packageAsStream.WriteToFile(@"C:\users\dev\desktop\compressed.pcc");
using var decompressedStream = MEPackage.GetDecompressedPackageStream(packageAsStream, true);
//decompressedStream.WriteToFile(@"C:\users\dev\desktop\decompressed.pcc");
using var finalStream = MixinHandler.ApplyMixins(decompressedStream, file.Value,
completedSingleApplicationCallback, failedApplicationCallback);
CLog.Information(@"Compressing package to mod directory: " + file.Key, Settings.LogModMakerCompiler);
finalStream.Position = 0;
var package = MEPackageHandler.OpenMEPackage(finalStream);
var outfile = Path.Combine(outdir, Path.GetFileName(file.Key));
package.save(outfile, false); // don't compress
//finalStream.WriteToFile(outfile);
//File.WriteAllBytes(outfile, finalStream.ToArray());
}
catch (Exception e)
{
var mixinsStr = string.Join(@", ", file.Value.Select(x => x.PatchName));
Log.Error($@"Error in mixin application for file {file.Key}: {e.Message}");
failedApplicationCallback(M3L.GetString(M3L.string_interp_errorApplyingMixinsForFile, mixinsStr, file.Key, e.Message));
}
}
}
else
{
//dlc
var dlcPackage = VanillaDatabaseService.FetchVanillaSFAR(dlcFolderName); //do not have to open file multiple times.
foreach (var file in mapping.Value)
{
try
{
using var packageAsStream =
VanillaDatabaseService.FetchFileFromVanillaSFAR(dlcFolderName, file.Key, forcedDLC: dlcPackage);
using var decompressedStream = MEPackage.GetDecompressedPackageStream(packageAsStream);
using var finalStream = MixinHandler.ApplyMixins(decompressedStream, file.Value, completedSingleApplicationCallback, failedApplicationCallback);
CLog.Information(@"Compressing package to mod directory: " + file.Key, Settings.LogModMakerCompiler);
finalStream.Position = 0;
var package = MEPackageHandler.OpenMEPackage(finalStream);
var outfile = Path.Combine(outdir, Path.GetFileName(file.Key));
package.save(outfile, true);
}
catch (Exception e)
{
var mixinsStr = string.Join(@", ", file.Value.Select(x => x.PatchName));
Log.Error($@"Error in mixin application for file {file.Key}: {e.Message}");
failedApplicationCallback(M3L.GetString(M3L.string_interp_errorApplyingMixinsForFile, mixinsStr, file.Key, e.Message));
}
//finalStream.WriteToFile(outfile);
}
}
});
MixinHandler.FreeME3TweaksPatchData();
//Generate moddesc
IniData ini = new IniData();
ini[@"ModManager"][@"cmmver"] = App.HighestSupportedModDesc.ToString(CultureInfo.InvariantCulture); //prevent commas
ini[@"ModInfo"][@"game"] = @"ME3";
ini[@"ModInfo"][@"modname"] = modname;
ini[@"ModInfo"][@"moddev"] = App.AppVersionHR;
ini[@"ModInfo"][@"moddesc"] = M3L.GetString(M3L.string_compiledFromTheFollowingMixins);
ini[@"ModInfo"][@"modver"] = @"1.0";
generateRepaceFilesMapping(ini, modpath);
File.WriteAllText(Path.Combine(modpath, @"moddesc.ini"), ini.ToString());
};
nbw.RunWorkerCompleted += (a, b) =>
{
if (b.Error != null)
{
Log.Error($@"Exception occured in {nbw.Name} thread: {b.Error.Message}");
}
OperationInProgress = false;
ClearMixinHandler();
if (failedApplications.Count > 0)
{
var ld = new ListDialog(failedApplications, M3L.GetString(M3L.string_failedToApplyAllMixins), M3L.GetString(M3L.string_theFollowingMixinsFailedToApply), mainwindow);
ld.ShowDialog();
}
if (modpath != null)
{
OnClosing(new DataEventArgs(modpath));
}
else
{
BottomLeftMessage = M3L.GetString(M3L.string_selectMixinsToCompile);
}
};
CompilePanelButton.IsOpen = false;
nbw.RunWorkerAsync();
}
public static void ApplyMixinsToModule(KeyValuePair <ModJob.JobHeader, Dictionary <string, List <Mixin> > > mapping, string modpath, Action completedSingleApplicationCallback, Action <string> failedApplicationCallback)
{
var dlcFolderName = ModMakerCompiler.ModmakerChunkNameToDLCFoldername(mapping.Key.ToString());
var outdir = Path.Combine(modpath, ModMakerCompiler.HeaderToDefaultFoldername(mapping.Key), @"CookedPCConsole");
Directory.CreateDirectory(outdir);
if (mapping.Key == ModJob.JobHeader.BASEGAME)
{
//basegame
foreach (var file in mapping.Value)
{
try
{
using var packageAsStream =
VanillaDatabaseService.FetchBasegameFile(MEGame.ME3,
Path.GetFileName(file.Key));
//packageAsStream.WriteToFile(@"C:\users\dev\desktop\compressed.pcc");
using var decompressedStream = MEPackage.GetDecompressedPackageStream(packageAsStream, false, true);
using var finalStream = MixinHandler.ApplyMixins(decompressedStream, file.Value, true, completedSingleApplicationCallback, failedApplicationCallback);
CLog.Information(@"Compressing package to mod directory: " + file.Key, Settings.LogModMakerCompiler);
finalStream.Position = 0;
var package = MEPackageHandler.OpenMEPackageFromStream(finalStream);
var outfile = Path.Combine(outdir, Path.GetFileName(file.Key));
package.Save(outfile, false, includeAdditionalPackagesToCook: false, includeDependencyTable: true); // don't compress, use mixin saving rules for basegame files
}
catch (Exception e)
{
var mixinsStr = string.Join(@", ", file.Value.Select(x => x.PatchName));
Log.Error($@"Error in mixin application for file {file.Key}: {e.Message}");
failedApplicationCallback(M3L.GetString(M3L.string_interp_errorApplyingMixinsForFile, mixinsStr, file.Key, e.Message));
}
}
}
else
{
//dlc
var dlcPackage = VanillaDatabaseService.FetchVanillaSFAR(dlcFolderName); //do not have to open file multiple times.
foreach (var file in mapping.Value)
{
try
{
using var packageAsStream = VanillaDatabaseService.FetchFileFromVanillaSFAR(dlcFolderName, file.Key, forcedDLC: dlcPackage);
//as file comes from backup, we don't need to decompress it, it will always be decompressed in sfar
using var finalStream = MixinHandler.ApplyMixins(packageAsStream, file.Value, true, completedSingleApplicationCallback, failedApplicationCallback);
var outfile = Path.Combine(outdir, Path.GetFileName(file.Key));
if (mapping.Key != ModJob.JobHeader.TESTPATCH)
{
// TestPatch is never unpacked. So there is not really point to
// compressing it's rather small files. The other DLC jobs likely will be packed still, but this will save some disk space.
CLog.Information($@"Compressing package to mod directory: {outfile}", Settings.LogModMakerCompiler);
finalStream.Position = 0;
var package = MEPackageHandler.OpenMEPackageFromStream(finalStream);
package.Save(outfile, true);
}
else
{
Log.Information($@"Writing patched file to disk: {outfile}");
finalStream.WriteToFile(outfile);
}
}
catch (Exception e)
{
var mixinsStr = string.Join(@", ", file.Value.Select(x => x.PatchName));
Log.Error($@"Error in mixin application for file {file.Key}: {e.Message}");
failedApplicationCallback(M3L.GetString(M3L.string_interp_errorApplyingMixinsForFile, mixinsStr, file.Key, e.Message));
}
//finalStream.WriteToFile(outfile);
}
}
}
public static bool AttemptMerge(MEPackage vanillaPackage, MEPackage modifiedVanillaPackage, MEPackage targetPackage)
{
PackageDelta vanillaToModifiedDelta = PackageDelta.CalculateDelta(vanillaPackage, modifiedVanillaPackage);
PackageDelta vanillaToTargetDelta = PackageDelta.CalculateDelta(vanillaPackage, targetPackage);
string loggingPrefix = targetPackage.FilePath == null ? targetPackage.FileSourceForDebugging : Path.GetFileName(targetPackage.FilePath);
//Check merge conditions
var nameConflicts = vanillaToModifiedDelta.NameDeltas.Keys.Intersect(vanillaToTargetDelta.NameDeltas.Keys).ToList();
var importConflicts = vanillaToModifiedDelta.ImportDeltas.Keys.Intersect(vanillaToTargetDelta.ImportDeltas.Keys).ToList();
var exportConflicts = vanillaToModifiedDelta.ExportDeltas.Keys.Intersect(vanillaToTargetDelta.ExportDeltas.Keys).ToList();
Log.Information($"[{loggingPrefix}] Performing three way merge pre-check");
//Name deltas
if (nameConflicts.Count > 0)
{
//need to check if the conflicts result in same value, in this case it would not be a conflict.
foreach (int nameIndex in nameConflicts)
{
var modifiedName = modifiedVanillaPackage.Names[nameIndex];
var targetName = targetPackage.Names[nameIndex];
if (modifiedName != targetName)
{
//Differing names in same spots.
Log.Information($@"[{loggingPrefix}] Cannot merge files: Name index {nameIndex} is different between modified and target.");
return(false);
}
}
}
if (importConflicts.Count > 0)
{
//todo
}
if (exportConflicts.Count > 0)
{
//hmm... this will be a tough one.
foreach (int exportTableIndex in exportConflicts)
{
//we will have to check sizes if we ever hope to have way to merge this
var modifiedData = modifiedVanillaPackage.Exports[exportTableIndex].Data;
var targetData = targetPackage.Exports[exportTableIndex].Data;
if (!modifiedData.SequenceEqual(targetData))
{
Log.Information($@"[{loggingPrefix}] Cannot merge files: Export table index {exportTableIndex} data is different between modified and target.");
return(false);
}
//We will have to ignore size here somehow...
//var modifiedHeader = modifiedVanillaPackage.Exports[exportTableIndex].Header;
//var targetHeader = targetPackage.Exports[exportTableIndex].Header;
}
}
//Merge is OK to perform
//Apply vanilla to modified delta to target package
foreach (var nameDelta in vanillaToModifiedDelta.NameDeltas)
{
if (nameDelta.Key >= targetPackage.NameCount)
{
//add it
Log.Information($"[{loggingPrefix}] Adding name {nameDelta.Value}");
targetPackage.addName(nameDelta.Value);
}
else
{
Log.Information($"[{loggingPrefix}] Updating name index {nameDelta.Key} to {nameDelta.Value}");
targetPackage.replaceName(nameDelta.Key, nameDelta.Value);
}
}
foreach (var exportDelta in vanillaToModifiedDelta.ExportDeltas)
{
if (exportDelta.Key >= targetPackage.ExportCount)
{
//add it
Log.Information($"[{loggingPrefix}] Adding export {exportDelta.Value.GetInstancedFullPath}");
targetPackage.addExport(exportDelta.Value); //not sure if this is possible
}
else
{
//gonna need this reviewed, not entirely sure this is OK to do
Log.Information($"[{loggingPrefix}] Updating export {exportDelta.Value.GetInstancedFullPath}");
targetPackage.Exports[exportDelta.Key].Data = exportDelta.Value.Data;
targetPackage.Exports[exportDelta.Key].Header = exportDelta.Value.Header;
}
}
foreach (var importDelta in vanillaToModifiedDelta.ImportDeltas)
{
if (importDelta.Key >= targetPackage.ImportCount)
{
//add it
Log.Information($"[{loggingPrefix}] Adding import {importDelta.Value.GetInstancedFullPath}");
targetPackage.addImport(importDelta.Value); //not sure if this is possible
}
else
{
Log.Information($"[{loggingPrefix}] Updating import {importDelta.Value.GetInstancedFullPath}");
//gonna need this reviewed, not entirely sure this is OK to do
//targetPackage.Imports[importDelta.Key].Data = importDelta.Value.Data;
targetPackage.Imports[importDelta.Key].Header = importDelta.Value.Header;
}
}
Log.Information($"[{loggingPrefix}] Finished three way merge");
return(true);
}
