private void CompileIntoGame() { NamedBackgroundWorker nbw = new NamedBackgroundWorker(@"MixinManager CompileIntoGameThread"); List <string> failedApplications = new List <string>(); nbw.DoWork += (a, b) => { BottomLeftMessage = M3L.GetString(M3L.string_compilingMixins); OperationInProgress = true; //DEBUG STUFF #if DEBUG int numCoresToApplyWith = 1; #else var numCoresToApplyWith = Environment.ProcessorCount; if (numCoresToApplyWith > 4) { numCoresToApplyWith = 4; //no more than 4 as this uses a lot of memory } #endif var mixins = AvailableOfficialMixins.Where(x => x.UISelectedForUse).ToList(); MixinHandler.LoadPatchDataForMixins(mixins); //before dynamic void failedApplicationCallback(string str) { failedApplications.Add(str); } var compilingListsPerModule = MixinHandler.GetMixinApplicationList(mixins, failedApplicationCallback); if (Enumerable.Any(failedApplications)) { //Error building list Log.Information(@"Aborting mixin install due to incompatible selection of mixins"); return; } ProgressBarMax = mixins.Count(); ProgressBarValue = 0; int numdone = 0; void completedSingleApplicationCallback() { var val = Interlocked.Increment(ref numdone); ProgressBarValue = val; } //Mixins are ready to be applied Parallel.ForEach(compilingListsPerModule, new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount > numCoresToApplyWith ? numCoresToApplyWith : Environment.ProcessorCount }, mapping => { var dlcFolderName = ModMakerCompiler.ModmakerChunkNameToDLCFoldername(mapping.Key.ToString()); //var outdir = Path.Combine(modpath, ModMakerCompiler.HeaderToDefaultFoldername(mapping.Key), @"CookedPCConsole"); //Directory.CreateDirectory(outdir); if (mapping.Key == ModJob.JobHeader.BASEGAME) { //basegame foreach (var file in mapping.Value) { try { using var vanillaPackageAsStream = VanillaDatabaseService.FetchBasegameFile(MEGame.ME3, Path.GetFileName(file.Key)); //packageAsStream.WriteToFile(@"C:\users\dev\desktop\compressed.pcc"); using var decompressedStream = MEPackage.GetDecompressedPackageStream(vanillaPackageAsStream, false, true); decompressedStream.Position = 0; var vanillaPackage = MEPackageHandler.OpenMEPackageFromStream(decompressedStream, $@"Vanilla - {Path.GetFileName(file.Key)}"); //decompressedStream.WriteToFile(@"C:\users\dev\desktop\decompressed.pcc"); using var mixinModifiedStream = MixinHandler.ApplyMixins(decompressedStream, file.Value, true, completedSingleApplicationCallback, failedApplicationCallback); mixinModifiedStream.Position = 0; var modifiedPackage = MEPackageHandler.OpenMEPackageFromStream(mixinModifiedStream, $@"Mixin Modified - {Path.GetFileName(file.Key)}"); // three way merge: get target stream var targetFile = Path.Combine(M3Directories.GetCookedPath(SelectedInstallTarget), Path.GetFileName(file.Key)); var targetPackage = MEPackageHandler.OpenMEPackage(targetFile); var merged = ThreeWayPackageMerge.AttemptMerge(vanillaPackage, modifiedPackage, targetPackage); if (merged) { targetPackage.Save(compress: true); Log.Information(@"Three way merge succeeded for " + targetFile); } else { Log.Error(@"Could not merge three way merge into " + targetFile); } //var outfile = Path.Combine(outdir, Path.GetFileName(file.Key)); //package.save(outfile, false); // don't compress //finalStream.WriteToFile(outfile); //File.WriteAllBytes(outfile, finalStream.ToArray()); } catch (Exception e) { var mixinsStr = string.Join(@", ", file.Value.Select(x => x.PatchName)); Log.Error($@"Error in mixin application for file {file.Key}: {e.Message}"); failedApplicationCallback(M3L.GetString(M3L.string_interp_errorApplyingMixinsForFile, mixinsStr, file.Key, e.Message)); } } } else { //dlc var dlcPackage = VanillaDatabaseService.FetchVanillaSFAR(dlcFolderName); //do not have to open file multiple times. var targetCookedPCDir = Path.Combine(M3Directories.GetDLCPath(SelectedInstallTarget), dlcFolderName, @"CookedPCConsole"); var sfar = mapping.Key == ModJob.JobHeader.TESTPATCH ? M3Directories.GetTestPatchSFARPath(SelectedInstallTarget) : Path.Combine(targetCookedPCDir, @"Default.sfar"); bool unpacked = new FileInfo(sfar).Length == 32; DLCPackage targetDLCPackage = unpacked ? null : new DLCPackage(sfar); //cache SFAR target foreach (var file in mapping.Value) { try { using var vanillaPackageAsStream = VanillaDatabaseService.FetchFileFromVanillaSFAR(dlcFolderName, file.Key, forcedDLC: dlcPackage); using var decompressedStream = MEPackage.GetDecompressedPackageStream(vanillaPackageAsStream); decompressedStream.Position = 0; var vanillaPackage = MEPackageHandler.OpenMEPackageFromStream(decompressedStream, $@"VanillaDLC - {Path.GetFileName(file.Key)}"); using var mixinModifiedStream = MixinHandler.ApplyMixins(decompressedStream, file.Value, true, completedSingleApplicationCallback, failedApplicationCallback); mixinModifiedStream.Position = 0; var modifiedPackage = MEPackageHandler.OpenMEPackageFromStream(mixinModifiedStream, $@"Mixin Modified - {Path.GetFileName(file.Key)}"); // three way merge: get target stream // must see if DLC is unpacked first MemoryStream targetFileStream = null; //Packed if (unpacked) { targetFileStream = new MemoryStream(File.ReadAllBytes(Path.Combine(targetCookedPCDir, file.Key))); } else { targetFileStream = VanillaDatabaseService.FetchFileFromVanillaSFAR(dlcFolderName, Path.GetFileName(file.Key), forcedDLC: targetDLCPackage); } var targetPackage = MEPackageHandler.OpenMEPackageFromStream(targetFileStream, $@"Target package {dlcFolderName} - {file.Key}, from SFAR: {unpacked}"); var merged = ThreeWayPackageMerge.AttemptMerge(vanillaPackage, modifiedPackage, targetPackage); if (merged) { if (unpacked) { targetPackage.Save(Path.Combine(targetCookedPCDir, file.Key)); Log.Information(@"Three way merge succeeded for " + targetPackage.FilePath); } else { var finalSTream = targetPackage.SaveToStream(false); // No compress. Not sure if we should support doing that though. targetDLCPackage.ReplaceEntry(finalSTream.ToArray(), targetDLCPackage.FindFileEntry(Path.GetFileName(file.Key))); Log.Information(@"Three way merge succeeded for " + targetPackage.FilePath); } } else { Log.Error(@"Could not three way merge into: " + targetFileStream); } } catch (Exception e) { var mixinsStr = string.Join(@", ", file.Value.Select(x => x.PatchName)); Log.Error($@"Error in mixin application for file {file.Key}: {e.Message}"); failedApplicationCallback(M3L.GetString(M3L.string_interp_errorApplyingMixinsForFile, mixinsStr, file.Key, e.Message)); } //finalStream.WriteToFile(outfile); } } }); MixinHandler.FreeME3TweaksPatchData(); var percent = 0; //this is used to save a localization BottomLeftMessage = M3L.GetString(M3L.string_interp_runningAutoTOCOnGamePercentX, percent); //Run autotoc void tocingUpdate(int percent) { BottomLeftMessage = M3L.GetString(M3L.string_interp_runningAutoTOCOnGamePercentX, percent); } AutoTOC.RunTOCOnGameTarget(SelectedInstallTarget, tocingUpdate); //Generate moddesc //IniData ini = new IniData(); //ini[@"ModManager"][@"cmmver"] = App.HighestSupportedModDesc.ToString(CultureInfo.InvariantCulture); //prevent commas //ini[@"ModInfo"][@"game"] = @"ME3"; //ini[@"ModInfo"][@"modname"] = modname; //ini[@"ModInfo"][@"moddev"] = App.AppVersionHR; //ini[@"ModInfo"][@"moddesc"] = M3L.GetString(M3L.string_compiledFromTheFollowingMixins); //ini[@"ModInfo"][@"modver"] = @"1.0"; //generateRepaceFilesMapping(ini, modpath); //File.WriteAllText(Path.Combine(modpath, @"moddesc.ini"), ini.ToString()); }; nbw.RunWorkerCompleted += (a, b) => { if (b.Error != null) { Log.Error($@"Exception occurred in {nbw.Name} thread: {b.Error.Message}"); } OperationInProgress = false; ClearMixinHandler(); if (failedApplications.Count > 0) { var ld = new ListDialog(failedApplications, M3L.GetString(M3L.string_failedToApplyAllMixins), M3L.GetString(M3L.string_theFollowingMixinsFailedToApply), mainwindow); ld.ShowDialog(); } /*if (modpath != null) * { * OnClosing(new DataEventArgs(modpath)); * } * else * {*/ BottomLeftMessage = M3L.GetString(M3L.string_mixinsInstalledMaybe); //} }; CompilePanelButton.IsOpen = false; nbw.RunWorkerAsync(); }
private void CompileAsNewMod() { NamedBackgroundWorker nbw = new NamedBackgroundWorker(@"MixinManager CompileAsNewModThread"); List <string> failedApplications = new List <string>(); var modname = NewModName; var modpath = Path.Combine(Utilities.GetME3ModsDirectory(), Utilities.SanitizePath(modname)); var result = M3L.ShowDialog(mainwindow, M3L.GetString(M3L.string_interp_dialogCreatingNewModWithExistingName, NewModName, modpath), M3L.GetString(M3L.string_modAlreadyExists), MessageBoxButton.YesNo, MessageBoxImage.Warning, MessageBoxResult.No); if (result == MessageBoxResult.No) { Log.Information(@"User has aborted mixin compilation due to same-named mod existing"); return; //abort. } nbw.DoWork += (a, b) => { BottomLeftMessage = M3L.GetString(M3L.string_compilingMixins); OperationInProgress = true; //DEBUG STUFF #if DEBUG int numCoresToApplyWith = 1; #else var numCoresToApplyWith = Environment.ProcessorCount; if (numCoresToApplyWith > 4) { numCoresToApplyWith = 4; //no more than 4 as this uses a lot of memory } #endif var mixins = AvailableOfficialMixins.Where(x => x.UISelectedForUse).ToList(); MixinHandler.LoadPatchDataForMixins(mixins); //before dynamic void failedApplicationCallback(string str) { failedApplications.Add(str); } var compilingListsPerModule = MixinHandler.GetMixinApplicationList(mixins, failedApplicationCallback); if (failedApplications.Any()) { //Error building list modpath = null; Log.Information(@"Aborting mixin compiling due to incompatible selection of mixins"); return; } if (Directory.Exists(modpath)) { Utilities.DeleteFilesAndFoldersRecursively(modpath); } ProgressBarMax = mixins.Count(); ProgressBarValue = 0; int numdone = 0; void completedSingleApplicationCallback() { var val = Interlocked.Increment(ref numdone); ProgressBarValue = val; } //Mixins are ready to be applied Parallel.ForEach(compilingListsPerModule, new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount > numCoresToApplyWith ? numCoresToApplyWith : Environment.ProcessorCount }, mapping => { var dlcFolderName = ModMakerCompiler.ModmakerChunkNameToDLCFoldername(mapping.Key.ToString()); var outdir = Path.Combine(modpath, ModMakerCompiler.HeaderToDefaultFoldername(mapping.Key), @"CookedPCConsole"); Directory.CreateDirectory(outdir); if (mapping.Key == ModJob.JobHeader.BASEGAME) { //basegame foreach (var file in mapping.Value) { try { using var packageAsStream = VanillaDatabaseService.FetchBasegameFile(Mod.MEGame.ME3, Path.GetFileName(file.Key)); //packageAsStream.WriteToFile(@"C:\users\dev\desktop\compressed.pcc"); using var decompressedStream = MEPackage.GetDecompressedPackageStream(packageAsStream, true); //decompressedStream.WriteToFile(@"C:\users\dev\desktop\decompressed.pcc"); using var finalStream = MixinHandler.ApplyMixins(decompressedStream, file.Value, completedSingleApplicationCallback, failedApplicationCallback); CLog.Information(@"Compressing package to mod directory: " + file.Key, Settings.LogModMakerCompiler); finalStream.Position = 0; var package = MEPackageHandler.OpenMEPackage(finalStream); var outfile = Path.Combine(outdir, Path.GetFileName(file.Key)); package.save(outfile, false); // don't compress //finalStream.WriteToFile(outfile); //File.WriteAllBytes(outfile, finalStream.ToArray()); } catch (Exception e) { var mixinsStr = string.Join(@", ", file.Value.Select(x => x.PatchName)); Log.Error($@"Error in mixin application for file {file.Key}: {e.Message}"); failedApplicationCallback(M3L.GetString(M3L.string_interp_errorApplyingMixinsForFile, mixinsStr, file.Key, e.Message)); } } } else { //dlc var dlcPackage = VanillaDatabaseService.FetchVanillaSFAR(dlcFolderName); //do not have to open file multiple times. foreach (var file in mapping.Value) { try { using var packageAsStream = VanillaDatabaseService.FetchFileFromVanillaSFAR(dlcFolderName, file.Key, forcedDLC: dlcPackage); using var decompressedStream = MEPackage.GetDecompressedPackageStream(packageAsStream); using var finalStream = MixinHandler.ApplyMixins(decompressedStream, file.Value, completedSingleApplicationCallback, failedApplicationCallback); CLog.Information(@"Compressing package to mod directory: " + file.Key, Settings.LogModMakerCompiler); finalStream.Position = 0; var package = MEPackageHandler.OpenMEPackage(finalStream); var outfile = Path.Combine(outdir, Path.GetFileName(file.Key)); package.save(outfile, true); } catch (Exception e) { var mixinsStr = string.Join(@", ", file.Value.Select(x => x.PatchName)); Log.Error($@"Error in mixin application for file {file.Key}: {e.Message}"); failedApplicationCallback(M3L.GetString(M3L.string_interp_errorApplyingMixinsForFile, mixinsStr, file.Key, e.Message)); } //finalStream.WriteToFile(outfile); } } }); MixinHandler.FreeME3TweaksPatchData(); //Generate moddesc IniData ini = new IniData(); ini[@"ModManager"][@"cmmver"] = App.HighestSupportedModDesc.ToString(CultureInfo.InvariantCulture); //prevent commas ini[@"ModInfo"][@"game"] = @"ME3"; ini[@"ModInfo"][@"modname"] = modname; ini[@"ModInfo"][@"moddev"] = App.AppVersionHR; ini[@"ModInfo"][@"moddesc"] = M3L.GetString(M3L.string_compiledFromTheFollowingMixins); ini[@"ModInfo"][@"modver"] = @"1.0"; generateRepaceFilesMapping(ini, modpath); File.WriteAllText(Path.Combine(modpath, @"moddesc.ini"), ini.ToString()); }; nbw.RunWorkerCompleted += (a, b) => { if (b.Error != null) { Log.Error($@"Exception occured in {nbw.Name} thread: {b.Error.Message}"); } OperationInProgress = false; ClearMixinHandler(); if (failedApplications.Count > 0) { var ld = new ListDialog(failedApplications, M3L.GetString(M3L.string_failedToApplyAllMixins), M3L.GetString(M3L.string_theFollowingMixinsFailedToApply), mainwindow); ld.ShowDialog(); } if (modpath != null) { OnClosing(new DataEventArgs(modpath)); } else { BottomLeftMessage = M3L.GetString(M3L.string_selectMixinsToCompile); } }; CompilePanelButton.IsOpen = false; nbw.RunWorkerAsync(); }
public static void ApplyMixinsToModule(KeyValuePair <ModJob.JobHeader, Dictionary <string, List <Mixin> > > mapping, string modpath, Action completedSingleApplicationCallback, Action <string> failedApplicationCallback) { var dlcFolderName = ModMakerCompiler.ModmakerChunkNameToDLCFoldername(mapping.Key.ToString()); var outdir = Path.Combine(modpath, ModMakerCompiler.HeaderToDefaultFoldername(mapping.Key), @"CookedPCConsole"); Directory.CreateDirectory(outdir); if (mapping.Key == ModJob.JobHeader.BASEGAME) { //basegame foreach (var file in mapping.Value) { try { using var packageAsStream = VanillaDatabaseService.FetchBasegameFile(MEGame.ME3, Path.GetFileName(file.Key)); //packageAsStream.WriteToFile(@"C:\users\dev\desktop\compressed.pcc"); using var decompressedStream = MEPackage.GetDecompressedPackageStream(packageAsStream, false, true); using var finalStream = MixinHandler.ApplyMixins(decompressedStream, file.Value, true, completedSingleApplicationCallback, failedApplicationCallback); CLog.Information(@"Compressing package to mod directory: " + file.Key, Settings.LogModMakerCompiler); finalStream.Position = 0; var package = MEPackageHandler.OpenMEPackageFromStream(finalStream); var outfile = Path.Combine(outdir, Path.GetFileName(file.Key)); package.Save(outfile, false, includeAdditionalPackagesToCook: false, includeDependencyTable: true); // don't compress, use mixin saving rules for basegame files } catch (Exception e) { var mixinsStr = string.Join(@", ", file.Value.Select(x => x.PatchName)); Log.Error($@"Error in mixin application for file {file.Key}: {e.Message}"); failedApplicationCallback(M3L.GetString(M3L.string_interp_errorApplyingMixinsForFile, mixinsStr, file.Key, e.Message)); } } } else { //dlc var dlcPackage = VanillaDatabaseService.FetchVanillaSFAR(dlcFolderName); //do not have to open file multiple times. foreach (var file in mapping.Value) { try { using var packageAsStream = VanillaDatabaseService.FetchFileFromVanillaSFAR(dlcFolderName, file.Key, forcedDLC: dlcPackage); //as file comes from backup, we don't need to decompress it, it will always be decompressed in sfar using var finalStream = MixinHandler.ApplyMixins(packageAsStream, file.Value, true, completedSingleApplicationCallback, failedApplicationCallback); var outfile = Path.Combine(outdir, Path.GetFileName(file.Key)); if (mapping.Key != ModJob.JobHeader.TESTPATCH) { // TestPatch is never unpacked. So there is not really point to // compressing it's rather small files. The other DLC jobs likely will be packed still, but this will save some disk space. CLog.Information($@"Compressing package to mod directory: {outfile}", Settings.LogModMakerCompiler); finalStream.Position = 0; var package = MEPackageHandler.OpenMEPackageFromStream(finalStream); package.Save(outfile, true); } else { Log.Information($@"Writing patched file to disk: {outfile}"); finalStream.WriteToFile(outfile); } } catch (Exception e) { var mixinsStr = string.Join(@", ", file.Value.Select(x => x.PatchName)); Log.Error($@"Error in mixin application for file {file.Key}: {e.Message}"); failedApplicationCallback(M3L.GetString(M3L.string_interp_errorApplyingMixinsForFile, mixinsStr, file.Key, e.Message)); } //finalStream.WriteToFile(outfile); } } }
public static bool AttemptMerge(MEPackage vanillaPackage, MEPackage modifiedVanillaPackage, MEPackage targetPackage) { PackageDelta vanillaToModifiedDelta = PackageDelta.CalculateDelta(vanillaPackage, modifiedVanillaPackage); PackageDelta vanillaToTargetDelta = PackageDelta.CalculateDelta(vanillaPackage, targetPackage); string loggingPrefix = targetPackage.FilePath == null ? targetPackage.FileSourceForDebugging : Path.GetFileName(targetPackage.FilePath); //Check merge conditions var nameConflicts = vanillaToModifiedDelta.NameDeltas.Keys.Intersect(vanillaToTargetDelta.NameDeltas.Keys).ToList(); var importConflicts = vanillaToModifiedDelta.ImportDeltas.Keys.Intersect(vanillaToTargetDelta.ImportDeltas.Keys).ToList(); var exportConflicts = vanillaToModifiedDelta.ExportDeltas.Keys.Intersect(vanillaToTargetDelta.ExportDeltas.Keys).ToList(); Log.Information($"[{loggingPrefix}] Performing three way merge pre-check"); //Name deltas if (nameConflicts.Count > 0) { //need to check if the conflicts result in same value, in this case it would not be a conflict. foreach (int nameIndex in nameConflicts) { var modifiedName = modifiedVanillaPackage.Names[nameIndex]; var targetName = targetPackage.Names[nameIndex]; if (modifiedName != targetName) { //Differing names in same spots. Log.Information($@"[{loggingPrefix}] Cannot merge files: Name index {nameIndex} is different between modified and target."); return(false); } } } if (importConflicts.Count > 0) { //todo } if (exportConflicts.Count > 0) { //hmm... this will be a tough one. foreach (int exportTableIndex in exportConflicts) { //we will have to check sizes if we ever hope to have way to merge this var modifiedData = modifiedVanillaPackage.Exports[exportTableIndex].Data; var targetData = targetPackage.Exports[exportTableIndex].Data; if (!modifiedData.SequenceEqual(targetData)) { Log.Information($@"[{loggingPrefix}] Cannot merge files: Export table index {exportTableIndex} data is different between modified and target."); return(false); } //We will have to ignore size here somehow... //var modifiedHeader = modifiedVanillaPackage.Exports[exportTableIndex].Header; //var targetHeader = targetPackage.Exports[exportTableIndex].Header; } } //Merge is OK to perform //Apply vanilla to modified delta to target package foreach (var nameDelta in vanillaToModifiedDelta.NameDeltas) { if (nameDelta.Key >= targetPackage.NameCount) { //add it Log.Information($"[{loggingPrefix}] Adding name {nameDelta.Value}"); targetPackage.addName(nameDelta.Value); } else { Log.Information($"[{loggingPrefix}] Updating name index {nameDelta.Key} to {nameDelta.Value}"); targetPackage.replaceName(nameDelta.Key, nameDelta.Value); } } foreach (var exportDelta in vanillaToModifiedDelta.ExportDeltas) { if (exportDelta.Key >= targetPackage.ExportCount) { //add it Log.Information($"[{loggingPrefix}] Adding export {exportDelta.Value.GetInstancedFullPath}"); targetPackage.addExport(exportDelta.Value); //not sure if this is possible } else { //gonna need this reviewed, not entirely sure this is OK to do Log.Information($"[{loggingPrefix}] Updating export {exportDelta.Value.GetInstancedFullPath}"); targetPackage.Exports[exportDelta.Key].Data = exportDelta.Value.Data; targetPackage.Exports[exportDelta.Key].Header = exportDelta.Value.Header; } } foreach (var importDelta in vanillaToModifiedDelta.ImportDeltas) { if (importDelta.Key >= targetPackage.ImportCount) { //add it Log.Information($"[{loggingPrefix}] Adding import {importDelta.Value.GetInstancedFullPath}"); targetPackage.addImport(importDelta.Value); //not sure if this is possible } else { Log.Information($"[{loggingPrefix}] Updating import {importDelta.Value.GetInstancedFullPath}"); //gonna need this reviewed, not entirely sure this is OK to do //targetPackage.Imports[importDelta.Key].Data = importDelta.Value.Data; targetPackage.Imports[importDelta.Key].Header = importDelta.Value.Header; } } Log.Information($"[{loggingPrefix}] Finished three way merge"); return(true); }