public static void MWLoadTest()
{
string fileESM = "C:\\Software\\Steam\\steamapps\\common\\Morrowind\\Data Files\\Morrowind.esm";
//string BM = "C:\\Software\\Steam\\steamapps\\common\\Morrowind\\Data Files\\Bloodmoon.esm";
var timer = Stopwatch.StartNew();
TES3 tes3 = TES3.TES3Load(fileESM, new List <string> {
"LAND", "LTEX"
});
//TES3 bm = TES3.TES3Load(BM, new List<string> { "LAND","LTEX" });
//tes3.TES3Save("C:/mapstuff/out.esp");
var heightmap = new TES3HeightMap(tes3);
var options = new ExportOptions()
{
HeightMap = true, VertexColorMap = true, TexturePlacementMap = true, ExportHeightAsRaw = true
};
heightmap.ReadMapData(@"C:/mapstuff/output", options);
//heightmap.ImportMapFromImage("C:/mapstuff/output");
//var test = new TES3HeightMap();
//test.ImportMapFromImage("C:/mapstuff/SEWorld", -50, -20);
//var options2 = new ExportOptions() { HeightMap = true, VertexColorMap = false, TexturePlacementMap = false, ExportHeightAsRaw = false };
//heightmap.ReadMapData(@"C:/mapstuff/output", options2);
timer.Stop();
Console.WriteLine($"Done in {timer.ElapsedMilliseconds} ms");
}
static void Main(string[] args)
{
#if DEBUG
Console.WriteLine("Press any key to continue...");
Console.ReadKey();
#endif
// Create our log.
Logger = new StreamWriter("TES3Merge.log", false)
{
AutoFlush = true
};
var version = Assembly.GetExecutingAssembly().GetName().Version;
Logger.WriteLine($"TES3Merge v0.5.");
// Main execution attempt.
#if DEBUG == false
try
#endif
{
// Load this application's configuration.
{
var parser = new FileIniDataParser();
string iniPath = $"{AppDomain.CurrentDomain.BaseDirectory}\\TES3Merge.ini";
Configuration = parser.ReadFile(iniPath);
}
// Determine what encoding to use.
try
{
var iniEncodingCode = Configuration["General"]["TextEncodingCode"];
if (int.TryParse(iniEncodingCode, out int newEncodingCode))
{
// TODO: Check a list of supported encoding codes.
if (newEncodingCode != 932 && (newEncodingCode < 1250 || newEncodingCode > 1252))
{
throw new Exception($"Encoding code '{newEncodingCode}' is not supported. See TES3Merge.ini for supported values.");
}
var encoding = Encoding.GetEncoding(newEncodingCode);
Logger.WriteLine($"Using encoding: {encoding.EncodingName}");
Utility.Common.TextEncodingCode = newEncodingCode;
}
else
{
throw new Exception($"Encoding code '{iniEncodingCode}' is not a valid integer. See TES3Merge.ini for supported values.");
}
}
catch (Exception e)
{
// Write the exception as a warning and set the default Windows-1252 encoding.
WriteToLogAndConsole($"WARNING: Could not resolve default text encoding code: {e.Message}");
Console.WriteLine("Default encoding of Windows-1252 (English) will be used.");
Utility.Common.TextEncodingCode = 1252;
}
// Find out where Morrowind lives.
string morrowindPath = GetMorrowindFolder();
if (morrowindPath == null)
{
WriteToLogAndConsole("ERROR: Could not resolve Morrowind directory. Install TES3Merge folder into the Morrowind installation folder.");
}
Logger.WriteLine($"Morrowind found at '{morrowindPath}'.");
// Create our merged object TES3 file.
TES3 mergedObjects = new TES3();
var mergedObjectsHeader = new TES3Lib.Records.TES3
{
HEDR = new TES3Lib.Subrecords.TES3.HEDR()
{
CompanyName = "TES3Merge",
Description = $"Automatic merge generated at {DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ssZ")}.",
Version = 1.3f,
}
};
mergedObjects.Records.Add(mergedObjectsHeader);
// Get a list of supported mergable object types.
List <string> supportedMergeTags = new List <string>
{
"ACTI",
"ALCH",
"APPA",
"ARMO",
"BODY",
"BOOK",
"BSGN",
//"CELL",
"CLAS",
"CLOT",
"CONT",
"CREA",
//"DIAL",
"DOOR",
"ENCH",
"FACT",
//"GLOB",
"GMST",
//"INFO",
"INGR",
//"LAND",
//"LEVC",
//"LEVI",
"LIGH",
"LOCK",
//"LTEX",
"MGEF",
"MISC",
"NPC_",
//"PGRD",
"PROB",
//"RACE",
//"REFR",
//"REGN",
"REPA",
//"SCPT",
"SKIL",
"SNDG",
"SOUN",
"SPEL",
"STAT",
"WEAP",
};
// Allow INI to remove types from merge.
foreach (var recordTypeConfig in Configuration["RecordTypes"])
{
bool.TryParse(recordTypeConfig.Value, out bool supported);
if (!supported)
{
supportedMergeTags.Remove(recordTypeConfig.KeyName);
}
}
Logger.WriteLine($"Supported record types: {string.Join(", ", supportedMergeTags)}");
// Get object ID filtering from INI.
List <KeyValuePair <string, bool> > objectIdFilters = new List <KeyValuePair <string, bool> >();
foreach (var kv in Configuration["ObjectFilters"])
{
bool.TryParse(kv.Value, out bool allow);
objectIdFilters.Add(new KeyValuePair <string, bool>(kv.KeyName.Trim('"'), allow));
}
// Collections for managing our objects.
Dictionary <string, Dictionary <string, List <TES3Lib.Base.Record> > > recordOverwriteMap = new Dictionary <string, Dictionary <string, List <TES3Lib.Base.Record> > >();
// Get the game file list from the ini file.
List <string> sortedMasters = new List <string>();
Dictionary <TES3, string> mapTES3ToFileNames = new Dictionary <TES3, string>();
Dictionary <TES3Lib.Base.Record, TES3> recordMasters = new Dictionary <TES3Lib.Base.Record, TES3>();
Console.WriteLine("Parsing content files...");
{
// Try to get INI information.
IniData data;
try
{
var parser = new FileIniDataParser();
data = parser.ReadFile($"{morrowindPath}\\Morrowind.ini");
}
catch (Exception firstTry)
{
try
{
// Try again with invalid line skipping.
var parser = new FileIniDataParser();
var config = parser.Parser.Configuration;
config.SkipInvalidLines = true;
config.AllowDuplicateKeys = true;
config.AllowDuplicateSections = true;
data = parser.ReadFile($"{morrowindPath}\\Morrowind.ini");
// If the first pass fails, be more forgiving, but let the user know their INI has issues.
Console.WriteLine("WARNING: Issues were found with your Morrowind.ini file. See TES3Merge.log for details.");
Logger.WriteLine($"WARNING: Could not parse Morrowind.ini with initial pass. Error: {firstTry.Message}");
}
catch (Exception secondTry)
{
Console.WriteLine("ERROR: Unrecoverable issues were found with your Morrowind.ini file. See TES3Merge.log for details.");
Logger.WriteLine($"ERROR: Could not parse Morrowind.ini with second pass. Error: {secondTry.Message}");
ShowCompletionPrompt();
return;
}
}
// Build a list of activated files.
HashSet <string> activatedMasters = new HashSet <string>();
for (int i = 0; i < 255; i++)
{
string gameFile = data["Game Files"]["GameFile" + i];
if (gameFile == null)
{
break;
}
if (gameFile == "Merged_Objects.esp" || gameFile == "Merged Objects.esp")
{
continue;
}
activatedMasters.Add(gameFile);
}
// Add all ESM files first, then ESP files.
foreach (var path in Directory.GetFiles($"{morrowindPath}\\Data Files", "*.esm", SearchOption.TopDirectoryOnly).OrderBy(p => File.GetLastWriteTime(p).Ticks))
{
var fileName = Path.GetFileName(path);
if (activatedMasters.Contains(fileName))
{
sortedMasters.Add(fileName);
}
}
foreach (var path in Directory.GetFiles($"{morrowindPath}\\Data Files", "*.esp", SearchOption.TopDirectoryOnly).OrderBy(p => File.GetLastWriteTime(p).Ticks))
{
var fileName = Path.GetFileName(path);
if (activatedMasters.Contains(fileName))
{
sortedMasters.Add(fileName);
}
}
// Go through and build a record list.
foreach (var sortedMaster in sortedMasters)
{
string fullGameFilePath = $"{morrowindPath}\\Data Files\\{sortedMaster}";
var lastWriteTime = File.GetLastWriteTime(fullGameFilePath);
Logger.WriteLine($"Parsing input file: {sortedMaster} @ {lastWriteTime}");
TES3 file = TES3.TES3Load(fullGameFilePath, supportedMergeTags);
mapTES3ToFileNames[file] = sortedMaster;
foreach (var record in file.Records)
{
if (record == null)
{
continue;
}
if (record.GetType().Equals(typeof(TES3Lib.Records.TES3)))
{
continue;
}
string editorId = record.GetEditorId().Replace("\0", string.Empty);
if (string.IsNullOrEmpty(editorId))
{
continue;
}
// Check against object filters.
bool allow = true;
string lowerId = editorId.ToLower();
foreach (var kv in objectIdFilters)
{
try
{
if (Regex.Match(lowerId, kv.Key).Success)
{
allow = kv.Value;
}
}
catch (Exception)
{
}
}
if (!allow)
{
continue;
}
if (!recordOverwriteMap.ContainsKey(record.Name))
{
recordOverwriteMap[record.Name] = new Dictionary <string, List <TES3Lib.Base.Record> >();
}
var map = recordOverwriteMap[record.Name];
if (!map.ContainsKey(editorId))
{
map[editorId] = new List <TES3Lib.Base.Record>();
}
map[editorId].Add(record);
recordMasters[record] = file;
}
}
}
// Check to see if we have any potential merges.
if (recordMasters.Count == 0)
{
WriteToLogAndConsole("No potential record merges found. Aborting.");
ShowCompletionPrompt();
return;
}
// Go through and build merged objects.
bool.TryParse(Configuration["General"]["DumpMergedRecordsToLog"], out bool dumpMergedRecordsToLog);
Console.WriteLine("Building merges...");
HashSet <string> usedMasters = new HashSet <string>();
foreach (var recordType in recordOverwriteMap.Keys)
{
var recordsMap = recordOverwriteMap[recordType];
foreach (string id in recordsMap.Keys)
{
var records = recordsMap[id];
if (records.Count > 2)
{
var firstRecord = records[0];
var lastRecord = records.Last();
var firstMaster = mapTES3ToFileNames[recordMasters[firstRecord]];
var lastMaster = mapTES3ToFileNames[recordMasters[lastRecord]];
HashSet <string> localUsedMasters = new HashSet <string>()
{
firstMaster, lastMaster
};
var lastSerialized = lastRecord.GetRawLoadedBytes();
TES3Lib.Base.Record newRecord = Activator.CreateInstance(lastRecord.GetType(), new object[] { lastSerialized }) as TES3Lib.Base.Record;
for (int i = records.Count - 2; i > 0; i--)
{
var record = records[i];
var master = mapTES3ToFileNames[recordMasters[record]];
if (newRecord.MergeWith(record, firstRecord))
{
localUsedMasters.Add(master);
}
}
var newSerialized = newRecord.SerializeRecord();
if (!lastSerialized.SequenceEqual(newSerialized))
{
Console.WriteLine($"Merged {newRecord.Name} record: {id}");
mergedObjects.Records.Add(newRecord);
foreach (string master in localUsedMasters)
{
usedMasters.Add(master);
}
string masterList = string.Join(", ", GetFilteredLoadList(sortedMasters, localUsedMasters).ToArray());
Logger.WriteLine($"Resolved conflicts for {firstRecord.Name} record '{id}' from mods: {masterList}");
if (dumpMergedRecordsToLog)
{
foreach (var record in records)
{
var master = mapTES3ToFileNames[recordMasters[record]];
Logger.WriteLine($">> {master}: {BitConverter.ToString(record.GetRawLoadedBytes()).Replace("-", "")}");
}
Logger.WriteLine($">> Merged Objects.esp: {BitConverter.ToString(newSerialized).Replace("-", "")}");
}
}
}
}
}
// Did we even merge anything?
if (usedMasters.Count == 0)
{
WriteToLogAndConsole("No merges were deemed necessary. Aborting.");
ShowCompletionPrompt();
return;
}
// Add the necessary masters.
Logger.WriteLine("Saving Merged Objects.esp ...");
mergedObjectsHeader.Masters = new List <(TES3Lib.Subrecords.TES3.MAST MAST, TES3Lib.Subrecords.TES3.DATA DATA)>();
foreach (var gameFile in GetFilteredLoadList(sortedMasters, usedMasters))
{
if (usedMasters.Contains(gameFile))
{
long size = new FileInfo($"{morrowindPath}\\Data Files\\{gameFile}").Length;
mergedObjectsHeader.Masters.Add((new TES3Lib.Subrecords.TES3.MAST {
Filename = $"{gameFile}\0"
}, new TES3Lib.Subrecords.TES3.DATA {
MasterDataSize = size
}));
}
}
// Save out the merged objects file.
mergedObjectsHeader.HEDR.NumRecords = mergedObjects.Records.Count - 1;
mergedObjects.TES3Save(morrowindPath + "\\Data Files\\Merged Objects.esp");
Logger.WriteLine($"Wrote {mergedObjects.Records.Count - 1} merged objects.");
ShowCompletionPrompt();
}
#if DEBUG == false
catch (Exception e)
{
Console.WriteLine("A serious error has occurred. Please post the TES3Merge.log file to GitHub: https://github.com/NullCascade/TES3Merge/issues");
Logger.WriteLine("An unhandled exception has occurred. Traceback:");
Logger.WriteLine(e.Message);
Logger.WriteLine(e.StackTrace);
ShowCompletionPrompt();
}
#endif
}
/// <summary>
/// Verifies all active esps in the current Morrowind directory
/// Parses all enabled records of the plugin and checks paths if the file exists
/// </summary>
/// <exception cref="Exception"></exception>
private static void Verify()
{
ArgumentNullException.ThrowIfNull(CurrentInstallation);
using var ssw = new ScopedStopwatch();
LoadConfig();
ArgumentNullException.ThrowIfNull(Configuration);
// get merge tags
var(supportedMergeTags, objectIdFilters) = GetMergeTags();
// Shorthand install access.
var sortedMasters = CurrentInstallation.GameFiles;
// Go through and build a record list.
var reportDict = new ConcurrentDictionary <string, Dictionary <string, List <string> > >();
WriteToLogAndConsole($"Parsing plugins ... ");
//foreach (var sortedMaster in sortedMasters)
Parallel.ForEach(sortedMasters, sortedMaster =>
{
// this can be enabled actually
if (Path.GetExtension(sortedMaster) == ".esm")
{
//continue;
return;
}
var map = new Dictionary <string, List <string> >();
// go through all records
WriteToLogAndConsole($"Parsing input file: {sortedMaster}");
var fullGameFilePath = Path.Combine(CurrentInstallation.RootDirectory, "Data Files", $"{sortedMaster}");
var file = TES3.TES3Load(fullGameFilePath, supportedMergeTags);
foreach (var record in file.Records)
{
#region checks
if (record is null)
{
continue;
}
if (record.GetType().Equals(typeof(TES3Lib.Records.TES3)))
{
continue;
}
var editorId = record.GetEditorId().Replace("\0", string.Empty);
if (string.IsNullOrEmpty(editorId))
{
continue;
}
// Check against object filters.
var allow = true;
var lowerId = editorId.ToLower();
foreach (var kv in objectIdFilters)
{
try
{
if (Regex.Match(lowerId, kv.Key).Success)
{
allow = kv.Value;
}
}
catch (Exception)
{
}
}
if (!allow)
{
continue;
}
#endregion
// verify here
GetPathsInRecord(record, map);
}
if (map.Count > 0)
{
reportDict.AddOrUpdate(sortedMaster, map, (key, oldValue) => map);
}
}
);
// pretty print
WriteToLogAndConsole($"\n------------------------------------");
WriteToLogAndConsole($"Results:\n");
foreach (var(plugin, val) in reportDict)
{
WriteToLogAndConsole($"\n{plugin} ({val.Count})");
foreach (var(recordID, list) in val)
{
foreach (var item in list)
{
//Console.WriteLine("{0,-20} {1,5}\n", "Name", "Hours");
WriteToLogAndConsole(string.Format("\t{0,-40} {1,5}", recordID, item));
}
}
}
// serialize to file
WriteToLogAndConsole($"\n");
var reportPath = Path.Combine(CurrentInstallation.RootDirectory, "Data Files", "report.json");
WriteToLogAndConsole($"Writing report to: {reportPath}");
{
using var fs = new FileStream(reportPath, FileMode.Create);
JsonSerializer.Serialize(fs, reportDict, new JsonSerializerOptions()
{
WriteIndented = true
});
}
}