private static void ConfigureLogger()
{
var folder = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments), "JWLMerge\\Logs");
try
{
if (!Directory.Exists(folder))
{
Directory.CreateDirectory(folder);
}
Log.Logger = new LoggerConfiguration()
.MinimumLevel.Debug()
.WriteTo.File(Path.Combine(folder, "log.txt"), rollingInterval: RollingInterval.Day, retainedFileCountLimit: 28)
.CreateLogger();
Log.Logger.Information("==== Launched ====");
Log.Logger.Information($"Version {VersionDetection.GetCurrentVersion()}");
}
catch (Exception ex)
{
// logging won't work but silently fails
EventTracker.Error(ex, "Logging cannot be configured");
// "no-op" logger
Log.Logger = new LoggerConfiguration().CreateLogger();
}
}
private void FixObjectTypes(object data, Type type, OpenTagCache srcInfo)
{
// The object type enum changed in 11.1.498295 because a new armor type was added at value 3.
// These are a bunch of hacks to fix this in most cases.
var oldObjectTypeField = type.GetField("ObjectTypeOld");
var newObjectTypeField = type.GetField("ObjectTypeNew");
if (oldObjectTypeField != null && newObjectTypeField != null)
{
if (VersionDetection.Compare(srcInfo.Version, EngineVersion.V11_1_498295_Live) < 0)
{
var oldType = (ObjectTypeValueOld)oldObjectTypeField.GetValue(data);
newObjectTypeField.SetValue(data, ConvertObjectType(oldType));
}
else
{
var newType = (ObjectTypeValueNew)newObjectTypeField.GetValue(data);
oldObjectTypeField.SetValue(data, ConvertObjectType(newType));
}
}
var phantom = data as PhysicsModel.PhantomType;
if (phantom != null)
{
// Remove the armor bit added at position 8 in flags
phantom.Flags = (uint)((phantom.Flags & ~0x1FFE00) | ((phantom.Flags & 0x1FFE00) >> 1));
}
var chmt = data as ChocolateMountainNew;
if (chmt != null && chmt.LightingVariables != null && chmt.LightingVariables.Count > 3)
{
chmt.LightingVariables.RemoveAt(3); // Remove armor data from chmt
}
}
/// <summary>
/// Creates a vertex stream for a given engine version.
/// </summary>
/// <param name="version">The engine version.</param>
/// <param name="stream">The base stream.</param>
/// <returns>The created vertex stream.</returns>
public static IVertexStream Create(EngineVersion version, Stream stream)
{
if (VersionDetection.Compare(version, EngineVersion.V1_235640_cert_ms25) >= 0)
{
return(new V1_235640.VertexStream(stream));
}
return(new V1_106708.VertexStream(stream));
}
private static void ConfigureLogger()
{
var logsDirectory = FileUtils.GetLogFolder();
Log.Logger = new LoggerConfiguration()
.MinimumLevel.ControlledBy(LogLevelSwitchService.LevelSwitch)
.WriteTo.File(Path.Combine(logsDirectory, "log-.txt"), retainedFileCountLimit: 28, rollingInterval: RollingInterval.Day)
.CreateLogger();
Log.Logger.Information("==== Launched ====");
Log.Logger.Information($"Version {VersionDetection.GetCurrentVersion()}");
}
private static TagStructureAttribute GetStructureAttribute(Type type, EngineVersion version)
{
// First match against any TagStructureAttributes that have version restrictions
var attrib = type.GetCustomAttributes(typeof(TagStructureAttribute), false)
.Cast <TagStructureAttribute>()
.Where(a => a.MinVersion != EngineVersion.Unknown || a.MaxVersion != EngineVersion.Unknown)
.FirstOrDefault(a => VersionDetection.IsBetween(version, a.MinVersion, a.MaxVersion));
// If nothing was found, find the first attribute without any version restrictions
return(attrib ?? type.GetCustomAttributes(typeof(TagStructureAttribute), false)
.Cast <TagStructureAttribute>()
.FirstOrDefault(a => a.MinVersion == EngineVersion.Unknown && a.MaxVersion == EngineVersion.Unknown));
}
private void ConfigureLogger()
{
string logsDirectory = FileUtils.GetLogFolder();
Log.Logger = new LoggerConfiguration()
.MinimumLevel.ControlledBy(LogLevelSwitchService.LevelSwitch)
.WriteTo.RollingFile(Path.Combine(logsDirectory, "log-{Date}.txt"), retainedFileCountLimit: 28)
#if DEBUG
.WriteTo.Console()
#endif
.CreateLogger();
Log.Logger.Information("==== Launched ====");
Log.Logger.Information($"Version {VersionDetection.GetCurrentVersion()}");
}
private void GetVersionData()
{
Task.Delay(2000).ContinueWith(_ =>
{
var latestVersion = VersionDetection.GetLatestReleaseVersion();
if (latestVersion != null)
{
if (latestVersion != VersionDetection.GetCurrentVersion())
{
// there is a new version....
IsNewVersionAvailable = true;
RaisePropertyChanged(nameof(IsNewVersionAvailable));
}
}
});
}
private ResourceLocation FixResourceLocation(ResourceLocation location, EngineVersion srcVersion, EngineVersion destVersion)
{
if (VersionDetection.Compare(destVersion, EngineVersion.V1_235640_cert_ms25) >= 0)
{
return(location);
}
switch (location)
{
case ResourceLocation.RenderModels:
return(ResourceLocation.Resources);
case ResourceLocation.Lightmaps:
return(ResourceLocation.Textures);
}
return(location);
}
private bool GetCurrentPropertyInfo()
{
// If the field has a TagFieldAttribute, use it, otherwise use the default
Attribute = Field.GetCustomAttributes(typeof(TagFieldAttribute), false).FirstOrDefault() as TagFieldAttribute ?? DefaultFieldAttribute;
if (Attribute.Offset >= Info.TotalSize)
{
throw new InvalidOperationException("Offset for property \"" + Field.Name + "\" is outside of its structure");
}
// Read version restrictions, if any
var minVersionAttrib = Field.GetCustomAttributes(typeof(MinVersionAttribute), false).FirstOrDefault() as MinVersionAttribute;
var maxVersionAttrib = Field.GetCustomAttributes(typeof(MaxVersionAttribute), false).FirstOrDefault() as MaxVersionAttribute;
MinVersion = (minVersionAttrib != null) ? minVersionAttrib.Version : EngineVersion.Unknown;
MaxVersion = (maxVersionAttrib != null) ? maxVersionAttrib.Version : EngineVersion.Unknown;
return(VersionDetection.IsBetween(Info.Version, MinVersion, MaxVersion));
}
/// <summary>
/// Display the program startup banner
/// </summary>
/// <remarks>Generated from https://www.coolgenerator.com/ascii-text-generator Font: Roman</remarks>
public static void Show()
{
const string banner = @"
oooooooooo. o8o oooo .o8 oooooooooo. .
`888' `Y8b `""' `888 ""888 `888' `Y8b .o8
888 888 oooo oooo oooo 888 .oooo888 888 888 .ooooo. .o888oo
888oooo888' `888 `888 `888 888 d88' `888 888oooo888' d88' `88b 888
888 `88b 888 888 888 888 888 888 888 `88b 888 888 888
888 .88P 888 888 888 888 888 888 888 .88P 888 888 888 .
o888bood8P' `V88V""V8P' o888o o888o `Y8bod88P"" o888bood8P' `Y8bod8P' ""888""
";
Console.WriteLine(banner);
string version = VersionDetection.ProgramVersion(typeof(StartupBanner));
Console.WriteLine(value: $"Starting version {version}...");
}
public override bool Execute(List <string> args)
{
if (args.Count < 2)
{
return(false);
}
var outputPath = args[0];
// Load each file and do version detection
var infos = new List <OpenTagCache>();
foreach (var path in args.Skip(1))
{
Console.WriteLine("Loading {0}...", path);
// Load the cache file
var info = new OpenTagCache {
CacheFile = new FileInfo(path)
};
using (var stream = info.OpenCacheRead())
info.Cache = new TagCache(stream);
// Do version detection, and don't accept the closest version
// because that might not work
EngineVersion closestVersion;
info.Version = VersionDetection.DetectVersion(info.Cache, out closestVersion);
if (info.Version == EngineVersion.Unknown)
{
Console.WriteLine("- Unrecognized version! Ignoring.");
continue;
}
info.Deserializer = new TagDeserializer(info.Version);
infos.Add(info);
}
var result = new TagVersionMap();
using (var baseStream = _info.OpenCacheRead())
{
// Get the scenario tags for this cache
Console.WriteLine("Finding base scenario tags...");
var baseScenarios = FindScenarios(_info, baseStream);
var baseVersion = _info.Version;
var baseTagData = new Dictionary <int, object>();
foreach (var scenario in baseScenarios)
{
baseTagData[scenario.Tag.Index] = scenario.Data;
}
// Now compare with each of the other caches
foreach (var info in infos)
{
using (var stream = info.OpenCacheRead())
{
Console.WriteLine("Finding scenario tags in {0}...", info.CacheFile.FullName);
// Get the scenario tags and connect them to the base tags
var scenarios = FindScenarios(info, stream);
var tagsToCompare = new Queue <QueuedTag>();
for (var i = 0; i < scenarios.Count; i++)
{
tagsToCompare.Enqueue(scenarios[i]);
if (i < baseScenarios.Count)
{
result.Add(baseVersion, baseScenarios[i].Tag.Index, info.Version, scenarios[i].Tag.Index);
}
}
// Process each tag in the queue, enqueuing all of its dependencies as well
while (tagsToCompare.Count > 0)
{
// Get the tag and its data
var tag = tagsToCompare.Dequeue();
TagPrinter.PrintTagShort(tag.Tag);
var data = tag.Data;
if (data == null)
{
// No data yet - deserialize it
var context = new TagSerializationContext(stream, info.Cache, info.StringIds, tag.Tag);
var type = TagStructureTypes.FindByGroupTag(tag.Tag.Group.Tag);
data = info.Deserializer.Deserialize(context, type);
}
// Now get the data for the base tag
var baseTag = result.Translate(info.Version, tag.Tag.Index, baseVersion);
if (baseTag == -1 || _info.Cache.Tags[baseTag].Group.Tag != tag.Tag.Group.Tag)
{
continue;
}
object baseData;
if (!baseTagData.TryGetValue(baseTag, out baseData))
{
// No data yet - deserialize it
var context = new TagSerializationContext(baseStream, _info.Cache, _info.StringIds, _info.Cache.Tags[baseTag]);
var type = TagStructureTypes.FindByGroupTag(tag.Tag.Group.Tag);
baseData = _info.Deserializer.Deserialize(context, type);
baseTagData[baseTag] = baseData;
}
// Compare the two blocks
CompareBlocks(baseData, baseVersion, data, info.Version, result, tagsToCompare);
}
}
}
}
// Write out the CSV
Console.WriteLine("Writing results...");
using (var writer = new StreamWriter(File.Open(outputPath, FileMode.Create, FileAccess.Write)))
result.WriteCsv(writer);
Console.WriteLine("Done!");
return(true);
}
private static void CompareBlocks(object leftData, EngineVersion leftVersion, object rightData, EngineVersion rightVersion, TagVersionMap result, Queue <QueuedTag> tagQueue)
{
if (leftData == null || rightData == null)
{
return;
}
var type = leftData.GetType();
if (type == typeof(TagInstance))
{
// If the objects are tags, then we've found a match
var leftTag = (TagInstance)leftData;
var rightTag = (TagInstance)rightData;
if (leftTag.Group.Tag != rightTag.Group.Tag)
{
return;
}
if (leftTag.IsInGroup("rmt2") || leftTag.IsInGroup("rmdf") || leftTag.IsInGroup("vtsh") || leftTag.IsInGroup("pixl") || leftTag.IsInGroup("rm ") || leftTag.IsInGroup("bitm"))
{
return;
}
var translated = result.Translate(leftVersion, leftTag.Index, rightVersion);
if (translated >= 0)
{
return;
}
result.Add(leftVersion, leftTag.Index, rightVersion, rightTag.Index);
tagQueue.Enqueue(new QueuedTag {
Tag = rightTag
});
}
else if (type.IsArray)
{
if (type.GetElementType().IsPrimitive)
{
return;
}
// If the objects are arrays, then loop through each element
var leftArray = (Array)leftData;
var rightArray = (Array)rightData;
if (leftArray.Length != rightArray.Length)
{
return; // If the sizes are different, we probably can't compare them
}
for (var i = 0; i < leftArray.Length; i++)
{
CompareBlocks(leftArray.GetValue(i), leftVersion, rightArray.GetValue(i), rightVersion, result, tagQueue);
}
}
else if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(List <>))
{
if (type.GenericTypeArguments[0].IsPrimitive)
{
return;
}
// If the objects are lists, then loop through each element
var countProperty = type.GetProperty("Count");
var leftCount = (int)countProperty.GetValue(leftData);
var rightCount = (int)countProperty.GetValue(rightData);
if (leftCount != rightCount)
{
return; // If the sizes are different, we probably can't compare them
}
var getItem = type.GetMethod("get_Item");
for (var i = 0; i < leftCount; i++)
{
var leftItem = getItem.Invoke(leftData, new object[] { i });
var rightItem = getItem.Invoke(rightData, new object[] { i });
CompareBlocks(leftItem, leftVersion, rightItem, rightVersion, result, tagQueue);
}
}
else if (type.GetCustomAttributes(typeof(TagStructureAttribute), false).Length > 0)
{
// The objects are structures
var left = new TagFieldEnumerator(new TagStructureInfo(leftData.GetType(), leftVersion));
var right = new TagFieldEnumerator(new TagStructureInfo(rightData.GetType(), rightVersion));
while (left.Next() && right.Next())
{
// Keep going on the left until the field is on the right
while (!VersionDetection.IsBetween(rightVersion, left.MinVersion, left.MaxVersion))
{
if (!left.Next())
{
return;
}
}
// Keep going on the right until the field is on the left
while (!VersionDetection.IsBetween(leftVersion, right.MinVersion, right.MaxVersion))
{
if (!right.Next())
{
return;
}
}
if (left.Field.MetadataToken != right.Field.MetadataToken)
{
throw new InvalidOperationException("WTF, left and right fields don't match!");
}
// Process the fields
var leftFieldData = left.Field.GetValue(leftData);
var rightFieldData = right.Field.GetValue(rightData);
CompareBlocks(leftFieldData, leftVersion, rightFieldData, rightVersion, result, tagQueue);
}
}
}
public override bool Execute(List <string> args)
{
if (args.Count != 4)
{
return(false);
}
var srcTag = ArgumentParser.ParseTagIndex(_info.Cache, args[0]);
if (srcTag == null)
{
return(false);
}
var csvPath = args[1];
var csvOutPath = args[2];
var targetDir = args[3];
// Load the CSV
Console.WriteLine("Reading {0}...", csvPath);
TagVersionMap tagMap;
using (var reader = new StreamReader(File.OpenRead(csvPath)))
tagMap = TagVersionMap.ParseCsv(reader);
// Load destination files
Console.WriteLine("Loading the target tags.dat...");
var destCachePath = Path.Combine(targetDir, "tags.dat");
var destInfo = new OpenTagCache {
CacheFile = new FileInfo(destCachePath)
};
using (var stream = destInfo.OpenCacheRead())
destInfo.Cache = new TagCache(stream);
// Do version detection
EngineVersion guessedVersion;
destInfo.Version = VersionDetection.DetectVersion(destInfo.Cache, out guessedVersion);
if (destInfo.Version == EngineVersion.Unknown)
{
Console.WriteLine("Unrecognized target version!");
return(true);
}
Console.WriteLine("- Detected version {0}", VersionDetection.GetVersionString(destInfo.Version));
if (_info.Version != EngineVersion.V11_1_498295_Live && destInfo.Version != EngineVersion.V1_106708_cert_ms23)
{
Console.Error.WriteLine("Conversion is only supported from 11.1.498295 Live to 1.106708 cert_ms23.");
return(true);
}
// Set up version-specific objects
destInfo.Serializer = new TagSerializer(destInfo.Version);
destInfo.Deserializer = new TagDeserializer(destInfo.Version);
StringIdResolverBase resolver;
if (VersionDetection.Compare(destInfo.Version, EngineVersion.V11_1_498295_Live) >= 0)
{
resolver = new V11_1_498295.StringIdResolver();
}
else
{
resolver = new V1_106708.StringIdResolver();
}
// Load stringIDs
Console.WriteLine("Loading the target string_ids.dat...");
var destStringIdsPath = Path.Combine(targetDir, "string_ids.dat");
destInfo.StringIdsFile = new FileInfo(destStringIdsPath);
using (var stream = destInfo.StringIdsFile.OpenRead())
destInfo.StringIds = new StringIdCache(stream, resolver);
// Load resources for the target build
Console.WriteLine("Loading target resources...");
var destResources = new ResourceDataManager();
destResources.LoadCachesFromDirectory(destInfo.CacheFile.DirectoryName);
// Load resources for our build
Console.WriteLine("Loading source resources...");
var srcResources = new ResourceDataManager();
srcResources.LoadCachesFromDirectory(_info.CacheFile.DirectoryName);
Console.WriteLine();
Console.WriteLine("CONVERTING FROM VERSION {0} TO {1}", VersionDetection.GetVersionString(_info.Version), VersionDetection.GetVersionString(destInfo.Version));
Console.WriteLine();
TagInstance resultTag;
using (Stream srcStream = _info.OpenCacheRead(), destStream = destInfo.OpenCacheReadWrite())
resultTag = ConvertTag(srcTag, _info, srcStream, srcResources, destInfo, destStream, destResources, tagMap);
Console.WriteLine();
Console.WriteLine("Repairing decal systems...");
FixDecalSystems(destInfo, resultTag.Index);
Console.WriteLine();
Console.WriteLine("Saving stringIDs...");
using (var stream = destInfo.StringIdsFile.Open(FileMode.Open, FileAccess.ReadWrite))
destInfo.StringIds.Save(stream);
Console.WriteLine("Writing {0}...", csvOutPath);
using (var stream = new StreamWriter(File.Open(csvOutPath, FileMode.Create, FileAccess.ReadWrite)))
tagMap.WriteCsv(stream);
// Uncomment this to add the new tag as a dependency to cfgt to make testing easier
/*using (var stream = destInfo.OpenCacheReadWrite())
* {
* destInfo.Cache.Tags[0].Dependencies.Add(resultTag.Index);
* destInfo.Cache.UpdateTag(stream, destInfo.Cache.Tags[0]);
* }*/
Console.WriteLine();
Console.WriteLine("All done! The converted tag is:");
TagPrinter.PrintTagShort(resultTag);
return(true);
}
private GeometryReference ConvertGeometry(GeometryReference geometry, OpenTagCache srcInfo, ResourceDataManager srcResources, OpenTagCache destInfo, ResourceDataManager destResources)
{
if (geometry == null || geometry.Resource == null || geometry.Resource.Index < 0)
{
return(geometry);
}
// The format changed starting with version 1.235640, so if both versions are on the same side then they can be converted normally
var srcCompare = VersionDetection.Compare(srcInfo.Version, EngineVersion.V1_235640_cert_ms25);
var destCompare = VersionDetection.Compare(destInfo.Version, EngineVersion.V1_235640_cert_ms25);
if ((srcCompare < 0 && destCompare < 0) || (srcCompare >= 0 && destCompare >= 0))
{
geometry.Resource = ConvertResource(geometry.Resource, srcInfo, srcResources, destInfo, destResources);
return(geometry);
}
Console.WriteLine("- Rebuilding geometry resource {0} in {1}...", geometry.Resource.Index, geometry.Resource.GetLocation());
using (MemoryStream inStream = new MemoryStream(), outStream = new MemoryStream())
{
// First extract the model data
srcResources.Extract(geometry.Resource, inStream);
// Now open source and destination vertex streams
inStream.Position = 0;
var inVertexStream = VertexStreamFactory.Create(srcInfo.Version, inStream);
var outVertexStream = VertexStreamFactory.Create(destInfo.Version, outStream);
// Deserialize the definition data
var resourceContext = new ResourceSerializationContext(geometry.Resource);
var definition = srcInfo.Deserializer.Deserialize <RenderGeometryResourceDefinition>(resourceContext);
// Convert each vertex buffer
foreach (var buffer in definition.VertexBuffers)
{
ConvertVertexBuffer(buffer.Definition, inStream, inVertexStream, outStream, outVertexStream);
}
// Copy each index buffer over
foreach (var buffer in definition.IndexBuffers)
{
if (buffer.Definition.Data.Size == 0)
{
continue;
}
inStream.Position = buffer.Definition.Data.Address.Offset;
buffer.Definition.Data.Address = new ResourceAddress(ResourceAddressType.Resource, (int)outStream.Position);
var bufferData = new byte[buffer.Definition.Data.Size];
inStream.Read(bufferData, 0, bufferData.Length);
outStream.Write(bufferData, 0, bufferData.Length);
StreamUtil.Align(outStream, 4);
}
// Update the definition data
destInfo.Serializer.Serialize(resourceContext, definition);
// Now inject the new resource data
var newLocation = FixResourceLocation(geometry.Resource.GetLocation(), srcInfo.Version, destInfo.Version);
outStream.Position = 0;
destResources.Add(geometry.Resource, newLocation, outStream);
}
return(geometry);
}
