public override bool Execute(List <string> args)
{
if (args.Count < 2)
{
return(false);
}
var outputPath = args[0];
// Load each file and do version detection
var infos = new List <OpenTagCache>();
foreach (var path in args.Skip(1))
{
Console.WriteLine("Loading {0}...", path);
// Load the cache file
var info = new OpenTagCache {
CacheFile = new FileInfo(path)
};
using (var stream = info.OpenCacheRead())
info.Cache = new TagCache(stream);
// Do version detection, and don't accept the closest version
// because that might not work
DefinitionSet closestVersion;
info.Version = Definition.Detect(info.Cache, out closestVersion);
if (info.Version == DefinitionSet.Unknown)
{
Console.WriteLine("- Unrecognized version! Ignoring.");
continue;
}
info.Deserializer = new TagDeserializer(info.Version);
infos.Add(info);
}
var result = new TagVersionMap();
using (var baseStream = _info.OpenCacheRead())
{
// Get the scenario tags for this cache
Console.WriteLine("Finding base scenario tags...");
var baseScenarios = FindScenarios(_info, baseStream);
var baseVersion = _info.Version;
var baseTagData = new Dictionary <int, object>();
foreach (var scenario in baseScenarios)
{
baseTagData[scenario.Tag.Index] = scenario.Data;
}
// Now compare with each of the other caches
foreach (var info in infos)
{
using (var stream = info.OpenCacheRead())
{
Console.WriteLine("Finding scenario tags in {0}...", info.CacheFile.FullName);
// Get the scenario tags and connect them to the base tags
var scenarios = FindScenarios(info, stream);
var tagsToCompare = new Queue <QueuedTag>();
for (var i = 0; i < scenarios.Count; i++)
{
tagsToCompare.Enqueue(scenarios[i]);
if (i < baseScenarios.Count)
{
result.Add(baseVersion, baseScenarios[i].Tag.Index, info.Version, scenarios[i].Tag.Index);
}
}
// Process each tag in the queue, enqueuing all of its dependencies as well
while (tagsToCompare.Count > 0)
{
// Get the tag and its data
var tag = tagsToCompare.Dequeue();
TagPrinter.PrintTagShort(tag.Tag);
var data = tag.Data;
if (data == null)
{
// No data yet - deserialize it
var context = new TagSerializationContext(stream, info.Cache, info.StringIDs, tag.Tag);
var type = TagStructureTypes.FindByGroupTag(tag.Tag.Group.Tag);
data = info.Deserializer.Deserialize(context, type);
}
// Now get the data for the base tag
var baseTag = result.Translate(info.Version, tag.Tag.Index, baseVersion);
if (baseTag == -1 || _info.Cache.Tags[baseTag].Group.Tag != tag.Tag.Group.Tag)
{
continue;
}
object baseData;
if (!baseTagData.TryGetValue(baseTag, out baseData))
{
// No data yet - deserialize it
var context = new TagSerializationContext(baseStream, _info.Cache, _info.StringIDs, _info.Cache.Tags[baseTag]);
var type = TagStructureTypes.FindByGroupTag(tag.Tag.Group.Tag);
baseData = _info.Deserializer.Deserialize(context, type);
baseTagData[baseTag] = baseData;
}
// Compare the two blocks
CompareBlocks(baseData, baseVersion, data, info.Version, result, tagsToCompare);
}
}
}
}
// Write out the CSV
Console.WriteLine("Writing results...");
using (var writer = new StreamWriter(File.Open(outputPath, FileMode.Create, FileAccess.Write)))
result.WriteCsv(writer);
Console.WriteLine("Done!");
return(true);
}
private static void CompareBlocks(object dataA, CacheVersion versionA, object dataB, CacheVersion versionB, TagVersionMap result, Queue <QueuedTag> tagQueue)
{
if (dataA == null || dataB == null)
{
return;
}
if (dataA is CachedTagInstance)
{
// If the objects are tags, then we've found a match
var tagA = dataA as CachedTagInstance;
var tagB = dataB as CachedTagInstance;
if (tagA.Group.Tag != tagB.Group.Tag)
{
return;
}
if (tagA.IsInGroup("rmt2") || tagA.IsInGroup("rmdf") || tagA.IsInGroup("vtsh") || tagA.IsInGroup("pixl") || tagA.IsInGroup("rm ") || tagA.IsInGroup("bitm"))
{
return;
}
var translated = result.Translate(versionA, tagA.Index, versionB);
if (translated >= 0)
{
return;
}
result.Add(versionA, tagA.Index, versionB, tagB.Index);
tagQueue.Enqueue(new QueuedTag {
Tag = tagB
});
}
else if (dataA is IList) // Compare lists and arrays
{
var collectionA = dataA as IList;
var collectionB = dataB as IList;
if (collectionA.Count == 0 || collectionA[0].GetType().IsPrimitive)
{
return;
}
// If the sizes are different, we probably can't compare them
if (collectionA.Count != collectionB.Count)
{
return;
}
// Compare each element
for (var i = 0; i < collectionA.Count; i++)
{
var itemA = collectionA[i];
var itemB = collectionB[i];
CompareBlocks(itemA, versionA, itemB, versionB, result, tagQueue);
}
}
else if (dataA is TagStructure)
{
var tagFieldInfosA = TagStructure.GetTagFieldEnumerable(dataA.GetType(), versionA);
var tagFieldInfosB = TagStructure.GetTagFieldEnumerable(dataB.GetType(), versionB);
// The objects are structures
for (int a = 0, b = 0; a < tagFieldInfosA.Count && b < tagFieldInfosB.Count; a++, b++)
{
var tagFieldInfoA = tagFieldInfosA[a];
var tagFieldInfoB = tagFieldInfosB[b];
// Keep going on the left until the field is on the right
while (!CacheVersionDetection.IsBetween(versionB, tagFieldInfoA.Attribute.MinVersion, tagFieldInfoA.Attribute.MaxVersion))
{
if (++a < tagFieldInfosA.Count)
{
tagFieldInfoA = tagFieldInfosA[a];
}
else
{
return;
}
}
// Keep going on the right until the field is on the left
while (!CacheVersionDetection.IsBetween(versionA, tagFieldInfoB.Attribute.MinVersion, tagFieldInfoB.Attribute.MaxVersion))
{
if (++b < tagFieldInfosB.Count)
{
tagFieldInfoB = tagFieldInfosB[b];
}
else
{
return;
}
}
if (tagFieldInfoA.MetadataToken != tagFieldInfoB.MetadataToken)
{
throw new InvalidOperationException("WTF, left and right fields don't match!");
}
// Process the fields
var fieldDataA = tagFieldInfoA.GetValue(dataA);
var fieldDataB = tagFieldInfoB.GetValue(dataB);
CompareBlocks(fieldDataA, versionA, fieldDataB, versionB, result, tagQueue);
}
}
}
private object ConvertStructure(object data, Type type, OpenTagCache srcInfo, Stream srcStream, ResourceDataManager srcResources, OpenTagCache destInfo, Stream destStream, ResourceDataManager destResources, TagVersionMap tagMap)
{
// Convert each field
var enumerator = new TagFieldEnumerator(new TagStructureInfo(type, destInfo.Version));
while (enumerator.Next())
{
var oldValue = enumerator.Field.GetValue(data);
var newValue = Convert(oldValue, srcInfo, srcStream, srcResources, destInfo, destStream, destResources, tagMap);
enumerator.Field.SetValue(data, newValue);
}
// Perform fixups
FixObjectTypes(data, type, srcInfo);
FixShaders(data);
var scenario = data as Scenario;
if (scenario != null)
{
FixScenario(scenario);
}
return(data);
}
public override bool Execute(List <string> args)
{
if (args.Count != 4)
{
return(false);
}
var srcTag = ArgumentParser.ParseTagIndex(_info, args[0]);
if (srcTag == null)
{
return(false);
}
var csvPath = args[1];
var csvOutPath = args[2];
var targetDir = args[3];
// Load the CSV
Console.WriteLine("Reading {0}...", csvPath);
TagVersionMap tagMap;
using (var reader = new StreamReader(File.OpenRead(csvPath)))
tagMap = TagVersionMap.ParseTagVersionMap(reader);
// Load destination files
Console.WriteLine("Loading the target tags.dat...");
var destCachePath = Path.Combine(targetDir, "tags.dat");
var destInfo = new OpenTagCache {
CacheFile = new FileInfo(destCachePath)
};
using (var stream = destInfo.OpenCacheRead())
destInfo.Cache = new TagCache(stream);
// Do version detection
DefinitionSet guessedVersion;
destInfo.Version = Definition.Detect(destInfo.Cache, out guessedVersion);
if (destInfo.Version == DefinitionSet.Unknown)
{
Console.WriteLine("Unrecognized target version!");
return(true);
}
Console.WriteLine("- Detected version {0}", Definition.GetVersionString(destInfo.Version));
if (_info.Version != DefinitionSet.HaloOnline498295 && destInfo.Version != DefinitionSet.HaloOnline106708)
{
Console.Error.WriteLine("Conversion is only supported from 11.1.498295 Live to 1.106708 cert_ms23.");
return(true);
}
// Set up version-specific objects
destInfo.Serializer = new TagSerializer(destInfo.Version);
destInfo.Deserializer = new TagDeserializer(destInfo.Version);
// Load stringIDs
Console.WriteLine("Loading the target string_ids.dat...");
var resolver = StringIDResolverFactory.Create(destInfo.Version);
var destStringIDsPath = Path.Combine(targetDir, "string_ids.dat");
destInfo.StringIDsFile = new FileInfo(destStringIDsPath);
using (var stream = destInfo.StringIDsFile.OpenRead())
destInfo.StringIDs = new StringIDCache(stream, resolver);
// Load resources for the target build
Console.WriteLine("Loading target resources...");
var destResources = new ResourceDataManager();
destResources.LoadCachesFromDirectory(destInfo.CacheFile.DirectoryName);
// Load resources for our build
Console.WriteLine("Loading source resources...");
var srcResources = new ResourceDataManager();
srcResources.LoadCachesFromDirectory(_info.CacheFile.DirectoryName);
Console.WriteLine();
Console.WriteLine("CONVERTING FROM VERSION {0} TO {1}", Definition.GetVersionString(_info.Version), Definition.GetVersionString(destInfo.Version));
Console.WriteLine();
TagInstance resultTag;
using (Stream srcStream = _info.OpenCacheRead(), destStream = destInfo.OpenCacheReadWrite())
resultTag = ConvertTag(srcTag, _info, srcStream, srcResources, destInfo, destStream, destResources, tagMap);
Console.WriteLine();
Console.WriteLine("Repairing decal systems...");
FixDecalSystems(destInfo, resultTag.Index);
Console.WriteLine();
Console.WriteLine("Saving stringIDs...");
using (var stream = destInfo.StringIDsFile.Open(FileMode.Open, FileAccess.ReadWrite))
destInfo.StringIDs.Save(stream);
Console.WriteLine("Writing {0}...", csvOutPath);
using (var stream = new StreamWriter(File.Open(csvOutPath, FileMode.Create, FileAccess.ReadWrite)))
tagMap.WriteCsv(stream);
// Uncomment this to add the new tag as a dependency to cfgt to make testing easier
/*using (var stream = destInfo.OpenCacheReadWrite())
* {
* destInfo.Cache.Tags[0].Dependencies.Add(resultTag.Index);
* destInfo.Cache.UpdateTag(stream, destInfo.Cache.Tags[0]);
* }*/
Console.WriteLine();
Console.WriteLine("All done! The converted tag is:");
TagPrinter.PrintTagShort(resultTag);
return(true);
}
private object Convert(object data, GameCacheContextHaloOnline srcCacheContext, Stream srcStream, GameCacheContextHaloOnline destCacheContext, Stream destStream, TagVersionMap tagMap)
{
switch (data)
{
case StringId stringId:
return(ConvertStringID(stringId, srcCacheContext, destCacheContext));
case null:
case string _:
case ValueType _:
return(data);
case CachedTagHaloOnline CachedTagHaloOnline:
return(ConvertTag(CachedTagHaloOnline, srcCacheContext, srcStream, destCacheContext, destStream, tagMap));
case PageableResource pageableResource:
return(ConvertResource(pageableResource, srcCacheContext, destCacheContext));
case RenderGeometry renderGeometry:
return(ConvertGeometry(renderGeometry, srcCacheContext, destCacheContext));
case GameObjectType gameObjectType:
return(ConvertGameObjectType(gameObjectType, srcCacheContext, destCacheContext));
case ObjectTypeFlags objectTypeFlags:
return(ConvertObjectTypeFlags(objectTypeFlags, srcCacheContext, destCacheContext));
case ScenarioObjectType scenarioObjectType:
return(ConvertScenarioObjectType(scenarioObjectType, srcCacheContext, destCacheContext));
case TagStructure tagStructure:
return(ConvertStructure(tagStructure, srcCacheContext, srcStream, destCacheContext, destStream, tagMap));
case IList collection:
return(ConvertCollection(collection, srcCacheContext, srcStream, destCacheContext, destStream, tagMap));
}
return(data);
}
private object ConvertList(object list, Type type, OpenTagCache srcInfo, Stream srcStream, ResourceDataManager srcResources, OpenTagCache destInfo, Stream destStream, ResourceDataManager destResources, TagVersionMap tagMap)
{
if (type.GenericTypeArguments[0].IsPrimitive)
{
return(list);
}
var count = (int)type.GetProperty("Count").GetValue(list);
var getItem = type.GetMethod("get_Item");
var setItem = type.GetMethod("set_Item");
for (var i = 0; i < count; i++)
{
var oldValue = getItem.Invoke(list, new object[] { i });
var newValue = Convert(oldValue, srcInfo, srcStream, srcResources, destInfo, destStream, destResources, tagMap);
setItem.Invoke(list, new object[] { i, newValue });
}
return(list);
}
private TagInstance ConvertTag(TagInstance srcTag, OpenTagCache srcInfo, Stream srcStream, ResourceDataManager srcResources, OpenTagCache destInfo, Stream destStream, ResourceDataManager destResources, TagVersionMap tagMap)
{
TagPrinter.PrintTagShort(srcTag);
// Uncomment this to use 0x101F for all shaders
/*if (srcTag.IsClass("rm "))
* return destInfo.Cache.Tags[0x101F];*/
// Check if the tag is in the map, and just return the translated tag if so
var destIndex = tagMap.Translate(srcInfo.Version, srcTag.Index, destInfo.Version);
if (destIndex >= 0)
{
Console.WriteLine("- Using already-known index {0:X4}", destIndex);
return(destInfo.Cache.Tags[destIndex]);
}
// Deserialize the tag from the source cache
var structureType = TagStructureTypes.FindByGroupTag(srcTag.Group.Tag);
var srcContext = new TagSerializationContext(srcStream, srcInfo.Cache, srcInfo.StringIDs, srcTag);
var tagData = srcInfo.Deserializer.Deserialize(srcContext, structureType);
// Uncomment this to use 0x101F in place of shaders that need conversion
/*if (tagData is RenderMethod)
* {
* var rm = (RenderMethod)tagData;
* foreach (var prop in rm.ShaderProperties)
* {
* if (tagMap.Translate(srcInfo.Version, prop.Template.Index, destInfo.Version) < 0)
* return destInfo.Cache.Tags[0x101F];
* }
* }*/
// Allocate a new tag and create a mapping for it
var newTag = destInfo.Cache.AllocateTag(srcTag.Group);
tagMap.Add(srcInfo.Version, srcTag.Index, destInfo.Version, newTag.Index);
if (srcTag.IsInGroup("decs") || srcTag.IsInGroup("rmd "))
{
_isDecalShader = true;
}
// Convert it
tagData = Convert(tagData, srcInfo, srcStream, srcResources, destInfo, destStream, destResources, tagMap);
if (srcTag.IsInGroup("decs") || srcTag.IsInGroup("rmd "))
{
_isDecalShader = false;
}
// Re-serialize into the destination cache
var destContext = new TagSerializationContext(destStream, destInfo.Cache, destInfo.StringIDs, newTag);
destInfo.Serializer.Serialize(destContext, tagData);
return(newTag);
}
private static void CompareBlocks(object leftData, EngineVersion leftVersion, object rightData, EngineVersion rightVersion, TagVersionMap result, Queue<QueuedTag> tagQueue)
{
if (leftData == null || rightData == null)
return;
var type = leftData.GetType();
if (type == typeof(TagInstance))
{
// If the objects are tags, then we've found a match
var leftTag = (TagInstance)leftData;
var rightTag = (TagInstance)rightData;
if (leftTag.Group.Tag != rightTag.Group.Tag)
return;
if (leftTag.IsInGroup("rmt2") || leftTag.IsInGroup("rmdf") || leftTag.IsInGroup("vtsh") || leftTag.IsInGroup("pixl") || leftTag.IsInGroup("rm ") || leftTag.IsInGroup("bitm"))
return;
var translated = result.Translate(leftVersion, leftTag.Index, rightVersion);
if (translated >= 0)
return;
result.Add(leftVersion, leftTag.Index, rightVersion, rightTag.Index);
tagQueue.Enqueue(new QueuedTag { Tag = rightTag });
}
else if (type.IsArray)
{
if (type.GetElementType().IsPrimitive)
return;
// If the objects are arrays, then loop through each element
var leftArray = (Array)leftData;
var rightArray = (Array)rightData;
if (leftArray.Length != rightArray.Length)
return; // If the sizes are different, we probably can't compare them
for (var i = 0; i < leftArray.Length; i++)
CompareBlocks(leftArray.GetValue(i), leftVersion, rightArray.GetValue(i), rightVersion, result, tagQueue);
}
else if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(List<>))
{
if (type.GenericTypeArguments[0].IsPrimitive)
return;
// If the objects are lists, then loop through each element
var countProperty = type.GetProperty("Count");
var leftCount = (int)countProperty.GetValue(leftData);
var rightCount = (int)countProperty.GetValue(rightData);
if (leftCount != rightCount)
return; // If the sizes are different, we probably can't compare them
var getItem = type.GetMethod("get_Item");
for (var i = 0; i < leftCount; i++)
{
var leftItem = getItem.Invoke(leftData, new object[] { i });
var rightItem = getItem.Invoke(rightData, new object[] { i });
CompareBlocks(leftItem, leftVersion, rightItem, rightVersion, result, tagQueue);
}
}
else if (type.GetCustomAttributes(typeof(TagStructureAttribute), false).Length > 0)
{
// The objects are structures
var left = new TagFieldEnumerator(new TagStructureInfo(leftData.GetType(), leftVersion));
var right = new TagFieldEnumerator(new TagStructureInfo(rightData.GetType(), rightVersion));
while (left.Next() && right.Next())
{
// Keep going on the left until the field is on the right
while (!VersionDetection.IsBetween(rightVersion, left.MinVersion, left.MaxVersion))
{
if (!left.Next())
return;
}
// Keep going on the right until the field is on the left
while (!VersionDetection.IsBetween(leftVersion, right.MinVersion, right.MaxVersion))
{
if (!right.Next())
return;
}
if (left.Field.MetadataToken != right.Field.MetadataToken)
throw new InvalidOperationException("WTF, left and right fields don't match!");
// Process the fields
var leftFieldData = left.Field.GetValue(leftData);
var rightFieldData = right.Field.GetValue(rightData);
CompareBlocks(leftFieldData, leftVersion, rightFieldData, rightVersion, result, tagQueue);
}
}
}
private T ConvertStructure <T>(T data, GameCacheContextHaloOnline srcCacheContext, Stream srcStream, GameCacheContextHaloOnline destCacheContext, Stream destStream, TagVersionMap tagMap) where T : TagStructure
{
// Convert each field
foreach (var tagFieldInfo in TagStructure.GetTagFieldEnumerable(data.GetType(), destCacheContext.Version))
{
var oldValue = tagFieldInfo.GetValue(data);
var newValue = Convert(oldValue, srcCacheContext, srcStream, destCacheContext, destStream, tagMap);
tagFieldInfo.SetValue(data, newValue);
}
// Perform fixups
FixShaders(data);
if (data is Scenario scenario)
{
FixScenario(scenario);
}
return(data);
}
public override object Execute(List <string> args)
{
if (args.Count != 4)
{
return(false);
}
if (!CacheContext.TryGetTag(args[0], out var srcTag))
{
return(false);
}
var csvPath = args[1];
var csvOutPath = args[2];
var targetDir = args[3];
// Load the CSV
Console.WriteLine("Reading {0}...", csvPath);
TagVersionMap tagMap;
using (var reader = new StreamReader(File.Exists(csvPath) ? File.OpenRead(csvPath) : File.Create(csvPath)))
tagMap = TagVersionMap.ParseTagVersionMap(reader);
// Load destination cache files
var destCacheContext = new GameCacheContextHaloOnline(new DirectoryInfo(targetDir));
using (var stream = destCacheContext.OpenTagCacheRead())
destCacheContext.TagCache = new TagCache(stream, destCacheContext.LoadTagNames());
Console.WriteLine();
Console.WriteLine("CONVERTING FROM VERSION {0} TO {1}", CacheVersionDetection.GetBuildName(CacheContext.Version), CacheVersionDetection.GetBuildName(destCacheContext.Version));
Console.WriteLine();
CachedTagHaloOnline resultTag;
using (Stream srcStream = CacheContext.TagCache.OpenTagCacheRead(), destStream = destCacheContext.OpenTagCacheReadWrite())
resultTag = ConvertTag(srcTag, CacheContext, srcStream, destCacheContext, destStream, tagMap);
Console.WriteLine();
Console.WriteLine("Repairing decal systems...");
if (CacheContext.Version != destCacheContext.Version)
{
FixDecalSystems(destCacheContext, resultTag.Index);
}
Console.WriteLine();
Console.WriteLine("Saving stringIDs...");
using (var stream = destCacheContext.OpenStringIdCacheReadWrite())
destCacheContext.StringIdCache.Save(stream);
Console.WriteLine("Writing {0}...", csvOutPath);
using (var stream = new StreamWriter(File.Open(csvOutPath, FileMode.Create, FileAccess.ReadWrite)))
tagMap.WriteCsv(stream);
// Uncomment this to add the new tag as a dependency to cfgt to make testing easier
/*using (var stream = destCacheContext.OpenCacheReadWrite())
* {
* destCacheContext.Cache.Tags[0].Dependencies.Add(resultTag.Index);
* destCacheContext.Cache.UpdateTag(stream, destCacheContext.Cache.Tags[0]);
* }*/
Console.WriteLine();
Console.WriteLine("All done! The converted tag is:");
TagPrinter.PrintTagShort(resultTag);
destCacheContext.SaveTagNames();
return(true);
}
private IList ConvertCollection(IList collection, GameCacheContextHaloOnline srcCacheContext, Stream srcStream, GameCacheContextHaloOnline destCacheContext, Stream destStream, TagVersionMap tagMap)
{
if (collection.Count == 0 || collection[0].GetType().IsPrimitive)
{
return(collection);
}
for (var i = 0; i < collection.Count; i++)
{
var oldValue = collection[i];
var newValue = Convert(oldValue, srcCacheContext, srcStream, destCacheContext, destStream, tagMap);
collection[i] = newValue;
}
return(collection);
}
private object ConvertList(object list, Type type, GameCacheContextHaloOnline srcCacheContext, Stream srcStream, GameCacheContextHaloOnline destCacheContext, Stream destStream, TagVersionMap tagMap)
{
if (type.GenericTypeArguments[0].IsPrimitive)
{
return(list);
}
var count = (int)type.GetProperty("Count").GetValue(list);
var getItem = type.GetMethod("get_Item");
var setItem = type.GetMethod("set_Item");
for (var i = 0; i < count; i++)
{
var oldValue = getItem.Invoke(list, new object[] { i });
var newValue = Convert(oldValue, srcCacheContext, srcStream, destCacheContext, destStream, tagMap);
setItem.Invoke(list, new object[] { i, newValue });
}
return(list);
}
private Array ConvertArray(Array array, GameCacheContextHaloOnline srcCacheContext, Stream srcStream, GameCacheContextHaloOnline destCacheContext, Stream destStream, TagVersionMap tagMap)
{
if (array.GetType().GetElementType().IsPrimitive)
{
return(array);
}
for (var i = 0; i < array.Length; i++)
{
var oldValue = array.GetValue(i);
var newValue = Convert(oldValue, srcCacheContext, srcStream, destCacheContext, destStream, tagMap);
array.SetValue(newValue, i);
}
return(array);
}
public override object Execute(List <string> args)
{
if (args.Count < 2)
{
return(false);
}
var outputPath = args[0];
// Load each file and do version detection
var infos = new List <HaloOnlineCacheContext>();
foreach (var path in args.Skip(1))
{
Console.WriteLine("Loading {0}...", path);
// Load the cache file
var cacheContext = new HaloOnlineCacheContext(new FileInfo(path).Directory);
infos.Add(cacheContext);
}
var result = new TagVersionMap();
using (var baseStream = CacheContext.OpenTagCacheRead())
{
// Get the scenario tags for this cache
Console.WriteLine("Finding base scenario tags...");
var baseScenarios = FindScenarios(CacheContext, baseStream);
var baseVersion = CacheContext.Version;
var baseTagData = new Dictionary <int, object>();
foreach (var scenario in baseScenarios)
{
baseTagData[scenario.Tag.Index] = scenario.Data;
}
// Now compare with each of the other caches
foreach (var info in infos)
{
using (var stream = info.OpenTagCacheRead())
{
Console.WriteLine("Finding scenario tags in {0}...", info.TagCacheFile.FullName);
// Get the scenario tags and connect them to the base tags
var scenarios = FindScenarios(info, stream);
var tagsToCompare = new Queue <QueuedTag>();
for (var i = 0; i < scenarios.Count; i++)
{
tagsToCompare.Enqueue(scenarios[i]);
if (i < baseScenarios.Count)
{
result.Add(baseVersion, baseScenarios[i].Tag.Index, info.Version, scenarios[i].Tag.Index);
}
}
// Process each tag in the queue, enqueuing all of its dependencies as well
while (tagsToCompare.Count > 0)
{
// Get the tag and its data
var tag = tagsToCompare.Dequeue();
TagPrinter.PrintTagShort(tag.Tag);
var data = tag.Data;
if (data == null)
{
// No data yet - deserialize it
var context = new TagSerializationContext(stream, info, tag.Tag);
var type = TagDefinition.Find(tag.Tag.Group.Tag);
data = info.Deserializer.Deserialize(context, type);
}
// Now get the data for the base tag
var baseTag = result.Translate(info.Version, tag.Tag.Index, baseVersion);
if (baseTag == -1 || CacheContext.TagCache.Index[baseTag].Group.Tag != tag.Tag.Group.Tag)
{
continue;
}
if (!baseTagData.TryGetValue(baseTag, out object baseData))
{
// No data yet - deserialize it
var context = new TagSerializationContext(baseStream, CacheContext, CacheContext.TagCache.Index[baseTag]);
var type = TagDefinition.Find(tag.Tag.Group.Tag);
baseData = CacheContext.Deserializer.Deserialize(context, type);
baseTagData[baseTag] = baseData;
}
// Compare the two blocks
CompareBlocks(baseData, baseVersion, data, info.Version, result, tagsToCompare);
}
}
}
}
// Write out the CSV
Console.WriteLine("Writing results...");
using (var writer = new StreamWriter(File.Open(outputPath, FileMode.Create, FileAccess.Write)))
result.WriteCsv(writer);
Console.WriteLine("Done!");
return(true);
}
private object Convert(object data, OpenTagCache srcInfo, Stream srcStream, ResourceDataManager srcResources, OpenTagCache destInfo, Stream destStream, ResourceDataManager destResources, TagVersionMap tagMap)
{
if (data == null)
{
return(null);
}
var type = data.GetType();
if (type.IsPrimitive)
{
return(data);
}
if (type == typeof(StringID))
{
return(ConvertStringID((StringID)data, srcInfo, destInfo));
}
if (type == typeof(TagInstance))
{
return(ConvertTag((TagInstance)data, srcInfo, srcStream, srcResources, destInfo, destStream, destResources, tagMap));
}
if (type == typeof(ResourceReference))
{
return(ConvertResource((ResourceReference)data, srcInfo, srcResources, destInfo, destResources));
}
if (type == typeof(GeometryReference))
{
return(ConvertGeometry((GeometryReference)data, srcInfo, srcResources, destInfo, destResources));
}
if (type.IsArray)
{
return(ConvertArray((Array)data, srcInfo, srcStream, srcResources, destInfo, destStream, destResources, tagMap));
}
if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(List <>))
{
return(ConvertList(data, type, srcInfo, srcStream, srcResources, destInfo, destStream, destResources, tagMap));
}
if (type.GetCustomAttributes(typeof(TagStructureAttribute), false).Length > 0)
{
return(ConvertStructure(data, type, srcInfo, srcStream, srcResources, destInfo, destStream, destResources, tagMap));
}
return(data);
}
private static void CompareBlocks(object leftData, EngineVersion leftVersion, object rightData, EngineVersion rightVersion, TagVersionMap result, Queue <QueuedTag> tagQueue)
{
if (leftData == null || rightData == null)
{
return;
}
var type = leftData.GetType();
if (type == typeof(TagInstance))
{
// If the objects are tags, then we've found a match
var leftTag = (TagInstance)leftData;
var rightTag = (TagInstance)rightData;
if (leftTag.Group.Tag != rightTag.Group.Tag)
{
return;
}
if (leftTag.IsInGroup("rmt2") || leftTag.IsInGroup("rmdf") || leftTag.IsInGroup("vtsh") || leftTag.IsInGroup("pixl") || leftTag.IsInGroup("rm ") || leftTag.IsInGroup("bitm"))
{
return;
}
var translated = result.Translate(leftVersion, leftTag.Index, rightVersion);
if (translated >= 0)
{
return;
}
result.Add(leftVersion, leftTag.Index, rightVersion, rightTag.Index);
tagQueue.Enqueue(new QueuedTag {
Tag = rightTag
});
}
else if (type.IsArray)
{
if (type.GetElementType().IsPrimitive)
{
return;
}
// If the objects are arrays, then loop through each element
var leftArray = (Array)leftData;
var rightArray = (Array)rightData;
if (leftArray.Length != rightArray.Length)
{
return; // If the sizes are different, we probably can't compare them
}
for (var i = 0; i < leftArray.Length; i++)
{
CompareBlocks(leftArray.GetValue(i), leftVersion, rightArray.GetValue(i), rightVersion, result, tagQueue);
}
}
else if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(List <>))
{
if (type.GenericTypeArguments[0].IsPrimitive)
{
return;
}
// If the objects are lists, then loop through each element
var countProperty = type.GetProperty("Count");
var leftCount = (int)countProperty.GetValue(leftData);
var rightCount = (int)countProperty.GetValue(rightData);
if (leftCount != rightCount)
{
return; // If the sizes are different, we probably can't compare them
}
var getItem = type.GetMethod("get_Item");
for (var i = 0; i < leftCount; i++)
{
var leftItem = getItem.Invoke(leftData, new object[] { i });
var rightItem = getItem.Invoke(rightData, new object[] { i });
CompareBlocks(leftItem, leftVersion, rightItem, rightVersion, result, tagQueue);
}
}
else if (type.GetCustomAttributes(typeof(TagStructureAttribute), false).Length > 0)
{
// The objects are structures
var left = new TagFieldEnumerator(new TagStructureInfo(leftData.GetType(), leftVersion));
var right = new TagFieldEnumerator(new TagStructureInfo(rightData.GetType(), rightVersion));
while (left.Next() && right.Next())
{
// Keep going on the left until the field is on the right
while (!VersionDetection.IsBetween(rightVersion, left.MinVersion, left.MaxVersion))
{
if (!left.Next())
{
return;
}
}
// Keep going on the right until the field is on the left
while (!VersionDetection.IsBetween(leftVersion, right.MinVersion, right.MaxVersion))
{
if (!right.Next())
{
return;
}
}
if (left.Field.MetadataToken != right.Field.MetadataToken)
{
throw new InvalidOperationException("WTF, left and right fields don't match!");
}
// Process the fields
var leftFieldData = left.Field.GetValue(leftData);
var rightFieldData = right.Field.GetValue(rightData);
CompareBlocks(leftFieldData, leftVersion, rightFieldData, rightVersion, result, tagQueue);
}
}
}
private Array ConvertArray(Array array, OpenTagCache srcInfo, Stream srcStream, ResourceDataManager srcResources, OpenTagCache destInfo, Stream destStream, ResourceDataManager destResources, TagVersionMap tagMap)
{
if (array.GetType().GetElementType().IsPrimitive)
{
return(array);
}
for (var i = 0; i < array.Length; i++)
{
var oldValue = array.GetValue(i);
var newValue = Convert(oldValue, srcInfo, srcStream, srcResources, destInfo, destStream, destResources, tagMap);
array.SetValue(newValue, i);
}
return(array);
}
public override bool Execute(List<string> args)
{
if (args.Count < 2)
return false;
var outputPath = args[0];
// Load each file and do version detection
var infos = new List<OpenTagCache>();
foreach (var path in args.Skip(1))
{
Console.WriteLine("Loading {0}...", path);
// Load the cache file
var info = new OpenTagCache { CacheFile = new FileInfo(path) };
using (var stream = info.OpenCacheRead())
info.Cache = new TagCache(stream);
// Do version detection, and don't accept the closest version
// because that might not work
EngineVersion closestVersion;
info.Version = VersionDetection.DetectVersion(info.Cache, out closestVersion);
if (info.Version == EngineVersion.Unknown)
{
Console.WriteLine("- Unrecognized version! Ignoring.");
continue;
}
info.Deserializer = new TagDeserializer(info.Version);
infos.Add(info);
}
var result = new TagVersionMap();
using (var baseStream = _info.OpenCacheRead())
{
// Get the scenario tags for this cache
Console.WriteLine("Finding base scenario tags...");
var baseScenarios = FindScenarios(_info, baseStream);
var baseVersion = _info.Version;
var baseTagData = new Dictionary<int, object>();
foreach (var scenario in baseScenarios)
baseTagData[scenario.Tag.Index] = scenario.Data;
// Now compare with each of the other caches
foreach (var info in infos)
{
using (var stream = info.OpenCacheRead())
{
Console.WriteLine("Finding scenario tags in {0}...", info.CacheFile.FullName);
// Get the scenario tags and connect them to the base tags
var scenarios = FindScenarios(info, stream);
var tagsToCompare = new Queue<QueuedTag>();
for (var i = 0; i < scenarios.Count; i++)
{
tagsToCompare.Enqueue(scenarios[i]);
result.Add(baseVersion, baseScenarios[i].Tag.Index, info.Version, scenarios[i].Tag.Index);
}
// Process each tag in the queue, enqueuing all of its dependencies as well
while (tagsToCompare.Count > 0)
{
// Get the tag and its data
var tag = tagsToCompare.Dequeue();
TagPrinter.PrintTagShort(tag.Tag);
var data = tag.Data;
if (data == null)
{
// No data yet - deserialize it
var context = new TagSerializationContext(stream, info.Cache, info.StringIds, tag.Tag);
var type = TagStructureTypes.FindByGroupTag(tag.Tag.Group.Tag);
data = info.Deserializer.Deserialize(context, type);
}
// Now get the data for the base tag
var baseTag = result.Translate(info.Version, tag.Tag.Index, baseVersion);
if (baseTag == -1 || _info.Cache.Tags[baseTag].Group.Tag != tag.Tag.Group.Tag)
continue;
object baseData;
if (!baseTagData.TryGetValue(baseTag, out baseData))
{
// No data yet - deserialize it
var context = new TagSerializationContext(baseStream, _info.Cache, _info.StringIds, _info.Cache.Tags[baseTag]);
var type = TagStructureTypes.FindByGroupTag(tag.Tag.Group.Tag);
baseData = _info.Deserializer.Deserialize(context, type);
baseTagData[baseTag] = baseData;
}
// Compare the two blocks
CompareBlocks(baseData, baseVersion, data, info.Version, result, tagsToCompare);
}
}
}
}
// Write out the CSV
Console.WriteLine("Writing results...");
using (var writer = new StreamWriter(File.Open(outputPath, FileMode.Create, FileAccess.Write)))
result.WriteCsv(writer);
Console.WriteLine("Done!");
return true;
}
private CachedTagHaloOnline ConvertTag(CachedTagHaloOnline srcTag, GameCacheContextHaloOnline srcCacheContext, Stream srcStream, GameCacheContextHaloOnline destCacheContext, Stream destStream, TagVersionMap tagMap)
{
TagPrinter.PrintTagShort(srcTag);
// Uncomment this to use 0x101F for all shaders
/*if (srcTag.IsClass("rm "))
* return destCacheContext.Cache.Tags[0x101F];*/
// Check if the tag is in the map, and just return the translated tag if so
var destIndex = tagMap.Translate(srcCacheContext.Version, srcTag.Index, destCacheContext.Version);
if (destIndex >= 0)
{
Console.WriteLine("- Using already-known index {0:X4}", destIndex);
return(destCacheContext.TagCache.Index[destIndex]);
}
// Deserialize the tag from the source cache
var tagData = srcCacheContext.Deserialize(srcStream, srcTag);
// Uncomment this to use 0x101F in place of shaders that need conversion
/*if (tagData is RenderMethod)
* {
* var rm = (RenderMethod)tagData;
* foreach (var prop in rm.ShaderProperties)
* {
* if (tagMap.Translate(srcCacheContext.Version, prop.Template.Index, destCacheContext.Version) < 0)
* return destCacheContext.Cache.Tags[0x101F];
* }
* }*/
// Allocate a new tag and create a mapping for it
CachedTagHaloOnline instance = null;
if (srcCacheContext.Version != destCacheContext.Version)
{
for (var i = 0; i < destCacheContext.TagCache.Index.Count; i++)
{
if (destCacheContext.TagCache.Index[i] == null)
{
destCacheContext.TagCache.Index[i] = instance = new CachedTagHaloOnline(i, TagGroup.Instances[srcTag.Group.Tag]);
break;
}
}
}
else
{
if (destCacheContext.TagCache.Index[srcTag.Index] != null)
{
if (destCacheContext.TagCache.Index[srcTag.Index].IsInGroup(srcTag.Group))
{
return(destCacheContext.TagCache.Index[srcTag.Index]);
}
}
else
{
destCacheContext.TagCache.Index[srcTag.Index] = instance = new CachedTagHaloOnline(srcTag.Index, TagGroup.Instances[srcTag.Group.Tag], srcTag.Name);
}
}
if (instance == null)
{
instance = destCacheContext.TagCache.AllocateTag(srcTag.Group);
}
tagMap.Add(srcCacheContext.Version, srcTag.Index, destCacheContext.Version, instance.Index);
if (srcTag.IsInGroup("decs") || srcTag.IsInGroup("rmd "))
{
IsDecalShader = true;
}
// Convert it
tagData = Convert(tagData, srcCacheContext, srcStream, destCacheContext, destStream, tagMap);
if (srcTag.IsInGroup("decs") || srcTag.IsInGroup("rmd "))
{
IsDecalShader = false;
}
// Re-serialize into the destination cache
destCacheContext.Serialize(destStream, instance, tagData);
return(instance);
}
