private TagInstance ConvertTag(TagInstance srcTag, OpenTagCache srcInfo, Stream srcStream, ResourceDataManager srcResources, OpenTagCache destInfo, Stream destStream, ResourceDataManager destResources, TagVersionMap tagMap) { TagPrinter.PrintTagShort(srcTag); // Uncomment this to use 0x101F for all shaders /*if (srcTag.IsClass("rm ")) * return destInfo.Cache.Tags[0x101F];*/ // Check if the tag is in the map, and just return the translated tag if so var destIndex = tagMap.Translate(srcInfo.Version, srcTag.Index, destInfo.Version); if (destIndex >= 0) { Console.WriteLine("- Using already-known index {0:X4}", destIndex); return(destInfo.Cache.Tags[destIndex]); } // Deserialize the tag from the source cache var structureType = TagStructureTypes.FindByGroupTag(srcTag.Group.Tag); var srcContext = new TagSerializationContext(srcStream, srcInfo.Cache, srcInfo.StringIDs, srcTag); var tagData = srcInfo.Deserializer.Deserialize(srcContext, structureType); // Uncomment this to use 0x101F in place of shaders that need conversion /*if (tagData is RenderMethod) * { * var rm = (RenderMethod)tagData; * foreach (var prop in rm.ShaderProperties) * { * if (tagMap.Translate(srcInfo.Version, prop.Template.Index, destInfo.Version) < 0) * return destInfo.Cache.Tags[0x101F]; * } * }*/ // Allocate a new tag and create a mapping for it var newTag = destInfo.Cache.AllocateTag(srcTag.Group); tagMap.Add(srcInfo.Version, srcTag.Index, destInfo.Version, newTag.Index); if (srcTag.IsInGroup("decs") || srcTag.IsInGroup("rmd ")) { _isDecalShader = true; } // Convert it tagData = Convert(tagData, srcInfo, srcStream, srcResources, destInfo, destStream, destResources, tagMap); if (srcTag.IsInGroup("decs") || srcTag.IsInGroup("rmd ")) { _isDecalShader = false; } // Re-serialize into the destination cache var destContext = new TagSerializationContext(destStream, destInfo.Cache, destInfo.StringIDs, newTag); destInfo.Serializer.Serialize(destContext, tagData); return(newTag); }
public override bool Execute(List<string> args) { if (args.Count < 2) return false; var outputPath = args[0]; // Load each file and do version detection var infos = new List<OpenTagCache>(); foreach (var path in args.Skip(1)) { Console.WriteLine("Loading {0}...", path); // Load the cache file var info = new OpenTagCache { CacheFile = new FileInfo(path) }; using (var stream = info.OpenCacheRead()) info.Cache = new TagCache(stream); // Do version detection, and don't accept the closest version // because that might not work EngineVersion closestVersion; info.Version = VersionDetection.DetectVersion(info.Cache, out closestVersion); if (info.Version == EngineVersion.Unknown) { Console.WriteLine("- Unrecognized version! Ignoring."); continue; } info.Deserializer = new TagDeserializer(info.Version); infos.Add(info); } var result = new TagVersionMap(); using (var baseStream = _info.OpenCacheRead()) { // Get the scenario tags for this cache Console.WriteLine("Finding base scenario tags..."); var baseScenarios = FindScenarios(_info, baseStream); var baseVersion = _info.Version; var baseTagData = new Dictionary<int, object>(); foreach (var scenario in baseScenarios) baseTagData[scenario.Tag.Index] = scenario.Data; // Now compare with each of the other caches foreach (var info in infos) { using (var stream = info.OpenCacheRead()) { Console.WriteLine("Finding scenario tags in {0}...", info.CacheFile.FullName); // Get the scenario tags and connect them to the base tags var scenarios = FindScenarios(info, stream); var tagsToCompare = new Queue<QueuedTag>(); for (var i = 0; i < scenarios.Count; i++) { tagsToCompare.Enqueue(scenarios[i]); result.Add(baseVersion, baseScenarios[i].Tag.Index, info.Version, scenarios[i].Tag.Index); } // Process each tag in the queue, enqueuing all of its dependencies as well while (tagsToCompare.Count > 0) { // Get the tag and its data var tag = tagsToCompare.Dequeue(); TagPrinter.PrintTagShort(tag.Tag); var data = tag.Data; if (data == null) { // No data yet - deserialize it var context = new TagSerializationContext(stream, info.Cache, info.StringIds, tag.Tag); var type = TagStructureTypes.FindByGroupTag(tag.Tag.Group.Tag); data = info.Deserializer.Deserialize(context, type); } // Now get the data for the base tag var baseTag = result.Translate(info.Version, tag.Tag.Index, baseVersion); if (baseTag == -1 || _info.Cache.Tags[baseTag].Group.Tag != tag.Tag.Group.Tag) continue; object baseData; if (!baseTagData.TryGetValue(baseTag, out baseData)) { // No data yet - deserialize it var context = new TagSerializationContext(baseStream, _info.Cache, _info.StringIds, _info.Cache.Tags[baseTag]); var type = TagStructureTypes.FindByGroupTag(tag.Tag.Group.Tag); baseData = _info.Deserializer.Deserialize(context, type); baseTagData[baseTag] = baseData; } // Compare the two blocks CompareBlocks(baseData, baseVersion, data, info.Version, result, tagsToCompare); } } } } // Write out the CSV Console.WriteLine("Writing results..."); using (var writer = new StreamWriter(File.Open(outputPath, FileMode.Create, FileAccess.Write))) result.WriteCsv(writer); Console.WriteLine("Done!"); return true; }
private static void CompareBlocks(object leftData, EngineVersion leftVersion, object rightData, EngineVersion rightVersion, TagVersionMap result, Queue<QueuedTag> tagQueue) { if (leftData == null || rightData == null) return; var type = leftData.GetType(); if (type == typeof(TagInstance)) { // If the objects are tags, then we've found a match var leftTag = (TagInstance)leftData; var rightTag = (TagInstance)rightData; if (leftTag.Group.Tag != rightTag.Group.Tag) return; if (leftTag.IsInGroup("rmt2") || leftTag.IsInGroup("rmdf") || leftTag.IsInGroup("vtsh") || leftTag.IsInGroup("pixl") || leftTag.IsInGroup("rm ") || leftTag.IsInGroup("bitm")) return; var translated = result.Translate(leftVersion, leftTag.Index, rightVersion); if (translated >= 0) return; result.Add(leftVersion, leftTag.Index, rightVersion, rightTag.Index); tagQueue.Enqueue(new QueuedTag { Tag = rightTag }); } else if (type.IsArray) { if (type.GetElementType().IsPrimitive) return; // If the objects are arrays, then loop through each element var leftArray = (Array)leftData; var rightArray = (Array)rightData; if (leftArray.Length != rightArray.Length) return; // If the sizes are different, we probably can't compare them for (var i = 0; i < leftArray.Length; i++) CompareBlocks(leftArray.GetValue(i), leftVersion, rightArray.GetValue(i), rightVersion, result, tagQueue); } else if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(List<>)) { if (type.GenericTypeArguments[0].IsPrimitive) return; // If the objects are lists, then loop through each element var countProperty = type.GetProperty("Count"); var leftCount = (int)countProperty.GetValue(leftData); var rightCount = (int)countProperty.GetValue(rightData); if (leftCount != rightCount) return; // If the sizes are different, we probably can't compare them var getItem = type.GetMethod("get_Item"); for (var i = 0; i < leftCount; i++) { var leftItem = getItem.Invoke(leftData, new object[] { i }); var rightItem = getItem.Invoke(rightData, new object[] { i }); CompareBlocks(leftItem, leftVersion, rightItem, rightVersion, result, tagQueue); } } else if (type.GetCustomAttributes(typeof(TagStructureAttribute), false).Length > 0) { // The objects are structures var left = new TagFieldEnumerator(new TagStructureInfo(leftData.GetType(), leftVersion)); var right = new TagFieldEnumerator(new TagStructureInfo(rightData.GetType(), rightVersion)); while (left.Next() && right.Next()) { // Keep going on the left until the field is on the right while (!VersionDetection.IsBetween(rightVersion, left.MinVersion, left.MaxVersion)) { if (!left.Next()) return; } // Keep going on the right until the field is on the left while (!VersionDetection.IsBetween(leftVersion, right.MinVersion, right.MaxVersion)) { if (!right.Next()) return; } if (left.Field.MetadataToken != right.Field.MetadataToken) throw new InvalidOperationException("WTF, left and right fields don't match!"); // Process the fields var leftFieldData = left.Field.GetValue(leftData); var rightFieldData = right.Field.GetValue(rightData); CompareBlocks(leftFieldData, leftVersion, rightFieldData, rightVersion, result, tagQueue); } } }
public override bool Execute(List <string> args) { if (args.Count < 2) { return(false); } var outputPath = args[0]; // Load each file and do version detection var infos = new List <OpenTagCache>(); foreach (var path in args.Skip(1)) { Console.WriteLine("Loading {0}...", path); // Load the cache file var info = new OpenTagCache { CacheFile = new FileInfo(path) }; using (var stream = info.OpenCacheRead()) info.Cache = new TagCache(stream); // Do version detection, and don't accept the closest version // because that might not work EngineVersion closestVersion; info.Version = VersionDetection.DetectVersion(info.Cache, out closestVersion); if (info.Version == EngineVersion.Unknown) { Console.WriteLine("- Unrecognized version! Ignoring."); continue; } info.Deserializer = new TagDeserializer(info.Version); infos.Add(info); } var result = new TagVersionMap(); using (var baseStream = _info.OpenCacheRead()) { // Get the scenario tags for this cache Console.WriteLine("Finding base scenario tags..."); var baseScenarios = FindScenarios(_info, baseStream); var baseVersion = _info.Version; var baseTagData = new Dictionary <int, object>(); foreach (var scenario in baseScenarios) { baseTagData[scenario.Tag.Index] = scenario.Data; } // Now compare with each of the other caches foreach (var info in infos) { using (var stream = info.OpenCacheRead()) { Console.WriteLine("Finding scenario tags in {0}...", info.CacheFile.FullName); // Get the scenario tags and connect them to the base tags var scenarios = FindScenarios(info, stream); var tagsToCompare = new Queue <QueuedTag>(); for (var i = 0; i < scenarios.Count; i++) { tagsToCompare.Enqueue(scenarios[i]); if (i < baseScenarios.Count) { result.Add(baseVersion, baseScenarios[i].Tag.Index, info.Version, scenarios[i].Tag.Index); } } // Process each tag in the queue, enqueuing all of its dependencies as well while (tagsToCompare.Count > 0) { // Get the tag and its data var tag = tagsToCompare.Dequeue(); TagPrinter.PrintTagShort(tag.Tag); var data = tag.Data; if (data == null) { // No data yet - deserialize it var context = new TagSerializationContext(stream, info.Cache, info.StringIds, tag.Tag); var type = TagStructureTypes.FindByGroupTag(tag.Tag.Group.Tag); data = info.Deserializer.Deserialize(context, type); } // Now get the data for the base tag var baseTag = result.Translate(info.Version, tag.Tag.Index, baseVersion); if (baseTag == -1 || _info.Cache.Tags[baseTag].Group.Tag != tag.Tag.Group.Tag) { continue; } object baseData; if (!baseTagData.TryGetValue(baseTag, out baseData)) { // No data yet - deserialize it var context = new TagSerializationContext(baseStream, _info.Cache, _info.StringIds, _info.Cache.Tags[baseTag]); var type = TagStructureTypes.FindByGroupTag(tag.Tag.Group.Tag); baseData = _info.Deserializer.Deserialize(context, type); baseTagData[baseTag] = baseData; } // Compare the two blocks CompareBlocks(baseData, baseVersion, data, info.Version, result, tagsToCompare); } } } } // Write out the CSV Console.WriteLine("Writing results..."); using (var writer = new StreamWriter(File.Open(outputPath, FileMode.Create, FileAccess.Write))) result.WriteCsv(writer); Console.WriteLine("Done!"); return(true); }
private static void CompareBlocks(object leftData, EngineVersion leftVersion, object rightData, EngineVersion rightVersion, TagVersionMap result, Queue <QueuedTag> tagQueue) { if (leftData == null || rightData == null) { return; } var type = leftData.GetType(); if (type == typeof(TagInstance)) { // If the objects are tags, then we've found a match var leftTag = (TagInstance)leftData; var rightTag = (TagInstance)rightData; if (leftTag.Group.Tag != rightTag.Group.Tag) { return; } if (leftTag.IsInGroup("rmt2") || leftTag.IsInGroup("rmdf") || leftTag.IsInGroup("vtsh") || leftTag.IsInGroup("pixl") || leftTag.IsInGroup("rm ") || leftTag.IsInGroup("bitm")) { return; } var translated = result.Translate(leftVersion, leftTag.Index, rightVersion); if (translated >= 0) { return; } result.Add(leftVersion, leftTag.Index, rightVersion, rightTag.Index); tagQueue.Enqueue(new QueuedTag { Tag = rightTag }); } else if (type.IsArray) { if (type.GetElementType().IsPrimitive) { return; } // If the objects are arrays, then loop through each element var leftArray = (Array)leftData; var rightArray = (Array)rightData; if (leftArray.Length != rightArray.Length) { return; // If the sizes are different, we probably can't compare them } for (var i = 0; i < leftArray.Length; i++) { CompareBlocks(leftArray.GetValue(i), leftVersion, rightArray.GetValue(i), rightVersion, result, tagQueue); } } else if (type.IsGenericType && type.GetGenericTypeDefinition() == typeof(List <>)) { if (type.GenericTypeArguments[0].IsPrimitive) { return; } // If the objects are lists, then loop through each element var countProperty = type.GetProperty("Count"); var leftCount = (int)countProperty.GetValue(leftData); var rightCount = (int)countProperty.GetValue(rightData); if (leftCount != rightCount) { return; // If the sizes are different, we probably can't compare them } var getItem = type.GetMethod("get_Item"); for (var i = 0; i < leftCount; i++) { var leftItem = getItem.Invoke(leftData, new object[] { i }); var rightItem = getItem.Invoke(rightData, new object[] { i }); CompareBlocks(leftItem, leftVersion, rightItem, rightVersion, result, tagQueue); } } else if (type.GetCustomAttributes(typeof(TagStructureAttribute), false).Length > 0) { // The objects are structures var left = new TagFieldEnumerator(new TagStructureInfo(leftData.GetType(), leftVersion)); var right = new TagFieldEnumerator(new TagStructureInfo(rightData.GetType(), rightVersion)); while (left.Next() && right.Next()) { // Keep going on the left until the field is on the right while (!VersionDetection.IsBetween(rightVersion, left.MinVersion, left.MaxVersion)) { if (!left.Next()) { return; } } // Keep going on the right until the field is on the left while (!VersionDetection.IsBetween(leftVersion, right.MinVersion, right.MaxVersion)) { if (!right.Next()) { return; } } if (left.Field.MetadataToken != right.Field.MetadataToken) { throw new InvalidOperationException("WTF, left and right fields don't match!"); } // Process the fields var leftFieldData = left.Field.GetValue(leftData); var rightFieldData = right.Field.GetValue(rightData); CompareBlocks(leftFieldData, leftVersion, rightFieldData, rightVersion, result, tagQueue); } } }
public override object Execute(List <string> args) { if (args.Count < 2) { return(false); } var outputPath = args[0]; // Load each file and do version detection var infos = new List <HaloOnlineCacheContext>(); foreach (var path in args.Skip(1)) { Console.WriteLine("Loading {0}...", path); // Load the cache file var cacheContext = new HaloOnlineCacheContext(new FileInfo(path).Directory); infos.Add(cacheContext); } var result = new TagVersionMap(); using (var baseStream = CacheContext.OpenTagCacheRead()) { // Get the scenario tags for this cache Console.WriteLine("Finding base scenario tags..."); var baseScenarios = FindScenarios(CacheContext, baseStream); var baseVersion = CacheContext.Version; var baseTagData = new Dictionary <int, object>(); foreach (var scenario in baseScenarios) { baseTagData[scenario.Tag.Index] = scenario.Data; } // Now compare with each of the other caches foreach (var info in infos) { using (var stream = info.OpenTagCacheRead()) { Console.WriteLine("Finding scenario tags in {0}...", info.TagCacheFile.FullName); // Get the scenario tags and connect them to the base tags var scenarios = FindScenarios(info, stream); var tagsToCompare = new Queue <QueuedTag>(); for (var i = 0; i < scenarios.Count; i++) { tagsToCompare.Enqueue(scenarios[i]); if (i < baseScenarios.Count) { result.Add(baseVersion, baseScenarios[i].Tag.Index, info.Version, scenarios[i].Tag.Index); } } // Process each tag in the queue, enqueuing all of its dependencies as well while (tagsToCompare.Count > 0) { // Get the tag and its data var tag = tagsToCompare.Dequeue(); TagPrinter.PrintTagShort(tag.Tag); var data = tag.Data; if (data == null) { // No data yet - deserialize it var context = new TagSerializationContext(stream, info, tag.Tag); var type = TagDefinition.Find(tag.Tag.Group.Tag); data = info.Deserializer.Deserialize(context, type); } // Now get the data for the base tag var baseTag = result.Translate(info.Version, tag.Tag.Index, baseVersion); if (baseTag == -1 || CacheContext.TagCache.Index[baseTag].Group.Tag != tag.Tag.Group.Tag) { continue; } if (!baseTagData.TryGetValue(baseTag, out object baseData)) { // No data yet - deserialize it var context = new TagSerializationContext(baseStream, CacheContext, CacheContext.TagCache.Index[baseTag]); var type = TagDefinition.Find(tag.Tag.Group.Tag); baseData = CacheContext.Deserializer.Deserialize(context, type); baseTagData[baseTag] = baseData; } // Compare the two blocks CompareBlocks(baseData, baseVersion, data, info.Version, result, tagsToCompare); } } } } // Write out the CSV Console.WriteLine("Writing results..."); using (var writer = new StreamWriter(File.Open(outputPath, FileMode.Create, FileAccess.Write))) result.WriteCsv(writer); Console.WriteLine("Done!"); return(true); }
private static void CompareBlocks(object dataA, CacheVersion versionA, object dataB, CacheVersion versionB, TagVersionMap result, Queue <QueuedTag> tagQueue) { if (dataA == null || dataB == null) { return; } if (dataA is CachedTagInstance) { // If the objects are tags, then we've found a match var tagA = dataA as CachedTagInstance; var tagB = dataB as CachedTagInstance; if (tagA.Group.Tag != tagB.Group.Tag) { return; } if (tagA.IsInGroup("rmt2") || tagA.IsInGroup("rmdf") || tagA.IsInGroup("vtsh") || tagA.IsInGroup("pixl") || tagA.IsInGroup("rm ") || tagA.IsInGroup("bitm")) { return; } var translated = result.Translate(versionA, tagA.Index, versionB); if (translated >= 0) { return; } result.Add(versionA, tagA.Index, versionB, tagB.Index); tagQueue.Enqueue(new QueuedTag { Tag = tagB }); } else if (dataA is IList) // Compare lists and arrays { var collectionA = dataA as IList; var collectionB = dataB as IList; if (collectionA.Count == 0 || collectionA[0].GetType().IsPrimitive) { return; } // If the sizes are different, we probably can't compare them if (collectionA.Count != collectionB.Count) { return; } // Compare each element for (var i = 0; i < collectionA.Count; i++) { var itemA = collectionA[i]; var itemB = collectionB[i]; CompareBlocks(itemA, versionA, itemB, versionB, result, tagQueue); } } else if (dataA is TagStructure) { var tagFieldInfosA = TagStructure.GetTagFieldEnumerable(dataA.GetType(), versionA); var tagFieldInfosB = TagStructure.GetTagFieldEnumerable(dataB.GetType(), versionB); // The objects are structures for (int a = 0, b = 0; a < tagFieldInfosA.Count && b < tagFieldInfosB.Count; a++, b++) { var tagFieldInfoA = tagFieldInfosA[a]; var tagFieldInfoB = tagFieldInfosB[b]; // Keep going on the left until the field is on the right while (!CacheVersionDetection.IsBetween(versionB, tagFieldInfoA.Attribute.MinVersion, tagFieldInfoA.Attribute.MaxVersion)) { if (++a < tagFieldInfosA.Count) { tagFieldInfoA = tagFieldInfosA[a]; } else { return; } } // Keep going on the right until the field is on the left while (!CacheVersionDetection.IsBetween(versionA, tagFieldInfoB.Attribute.MinVersion, tagFieldInfoB.Attribute.MaxVersion)) { if (++b < tagFieldInfosB.Count) { tagFieldInfoB = tagFieldInfosB[b]; } else { return; } } if (tagFieldInfoA.MetadataToken != tagFieldInfoB.MetadataToken) { throw new InvalidOperationException("WTF, left and right fields don't match!"); } // Process the fields var fieldDataA = tagFieldInfoA.GetValue(dataA); var fieldDataB = tagFieldInfoB.GetValue(dataB); CompareBlocks(fieldDataA, versionA, fieldDataB, versionB, result, tagQueue); } } }
private CachedTagHaloOnline ConvertTag(CachedTagHaloOnline srcTag, GameCacheContextHaloOnline srcCacheContext, Stream srcStream, GameCacheContextHaloOnline destCacheContext, Stream destStream, TagVersionMap tagMap) { TagPrinter.PrintTagShort(srcTag); // Uncomment this to use 0x101F for all shaders /*if (srcTag.IsClass("rm ")) * return destCacheContext.Cache.Tags[0x101F];*/ // Check if the tag is in the map, and just return the translated tag if so var destIndex = tagMap.Translate(srcCacheContext.Version, srcTag.Index, destCacheContext.Version); if (destIndex >= 0) { Console.WriteLine("- Using already-known index {0:X4}", destIndex); return(destCacheContext.TagCache.Index[destIndex]); } // Deserialize the tag from the source cache var tagData = srcCacheContext.Deserialize(srcStream, srcTag); // Uncomment this to use 0x101F in place of shaders that need conversion /*if (tagData is RenderMethod) * { * var rm = (RenderMethod)tagData; * foreach (var prop in rm.ShaderProperties) * { * if (tagMap.Translate(srcCacheContext.Version, prop.Template.Index, destCacheContext.Version) < 0) * return destCacheContext.Cache.Tags[0x101F]; * } * }*/ // Allocate a new tag and create a mapping for it CachedTagHaloOnline instance = null; if (srcCacheContext.Version != destCacheContext.Version) { for (var i = 0; i < destCacheContext.TagCache.Index.Count; i++) { if (destCacheContext.TagCache.Index[i] == null) { destCacheContext.TagCache.Index[i] = instance = new CachedTagHaloOnline(i, TagGroup.Instances[srcTag.Group.Tag]); break; } } } else { if (destCacheContext.TagCache.Index[srcTag.Index] != null) { if (destCacheContext.TagCache.Index[srcTag.Index].IsInGroup(srcTag.Group)) { return(destCacheContext.TagCache.Index[srcTag.Index]); } } else { destCacheContext.TagCache.Index[srcTag.Index] = instance = new CachedTagHaloOnline(srcTag.Index, TagGroup.Instances[srcTag.Group.Tag], srcTag.Name); } } if (instance == null) { instance = destCacheContext.TagCache.AllocateTag(srcTag.Group); } tagMap.Add(srcCacheContext.Version, srcTag.Index, destCacheContext.Version, instance.Index); if (srcTag.IsInGroup("decs") || srcTag.IsInGroup("rmd ")) { IsDecalShader = true; } // Convert it tagData = Convert(tagData, srcCacheContext, srcStream, destCacheContext, destStream, tagMap); if (srcTag.IsInGroup("decs") || srcTag.IsInGroup("rmd ")) { IsDecalShader = false; } // Re-serialize into the destination cache destCacheContext.Serialize(destStream, instance, tagData); return(instance); }