private void FixDecalSystems(GameCacheContextHaloOnline destCacheContext, int firstNewIndex)
{
// decs tags need to be updated to use the old rmdf for decals,
// because the decal planes seem to be generated by the engine and
// therefore need to use the old vertex format.
//
// This could probably be done as a post-processing step in
// ConvertStructure to avoid the extra deserialize-reserialize
// pass, but we'd have to store the rmdf somewhere and frankly I'm
// too lazy to do that...
var firstDecalSystemTag = destCacheContext.TagCache.FindFirstInGroup("decs");
if (firstDecalSystemTag == null)
{
return;
}
using (var stream = destCacheContext.OpenTagCacheReadWrite())
{
var firstDecalSystem = destCacheContext.Deserialize <DecalSystem>(stream, firstDecalSystemTag);
foreach (var decalSystemTag in destCacheContext.TagCache.Index.FindAllInGroup("decs").Where(t => t.Index >= firstNewIndex))
{
TagPrinter.PrintTagShort(decalSystemTag);
var decalSystem = destCacheContext.Deserialize <DecalSystem>(stream, decalSystemTag);
foreach (var system in decalSystem.Decal)
{
system.RenderMethod.BaseRenderMethod = firstDecalSystem.Decal[0].RenderMethod.BaseRenderMethod;
}
destCacheContext.Serialize(stream, decalSystemTag, decalSystem);
}
}
}
private ObjectTypeFlags ConvertObjectTypeFlags(ObjectTypeFlags data, GameCacheContextHaloOnline srcCacheContext, GameCacheContextHaloOnline destCacheContext)
{
if (destCacheContext.Version < CacheVersion.HaloOnline449175)
{
if (!Enum.TryParse(data.HaloOnline.ToString(), out data.Halo3ODST))
{
throw new FormatException(destCacheContext.Version.ToString());
}
}
return(data);
}
private GameObjectType ConvertGameObjectType(GameObjectType objectType, GameCacheContextHaloOnline srcCacheContext, GameCacheContextHaloOnline destCacheContext)
{
if (srcCacheContext.Version >= CacheVersion.HaloOnline498295)
{
if (Enum.TryParse <GameObjectTypeHalo3ODST>(objectType.HaloOnline.ToString(), out var result))
{
objectType.Halo3ODST = result;
}
}
return(objectType);
}
private Array ConvertArray(Array array, GameCacheContextHaloOnline srcCacheContext, Stream srcStream, GameCacheContextHaloOnline destCacheContext, Stream destStream, TagVersionMap tagMap)
{
if (array.GetType().GetElementType().IsPrimitive)
{
return(array);
}
for (var i = 0; i < array.Length; i++)
{
var oldValue = array.GetValue(i);
var newValue = Convert(oldValue, srcCacheContext, srcStream, destCacheContext, destStream, tagMap);
array.SetValue(newValue, i);
}
return(array);
}
public PortArmorVariantCommand(GameCacheContextHaloOnline cacheContext, GameCache blamCache) :
base(true,
"PortArmorVariant",
"Ports an mp_masterchief armor variant.",
"PortArmorVariant <Spartan | Elite> <Variant Name> [Scenery] [Replace: <Tag>] [Regions: <Region 1> <Region 2> ... <Region N>]",
"Ports an mp_masterchief armor variant.")
{
CacheContext = cacheContext;
BlamCache = blamCache;
GeometryConverter = new RenderGeometryConverter(cacheContext, blamCache);
}
private IList ConvertCollection(IList collection, GameCacheContextHaloOnline srcCacheContext, Stream srcStream, GameCacheContextHaloOnline destCacheContext, Stream destStream, TagVersionMap tagMap)
{
if (collection.Count == 0 || collection[0].GetType().IsPrimitive)
{
return(collection);
}
for (var i = 0; i < collection.Count; i++)
{
var oldValue = collection[i];
var newValue = Convert(oldValue, srcCacheContext, srcStream, destCacheContext, destStream, tagMap);
collection[i] = newValue;
}
return(collection);
}
public ConvertTagCommand(GameCacheContextHaloOnline info)
: base(false,
"ConvertTag",
"Convert a tag and its dependencies to another engine version",
"ConvertTag <tag> <input csv> <output csv> <target directory>",
"The tag map CSV should be generated using the \"MatchTags\" command.\n" +
"If a tag is listed in the CSV file, it will not be converted.\n" +
"The output CSV file is used for converting multiple maps.\n" +
"Subsequent convert commands should use the new CSV.\n" +
"The target directory should be the maps folder for the target engine.")
{
CacheContext = info;
}
private PageableResource ConvertResource(PageableResource resource, GameCacheContextHaloOnline srcCacheContext, GameCacheContextHaloOnline destCacheContext)
{
if (resource == null || resource.Page.Index < 0 || !resource.GetLocation(out var location))
{
return(null);
}
Console.WriteLine("- Copying resource {0} in {1}...", resource.Page.Index, location);
var data = srcCacheContext.ExtractRawResource(resource);
var newLocation = FixResourceLocation(location, srcCacheContext.Version, destCacheContext.Version);
resource.ChangeLocation(newLocation);
destCacheContext.AddRawResource(resource, data);
return(resource);
}
private object ConvertList(object list, Type type, GameCacheContextHaloOnline srcCacheContext, Stream srcStream, GameCacheContextHaloOnline destCacheContext, Stream destStream, TagVersionMap tagMap)
{
if (type.GenericTypeArguments[0].IsPrimitive)
{
return(list);
}
var count = (int)type.GetProperty("Count").GetValue(list);
var getItem = type.GetMethod("get_Item");
var setItem = type.GetMethod("set_Item");
for (var i = 0; i < count; i++)
{
var oldValue = getItem.Invoke(list, new object[] { i });
var newValue = Convert(oldValue, srcCacheContext, srcStream, destCacheContext, destStream, tagMap);
setItem.Invoke(list, new object[] { i, newValue });
}
return(list);
}
private StringId ConvertStringID(StringId stringId, GameCacheContextHaloOnline srcCacheContext, GameCacheContextHaloOnline destCacheContext)
{
if (stringId == StringId.Invalid)
{
return(stringId);
}
var srcString = srcCacheContext.StringTable.GetString(stringId);
if (srcString == null)
{
return(StringId.Invalid);
}
var destStringID = destCacheContext.StringTable.GetStringId(srcString);
if (destStringID == StringId.Invalid)
{
destStringID = destCacheContext.StringIdCache.AddString(srcString);
}
return(destStringID);
}
private T ConvertStructure <T>(T data, GameCacheContextHaloOnline srcCacheContext, Stream srcStream, GameCacheContextHaloOnline destCacheContext, Stream destStream, TagVersionMap tagMap) where T : TagStructure
{
// Convert each field
foreach (var tagFieldInfo in TagStructure.GetTagFieldEnumerable(data.GetType(), destCacheContext.Version))
{
var oldValue = tagFieldInfo.GetValue(data);
var newValue = Convert(oldValue, srcCacheContext, srcStream, destCacheContext, destStream, tagMap);
tagFieldInfo.SetValue(data, newValue);
}
// Perform fixups
FixShaders(data);
if (data is Scenario scenario)
{
FixScenario(scenario);
}
return(data);
}
private object Convert(object data, GameCacheContextHaloOnline srcCacheContext, Stream srcStream, GameCacheContextHaloOnline destCacheContext, Stream destStream, TagVersionMap tagMap)
{
switch (data)
{
case StringId stringId:
return(ConvertStringID(stringId, srcCacheContext, destCacheContext));
case null:
case string _:
case ValueType _:
return(data);
case CachedTagHaloOnline CachedTagHaloOnline:
return(ConvertTag(CachedTagHaloOnline, srcCacheContext, srcStream, destCacheContext, destStream, tagMap));
case PageableResource pageableResource:
return(ConvertResource(pageableResource, srcCacheContext, destCacheContext));
case RenderGeometry renderGeometry:
return(ConvertGeometry(renderGeometry, srcCacheContext, destCacheContext));
case GameObjectType gameObjectType:
return(ConvertGameObjectType(gameObjectType, srcCacheContext, destCacheContext));
case ObjectTypeFlags objectTypeFlags:
return(ConvertObjectTypeFlags(objectTypeFlags, srcCacheContext, destCacheContext));
case ScenarioObjectType scenarioObjectType:
return(ConvertScenarioObjectType(scenarioObjectType, srcCacheContext, destCacheContext));
case TagStructure tagStructure:
return(ConvertStructure(tagStructure, srcCacheContext, srcStream, destCacheContext, destStream, tagMap));
case IList collection:
return(ConvertCollection(collection, srcCacheContext, srcStream, destCacheContext, destStream, tagMap));
}
return(data);
}
private void ConvertVertexBuffer(GameCacheContextHaloOnline srcCacheContext, GameCacheContextHaloOnline destCacheContext, VertexBufferDefinition buffer, MemoryStream inStream, IVertexStream inVertexStream, MemoryStream outStream, IVertexStream outVertexStream)
{
if (buffer.Data.Size == 0)
{
return;
}
var count = buffer.Count;
var startPos = (int)outStream.Position;
inStream.Position = buffer.Data.Address.Offset;
buffer.Data.Address = new CacheAddress(CacheAddressType.Data, startPos);
switch (buffer.Format)
{
case VertexBufferFormat.World:
ConvertVertices(count, inVertexStream.ReadWorldVertex, v =>
{
if (srcCacheContext.Version > CacheVersion.HaloOnline235640)
{
v.Binormal = new RealVector3d(v.Position.W, v.Tangent.W, 0); // Converted shaders use this
}
outVertexStream.WriteWorldVertex(v);
});
break;
case VertexBufferFormat.Rigid:
ConvertVertices(count, inVertexStream.ReadRigidVertex, v =>
{
if (srcCacheContext.Version > CacheVersion.HaloOnline235640)
{
v.Binormal = new RealVector3d(v.Position.W, v.Tangent.W, 0); // Converted shaders use this
}
outVertexStream.WriteRigidVertex(v);
});
break;
case VertexBufferFormat.Skinned:
ConvertVertices(count, inVertexStream.ReadSkinnedVertex, v =>
{
if (srcCacheContext.Version > CacheVersion.HaloOnline235640)
{
v.Binormal = new RealVector3d(v.Position.W, v.Tangent.W, 0); // Converted shaders use this
}
outVertexStream.WriteSkinnedVertex(v);
});
break;
case VertexBufferFormat.StaticPerPixel:
ConvertVertices(count, inVertexStream.ReadStaticPerPixelData, outVertexStream.WriteStaticPerPixelData);
break;
case VertexBufferFormat.StaticPerVertex:
ConvertVertices(count, inVertexStream.ReadStaticPerVertexData, outVertexStream.WriteStaticPerVertexData);
break;
case VertexBufferFormat.AmbientPrt:
ConvertVertices(count, inVertexStream.ReadAmbientPrtData, outVertexStream.WriteAmbientPrtData);
break;
case VertexBufferFormat.LinearPrt:
ConvertVertices(count, inVertexStream.ReadLinearPrtData, outVertexStream.WriteLinearPrtData);
break;
case VertexBufferFormat.QuadraticPrt:
ConvertVertices(count, inVertexStream.ReadQuadraticPrtData, outVertexStream.WriteQuadraticPrtData);
break;
case VertexBufferFormat.StaticPerVertexColor:
ConvertVertices(count, inVertexStream.ReadStaticPerVertexColorData, outVertexStream.WriteStaticPerVertexColorData);
break;
case VertexBufferFormat.Decorator:
ConvertVertices(count, inVertexStream.ReadDecoratorVertex, outVertexStream.WriteDecoratorVertex);
break;
case VertexBufferFormat.World2:
buffer.Format = VertexBufferFormat.World;
goto default;
default:
// Just copy the raw buffer over and pray that it works...
var bufferData = new byte[buffer.Data.Size];
inStream.Read(bufferData, 0, bufferData.Length);
outStream.Write(bufferData, 0, bufferData.Length);
break;
}
buffer.Data.Size = (int)outStream.Position - startPos;
buffer.VertexSize = (short)(buffer.Data.Size / buffer.Count);
}
public override object Execute(List <string> args)
{
if (args.Count != 4)
{
return(false);
}
if (!CacheContext.TryGetTag(args[0], out var srcTag))
{
return(false);
}
var csvPath = args[1];
var csvOutPath = args[2];
var targetDir = args[3];
// Load the CSV
Console.WriteLine("Reading {0}...", csvPath);
TagVersionMap tagMap;
using (var reader = new StreamReader(File.Exists(csvPath) ? File.OpenRead(csvPath) : File.Create(csvPath)))
tagMap = TagVersionMap.ParseTagVersionMap(reader);
// Load destination cache files
var destCacheContext = new GameCacheContextHaloOnline(new DirectoryInfo(targetDir));
using (var stream = destCacheContext.OpenTagCacheRead())
destCacheContext.TagCache = new TagCache(stream, destCacheContext.LoadTagNames());
Console.WriteLine();
Console.WriteLine("CONVERTING FROM VERSION {0} TO {1}", CacheVersionDetection.GetBuildName(CacheContext.Version), CacheVersionDetection.GetBuildName(destCacheContext.Version));
Console.WriteLine();
CachedTagHaloOnline resultTag;
using (Stream srcStream = CacheContext.TagCache.OpenTagCacheRead(), destStream = destCacheContext.OpenTagCacheReadWrite())
resultTag = ConvertTag(srcTag, CacheContext, srcStream, destCacheContext, destStream, tagMap);
Console.WriteLine();
Console.WriteLine("Repairing decal systems...");
if (CacheContext.Version != destCacheContext.Version)
{
FixDecalSystems(destCacheContext, resultTag.Index);
}
Console.WriteLine();
Console.WriteLine("Saving stringIDs...");
using (var stream = destCacheContext.OpenStringIdCacheReadWrite())
destCacheContext.StringIdCache.Save(stream);
Console.WriteLine("Writing {0}...", csvOutPath);
using (var stream = new StreamWriter(File.Open(csvOutPath, FileMode.Create, FileAccess.ReadWrite)))
tagMap.WriteCsv(stream);
// Uncomment this to add the new tag as a dependency to cfgt to make testing easier
/*using (var stream = destCacheContext.OpenCacheReadWrite())
* {
* destCacheContext.Cache.Tags[0].Dependencies.Add(resultTag.Index);
* destCacheContext.Cache.UpdateTag(stream, destCacheContext.Cache.Tags[0]);
* }*/
Console.WriteLine();
Console.WriteLine("All done! The converted tag is:");
TagPrinter.PrintTagShort(resultTag);
destCacheContext.SaveTagNames();
return(true);
}
private RenderGeometry ConvertGeometry(RenderGeometry geometry, GameCacheContextHaloOnline srcCacheContext, GameCacheContextHaloOnline destCacheContext)
{
if (geometry == null || geometry.Resource.HaloOnlinePageableResource == null || geometry.Resource.HaloOnlinePageableResource.Page.Index < 0 || !geometry.Resource.HaloOnlinePageableResource.GetLocation(out var location))
{
return(geometry);
}
// The format changed starting with version 1.235640, so if both versions are on the same side then they can be converted normally
var srcCompare = CacheVersionDetection.Compare(srcCacheContext.Version, CacheVersion.HaloOnline235640);
var destCompare = CacheVersionDetection.Compare(destCacheContext.Version, CacheVersion.HaloOnline235640);
if ((srcCompare < 0 && destCompare < 0) || (srcCompare >= 0 && destCompare >= 0))
{
geometry.Resource.HaloOnlinePageableResource = ConvertResource(geometry.Resource.HaloOnlinePageableResource, srcCacheContext, destCacheContext);
return(geometry);
}
Console.WriteLine("- Rebuilding geometry resource {0} in {1}...", geometry.Resource.HaloOnlinePageableResource.Page.Index, location);
using (MemoryStream inStream = new MemoryStream(), outStream = new MemoryStream())
{
// First extract the model data
srcCacheContext.ExtractResource(geometry.Resource.HaloOnlinePageableResource, inStream);
// Now open source and destination vertex streams
inStream.Position = 0;
var inVertexStream = VertexStreamFactory.Create(srcCacheContext.Version, inStream);
var outVertexStream = VertexStreamFactory.Create(destCacheContext.Version, outStream);
// Deserialize the definition data
var resourceContext = new ResourceSerializationContext(CacheContext, geometry.Resource.HaloOnlinePageableResource);
var definition = srcCacheContext.Deserializer.Deserialize <RenderGeometryApiResourceDefinition>(resourceContext);
// Convert each vertex buffer
foreach (var buffer in definition.VertexBuffers)
{
ConvertVertexBuffer(srcCacheContext, destCacheContext, buffer.Definition, inStream, inVertexStream, outStream, outVertexStream);
}
// Copy each index buffer over
foreach (var buffer in definition.IndexBuffers)
{
if (buffer.Definition.Data.Size == 0)
{
continue;
}
inStream.Position = buffer.Definition.Data.Address.Offset;
buffer.Definition.Data.Address = new CacheAddress(CacheAddressType.Data, (int)outStream.Position);
var bufferData = new byte[buffer.Definition.Data.Size];
inStream.Read(bufferData, 0, bufferData.Length);
outStream.Write(bufferData, 0, bufferData.Length);
StreamUtil.Align(outStream, 4);
}
// Update the definition data
destCacheContext.Serializer.Serialize(resourceContext, definition);
// Now inject the new resource data
var newLocation = FixResourceLocation(location, srcCacheContext.Version, destCacheContext.Version);
outStream.Position = 0;
geometry.Resource.HaloOnlinePageableResource.ChangeLocation(newLocation);
destCacheContext.AddResource(geometry.Resource.HaloOnlinePageableResource, outStream);
}
return(geometry);
}
private CachedTagHaloOnline ConvertTag(CachedTagHaloOnline srcTag, GameCacheContextHaloOnline srcCacheContext, Stream srcStream, GameCacheContextHaloOnline destCacheContext, Stream destStream, TagVersionMap tagMap)
{
TagPrinter.PrintTagShort(srcTag);
// Uncomment this to use 0x101F for all shaders
/*if (srcTag.IsClass("rm "))
* return destCacheContext.Cache.Tags[0x101F];*/
// Check if the tag is in the map, and just return the translated tag if so
var destIndex = tagMap.Translate(srcCacheContext.Version, srcTag.Index, destCacheContext.Version);
if (destIndex >= 0)
{
Console.WriteLine("- Using already-known index {0:X4}", destIndex);
return(destCacheContext.TagCache.Index[destIndex]);
}
// Deserialize the tag from the source cache
var tagData = srcCacheContext.Deserialize(srcStream, srcTag);
// Uncomment this to use 0x101F in place of shaders that need conversion
/*if (tagData is RenderMethod)
* {
* var rm = (RenderMethod)tagData;
* foreach (var prop in rm.ShaderProperties)
* {
* if (tagMap.Translate(srcCacheContext.Version, prop.Template.Index, destCacheContext.Version) < 0)
* return destCacheContext.Cache.Tags[0x101F];
* }
* }*/
// Allocate a new tag and create a mapping for it
CachedTagHaloOnline instance = null;
if (srcCacheContext.Version != destCacheContext.Version)
{
for (var i = 0; i < destCacheContext.TagCache.Index.Count; i++)
{
if (destCacheContext.TagCache.Index[i] == null)
{
destCacheContext.TagCache.Index[i] = instance = new CachedTagHaloOnline(i, TagGroup.Instances[srcTag.Group.Tag]);
break;
}
}
}
else
{
if (destCacheContext.TagCache.Index[srcTag.Index] != null)
{
if (destCacheContext.TagCache.Index[srcTag.Index].IsInGroup(srcTag.Group))
{
return(destCacheContext.TagCache.Index[srcTag.Index]);
}
}
else
{
destCacheContext.TagCache.Index[srcTag.Index] = instance = new CachedTagHaloOnline(srcTag.Index, TagGroup.Instances[srcTag.Group.Tag], srcTag.Name);
}
}
if (instance == null)
{
instance = destCacheContext.TagCache.AllocateTag(srcTag.Group);
}
tagMap.Add(srcCacheContext.Version, srcTag.Index, destCacheContext.Version, instance.Index);
if (srcTag.IsInGroup("decs") || srcTag.IsInGroup("rmd "))
{
IsDecalShader = true;
}
// Convert it
tagData = Convert(tagData, srcCacheContext, srcStream, destCacheContext, destStream, tagMap);
if (srcTag.IsInGroup("decs") || srcTag.IsInGroup("rmd "))
{
IsDecalShader = false;
}
// Re-serialize into the destination cache
destCacheContext.Serialize(destStream, instance, tagData);
return(instance);
}
